refactor: one directory per pipeline stage, and an assembler to drive them

The repository now reads as the pipeline it is: crawler fetches, parser
converts, assembler verifies and publishes, with data/ and web/ as the stores
they hand work through. go-parser is renamed parser now that there is no other.

The assembler replaces build-db.js and assemble-site.js. It compiles the
parser, builds and verifies each database, compresses it, runs the Vite build
and assembles _site — one command, and the only place that knows the order.

It also closes a real hole: nothing previously asserted that a database reached
the site. An empty staging directory assembled happily, so every page rendered,
every query 404d and CI stayed green. The row-count and size guards could not
catch that, since they only run when a database was built at all.

Removing Node from the root forced the dataset list out of web/src/datasets.js,
which the assembler cannot import. datasets.json is now the registry both sides
read — JSON because Go and the browser both parse it without a dependency —
while presentation stays in the web app, keyed by id and cross-checked against
the registry so a half-added dataset fails instead of half-working.

Guards verified by making each one fail: a missing database, and an expected
row count one higher than the truth.

parser/internal/reader/fidelity_test.go

@@ -0,0 +1,128 @@
+package reader_test
+
+import (
+	"bufio"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/tiennm99/thptqg/parser/internal/reader"
+)
+
+// TestReaderFidelity asserts the Go reader reproduces calamine byte-for-byte on
+// every real input file.
+//
+// The oracle is a committed SHA-256 per file over a canonical cell dump. The
+// dumps themselves are real student names and birthdates, so only the hashes are
+// committed — regenerate the dumps from the Rust side on demand
+// (parser/examples/dump_cells.rs), which is possible because parser/ still
+// builds.
+//
+// The canonical form carries geometry and rendered cell values. The calamine
+// Data variant is excluded on purpose: Data::Empty and Data::String("") both
+// render "" and both count as blank in is_all_blank and transform, so the
+// distinction cannot reach the database.
+// Runs by default so CI and `go test ./...` keep the full guarantee; skipped
+// under -short, which is how to iterate without paying ~77s to re-read 418 MB.
+func TestReaderFidelity(t *testing.T) {
+	if testing.Short() {
+		t.Skip("-short: skipping the 299-file corpus sweep")
+	}
+	root := repoRoot(t)
+	manifest := filepath.Join(root, "parser", "testdata", "reader-fidelity-hashes.tsv")
+
+	f, err := os.Open(manifest)
+	if err != nil {
+		t.Fatalf("open manifest: %v", err)
+	}
+	defer f.Close()
+
+	var checked int
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		line := sc.Text()
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		rel, want, ok := strings.Cut(line, "\t")
+		if !ok {
+			t.Fatalf("malformed manifest line: %q", line)
+		}
+		path := filepath.Join(root, rel)
+		if _, err := os.Stat(path); err != nil {
+			t.Skipf("input data not present (%s); skipping fidelity suite", rel)
+		}
+
+		checked++
+		t.Run(rel, func(t *testing.T) {
+			t.Parallel()
+			got, err := canonicalHash(path)
+			if err != nil {
+				t.Fatalf("hash %s: %v", rel, err)
+			}
+			if got != want {
+				t.Errorf("cell dump diverges from calamine\n  want %s\n  got  %s", want, got)
+			}
+		})
+	}
+	if err := sc.Err(); err != nil {
+		t.Fatalf("read manifest: %v", err)
+	}
+	if checked == 0 {
+		t.Fatal("manifest contained no entries")
+	}
+}
+
+// canonicalHash renders one file in the canonical form and hashes it. Kept
+// byte-identical to the awk canonicalisation used to build the manifest.
+func canonicalHash(path string) (string, error) {
+	wb, err := reader.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer wb.Close()
+
+	h := sha256.New()
+	sheets := wb.Sheets()
+	fmt.Fprintf(h, "SHEETCOUNT\t%d\n", len(sheets))
+	for _, sh := range sheets {
+		fmt.Fprintf(h, "SHEET\t%d\t%s\t%d\t%d\n", sh.Index, escape(sh.Name), sh.Height, sh.Width)
+		err := wb.EachRow(sh.Index, func(s reader.Sheet, rowIdx int, row []reader.Cell) error {
+			fmt.Fprintf(h, "ROW\t%d\t%d\t%d\n", s.Index, rowIdx, len(row))
+			for c, cell := range row {
+				fmt.Fprintf(h, "CELL\t%d\t%d\t%d\t%s\n", s.Index, rowIdx, c, escape(cell.Str))
+			}
+			return nil
+		})
+		if err != nil {
+			return "", err
+		}
+	}
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+func escape(s string) string {
+	return strings.NewReplacer("\\", `\\`, "\t", `\t`, "\n", `\n`, "\r", `\r`).Replace(s)
+}
+
+// repoRoot walks up from the test's working directory to the directory holding
+// the data/ corpus.
+func repoRoot(t *testing.T) string {
+	t.Helper()
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("getwd: %v", err)
+	}
+	for i := 0; i < 6; i++ {
+		if _, err := os.Stat(filepath.Join(dir, "data")); err == nil {
+			return dir
+		}
+		dir = filepath.Dir(dir)
+	}
+	t.Fatal("could not locate repo root (no data/ directory found)")
+	return ""
+}

parser/internal/reader/reader.go

@@ -0,0 +1,79 @@
+// Package reader wraps the two spreadsheet libraries behind one streaming
+// contract, mirroring parser/src/reader.rs (which wraps calamine's
+// open_workbook_auto for both formats).
+//
+// The contract is deliberately row-streaming rather than whole-workbook
+// materialising: data/2017/ha-noi.xls alone holds 72k rows across two sheets,
+// and the Rust original keeps at most one sheet range live at a time.
+package reader
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+// Cell is one spreadsheet cell rendered the way calamine's Data::to_string()
+// renders it.
+//
+// IsEmpty tracks calamine's Data::Empty variant separately from a string cell
+// that happens to be empty. Rust distinguishes them at reader.rs:42, but both
+// render "" and both count as blank in is_all_blank and in transform, so the
+// flag is diagnostic only — never compare on it.
+type Cell struct {
+	Str     string
+	IsEmpty bool
+}
+
+// Sheet carries a sheet's identity and geometry. Height and Width describe the
+// used range, matching calamine's Range::height()/width(); every used range in
+// the corpus starts at (0,0), verified across all 299 files.
+type Sheet struct {
+	Index  int
+	Name   string
+	Height int
+	Width  int
+}
+
+// RowFunc receives each row of each sheet. Rows are padded to the sheet's used
+// width — width is load-bearing because every column read downstream is
+// positional with an unwrap_or_default() equivalent, so a short row silently
+// NULLs its tail columns.
+type RowFunc func(sheet Sheet, rowIdx int, row []Cell) error
+
+// Workbook is one opened spreadsheet.
+type Workbook interface {
+	// Sheets returns sheet identity and geometry in workbook order.
+	Sheets() []Sheet
+	// EachRow streams every row of the given sheet in order.
+	EachRow(sheetIdx int, fn RowFunc) error
+	Close() error
+}
+
+// Open dispatches on file extension, mirroring calamine's open_workbook_auto.
+func Open(path string) (Workbook, error) {
+	switch strings.ToLower(filepath.Ext(path)) {
+	case ".xls":
+		return openXLS(path)
+	case ".xlsx", ".xlsm":
+		return openXLSX(path)
+	default:
+		return nil, fmt.Errorf("unsupported extension: %s", path)
+	}
+}
+
+// padRow extends row to width with empty cells, and truncates if longer.
+func padRow(row []Cell, width int) []Cell {
+	if len(row) == width {
+		return row
+	}
+	if len(row) > width {
+		return row[:width]
+	}
+	out := make([]Cell, width)
+	copy(out, row)
+	for i := len(row); i < width; i++ {
+		out[i] = Cell{IsEmpty: true}
+	}
+	return out
+}

parser/internal/reader/xls.go

@@ -0,0 +1,119 @@
+package reader
+
+import (
+	"fmt"
+
+	"github.com/pbnjay/grate"
+	// Registers the BIFF backend with grate.Open.
+	_ "github.com/pbnjay/grate/xls"
+)
+
+// xlsWorkbook reads legacy BIFF through pbnjay/grate.
+//
+// grate replaced extrame/xls, which was measured against calamine ground truth
+// and found to corrupt 69% of cells and drop a further 28%: undecoded UTF-16LE
+// and BIFF record framing leaked into cell values, content moved between rows
+// and columns, and tail rows came back blank. That was charset-independent and
+// unfixable from the outside. grate reproduces calamine exactly on the same
+// files.
+//
+// The one normalisation grate needs is trailing blank rows: it yields rows past
+// the end of calamine's used range (one for a populated sheet, two for an empty
+// one), so trailing all-blank rows are trimmed. Note this is the opposite of
+// the xlsx path, where excelize already trims and calamine keeps a 1x1 empty
+// range — in both cases the rule is "match calamine's used range".
+type xlsWorkbook struct {
+	sheets []Sheet
+	rows   [][][]Cell
+}
+
+func openXLS(path string) (Workbook, error) {
+	wb, err := grate.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("grate open %s: %w", path, err)
+	}
+	defer wb.Close()
+
+	names, err := wb.List()
+	if err != nil {
+		return nil, fmt.Errorf("grate list %s: %w", path, err)
+	}
+
+	out := &xlsWorkbook{}
+	for idx, name := range names {
+		sh, err := wb.Get(name)
+		if err != nil {
+			return nil, fmt.Errorf("grate get %s/%s: %w", path, name, err)
+		}
+
+		var raw [][]string
+		for sh.Next() {
+			row := sh.Strings()
+			cp := make([]string, len(row))
+			for i, v := range row {
+				cp[i] = demergeMarker(v)
+			}
+			raw = append(raw, cp)
+		}
+
+		// Trim to calamine's used range.
+		height := len(raw)
+		for height > 0 && rowAllBlank(raw[height-1]) {
+			height--
+		}
+		raw = raw[:height]
+
+		width := 0
+		for _, r := range raw {
+			if len(r) > width {
+				width = len(r)
+			}
+		}
+
+		cells := make([][]Cell, len(raw))
+		for i, r := range raw {
+			row := make([]Cell, len(r))
+			for j, v := range r {
+				row[j] = Cell{Str: v, IsEmpty: v == ""}
+			}
+			cells[i] = padRow(row, width)
+		}
+
+		out.sheets = append(out.sheets, Sheet{Index: idx, Name: name, Height: height, Width: width})
+		out.rows = append(out.rows, cells)
+	}
+	return out, nil
+}
+
+// demergeMarker blanks grate's merged-cell continuation markers.
+//
+// grate fills the cells covered by a merge with sentinel runes; calamine
+// reports them as empty. In this corpus they occur only in the merged title
+// block of the 2016 spreadsheets (19 cells in rows 0-2 of one file). Only an
+// exact whole-value match is blanked, so a real cell that merely contains an
+// arrow is untouched.
+func demergeMarker(v string) string {
+	switch v {
+	case grate.ContinueColumnMerged, grate.EndColumnMerged,
+		grate.ContinueRowMerged, grate.EndRowMerged:
+		return ""
+	}
+	return v
+}
+
+func (w *xlsWorkbook) Sheets() []Sheet { return w.sheets }
+
+func (w *xlsWorkbook) EachRow(sheetIdx int, fn RowFunc) error {
+	if sheetIdx < 0 || sheetIdx >= len(w.sheets) {
+		return fmt.Errorf("sheet index %d out of range", sheetIdx)
+	}
+	sh := w.sheets[sheetIdx]
+	for i, row := range w.rows[sheetIdx] {
+		if err := fn(sh, i, row); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (w *xlsWorkbook) Close() error { return nil }

parser/internal/reader/xlsx.go

@@ -0,0 +1,105 @@
+package reader
+
+import (
+	"fmt"
+
+	"github.com/xuri/excelize/v2"
+)
+
+// xlsxWorkbook reads OOXML through excelize.
+//
+// Two excelize behaviours must be corrected to match calamine:
+//
+//  1. GetRows applies the cell number format by default, while calamine renders
+//     the underlying value. RawCellValue: true disables that.
+//  2. GetRows trims trailing blank cells, so rows are ragged; calamine returns a
+//     rectangular used range. Rows are padded back out to the sheet width.
+type xlsxWorkbook struct {
+	f      *excelize.File
+	sheets []Sheet
+	rows   [][][]Cell // [sheetIdx][rowIdx][colIdx]
+}
+
+func openXLSX(path string) (Workbook, error) {
+	f, err := excelize.OpenFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("excelize open %s: %w", path, err)
+	}
+
+	wb := &xlsxWorkbook{f: f}
+	crFixups := buildCRFixups(path)
+	for idx, name := range f.GetSheetList() {
+		raw, err := f.GetRows(name, excelize.Options{RawCellValue: true})
+		if err != nil {
+			f.Close()
+			return nil, fmt.Errorf("excelize GetRows %s/%s: %w", path, name, err)
+		}
+
+		// Do NOT trim trailing blank rows: calamine's used range keeps them, and
+		// excelize's GetRows already drops trailing fully-empty rows itself.
+		//
+		// One correction is needed. 63 sheets in 2017-old and 53 in 2017-old2
+		// hold a single empty shared-string cell at A1; calamine reports those
+		// as a 1x1 range, while GetRows returns nothing. A genuinely empty sheet
+		// (230 of them in 2016) is height 0 on both sides. GetCellType tells the
+		// two apart: the empty-shared-string cell exists in the XML and types as
+		// CellTypeSharedString, an absent cell types as CellTypeUnset.
+		if len(raw) == 0 {
+			if t, terr := f.GetCellType(name, "A1"); terr == nil && t != excelize.CellTypeUnset {
+				raw = [][]string{{""}}
+			}
+		}
+		height := len(raw)
+
+		width := 0
+		for _, r := range raw {
+			if len(r) > width {
+				width = len(r)
+			}
+		}
+
+		cells := make([][]Cell, len(raw))
+		for i, r := range raw {
+			row := make([]Cell, len(r))
+			for j, v := range r {
+				if fixed, ok := crFixups[v]; ok {
+					v = fixed
+				} else {
+					v = normalizeNumeric(f, name, j, i, v)
+				}
+				row[j] = Cell{Str: v, IsEmpty: v == ""}
+			}
+			cells[i] = padRow(row, width)
+		}
+
+		wb.sheets = append(wb.sheets, Sheet{Index: idx, Name: name, Height: height, Width: width})
+		wb.rows = append(wb.rows, cells)
+	}
+	return wb, nil
+}
+
+func rowAllBlank(r []string) bool {
+	for _, v := range r {
+		if v != "" {
+			return false
+		}
+	}
+	return true
+}
+
+func (w *xlsxWorkbook) Sheets() []Sheet { return w.sheets }
+
+func (w *xlsxWorkbook) EachRow(sheetIdx int, fn RowFunc) error {
+	if sheetIdx < 0 || sheetIdx >= len(w.sheets) {
+		return fmt.Errorf("sheet index %d out of range", sheetIdx)
+	}
+	sh := w.sheets[sheetIdx]
+	for i, row := range w.rows[sheetIdx] {
+		if err := fn(sh, i, row); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (w *xlsxWorkbook) Close() error { return w.f.Close() }

parser/internal/reader/xlsx_fixups.go

@@ -0,0 +1,155 @@
+package reader
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/xml"
+	"io"
+	"strconv"
+
+	"github.com/xuri/excelize/v2"
+)
+
+// normalizeNumeric reproduces calamine's rendering of a numeric cell.
+//
+// calamine parses a numeric cell to f64 and renders it with Rust's f64 Display,
+// so the stored literal "6.0" becomes "6". RawCellValue hands back the literal.
+//
+// The cell type must be consulted, not guessed: "01063476", "6.00" and "NAN" are
+// all shared strings that survive ParseFloat, and renumbering them would drop a
+// leading zero, drop a trailing zero, or recase NaN. GetCellType is only called
+// when re-rendering would actually change the text, which keeps it off the hot
+// path for the ~99% of cells that are already canonical or plainly non-numeric.
+func normalizeNumeric(f *excelize.File, sheet string, col, row int, v string) string {
+	if v == "" {
+		return v
+	}
+	fv, err := strconv.ParseFloat(v, 64)
+	if err != nil {
+		return v
+	}
+	out := strconv.FormatFloat(fv, 'f', -1, 64)
+	if out == v {
+		return v
+	}
+	axis, err := excelize.CoordinatesToCellName(col+1, row+1)
+	if err != nil {
+		return v
+	}
+	// OOXML omits the t attribute on numeric cells, and excelize has no map
+	// entry for an empty t, so a plain number reports CellTypeUnset rather than
+	// CellTypeNumber. Unset is only reachable here for a cell that exists and
+	// parsed as a float — an absent cell is "" and returned above — so both
+	// values mean "numeric". Shared strings report CellTypeSharedString and are
+	// left alone, which is what protects "01063476", "6.00" and "NAN".
+	if t, err := f.GetCellType(sheet, axis); err == nil &&
+		(t == excelize.CellTypeNumber || t == excelize.CellTypeUnset) {
+		return out
+	}
+	return v
+}
+
+// buildCRFixups maps a shared string's line-ending-normalised form back to its
+// raw form, for the strings that contain a carriage return.
+//
+// Go's encoding/xml performs the line-ending normalisation the XML 1.0 spec
+// mandates (CRLF and lone CR both become LF), so excelize returns "a\nb" where
+// calamine — which reads the raw bytes — returns "a\r\nb". That difference
+// reaches the database: in one 2016 file it affects 2,233 TEN_CUMTHI values,
+// which populate the ten_cum_thi column.
+//
+// The trick is that character references are exempt from that normalisation, so
+// rewriting literal CR bytes to 
 before decoding round-trips them intact.
+//
+// Returns nil when the file has no CR at all, which is the common case.
+func buildCRFixups(path string) map[string]string {
+	zr, err := zip.OpenReader(path)
+	if err != nil {
+		return nil
+	}
+	defer zr.Close()
+
+	var entry *zip.File
+	for _, zf := range zr.File {
+		if zf.Name == "xl/sharedStrings.xml" {
+			entry = zf
+			break
+		}
+	}
+	if entry == nil {
+		return nil
+	}
+	rc, err := entry.Open()
+	if err != nil {
+		return nil
+	}
+	defer rc.Close()
+	data, err := io.ReadAll(rc)
+	if err != nil || !bytes.ContainsRune(data, '\r') {
+		return nil
+	}
+	data = bytes.ReplaceAll(data, []byte{'\r'}, []byte("
"))
+
+	fixups := make(map[string]string)
+	dec := xml.NewDecoder(bytes.NewReader(data))
+	var cur bytes.Buffer
+	inSI, inT := false, false
+	for {
+		tok, err := dec.Token()
+		if err != nil {
+			break
+		}
+		switch t := tok.(type) {
+		case xml.StartElement:
+			switch t.Name.Local {
+			case "si":
+				inSI, cur = true, bytes.Buffer{}
+			case "t":
+				inT = true
+			}
+		case xml.CharData:
+			if inSI && inT {
+				cur.Write(t)
+			}
+		case xml.EndElement:
+			switch t.Name.Local {
+			case "t":
+				inT = false
+			case "si":
+				if inSI {
+					raw := cur.String()
+					if norm := normalizeEOL(raw); norm != raw {
+						fixups[norm] = raw
+					}
+				}
+				inSI = false
+			}
+		}
+	}
+	if len(fixups) == 0 {
+		return nil
+	}
+	return fixups
+}
+
+// normalizeEOL applies XML 1.0 line-ending normalisation: CRLF and lone CR
+// both collapse to LF. This is what encoding/xml does to the raw bytes, so it
+// reproduces the text excelize hands back.
+func normalizeEOL(s string) string {
+	if !bytes.ContainsRune([]byte(s), '\r') {
+		return s
+	}
+	var b bytes.Buffer
+	b.Grow(len(s))
+	for i := 0; i < len(s); i++ {
+		if s[i] == '\r' {
+			if i+1 < len(s) && s[i+1] == '\n' {
+				continue // CRLF: the LF is emitted on the next pass
+			}
+			b.WriteByte('\n') // lone CR
+			continue
+		}
+		b.WriteByte(s[i])
+	}
+	return b.String()
+}