refactor: one directory per pipeline stage, and an assembler to drive them

The repository now reads as the pipeline it is: crawler fetches, parser
converts, assembler verifies and publishes, with data/ and web/ as the stores
they hand work through. go-parser is renamed parser now that there is no other.

The assembler replaces build-db.js and assemble-site.js. It compiles the
parser, builds and verifies each database, compresses it, runs the Vite build
and assembles _site — one command, and the only place that knows the order.

It also closes a real hole: nothing previously asserted that a database reached
the site. An empty staging directory assembled happily, so every page rendered,
every query 404d and CI stayed green. The row-count and size guards could not
catch that, since they only run when a database was built at all.

Removing Node from the root forced the dataset list out of web/src/datasets.js,
which the assembler cannot import. datasets.json is now the registry both sides
read — JSON because Go and the browser both parse it without a dependency —
while presentation stays in the web app, keyed by id and cross-checked against
the registry so a half-added dataset fails instead of half-working.

Guards verified by making each one fail: a missing database, and an expected
row count one higher than the truth.

assembler/cmd/assemble/main.go

@@ -0,0 +1,129 @@
+// Command assemble turns source data and the web app into the directory
+// GitHub Pages publishes.
+//
+//	assemble            # databases, then the site
+//	assemble db         # databases only (add ids to limit: assemble db 2017)
+//	assemble site       # web build and _site only, reusing staged databases
+//
+// It sequences the other stages rather than doing their work: the parser reads
+// spreadsheets, Vite bundles the app, and this decides what runs, checks what
+// came out, and refuses to publish anything that looks wrong.
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/tiennm99/thptqg/assembler/internal/databases"
+	"github.com/tiennm99/thptqg/assembler/internal/registry"
+	"github.com/tiennm99/thptqg/assembler/internal/site"
+)
+
+func main() {
+	if err := run(os.Args[1:]); err != nil {
+		fmt.Fprintf(os.Stderr, "assemble: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func usage() {
+	fmt.Fprint(os.Stderr, `usage: assemble [step] [dataset...]
+
+Steps:
+  (none)   databases, then the site
+  db       build, verify and compress the databases
+  site     build the web app and assemble _site
+
+Naming datasets limits the db step to those; the site step always covers all of
+them, since a partial site would publish links to databases it did not build.
+`)
+}
+
+func run(args []string) error {
+	step := ""
+	if len(args) > 0 {
+		switch args[0] {
+		case "db", "site":
+			step, args = args[0], args[1:]
+		case "-h", "--help":
+			usage()
+			return nil
+		default:
+			// Bare dataset ids are a natural thing to type; treat them as the
+			// db step rather than rejecting them.
+			step = "db"
+		}
+	}
+
+	root, err := repoRoot()
+	if err != nil {
+		return err
+	}
+
+	all, err := registry.Load(root)
+	if err != nil {
+		return err
+	}
+
+	if step == "" || step == "db" {
+		selected, err := registry.Select(all, args)
+		if err != nil {
+			return err
+		}
+		if err := buildDatabases(root, all, selected); err != nil {
+			return err
+		}
+	}
+
+	if step == "" || step == "site" {
+		sp := site.DefaultPaths(root)
+		if err := site.BuildWeb(sp); err != nil {
+			return err
+		}
+		if err := site.Assemble(sp, all); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func buildDatabases(root string, all, selected []registry.Dataset) error {
+	p := databases.DefaultPaths(root)
+
+	// Sweep first: a dataset dropped from the registry leaves its .db.gz behind,
+	// and the site assembly copies the staging directory wholesale.
+	if err := databases.Clean(p, all); err != nil {
+		return err
+	}
+
+	bin, err := databases.BuildParser(p)
+	if err != nil {
+		return err
+	}
+	for _, d := range selected {
+		if err := databases.Build(p, bin, d); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// repoRoot walks up from the working directory to the directory holding
+// datasets.json, so the command works from anywhere in the tree.
+func repoRoot() (string, error) {
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	for {
+		if _, err := os.Stat(filepath.Join(dir, "datasets.json")); err == nil {
+			return dir, nil
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			return "", fmt.Errorf("no datasets.json found in any parent of the working directory")
+		}
+		dir = parent
+	}
+}

assembler/go.mod

@@ -0,0 +1,17 @@
+module github.com/tiennm99/thptqg/assembler
+
+go 1.26.5
+
+require modernc.org/sqlite v1.56.0
+
+require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/mattn/go-isatty v0.0.24 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	modernc.org/libc v1.74.4 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
+)

assembler/go.sum

@@ -0,0 +1,50 @@
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
+github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
+modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
+modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
+modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
+modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k=
+modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0=
+modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

assembler/internal/databases/databases.go

@@ -0,0 +1,217 @@
+// Package databases builds, verifies and compresses one SQLite file per
+// dataset.
+//
+// VERIFICATION IS THE POINT OF THIS PACKAGE, not an extra.
+//
+// Nothing between the parser and the published site otherwise asserts that a
+// database has data in it. The parser logs a file-level failure and continues,
+// returns success regardless, and finishes cleanly even at zero rows; the site
+// assembly only inspects filenames. So a reader that silently under-produced
+// would publish a truncated dataset with green CI and no red signal anywhere.
+//
+// The guards below close that: a build whose row count does not match the
+// registry, or whose artifact is implausibly small, fails the pipeline.
+package databases
+
+import (
+	"compress/gzip"
+	"database/sql"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	_ "modernc.org/sqlite" // pure-Go driver: the pipeline stays cgo-free
+
+	"github.com/tiennm99/thptqg/assembler/internal/registry"
+)
+
+// driverName is modernc.org/sqlite's registered name.
+const driverName = "sqlite"
+
+// minSizeRatio: a gzipped database far below its usual size means a truncated
+// build, even if the row count somehow passed.
+const minSizeRatio = 0.9
+
+// Paths locates the pieces this package needs.
+type Paths struct {
+	// Root is the repository root.
+	Root string
+	// Parser is the parser module directory.
+	Parser string
+	// OutDir is where the databases are staged — the directory Vite publishes.
+	OutDir string
+}
+
+// DefaultPaths derives the standard layout from the repository root.
+func DefaultPaths(root string) Paths {
+	return Paths{
+		Root:   root,
+		Parser: filepath.Join(root, "parser"),
+		OutDir: filepath.Join(root, ".build", "public", "db"),
+	}
+}
+
+// BuildParser compiles the parser binary and returns its path.
+//
+// Compiling here rather than expecting a prebuilt binary keeps the pipeline one
+// command. Go caches the work, so repeat runs cost almost nothing.
+func BuildParser(p Paths) (string, error) {
+	bin := filepath.Join(p.Parser, "bin", "xlsxread")
+	cmd := exec.Command("go", "-C", p.Parser, "build", "-o", "bin/xlsxread", "./cmd/xlsxread")
+	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("compiling the parser: %w", err)
+	}
+	return bin, nil
+}
+
+// Build runs the parser for one dataset, verifies the result and compresses it.
+//
+// Only the .gz survives: shipping a 100+ MB uncompressed database is made
+// structurally impossible rather than left to a cleanup step.
+func Build(p Paths, bin string, d registry.Dataset) error {
+	if err := os.MkdirAll(p.OutDir, 0o755); err != nil {
+		return err
+	}
+	db := filepath.Join(p.OutDir, d.ID+".db")
+
+	cmd := exec.Command(bin,
+		"build",
+		"--schema", filepath.Join(p.Parser, "configs", d.ID+".yml"),
+		"--input", filepath.Join(p.Root, "data", d.ID),
+		"--output", db,
+	)
+	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("%s: parser failed: %w", d.ID, err)
+	}
+
+	rows, err := countRows(db)
+	if err != nil {
+		return fmt.Errorf("%s: %w", d.ID, err)
+	}
+	if rows != d.ExpectedRows {
+		return fmt.Errorf(
+			"%s: row count %d, expected %d\nRefusing to publish — the build did not reproduce the known dataset",
+			d.ID, rows, d.ExpectedRows)
+	}
+	fmt.Printf("  ✓ %s: %d rows (matches expected)\n", d.ID, rows)
+
+	gz, size, err := compress(db)
+	if err != nil {
+		return fmt.Errorf("%s: %w", d.ID, err)
+	}
+
+	sizeMb := float64(size) / 1024 / 1024
+	if min := d.DbSizeMb * minSizeRatio; sizeMb < min {
+		return fmt.Errorf(
+			"%s: %.1f MB is below %.1f MB (%.0f%% of the expected %.0f MB)\n"+
+				"Refusing to publish — the artifact looks truncated",
+			d.ID, sizeMb, min, minSizeRatio*100, d.DbSizeMb)
+	}
+
+	fmt.Printf("  → %s (%.1f MB)\n\n", filepath.Base(gz), sizeMb)
+	return nil
+}
+
+// countRows opens the database read-only and counts what was written.
+func countRows(path string) (int64, error) {
+	conn, err := sql.Open(driverName, "file:"+path+"?mode=ro")
+	if err != nil {
+		return 0, err
+	}
+	defer conn.Close()
+
+	var n int64
+	if err := conn.QueryRow("SELECT COUNT(*) FROM student").Scan(&n); err != nil {
+		return 0, fmt.Errorf("counting rows: %w", err)
+	}
+	return n, nil
+}
+
+// compress gzips path to path+".gz" and removes the original, returning the
+// compressed path and its size.
+//
+// The source is deleted only after the compressed file is closed successfully,
+// so a failure part-way through leaves the database rather than losing it.
+func compress(path string) (string, int64, error) {
+	in, err := os.Open(path)
+	if err != nil {
+		return "", 0, err
+	}
+	defer in.Close()
+
+	gzPath := path + ".gz"
+	out, err := os.Create(gzPath)
+	if err != nil {
+		return "", 0, err
+	}
+
+	zw, err := gzip.NewWriterLevel(out, gzip.BestCompression)
+	if err != nil {
+		out.Close()
+		return "", 0, err
+	}
+	if _, err := io.Copy(zw, in); err != nil {
+		zw.Close()
+		out.Close()
+		os.Remove(gzPath)
+		return "", 0, err
+	}
+	if err := zw.Close(); err != nil {
+		out.Close()
+		os.Remove(gzPath)
+		return "", 0, err
+	}
+	if err := out.Close(); err != nil {
+		os.Remove(gzPath)
+		return "", 0, err
+	}
+
+	if err := in.Close(); err != nil {
+		return "", 0, err
+	}
+	if err := os.Remove(path); err != nil {
+		return "", 0, fmt.Errorf("removing the uncompressed database: %w", err)
+	}
+
+	st, err := os.Stat(gzPath)
+	if err != nil {
+		return "", 0, err
+	}
+	return gzPath, st.Size(), nil
+}
+
+// Clean removes staged artifacts for datasets that are no longer in the
+// registry. Without this a removed dataset's .db.gz lingers in the staging
+// directory, and the site assembly copies that directory wholesale — so the
+// dead database would be published again.
+func Clean(p Paths, keep []registry.Dataset) error {
+	entries, err := os.ReadDir(p.OutDir)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	wanted := make(map[string]bool, len(keep)*2)
+	for _, d := range keep {
+		wanted[d.ID+".db"] = true
+		wanted[d.ID+".db.gz"] = true
+	}
+
+	for _, e := range entries {
+		if e.IsDir() || wanted[e.Name()] {
+			continue
+		}
+		full := filepath.Join(p.OutDir, e.Name())
+		if err := os.Remove(full); err != nil {
+			return err
+		}
+		fmt.Printf("  removed stale artifact %s\n", e.Name())
+	}
+	return nil
+}

assembler/internal/databases/databases_test.go

@@ -0,0 +1,108 @@
+package databases
+
+import (
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+
+	"github.com/tiennm99/thptqg/assembler/internal/registry"
+)
+
+// TestCompressRoundTripsAndRemovesTheSource: only the .gz may survive, so that
+// shipping a 100+ MB uncompressed database is structurally impossible rather
+// than left to a cleanup step.
+func TestCompressRoundTripsAndRemovesTheSource(t *testing.T) {
+	dir := t.TempDir()
+	src := filepath.Join(dir, "2016.db")
+	body := []byte("pretend this is a SQLite file")
+	if err := os.WriteFile(src, body, 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	gzPath, size, err := compress(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if gzPath != src+".gz" || size <= 0 {
+		t.Fatalf("gzPath=%q size=%d", gzPath, size)
+	}
+	if _, err := os.Stat(src); !os.IsNotExist(err) {
+		t.Error("the uncompressed database must not survive")
+	}
+
+	f, err := os.Open(gzPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	zr, err := gzip.NewReader(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := io.ReadAll(zr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != string(body) {
+		t.Errorf("round-trip gave %q", got)
+	}
+}
+
+func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) {
+	dir := t.TempDir()
+	for _, name := range []string{
+		"2016.db.gz", "2017.db.gz",
+		"2017-old.db.gz",  // dropped from the registry
+		"2017-old2.db.gz", // dropped from the registry
+		"2016.db-journal", // interrupted run
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	p := Paths{OutDir: dir}
+	keep := []registry.Dataset{{ID: "2016"}, {ID: "2017"}}
+	if err := Clean(p, keep); err != nil {
+		t.Fatal(err)
+	}
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var left []string
+	for _, e := range entries {
+		left = append(left, e.Name())
+	}
+	slices.Sort(left)
+
+	want := []string{"2016.db.gz", "2017.db.gz"}
+	if !slices.Equal(left, want) {
+		t.Errorf("left %v, want %v", left, want)
+	}
+}
+
+// TestCleanToleratesAnAbsentStagingDirectory: a fresh checkout has never built
+// anything, and that is not an error.
+func TestCleanToleratesAnAbsentStagingDirectory(t *testing.T) {
+	p := Paths{OutDir: filepath.Join(t.TempDir(), "never-created")}
+	if err := Clean(p, nil); err != nil {
+		t.Errorf("Clean on a missing directory should succeed, got %v", err)
+	}
+}
+
+func TestDefaultPaths(t *testing.T) {
+	p := DefaultPaths("/repo")
+	if p.Parser != filepath.Join("/repo", "parser") {
+		t.Errorf("Parser = %q", p.Parser)
+	}
+	// The staging directory must be the one Vite publishes, or the databases
+	// never reach the site.
+	if p.OutDir != filepath.Join("/repo", ".build", "public", "db") {
+		t.Errorf("OutDir = %q", p.OutDir)
+	}
+}

assembler/internal/registry/registry.go

@@ -0,0 +1,80 @@
+// Package registry reads the repository-root datasets.json.
+//
+// That file is the one place every stage agrees on what exists. The Vite app
+// reads it too, which is why it is JSON: Go and the browser both parse it
+// without a dependency.
+package registry
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// Dataset is one entry in the registry.
+type Dataset struct {
+	ID string `json:"id"`
+	// ExpectedRows is exact. The inputs are frozen historical exam results, so
+	// a deviation of even one row means something changed that nobody intended.
+	ExpectedRows int64 `json:"expectedRows"`
+	// DbSizeMb is the usual size of the gzipped database, used to catch a
+	// build that produced a plausible row count but a truncated artifact.
+	DbSizeMb float64 `json:"dbSizeMb"`
+}
+
+type file struct {
+	Datasets []Dataset `json:"datasets"`
+}
+
+// Load reads datasets.json from the repository root.
+func Load(root string) ([]Dataset, error) {
+	path := filepath.Join(root, "datasets.json")
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read the dataset registry: %w", err)
+	}
+
+	var f file
+	if err := json.Unmarshal(b, &f); err != nil {
+		return nil, fmt.Errorf("%s: %w", path, err)
+	}
+	if len(f.Datasets) == 0 {
+		return nil, fmt.Errorf("%s declares no datasets", path)
+	}
+
+	for _, d := range f.Datasets {
+		switch {
+		case d.ID == "":
+			return nil, fmt.Errorf("%s: a dataset has no id", path)
+		case d.ExpectedRows <= 0:
+			return nil, fmt.Errorf("%s: %s has no expectedRows; the build guard needs it", path, d.ID)
+		case d.DbSizeMb <= 0:
+			return nil, fmt.Errorf("%s: %s has no dbSizeMb; the size guard needs it", path, d.ID)
+		}
+	}
+	return f.Datasets, nil
+}
+
+// Select returns the named datasets, or all of them when none are named.
+func Select(all []Dataset, ids []string) ([]Dataset, error) {
+	if len(ids) == 0 {
+		return all, nil
+	}
+	byID := make(map[string]Dataset, len(all))
+	known := make([]string, 0, len(all))
+	for _, d := range all {
+		byID[d.ID] = d
+		known = append(known, d.ID)
+	}
+
+	out := make([]Dataset, 0, len(ids))
+	for _, id := range ids {
+		d, ok := byID[id]
+		if !ok {
+			return nil, fmt.Errorf("unknown dataset %q (known: %v)", id, known)
+		}
+		out = append(out, d)
+	}
+	return out, nil
+}

assembler/internal/registry/registry_test.go

@@ -0,0 +1,93 @@
+package registry
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func writeRegistry(t *testing.T, body string) string {
+	t.Helper()
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "datasets.json"), []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return dir
+}
+
+func TestLoadsTheRealRegistry(t *testing.T) {
+	root := filepath.Join("..", "..", "..")
+	got, err := Load(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) == 0 {
+		t.Fatal("no datasets")
+	}
+	for _, d := range got {
+		if d.ID == "" || d.ExpectedRows <= 0 || d.DbSizeMb <= 0 {
+			t.Errorf("incomplete entry: %+v", d)
+		}
+		// Every declared dataset must have somewhere to read from and a parse
+		// config, or the build fails much later with a worse message.
+		for _, p := range []string{
+			filepath.Join(root, "data", d.ID),
+			filepath.Join(root, "parser", "configs", d.ID+".yml"),
+		} {
+			if _, err := os.Stat(p); err != nil {
+				t.Errorf("%s: missing %s", d.ID, p)
+			}
+		}
+	}
+}
+
+// TestIncompleteEntriesAreRejected: a dataset missing either guard figure would
+// otherwise publish unverified. Both are load-bearing, so neither may default.
+func TestIncompleteEntriesAreRejected(t *testing.T) {
+	for name, body := range map[string]string{
+		"no expectedRows": `{"datasets":[{"id":"x","dbSizeMb":1}]}`,
+		"no dbSizeMb":     `{"datasets":[{"id":"x","expectedRows":1}]}`,
+		"no id":           `{"datasets":[{"expectedRows":1,"dbSizeMb":1}]}`,
+		"zero rows":       `{"datasets":[{"id":"x","expectedRows":0,"dbSizeMb":1}]}`,
+		"empty":           `{"datasets":[]}`,
+	} {
+		t.Run(name, func(t *testing.T) {
+			if _, err := Load(writeRegistry(t, body)); err == nil {
+				t.Error("expected an error")
+			}
+		})
+	}
+}
+
+func TestMissingAndMalformedRegistry(t *testing.T) {
+	if _, err := Load(t.TempDir()); err == nil {
+		t.Error("a missing registry must fail")
+	}
+	if _, err := Load(writeRegistry(t, "{not json")); err == nil {
+		t.Error("a malformed registry must fail")
+	}
+}
+
+func TestSelect(t *testing.T) {
+	all := []Dataset{{ID: "2016"}, {ID: "2017"}}
+
+	got, err := Select(all, nil)
+	if err != nil || len(got) != 2 {
+		t.Errorf("no ids should select everything: %v %v", got, err)
+	}
+
+	got, err = Select(all, []string{"2017"})
+	if err != nil || len(got) != 1 || got[0].ID != "2017" {
+		t.Errorf("Select(2017) = %v, %v", got, err)
+	}
+
+	// A typo must not silently build nothing.
+	_, err = Select(all, []string{"2018"})
+	if err == nil {
+		t.Fatal("an unknown id must fail")
+	}
+	if !strings.Contains(err.Error(), "2018") || !strings.Contains(err.Error(), "2016") {
+		t.Errorf("the error should name the bad id and the known ones, got: %v", err)
+	}
+}

assembler/internal/site/site.go

@@ -0,0 +1,210 @@
+// Package site turns the web app and the staged databases into the directory
+// GitHub Pages publishes.
+//
+// The app resolves its dataset from the URL, so every page is the same
+// index.html. Because Vite's `base` is absolute (/thptqg/), that file references
+// /thptqg/assets/... no matter which directory it is served from — so copying it
+// to each dataset path produces a real static file at every URL.
+//
+// GitHub Pages serves those as directory indexes, which is why this needs no
+// SPA 404-fallback redirect. That matters beyond tidiness: the usual fallback
+// rewrites the URL and would interfere with the ?q= deep links the app relies
+// on.
+package site
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/tiennm99/thptqg/assembler/internal/registry"
+)
+
+// Paths locates the pieces this package needs.
+type Paths struct {
+	Root string
+	// Web is the Vite project directory.
+	Web string
+	// Dist is where Vite emits, inside the web workspace.
+	Dist string
+	// Site is the artifact the deploy action uploads, at the repository root.
+	Site string
+}
+
+// DefaultPaths derives the standard layout from the repository root.
+func DefaultPaths(root string) Paths {
+	web := filepath.Join(root, "web")
+	return Paths{
+		Root: root,
+		Web:  web,
+		Dist: filepath.Join(web, "dist"),
+		Site: filepath.Join(root, "_site"),
+	}
+}
+
+// BuildWeb runs the Vite build.
+//
+// Shelling out to npm is not a wart: Vite is a Node tool, and web/ is the only
+// npm project left in the repository. This stage owns the sequencing, not the
+// bundling.
+func BuildWeb(p Paths) error {
+	cmd := exec.Command("npm", "run", "build")
+	cmd.Dir = p.Web
+	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("vite build: %w", err)
+	}
+	return nil
+}
+
+// Assemble copies the build to one directory per dataset and checks the result.
+func Assemble(p Paths, datasets []registry.Dataset) error {
+	index := filepath.Join(p.Dist, "index.html")
+	if _, err := os.Stat(index); err != nil {
+		return fmt.Errorf("no build found at %s — run the web build first", p.Dist)
+	}
+
+	if err := os.RemoveAll(p.Site); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(p.Site, 0o755); err != nil {
+		return err
+	}
+
+	// Base build: index.html, assets/, and the gzipped databases from publicDir.
+	if err := copyTree(p.Dist, p.Site); err != nil {
+		return err
+	}
+
+	// Unknown paths render the hub rather than the default Pages 404.
+	if err := copyFile(index, filepath.Join(p.Site, "404.html")); err != nil {
+		return err
+	}
+
+	// One entry point per dataset.
+	for _, d := range datasets {
+		dir := filepath.Join(p.Site, d.ID)
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			return err
+		}
+		if err := copyFile(index, filepath.Join(dir, "index.html")); err != nil {
+			return err
+		}
+	}
+
+	if err := checkDatabasesPresent(p.Site, datasets); err != nil {
+		return err
+	}
+	if err := checkNoRawDatabases(p.Site); err != nil {
+		return err
+	}
+
+	fmt.Printf("assembled %s\n", p.Site)
+	fmt.Printf("  /thptqg/\n  /thptqg/404.html\n")
+	for _, d := range datasets {
+		fmt.Printf("  /thptqg/%s\n", d.ID)
+	}
+	return nil
+}
+
+// checkDatabasesPresent: every dataset must have shipped its database.
+//
+// Without this the site assembles happily with an empty db/ directory — every
+// page renders, every query 404s, and CI stays green. That is the failure this
+// catches; the size and row-count guards only run when a database was built at
+// all.
+func checkDatabasesPresent(siteDir string, datasets []registry.Dataset) error {
+	var missing []string
+	for _, d := range datasets {
+		gz := filepath.Join(siteDir, "db", d.ID+".db.gz")
+		st, err := os.Stat(gz)
+		if err != nil || st.Size() == 0 {
+			missing = append(missing, d.ID+".db.gz")
+		}
+	}
+	if len(missing) > 0 {
+		return fmt.Errorf(
+			"no database in the site output for: %s\n"+
+				"Every page would render and every query would 404. Build the databases first",
+			strings.Join(missing, ", "))
+	}
+	return nil
+}
+
+// rawDatabase matches an uncompressed SQLite artifact, including the temporary
+// files SQLite leaves mid-build.
+var rawDatabase = regexp.MustCompile(`\.db(-journal|-wal|-shm)?$`)
+
+// checkNoRawDatabases rejects an uncompressed database that reached the output.
+//
+// The build gzips without keeping the source, so none should exist — but the
+// staging directory is copied wholesale, and a leftover from an interrupted run
+// would go straight through. A raw database is 100+ MB.
+func checkNoRawDatabases(siteDir string) error {
+	var stray []string
+	err := filepath.WalkDir(siteDir, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.IsDir() && rawDatabase.MatchString(d.Name()) {
+			stray = append(stray, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	if len(stray) > 0 {
+		var b strings.Builder
+		b.WriteString("uncompressed database artefact(s) found in the site output:\n")
+		for _, f := range stray {
+			st, _ := os.Stat(f)
+			fmt.Fprintf(&b, "  %s (%.1f MB)\n", f, float64(st.Size())/1048576)
+		}
+		b.WriteString("remove them from the staging directory and re-run")
+		return fmt.Errorf("%s", b.String())
+	}
+	return nil
+}
+
+func copyTree(src, dst string) error {
+	return filepath.WalkDir(src, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		target := filepath.Join(dst, rel)
+		if d.IsDir() {
+			return os.MkdirAll(target, 0o755)
+		}
+		return copyFile(path, target)
+	})
+}
+
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
+		return err
+	}
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(out, in); err != nil {
+		out.Close()
+		return err
+	}
+	return out.Close()
+}

assembler/internal/site/site_test.go

@@ -0,0 +1,137 @@
+package site
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/tiennm99/thptqg/assembler/internal/registry"
+)
+
+var datasets = []registry.Dataset{{ID: "2016"}, {ID: "2017"}}
+
+// fakeBuild stands in for a Vite build: an index.html, an asset, and whatever
+// databases the caller wants staged.
+func fakeBuild(t *testing.T, dbs ...string) Paths {
+	t.Helper()
+	root := t.TempDir()
+	dist := filepath.Join(root, "web", "dist")
+	write(t, filepath.Join(dist, "index.html"), "<html>app</html>")
+	write(t, filepath.Join(dist, "assets", "index.js"), "console.log(1)")
+	for _, name := range dbs {
+		write(t, filepath.Join(dist, "db", name), "gzipped-bytes")
+	}
+	return Paths{Root: root, Web: filepath.Join(root, "web"), Dist: dist, Site: filepath.Join(root, "_site")}
+}
+
+func write(t *testing.T, path, body string) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestAssembleProducesAPageForEveryDataset(t *testing.T) {
+	p := fakeBuild(t, "2016.db.gz", "2017.db.gz")
+	if err := Assemble(p, datasets); err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{
+		"index.html",
+		"404.html",
+		filepath.Join("2016", "index.html"),
+		filepath.Join("2017", "index.html"),
+		filepath.Join("assets", "index.js"),
+		filepath.Join("db", "2016.db.gz"),
+	} {
+		if _, err := os.Stat(filepath.Join(p.Site, want)); err != nil {
+			t.Errorf("missing from the artifact: %s", want)
+		}
+	}
+}
+
+// TestMissingDatabaseFailsTheBuild is the guard that closes the widest hole.
+//
+// Without it the site assembles happily with an empty db/ directory: every page
+// renders, every query 404s, and CI stays green. The row-count and size guards
+// cannot catch this — they only run when a database was built at all.
+func TestMissingDatabaseFailsTheBuild(t *testing.T) {
+	p := fakeBuild(t, "2016.db.gz") // 2017 never built
+	err := Assemble(p, datasets)
+	if err == nil {
+		t.Fatal("expected an error when a database is missing")
+	}
+	if !strings.Contains(err.Error(), "2017.db.gz") {
+		t.Errorf("the error should name the missing database, got: %v", err)
+	}
+}
+
+// TestEmptyDatabaseFailsTheBuild: a zero-byte file satisfies "exists" but is
+// not a database.
+func TestEmptyDatabaseFailsTheBuild(t *testing.T) {
+	p := fakeBuild(t, "2016.db.gz", "2017.db.gz")
+	write(t, filepath.Join(p.Dist, "db", "2017.db.gz"), "")
+	if err := Assemble(p, datasets); err == nil {
+		t.Fatal("expected an error for a zero-byte database")
+	}
+}
+
+// TestRawDatabaseFailsTheBuild: the compression step deletes its source, so a
+// raw .db here means an interrupted run left one behind — and it is 100+ MB.
+func TestRawDatabaseFailsTheBuild(t *testing.T) {
+	for _, name := range []string{"2016.db", "2016.db-journal", "2016.db-wal", "2016.db-shm"} {
+		t.Run(name, func(t *testing.T) {
+			p := fakeBuild(t, "2016.db.gz", "2017.db.gz")
+			write(t, filepath.Join(p.Dist, "db", name), "raw sqlite")
+			err := Assemble(p, datasets)
+			if err == nil {
+				t.Fatalf("expected an error for %s", name)
+			}
+			if !strings.Contains(err.Error(), "uncompressed") {
+				t.Errorf("unexpected error: %v", err)
+			}
+		})
+	}
+}
+
+// TestGzipIsNotMistakenForRaw: the reject pattern is anchored, so a .db.gz must
+// pass. Getting this wrong would fail every build.
+func TestGzipIsNotMistakenForRaw(t *testing.T) {
+	if rawDatabase.MatchString("2016.db.gz") {
+		t.Error("a .db.gz must not be treated as an uncompressed database")
+	}
+	for _, name := range []string{"2016.db", "x.db-journal", "x.db-wal", "x.db-shm"} {
+		if !rawDatabase.MatchString(name) {
+			t.Errorf("%s should be treated as an uncompressed artifact", name)
+		}
+	}
+}
+
+func TestAssembleRejectsAMissingBuild(t *testing.T) {
+	root := t.TempDir()
+	p := Paths{Root: root, Web: root, Dist: filepath.Join(root, "dist"), Site: filepath.Join(root, "_site")}
+	if err := Assemble(p, datasets); err == nil {
+		t.Fatal("expected an error when there is no Vite build")
+	}
+}
+
+// TestAssembleIsIdempotent: the site directory is rebuilt from scratch, so a
+// previous run's leftovers cannot survive into the artifact.
+func TestAssembleIsIdempotent(t *testing.T) {
+	p := fakeBuild(t, "2016.db.gz", "2017.db.gz")
+	if err := Assemble(p, datasets); err != nil {
+		t.Fatal(err)
+	}
+	stale := filepath.Join(p.Site, "2015", "index.html")
+	write(t, stale, "old dataset")
+	if err := Assemble(p, datasets); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(stale); !os.IsNotExist(err) {
+		t.Error("a directory from a previous run survived into the new artifact")
+	}
+}