refactor: one directory per pipeline stage, and an assembler to drive them

The repository now reads as the pipeline it is: crawler fetches, parser
converts, assembler verifies and publishes, with data/ and web/ as the stores
they hand work through. go-parser is renamed parser now that there is no other.

The assembler replaces build-db.js and assemble-site.js. It compiles the
parser, builds and verifies each database, compresses it, runs the Vite build
and assembles _site — one command, and the only place that knows the order.

It also closes a real hole: nothing previously asserted that a database reached
the site. An empty staging directory assembled happily, so every page rendered,
every query 404d and CI stayed green. The row-count and size guards could not
catch that, since they only run when a database was built at all.

Removing Node from the root forced the dataset list out of web/src/datasets.js,
which the assembler cannot import. datasets.json is now the registry both sides
read — JSON because Go and the browser both parse it without a dependency —
while presentation stays in the web app, keyed by id and cross-checked against
the registry so a half-added dataset fails instead of half-working.

Guards verified by making each one fail: a missing database, and an expected
row count one higher than the truth.

assembler/internal/databases/databases.go

@@ -0,0 +1,217 @@
+// Package databases builds, verifies and compresses one SQLite file per
+// dataset.
+//
+// VERIFICATION IS THE POINT OF THIS PACKAGE, not an extra.
+//
+// Nothing between the parser and the published site otherwise asserts that a
+// database has data in it. The parser logs a file-level failure and continues,
+// returns success regardless, and finishes cleanly even at zero rows; the site
+// assembly only inspects filenames. So a reader that silently under-produced
+// would publish a truncated dataset with green CI and no red signal anywhere.
+//
+// The guards below close that: a build whose row count does not match the
+// registry, or whose artifact is implausibly small, fails the pipeline.
+package databases
+
+import (
+	"compress/gzip"
+	"database/sql"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	_ "modernc.org/sqlite" // pure-Go driver: the pipeline stays cgo-free
+
+	"github.com/tiennm99/thptqg/assembler/internal/registry"
+)
+
+// driverName is modernc.org/sqlite's registered name.
+const driverName = "sqlite"
+
+// minSizeRatio: a gzipped database far below its usual size means a truncated
+// build, even if the row count somehow passed.
+const minSizeRatio = 0.9
+
+// Paths locates the pieces this package needs.
+type Paths struct {
+	// Root is the repository root.
+	Root string
+	// Parser is the parser module directory.
+	Parser string
+	// OutDir is where the databases are staged — the directory Vite publishes.
+	OutDir string
+}
+
+// DefaultPaths derives the standard layout from the repository root.
+func DefaultPaths(root string) Paths {
+	return Paths{
+		Root:   root,
+		Parser: filepath.Join(root, "parser"),
+		OutDir: filepath.Join(root, ".build", "public", "db"),
+	}
+}
+
+// BuildParser compiles the parser binary and returns its path.
+//
+// Compiling here rather than expecting a prebuilt binary keeps the pipeline one
+// command. Go caches the work, so repeat runs cost almost nothing.
+func BuildParser(p Paths) (string, error) {
+	bin := filepath.Join(p.Parser, "bin", "xlsxread")
+	cmd := exec.Command("go", "-C", p.Parser, "build", "-o", "bin/xlsxread", "./cmd/xlsxread")
+	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("compiling the parser: %w", err)
+	}
+	return bin, nil
+}
+
+// Build runs the parser for one dataset, verifies the result and compresses it.
+//
+// Only the .gz survives: shipping a 100+ MB uncompressed database is made
+// structurally impossible rather than left to a cleanup step.
+func Build(p Paths, bin string, d registry.Dataset) error {
+	if err := os.MkdirAll(p.OutDir, 0o755); err != nil {
+		return err
+	}
+	db := filepath.Join(p.OutDir, d.ID+".db")
+
+	cmd := exec.Command(bin,
+		"build",
+		"--schema", filepath.Join(p.Parser, "configs", d.ID+".yml"),
+		"--input", filepath.Join(p.Root, "data", d.ID),
+		"--output", db,
+	)
+	cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("%s: parser failed: %w", d.ID, err)
+	}
+
+	rows, err := countRows(db)
+	if err != nil {
+		return fmt.Errorf("%s: %w", d.ID, err)
+	}
+	if rows != d.ExpectedRows {
+		return fmt.Errorf(
+			"%s: row count %d, expected %d\nRefusing to publish — the build did not reproduce the known dataset",
+			d.ID, rows, d.ExpectedRows)
+	}
+	fmt.Printf("  ✓ %s: %d rows (matches expected)\n", d.ID, rows)
+
+	gz, size, err := compress(db)
+	if err != nil {
+		return fmt.Errorf("%s: %w", d.ID, err)
+	}
+
+	sizeMb := float64(size) / 1024 / 1024
+	if min := d.DbSizeMb * minSizeRatio; sizeMb < min {
+		return fmt.Errorf(
+			"%s: %.1f MB is below %.1f MB (%.0f%% of the expected %.0f MB)\n"+
+				"Refusing to publish — the artifact looks truncated",
+			d.ID, sizeMb, min, minSizeRatio*100, d.DbSizeMb)
+	}
+
+	fmt.Printf("  → %s (%.1f MB)\n\n", filepath.Base(gz), sizeMb)
+	return nil
+}
+
+// countRows opens the database read-only and counts what was written.
+func countRows(path string) (int64, error) {
+	conn, err := sql.Open(driverName, "file:"+path+"?mode=ro")
+	if err != nil {
+		return 0, err
+	}
+	defer conn.Close()
+
+	var n int64
+	if err := conn.QueryRow("SELECT COUNT(*) FROM student").Scan(&n); err != nil {
+		return 0, fmt.Errorf("counting rows: %w", err)
+	}
+	return n, nil
+}
+
+// compress gzips path to path+".gz" and removes the original, returning the
+// compressed path and its size.
+//
+// The source is deleted only after the compressed file is closed successfully,
+// so a failure part-way through leaves the database rather than losing it.
+func compress(path string) (string, int64, error) {
+	in, err := os.Open(path)
+	if err != nil {
+		return "", 0, err
+	}
+	defer in.Close()
+
+	gzPath := path + ".gz"
+	out, err := os.Create(gzPath)
+	if err != nil {
+		return "", 0, err
+	}
+
+	zw, err := gzip.NewWriterLevel(out, gzip.BestCompression)
+	if err != nil {
+		out.Close()
+		return "", 0, err
+	}
+	if _, err := io.Copy(zw, in); err != nil {
+		zw.Close()
+		out.Close()
+		os.Remove(gzPath)
+		return "", 0, err
+	}
+	if err := zw.Close(); err != nil {
+		out.Close()
+		os.Remove(gzPath)
+		return "", 0, err
+	}
+	if err := out.Close(); err != nil {
+		os.Remove(gzPath)
+		return "", 0, err
+	}
+
+	if err := in.Close(); err != nil {
+		return "", 0, err
+	}
+	if err := os.Remove(path); err != nil {
+		return "", 0, fmt.Errorf("removing the uncompressed database: %w", err)
+	}
+
+	st, err := os.Stat(gzPath)
+	if err != nil {
+		return "", 0, err
+	}
+	return gzPath, st.Size(), nil
+}
+
+// Clean removes staged artifacts for datasets that are no longer in the
+// registry. Without this a removed dataset's .db.gz lingers in the staging
+// directory, and the site assembly copies that directory wholesale — so the
+// dead database would be published again.
+func Clean(p Paths, keep []registry.Dataset) error {
+	entries, err := os.ReadDir(p.OutDir)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	wanted := make(map[string]bool, len(keep)*2)
+	for _, d := range keep {
+		wanted[d.ID+".db"] = true
+		wanted[d.ID+".db.gz"] = true
+	}
+
+	for _, e := range entries {
+		if e.IsDir() || wanted[e.Name()] {
+			continue
+		}
+		full := filepath.Join(p.OutDir, e.Name())
+		if err := os.Remove(full); err != nil {
+			return err
+		}
+		fmt.Printf("  removed stale artifact %s\n", e.Name())
+	}
+	return nil
+}

assembler/internal/databases/databases_test.go

@@ -0,0 +1,108 @@
+package databases
+
+import (
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+
+	"github.com/tiennm99/thptqg/assembler/internal/registry"
+)
+
+// TestCompressRoundTripsAndRemovesTheSource: only the .gz may survive, so that
+// shipping a 100+ MB uncompressed database is structurally impossible rather
+// than left to a cleanup step.
+func TestCompressRoundTripsAndRemovesTheSource(t *testing.T) {
+	dir := t.TempDir()
+	src := filepath.Join(dir, "2016.db")
+	body := []byte("pretend this is a SQLite file")
+	if err := os.WriteFile(src, body, 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	gzPath, size, err := compress(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if gzPath != src+".gz" || size <= 0 {
+		t.Fatalf("gzPath=%q size=%d", gzPath, size)
+	}
+	if _, err := os.Stat(src); !os.IsNotExist(err) {
+		t.Error("the uncompressed database must not survive")
+	}
+
+	f, err := os.Open(gzPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	zr, err := gzip.NewReader(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := io.ReadAll(zr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != string(body) {
+		t.Errorf("round-trip gave %q", got)
+	}
+}
+
+func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) {
+	dir := t.TempDir()
+	for _, name := range []string{
+		"2016.db.gz", "2017.db.gz",
+		"2017-old.db.gz",  // dropped from the registry
+		"2017-old2.db.gz", // dropped from the registry
+		"2016.db-journal", // interrupted run
+	} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	p := Paths{OutDir: dir}
+	keep := []registry.Dataset{{ID: "2016"}, {ID: "2017"}}
+	if err := Clean(p, keep); err != nil {
+		t.Fatal(err)
+	}
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var left []string
+	for _, e := range entries {
+		left = append(left, e.Name())
+	}
+	slices.Sort(left)
+
+	want := []string{"2016.db.gz", "2017.db.gz"}
+	if !slices.Equal(left, want) {
+		t.Errorf("left %v, want %v", left, want)
+	}
+}
+
+// TestCleanToleratesAnAbsentStagingDirectory: a fresh checkout has never built
+// anything, and that is not an error.
+func TestCleanToleratesAnAbsentStagingDirectory(t *testing.T) {
+	p := Paths{OutDir: filepath.Join(t.TempDir(), "never-created")}
+	if err := Clean(p, nil); err != nil {
+		t.Errorf("Clean on a missing directory should succeed, got %v", err)
+	}
+}
+
+func TestDefaultPaths(t *testing.T) {
+	p := DefaultPaths("/repo")
+	if p.Parser != filepath.Join("/repo", "parser") {
+		t.Errorf("Parser = %q", p.Parser)
+	}
+	// The staging directory must be the one Vite publishes, or the databases
+	// never reach the site.
+	if p.OutDir != filepath.Join("/repo", ".build", "public", "db") {
+		t.Errorf("OutDir = %q", p.OutDir)
+	}
+}