refactor: one directory per pipeline stage, and an assembler to drive them

The repository now reads as the pipeline it is: crawler fetches, parser
converts, assembler verifies and publishes, with data/ and web/ as the stores
they hand work through. go-parser is renamed parser now that there is no other.

The assembler replaces build-db.js and assemble-site.js. It compiles the
parser, builds and verifies each database, compresses it, runs the Vite build
and assembles _site — one command, and the only place that knows the order.

It also closes a real hole: nothing previously asserted that a database reached
the site. An empty staging directory assembled happily, so every page rendered,
every query 404d and CI stayed green. The row-count and size guards could not
catch that, since they only run when a database was built at all.

Removing Node from the root forced the dataset list out of web/src/datasets.js,
which the assembler cannot import. datasets.json is now the registry both sides
read — JSON because Go and the browser both parse it without a dependency —
while presentation stays in the web app, keyed by id and cross-checked against
the registry so a half-added dataset fails instead of half-working.

Guards verified by making each one fail: a missing database, and an expected
row count one higher than the truth.
This commit is contained in:
tiennm99 committed 2026-08-13 22:50:05 +07:00
1 parent b04f9844f9
commit c359a0b444
61 files changed
+1341 -480

No files matched your search

+129
View File
@@ -0,0 +1,129 @@
// Command assemble turns source data and the web app into the directory
// GitHub Pages publishes.
//
// assemble # databases, then the site
// assemble db # databases only (add ids to limit: assemble db 2017)
// assemble site # web build and _site only, reusing staged databases
//
// It sequences the other stages rather than doing their work: the parser reads
// spreadsheets, Vite bundles the app, and this decides what runs, checks what
// came out, and refuses to publish anything that looks wrong.
package main
import (
"fmt"
"os"
"path/filepath"
"github.com/tiennm99/thptqg/assembler/internal/databases"
"github.com/tiennm99/thptqg/assembler/internal/registry"
"github.com/tiennm99/thptqg/assembler/internal/site"
)
func main() {
if err := run(os.Args[1:]); err != nil {
fmt.Fprintf(os.Stderr, "assemble: %v\n", err)
os.Exit(1)
}
}
func usage() {
fmt.Fprint(os.Stderr, `usage: assemble [step] [dataset...]
Steps:
(none) databases, then the site
db build, verify and compress the databases
site build the web app and assemble _site
Naming datasets limits the db step to those; the site step always covers all of
them, since a partial site would publish links to databases it did not build.
`)
}
func run(args []string) error {
step := ""
if len(args) > 0 {
switch args[0] {
case "db", "site":
step, args = args[0], args[1:]
case "-h", "--help":
usage()
return nil
default:
// Bare dataset ids are a natural thing to type; treat them as the
// db step rather than rejecting them.
step = "db"
}
}
root, err := repoRoot()
if err != nil {
return err
}
all, err := registry.Load(root)
if err != nil {
return err
}
if step == "" || step == "db" {
selected, err := registry.Select(all, args)
if err != nil {
return err
}
if err := buildDatabases(root, all, selected); err != nil {
return err
}
}
if step == "" || step == "site" {
sp := site.DefaultPaths(root)
if err := site.BuildWeb(sp); err != nil {
return err
}
if err := site.Assemble(sp, all); err != nil {
return err
}
}
return nil
}
func buildDatabases(root string, all, selected []registry.Dataset) error {
p := databases.DefaultPaths(root)
// Sweep first: a dataset dropped from the registry leaves its .db.gz behind,
// and the site assembly copies the staging directory wholesale.
if err := databases.Clean(p, all); err != nil {
return err
}
bin, err := databases.BuildParser(p)
if err != nil {
return err
}
for _, d := range selected {
if err := databases.Build(p, bin, d); err != nil {
return err
}
}
return nil
}
// repoRoot walks up from the working directory to the directory holding
// datasets.json, so the command works from anywhere in the tree.
func repoRoot() (string, error) {
dir, err := os.Getwd()
if err != nil {
return "", err
}
for {
if _, err := os.Stat(filepath.Join(dir, "datasets.json")); err == nil {
return dir, nil
}
parent := filepath.Dir(dir)
if parent == dir {
return "", fmt.Errorf("no datasets.json found in any parent of the working directory")
}
dir = parent
}
}