refactor: one directory per pipeline stage, and an assembler to drive them

The repository now reads as the pipeline it is: crawler fetches, parser
converts, assembler verifies and publishes, with data/ and web/ as the stores
they hand work through. go-parser is renamed parser now that there is no other.

The assembler replaces build-db.js and assemble-site.js. It compiles the
parser, builds and verifies each database, compresses it, runs the Vite build
and assembles _site — one command, and the only place that knows the order.

It also closes a real hole: nothing previously asserted that a database reached
the site. An empty staging directory assembled happily, so every page rendered,
every query 404d and CI stayed green. The row-count and size guards could not
catch that, since they only run when a database was built at all.

Removing Node from the root forced the dataset list out of web/src/datasets.js,
which the assembler cannot import. datasets.json is now the registry both sides
read — JSON because Go and the browser both parse it without a dependency —
while presentation stays in the web app, keyed by id and cross-checked against
the registry so a half-added dataset fails instead of half-working.

Guards verified by making each one fail: a missing database, and an expected
row count one higher than the truth.
This commit is contained in:
tiennm99 committed 2026-08-13 22:50:05 +07:00
1 parent b04f9844f9
commit c359a0b444
61 files changed
+1341 -480

No files matched your search

+88
View File
@@ -0,0 +1,88 @@
// Command dumpcells emits the canonical cell rendering of a spreadsheet, for
// comparison against the Rust/calamine ground truth produced by
// parser/examples/dump_cells.rs.
//
// The canonical stream carries geometry and rendered cell values only. The
// calamine Data variant is deliberately excluded: Data::Empty and
// Data::String("") both render "" and both count as blank everywhere
// downstream, so the distinction cannot affect the database.
//
// Usage: dumpcells <spreadsheet> [out-file]
package main
import (
"bufio"
"fmt"
"os"
"strings"
"github.com/tiennm99/thptqg/parser/internal/reader"
)
// escape mirrors the Rust dumper so field separators can never break the format.
func escape(s string) string {
var b strings.Builder
b.Grow(len(s))
for _, ch := range s {
switch ch {
case '\\':
b.WriteString(`\\`)
case '\t':
b.WriteString(`\t`)
case '\n':
b.WriteString(`\n`)
case '\r':
b.WriteString(`\r`)
default:
b.WriteRune(ch)
}
}
return b.String()
}
func main() {
if len(os.Args) < 2 {
fmt.Fprintln(os.Stderr, "usage: dumpcells <spreadsheet> [out-file]")
os.Exit(2)
}
path := os.Args[1]
out := os.Stdout
if len(os.Args) > 2 {
f, err := os.Create(os.Args[2])
if err != nil {
fmt.Fprintf(os.Stderr, "create: %v\n", err)
os.Exit(1)
}
defer f.Close()
out = f
}
w := bufio.NewWriterSize(out, 1<<20)
defer w.Flush()
wb, err := reader.Open(path)
if err != nil {
fmt.Fprintf(os.Stderr, "open: %v\n", err)
os.Exit(1)
}
defer wb.Close()
sheets := wb.Sheets()
fmt.Fprintf(w, "FILE\t%s\n", escape(path))
fmt.Fprintf(w, "SHEETCOUNT\t%d\n", len(sheets))
for _, sh := range sheets {
fmt.Fprintf(w, "SHEET\t%d\t%s\t%d\t%d\n", sh.Index, escape(sh.Name), sh.Height, sh.Width)
err := wb.EachRow(sh.Index, func(s reader.Sheet, rowIdx int, row []reader.Cell) error {
fmt.Fprintf(w, "ROW\t%d\t%d\t%d\n", s.Index, rowIdx, len(row))
for c, cell := range row {
fmt.Fprintf(w, "CELL\t%d\t%d\t%d\t%s\n", s.Index, rowIdx, c, escape(cell.Str))
}
return nil
})
if err != nil {
fmt.Fprintf(os.Stderr, "rows: %v\n", err)
os.Exit(1)
}
}
}
+115
View File
@@ -0,0 +1,115 @@
// Command xlsxread reads .xls/.xlsx files and builds SQLite databases for the
// thptqg datasets.
//
// The CLI contract is fixed by parser/scripts/build-db.js and must match the
// Rust binary exactly:
//
// xlsxread build --schema <config.yml> --input <dir> --output <db>
// xlsxread audit --schema <config.yml> --input <dir> --db <db>
//
// Implemented with the standard flag package rather than a CLI framework: two
// subcommands with three flags each do not justify a dependency.
package main
import (
"flag"
"fmt"
"os"
"github.com/tiennm99/thptqg/parser/internal/audit"
"github.com/tiennm99/thptqg/parser/internal/config"
"github.com/tiennm99/thptqg/parser/internal/ingest"
)
func usage() {
fmt.Fprint(os.Stderr, `xlsxread — read .xls/.xlsx files and build SQLite databases for thptqg datasets
Usage:
xlsxread build --schema <config.yml> --input <dir> --output <db>
xlsxread audit --schema <config.yml> --input <dir> --db <db>
`)
}
func main() {
if len(os.Args) < 2 {
usage()
os.Exit(2)
}
switch os.Args[1] {
case "build":
runBuild(os.Args[2:])
case "audit":
runAudit(os.Args[2:])
case "-h", "--help", "help":
usage()
default:
fmt.Fprintf(os.Stderr, "unknown subcommand %q\n\n", os.Args[1])
usage()
os.Exit(2)
}
}
func runBuild(args []string) {
fs := flag.NewFlagSet("build", flag.ExitOnError)
schemaPath := fs.String("schema", "", "path to the dataset YAML config file")
inputDir := fs.String("input", "", "directory containing the .xls / .xlsx source files")
outputPath := fs.String("output", "", "output SQLite database path")
fs.Parse(args)
if *schemaPath == "" || *inputDir == "" || *outputPath == "" {
fmt.Fprintln(os.Stderr, "build requires --schema, --input and --output")
os.Exit(2)
}
cfg, err := config.Load(*schemaPath)
if err != nil {
fatalf("Failed to load config: %v", err)
}
// The 2016 dataset selects its column layout per file at runtime; every other
// dataset uses the fixed columns: mapping (main.rs:63-67).
if cfg.FormatDetection != nil && *cfg.FormatDetection == "thptqg2016" {
if err := ingest.Detect2016(cfg, *inputDir, *outputPath); err != nil {
fatalf("%v", err)
}
return
}
if err := ingest.Standard(cfg, *inputDir, *outputPath); err != nil {
fatalf("%v", err)
}
}
func runAudit(args []string) {
fs := flag.NewFlagSet("audit", flag.ExitOnError)
schemaPath := fs.String("schema", "", "path to the dataset YAML config file")
inputDir := fs.String("input", "", "directory containing the .xlsx source files")
dbPath := fs.String("db", "", "SQLite database to compare against")
fs.Parse(args)
if *schemaPath == "" || *inputDir == "" || *dbPath == "" {
fmt.Fprintln(os.Stderr, "audit requires --schema, --input and --db")
os.Exit(2)
}
cfg, err := config.Load(*schemaPath)
if err != nil {
fatalf("Failed to load config: %v", err)
}
res, err := audit.Run(*inputDir, *dbPath, cfg)
if err != nil {
fatalf("%v", err)
}
audit.PrintReport(res)
// A mismatch is a non-zero exit even though the audit itself succeeded
// (main.rs:46-48) — CI treats it as a failure signal.
if !res.Matched {
os.Exit(1)
}
}
func fatalf(format string, a ...any) {
fmt.Fprintf(os.Stderr, "Error: "+format+"\n", a...)
os.Exit(1)
}