mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-15 22:15:13 +00:00
refactor: one directory per pipeline stage, and an assembler to drive them
The repository now reads as the pipeline it is: crawler fetches, parser converts, assembler verifies and publishes, with data/ and web/ as the stores they hand work through. go-parser is renamed parser now that there is no other. The assembler replaces build-db.js and assemble-site.js. It compiles the parser, builds and verifies each database, compresses it, runs the Vite build and assembles _site — one command, and the only place that knows the order. It also closes a real hole: nothing previously asserted that a database reached the site. An empty staging directory assembled happily, so every page rendered, every query 404d and CI stayed green. The row-count and size guards could not catch that, since they only run when a database was built at all. Removing Node from the root forced the dataset list out of web/src/datasets.js, which the assembler cannot import. datasets.json is now the registry both sides read — JSON because Go and the browser both parse it without a dependency — while presentation stays in the web app, keyed by id and cross-checked against the registry so a half-added dataset fails instead of half-working. Guards verified by making each one fail: a missing database, and an expected row count one higher than the truth.
This commit is contained in:
1 parent
b04f9844f9
commit
c359a0b444
61 files changed
+1341
-480
No files matched your search
@@ -0,0 +1,88 @@
|
||||
// Command dumpcells emits the canonical cell rendering of a spreadsheet, for
|
||||
// comparison against the Rust/calamine ground truth produced by
|
||||
// parser/examples/dump_cells.rs.
|
||||
//
|
||||
// The canonical stream carries geometry and rendered cell values only. The
|
||||
// calamine Data variant is deliberately excluded: Data::Empty and
|
||||
// Data::String("") both render "" and both count as blank everywhere
|
||||
// downstream, so the distinction cannot affect the database.
|
||||
//
|
||||
// Usage: dumpcells <spreadsheet> [out-file]
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/tiennm99/thptqg/parser/internal/reader"
|
||||
)
|
||||
|
||||
// escape mirrors the Rust dumper so field separators can never break the format.
|
||||
func escape(s string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(s))
|
||||
for _, ch := range s {
|
||||
switch ch {
|
||||
case '\\':
|
||||
b.WriteString(`\\`)
|
||||
case '\t':
|
||||
b.WriteString(`\t`)
|
||||
case '\n':
|
||||
b.WriteString(`\n`)
|
||||
case '\r':
|
||||
b.WriteString(`\r`)
|
||||
default:
|
||||
b.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: dumpcells <spreadsheet> [out-file]")
|
||||
os.Exit(2)
|
||||
}
|
||||
path := os.Args[1]
|
||||
|
||||
out := os.Stdout
|
||||
if len(os.Args) > 2 {
|
||||
f, err := os.Create(os.Args[2])
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "create: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer f.Close()
|
||||
out = f
|
||||
}
|
||||
w := bufio.NewWriterSize(out, 1<<20)
|
||||
defer w.Flush()
|
||||
|
||||
wb, err := reader.Open(path)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "open: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer wb.Close()
|
||||
|
||||
sheets := wb.Sheets()
|
||||
fmt.Fprintf(w, "FILE\t%s\n", escape(path))
|
||||
fmt.Fprintf(w, "SHEETCOUNT\t%d\n", len(sheets))
|
||||
|
||||
for _, sh := range sheets {
|
||||
fmt.Fprintf(w, "SHEET\t%d\t%s\t%d\t%d\n", sh.Index, escape(sh.Name), sh.Height, sh.Width)
|
||||
err := wb.EachRow(sh.Index, func(s reader.Sheet, rowIdx int, row []reader.Cell) error {
|
||||
fmt.Fprintf(w, "ROW\t%d\t%d\t%d\n", s.Index, rowIdx, len(row))
|
||||
for c, cell := range row {
|
||||
fmt.Fprintf(w, "CELL\t%d\t%d\t%d\t%s\n", s.Index, rowIdx, c, escape(cell.Str))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "rows: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// Command xlsxread reads .xls/.xlsx files and builds SQLite databases for the
|
||||
// thptqg datasets.
|
||||
//
|
||||
// The CLI contract is fixed by parser/scripts/build-db.js and must match the
|
||||
// Rust binary exactly:
|
||||
//
|
||||
// xlsxread build --schema <config.yml> --input <dir> --output <db>
|
||||
// xlsxread audit --schema <config.yml> --input <dir> --db <db>
|
||||
//
|
||||
// Implemented with the standard flag package rather than a CLI framework: two
|
||||
// subcommands with three flags each do not justify a dependency.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/tiennm99/thptqg/parser/internal/audit"
|
||||
"github.com/tiennm99/thptqg/parser/internal/config"
|
||||
"github.com/tiennm99/thptqg/parser/internal/ingest"
|
||||
)
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `xlsxread — read .xls/.xlsx files and build SQLite databases for thptqg datasets
|
||||
|
||||
Usage:
|
||||
xlsxread build --schema <config.yml> --input <dir> --output <db>
|
||||
xlsxread audit --schema <config.yml> --input <dir> --db <db>
|
||||
`)
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
switch os.Args[1] {
|
||||
case "build":
|
||||
runBuild(os.Args[2:])
|
||||
case "audit":
|
||||
runAudit(os.Args[2:])
|
||||
case "-h", "--help", "help":
|
||||
usage()
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown subcommand %q\n\n", os.Args[1])
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func runBuild(args []string) {
|
||||
fs := flag.NewFlagSet("build", flag.ExitOnError)
|
||||
schemaPath := fs.String("schema", "", "path to the dataset YAML config file")
|
||||
inputDir := fs.String("input", "", "directory containing the .xls / .xlsx source files")
|
||||
outputPath := fs.String("output", "", "output SQLite database path")
|
||||
fs.Parse(args)
|
||||
|
||||
if *schemaPath == "" || *inputDir == "" || *outputPath == "" {
|
||||
fmt.Fprintln(os.Stderr, "build requires --schema, --input and --output")
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cfg, err := config.Load(*schemaPath)
|
||||
if err != nil {
|
||||
fatalf("Failed to load config: %v", err)
|
||||
}
|
||||
|
||||
// The 2016 dataset selects its column layout per file at runtime; every other
|
||||
// dataset uses the fixed columns: mapping (main.rs:63-67).
|
||||
if cfg.FormatDetection != nil && *cfg.FormatDetection == "thptqg2016" {
|
||||
if err := ingest.Detect2016(cfg, *inputDir, *outputPath); err != nil {
|
||||
fatalf("%v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := ingest.Standard(cfg, *inputDir, *outputPath); err != nil {
|
||||
fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func runAudit(args []string) {
|
||||
fs := flag.NewFlagSet("audit", flag.ExitOnError)
|
||||
schemaPath := fs.String("schema", "", "path to the dataset YAML config file")
|
||||
inputDir := fs.String("input", "", "directory containing the .xlsx source files")
|
||||
dbPath := fs.String("db", "", "SQLite database to compare against")
|
||||
fs.Parse(args)
|
||||
|
||||
if *schemaPath == "" || *inputDir == "" || *dbPath == "" {
|
||||
fmt.Fprintln(os.Stderr, "audit requires --schema, --input and --db")
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cfg, err := config.Load(*schemaPath)
|
||||
if err != nil {
|
||||
fatalf("Failed to load config: %v", err)
|
||||
}
|
||||
res, err := audit.Run(*inputDir, *dbPath, cfg)
|
||||
if err != nil {
|
||||
fatalf("%v", err)
|
||||
}
|
||||
audit.PrintReport(res)
|
||||
|
||||
// A mismatch is a non-zero exit even though the audit itself succeeded
|
||||
// (main.rs:46-48) — CI treats it as a failure signal.
|
||||
if !res.Matched {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func fatalf(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, "Error: "+format+"\n", a...)
|
||||
os.Exit(1)
|
||||
}
|
||||
Reference in new issue
Block a user