mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-17 06:15:15 +00:00
refactor: one directory per pipeline stage, and an assembler to drive them
The repository now reads as the pipeline it is: crawler fetches, parser converts, assembler verifies and publishes, with data/ and web/ as the stores they hand work through. go-parser is renamed parser now that there is no other. The assembler replaces build-db.js and assemble-site.js. It compiles the parser, builds and verifies each database, compresses it, runs the Vite build and assembles _site — one command, and the only place that knows the order. It also closes a real hole: nothing previously asserted that a database reached the site. An empty staging directory assembled happily, so every page rendered, every query 404d and CI stayed green. The row-count and size guards could not catch that, since they only run when a database was built at all. Removing Node from the root forced the dataset list out of web/src/datasets.js, which the assembler cannot import. datasets.json is now the registry both sides read — JSON because Go and the browser both parse it without a dependency — while presentation stays in the web app, keyed by id and cross-checked against the registry so a half-added dataset fails instead of half-working. Guards verified by making each one fail: a missing database, and an expected row count one higher than the truth.
This commit is contained in:
1 parent
b04f9844f9
commit
c359a0b444
61 files changed
+1341
-480
No files matched your search
@@ -0,0 +1,129 @@
|
||||
// Command assemble turns source data and the web app into the directory
|
||||
// GitHub Pages publishes.
|
||||
//
|
||||
// assemble # databases, then the site
|
||||
// assemble db # databases only (add ids to limit: assemble db 2017)
|
||||
// assemble site # web build and _site only, reusing staged databases
|
||||
//
|
||||
// It sequences the other stages rather than doing their work: the parser reads
|
||||
// spreadsheets, Vite bundles the app, and this decides what runs, checks what
|
||||
// came out, and refuses to publish anything that looks wrong.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/tiennm99/thptqg/assembler/internal/databases"
|
||||
"github.com/tiennm99/thptqg/assembler/internal/registry"
|
||||
"github.com/tiennm99/thptqg/assembler/internal/site"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:]); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "assemble: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `usage: assemble [step] [dataset...]
|
||||
|
||||
Steps:
|
||||
(none) databases, then the site
|
||||
db build, verify and compress the databases
|
||||
site build the web app and assemble _site
|
||||
|
||||
Naming datasets limits the db step to those; the site step always covers all of
|
||||
them, since a partial site would publish links to databases it did not build.
|
||||
`)
|
||||
}
|
||||
|
||||
func run(args []string) error {
|
||||
step := ""
|
||||
if len(args) > 0 {
|
||||
switch args[0] {
|
||||
case "db", "site":
|
||||
step, args = args[0], args[1:]
|
||||
case "-h", "--help":
|
||||
usage()
|
||||
return nil
|
||||
default:
|
||||
// Bare dataset ids are a natural thing to type; treat them as the
|
||||
// db step rather than rejecting them.
|
||||
step = "db"
|
||||
}
|
||||
}
|
||||
|
||||
root, err := repoRoot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
all, err := registry.Load(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if step == "" || step == "db" {
|
||||
selected, err := registry.Select(all, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := buildDatabases(root, all, selected); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if step == "" || step == "site" {
|
||||
sp := site.DefaultPaths(root)
|
||||
if err := site.BuildWeb(sp); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := site.Assemble(sp, all); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildDatabases(root string, all, selected []registry.Dataset) error {
|
||||
p := databases.DefaultPaths(root)
|
||||
|
||||
// Sweep first: a dataset dropped from the registry leaves its .db.gz behind,
|
||||
// and the site assembly copies the staging directory wholesale.
|
||||
if err := databases.Clean(p, all); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bin, err := databases.BuildParser(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, d := range selected {
|
||||
if err := databases.Build(p, bin, d); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// repoRoot walks up from the working directory to the directory holding
|
||||
// datasets.json, so the command works from anywhere in the tree.
|
||||
func repoRoot() (string, error) {
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(dir, "datasets.json")); err == nil {
|
||||
return dir, nil
|
||||
}
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
return "", fmt.Errorf("no datasets.json found in any parent of the working directory")
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user