thptqg/go-parser/scripts/build-db.js

#!/usr/bin/env node
/**
 * Build the SQLite database for one or all datasets, verify it, then gzip it.
 *
 * The dataset list comes from src/datasets.js so it is written in exactly one
 * place.
 *
 * Output goes to .build/public/db/ — the directory Vite copies as its publicDir.
 * Only the .gz survives: shipping a 100+ MB uncompressed database is made
 * structurally impossible rather than left to a cleanup step.
 *
 * VERIFICATION IS THE POINT OF THIS SCRIPT, not an extra.
 *
 * Until now nothing between the parser and the public site asserted that a
 * database actually had data in it. The parser logs a file-level failure and
 * continues, returns success regardless, and finishes cleanly even at zero rows;
 * this script gzipped whatever it got; and scripts/assemble-site.js only greps
 * *filenames* for stray .db files. So a reader that silently under-produced
 * would publish a truncated dataset with green CI and no red signal anywhere.
 *
 * The guard below closes that: a build whose row count does not match the known
 * figure, or whose artifact is implausibly small, fails the pipeline.
 *
 * Usage:
 *   node go-parser/scripts/build-db.js            # all four datasets
 *   node go-parser/scripts/build-db.js 2017-old   # just one
 */

import { execFileSync } from "node:child_process";
import { mkdirSync, rmSync, existsSync, statSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { DatabaseSync } from "node:sqlite";

import { DATASET_IDS, DATASETS } from "../../src/datasets.js";

const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
const BIN = resolve(ROOT, "go-parser/bin/xlsxread");
const OUT_DIR = resolve(ROOT, ".build/public/db");

/**
 * Known-good row counts, from docs/data-pipeline.md.
 *
 * The inputs are frozen historical exam results, so these are exact, not
 * approximate. A deviation of even one row means something changed that nobody
 * intended — treat it as a build failure, not a warning.
 */
const EXPECTED_ROWS = {
  "2016": 877461,
  "2017": 861068,
  "2017-old": 847348,
  "2017-old2": 679764,
};

/** A gzipped database far below its usual size means a truncated build. */
const MIN_SIZE_RATIO = 0.9;

const requested = process.argv.slice(2);
const unknown = requested.filter((id) => !DATASET_IDS.includes(id));
if (unknown.length) {
  console.error(`unknown dataset(s): ${unknown.join(", ")}`);
  console.error(`known: ${DATASET_IDS.join(", ")}`);
  process.exit(2);
}
const targets = requested.length ? requested : DATASET_IDS;

if (!existsSync(BIN)) {
  console.error(`parser binary not found at ${BIN}`);
  console.error("run: npm run build:go");
  process.exit(1);
}

mkdirSync(OUT_DIR, { recursive: true });

for (const id of targets) {
  const db = resolve(OUT_DIR, `${id}.db`);

  // execFileSync throws on a non-zero exit, so a parser failure aborts the run.
  execFileSync(
    BIN,
    [
      "build",
      "--schema",
      resolve(ROOT, `go-parser/configs/${id}.yml`),
      "--input",
      resolve(ROOT, `data/${id}`),
      "--output",
      db,
    ],
    { stdio: "inherit" },
  );

  // --- guard: the database must contain what it is supposed to contain ---
  const expected = EXPECTED_ROWS[id];
  if (expected === undefined) {
    console.error(`no expected row count recorded for ${id}; add one to EXPECTED_ROWS`);
    process.exit(1);
  }
  const conn = new DatabaseSync(db, { readOnly: true });
  const actual = conn.prepare("SELECT COUNT(*) c FROM student").get().c;
  conn.close();
  if (actual !== expected) {
    console.error(`\n${id}: row count ${actual}, expected ${expected}`);
    console.error("Refusing to publish — the build did not reproduce the known dataset.");
    process.exit(1);
  }
  console.log(`  ✓ ${id}: ${actual} rows (matches expected)`);

  // -9 without -k: the raw .db must not reach the published artifact.
  rmSync(`${db}.gz`, { force: true });
  execFileSync("gzip", ["-9", db], { stdio: "inherit" });

  const gz = `${db}.gz`;
  const sizeMb = statSync(gz).size / 1024 / 1024;
  const nominal = DATASETS.find((d) => d.id === id)?.dbSizeMb;
  if (nominal && sizeMb < nominal * MIN_SIZE_RATIO) {
    console.error(
      `\n${id}: ${sizeMb.toFixed(1)} MB is below ${(nominal * MIN_SIZE_RATIO).toFixed(1)} MB ` +
        `(${MIN_SIZE_RATIO * 100}% of the expected ${nominal} MB)`,
    );
    console.error("Refusing to publish — the artifact looks truncated.");
    process.exit(1);
  }

  console.log(`  → db/${id}.db.gz (${sizeMb.toFixed(1)} MB)\n`);
}