refactor: collapse the two projects into one tree and move to npm

The repo held two near-duplicate projects. 2016/ and 2017/ each carried their own React frontend, their own copy of the same Rust crate, and their own package manager setup. 2016/tools/sync-from-thptqg2017.sh existed purely to copy the parser source between them. New layout: index.html + src/ the 2017 frontend, now the only one data/<id>/ 2016, 2017, 2017-old, 2017-old2 parser/ the single Rust crate, configs renamed to <id>.toml docs/ both projects' docs, 2016 copies suffixed -2016-legacy pending the merge pass <id> is now one identifier end to end: data/<id>/ feeds parser/configs/<id>.toml and produces db/<id>.db.gz. pnpm gives way to npm. pnpm-workspace.yaml existed only to whitelist better-sqlite3's native build, which npm permits by default, so it has no equivalent and is simply gone. Lockfiles cannot be converted; package-lock.json is generated fresh. The migration direction is safe — pnpm's strict layout forbids phantom dependencies, so anything that resolved under pnpm resolves under npm's flat tree. Adds parser/scripts/build-db.js and src/datasets.js: the four dataset IDs are declared once and read by both the build tooling and (from the next phase) the frontend. Follow-on fixes the move made necessary: - eslint's Node-globals override pointed at scripts/, now parser/scripts/ - crawl-baotintuc.js wrote to <root>/data, now data/2017 - golden tests loaded configs by their old thptqg*-data.toml names Drops the #[ignore]d Rust-vs-Node golden test. It shelled out to pnpm to run scripts/build-database.js, a file removed when the parser was ported to Rust, so it could never pass. check-duplicates.js and diff-datasets.js were already broken before this change and are annotated as such rather than half-fixed. 63 Rust tests pass and clippy is clean from the new location.
2026-10-12 18:17:00 +00:00 · 2026-08-13 11:27:01 +07:00
1 parent 82db8295bb
commit 6ff2ed99ec
@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+/**
+ * Build the SQLite database for one or all datasets, then gzip it.
+ *
+ * Replaces the six per-dataset npm scripts the two old projects carried. The
+ * dataset list comes from src/datasets.js so it is written in exactly one place.
+ *
+ * Output goes to .build/public/db/ — the directory Vite copies as its publicDir.
+ * Only the .gz survives: shipping a 100+ MB uncompressed database is made
+ * structurally impossible rather than left to a cleanup step.
+ *
+ * Usage:
+ *   node parser/scripts/build-db.js            # all four datasets
+ *   node parser/scripts/build-db.js 2017-old   # just one
+ */
+
+import { execFileSync } from "node:child_process";
+import { mkdirSync, rmSync, existsSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { DATASET_IDS } from "../../src/datasets.js";
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
+const BIN = resolve(ROOT, "parser/target/release/xlsxread");
+const OUT_DIR = resolve(ROOT, ".build/public/db");
+
+const requested = process.argv.slice(2);
+const unknown = requested.filter((id) => !DATASET_IDS.includes(id));
+if (unknown.length) {
+  console.error(`unknown dataset(s): ${unknown.join(", ")}`);
+  console.error(`known: ${DATASET_IDS.join(", ")}`);
+  process.exit(2);
+}
+const targets = requested.length ? requested : DATASET_IDS;
+
+if (!existsSync(BIN)) {
+  console.error(`parser binary not found at ${BIN}`);
+  console.error("run: npm run build:rust");
+  process.exit(1);
+}
+
+mkdirSync(OUT_DIR, { recursive: true });
+
+for (const id of targets) {
+  const db = resolve(OUT_DIR, `${id}.db`);
+
+  execFileSync(
+    BIN,
+    [
+      "build",
+      "--schema",
+      resolve(ROOT, `parser/configs/${id}.toml`),
+      "--input",
+      resolve(ROOT, `data/${id}`),
+      "--output",
+      db,
+    ],
+    { stdio: "inherit" },
+  );
+
+  // -9 without -k: the raw .db must not reach the published artifact.
+  rmSync(`${db}.gz`, { force: true });
+  execFileSync("gzip", ["-9", db], { stdio: "inherit" });
+  console.log(`  → db/${id}.db.gz\n`);
+}
@@ -0,0 +1,30 @@
+// One-off audit: detect content-identical Excel files via md5.
+//
+// BROKEN as committed — `dirs` below is a hardcoded Windows path from the
+// original author's machine. Predates the repo unification; left unchanged
+// rather than half-fixed. Point `dirs` at data/<dataset> to use it.
+import crypto from "crypto";
+import fs from "fs";
+import path from "path";
+
+const dirs = ["D:/tiennm99/thptqg2017/data"];
+
+const byHash = {};
+for (const d of dirs) {
+  for (const f of fs.readdirSync(d)) {
+    const full = path.join(d, f);
+    if (!fs.statSync(full).isFile() || !f.endsWith(".xlsx")) continue;
+    const h = crypto.createHash("md5").update(fs.readFileSync(full)).digest("hex");
+    (byHash[h] ||= []).push(full);
+  }
+}
+
+const total = Object.values(byHash).reduce((s, a) => s + a.length, 0);
+const dupes = Object.entries(byHash).filter(([, a]) => a.length > 1);
+console.log(`Total files: ${total}`);
+console.log(`Unique by md5: ${Object.keys(byHash).length}`);
+console.log(`Duplicate groups: ${dupes.length}`);
+for (const [h, a] of dupes) {
+  console.log(`  ${h.slice(0, 12)}:`);
+  a.forEach((p) => console.log(`    ${p}`));
+}
@@ -0,0 +1,137 @@
+// Crawl Excel score files for all 63 provinces from baotintuc.vn article:
+// https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm
+// Saves to data/2017/<ascii-kebab-name>.xls (63 files). Idempotent: skips already-downloaded files.
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const OUT_DIR = path.join(__dirname, "..", "..", "data", "2017");
+
+const LINKS = [
+  ["An Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/17/57/Angiang.xls"],
+  ["Bac Lieu", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Baclieu.xls"],
+  ["Ba Ria - Vung Tau", "https://cdnmedia.baotintuc.vn/2017/07/06/08/17/1BaRiaVungTau.xls"],
+  ["Bac Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacGiang.xls"],
+  ["Bac Kan", "https://cdnmedia.baotintuc.vn/2017/07/06/08/27/BacKan.xls"],
+  ["Bac Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacNinh.xls"],
+  ["Ben Tre", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BenTre.xls"],
+  ["Binh Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BinhDuong.xls"],
+  ["Binh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/BinhThuan.xls"],
+  ["Binh Phuoc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/BinhPhuoc.xls"],
+  ["Binh Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Binhdinh.xls"],
+  ["Ca Mau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Camau.xls"],
+  ["Cao Bang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Caobang.xls"],
+  ["Can Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/13/07/Cantho.xls"],
+  ["Da Nang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/03/Danang.xls"],
+  ["Dak Nong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/DakNong.xls"],
+  ["Dak Lak", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Daklak.xls"],
+  ["Dong Nai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/08/dongnai.xls"],
+  ["Dong Thap", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Dongthap.xls"],
+  ["Dien Bien", "https://cdnmedia.baotintuc.vn/2017/07/06/09/08/DienBien.xls"],
+  ["Gia Lai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/09/Gia-Lai.xls"],
+  ["Ha Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/18/Hagiang.xls"],
+  ["Ha Noi", "https://cdnmedia.baotintuc.vn/2017/07/07/08/16/HaNoi.xls"],
+  ["Ha Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/07/Hanam.xls"],
+  ["Ha Tinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaTinh.xls"],
+  ["Hai Phong", "https://cdnmedia.baotintuc.vn/2017/07/06/08/18/23HaiPhong.xls"],
+  ["Hai Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaiDuong.xls"],
+  ["Hau Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Haugiang.xls"],
+  ["Ho Chi Minh", "https://cdnmedia.baotintuc.vn/2017/07/06/08/26/HCM.xls"],
+  ["Hoa Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/06/HoaBinh.xls"],
+  ["Hung Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/08/25/HungYen.xls"],
+  ["Khanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/18/04/Khanhhoa.xls"],
+  ["Kien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/08/28/KienGiang.xls"],
+  ["Kon Tum", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/KonTum.xls"],
+  ["Nam Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/13NamDinh.xls"],
+  ["Nghe An", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/14NgheAn.xls"],
+  ["Ninh Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/37/NinhBinh.xls"],
+  ["Ninh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Ninhthuan.xls"],
+  ["Lao Cai", "https://cdnmedia.baotintuc.vn/2017/07/06/08/52/11LaoCai.xls"],
+  ["Lai Chau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/33/LaiChau.xls"],
+  ["Lang Son", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/Langson.xls"],
+  ["Lam Dong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/00/LamDong.xls"],
+  ["Long An", "https://cdnmedia.baotintuc.vn/2017/07/06/08/53/12LongAn.xls"],
+  ["Quang Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/QuangBinh.xls"],
+  ["Quang Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNam.xls"],
+  ["Quang Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Quangninh.xls"],
+  ["Quang Ngai", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNgai.xls"],
+  ["Quang Tri", "https://cdnmedia.baotintuc.vn/2017/07/06/09/25/QuangTri.xls"],
+  ["Phu Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/PhuTho.xls"],
+  ["Phu Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/PhuYen.xls"],
+  ["Son La", "https://cdnmedia.baotintuc.vn/2017/07/06/18/01/Sonla.xls"],
+  ["Soc Trang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Soctrang.xls"],
+  ["Tay Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/26/TayNinh.xls"],
+  ["Thai Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/ThaiBinh.xls"],
+  ["Thai Nguyen", "https://cdnmedia.baotintuc.vn/2017/07/06/13/35/ThaiNguyen.xls"],
+  ["Thanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thanhoa.xls"],
+  ["Tra Vinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/TraVinh.xls"],
+  ["Thua Thien Hue", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thuathienhue.xls"],
+  ["Tien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/13/11/tiengiang.xls"],
+  ["Tuyen Quang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/39/TuyenQuang.xls"],
+  ["Vinh Phuc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/40/VinhPhuc.xls"],
+  ["Vinh Long", "https://cdnmedia.baotintuc.vn/2017/07/06/13/13/vinhlong.xls"],
+  ["Yen Bai", "https://cdnmedia.baotintuc.vn/2017/07/06/18/07/Yenbai.xls"],
+];
+
+function slug(name) {
+  return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
+}
+
+async function downloadOne(name, url) {
+  const outPath = path.join(OUT_DIR, `${slug(name)}.xls`);
+  if (fs.existsSync(outPath) && fs.statSync(outPath).size > 0) {
+    return { name, url, outPath, status: "skip", size: fs.statSync(outPath).size };
+  }
+  const res = await fetch(url, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
+      Referer:
+        "https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm",
+    },
+  });
+  if (!res.ok) {
+    return { name, url, outPath, status: "fail", httpStatus: res.status };
+  }
+  const buf = Buffer.from(await res.arrayBuffer());
+  fs.writeFileSync(outPath, buf);
+  return { name, url, outPath, status: "ok", size: buf.length };
+}
+
+async function main() {
+  fs.mkdirSync(OUT_DIR, { recursive: true });
+  console.log(`Downloading ${LINKS.length} files to ${OUT_DIR}...`);
+
+  const CONCURRENCY = 6;
+  const results = [];
+  let i = 0;
+  async function worker() {
+    while (i < LINKS.length) {
+      const idx = i++;
+      const [name, url] = LINKS[idx];
+      try {
+        const r = await downloadOne(name, url);
+        results.push(r);
+        const tag = r.status === "ok" ? "✓" : r.status === "skip" ? "·" : "✗";
+        const sz = r.size ? `${(r.size / 1024).toFixed(0)} KB` : "";
+        console.log(`  ${tag} [${idx + 1}/${LINKS.length}] ${name.padEnd(20)} ${sz} ${r.status === "fail" ? "HTTP " + r.httpStatus : ""}`);
+      } catch (err) {
+        console.log(`  ✗ [${idx + 1}/${LINKS.length}] ${name}: ${err.message}`);
+        results.push({ name, url, status: "error", message: err.message });
+      }
+    }
+  }
+  await Promise.all(Array.from({ length: CONCURRENCY }, worker));
+
+  const ok = results.filter((r) => r.status === "ok").length;
+  const skip = results.filter((r) => r.status === "skip").length;
+  const fail = results.filter((r) => r.status === "fail" || r.status === "error");
+  console.log(`\nDone. ok=${ok} skip=${skip} fail=${fail.length}`);
+  if (fail.length) {
+    console.log("Failed:", fail.map((f) => f.name).join(", "));
+    process.exit(1);
+  }
+}
+
+main();
@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+/**
+ * Dump per-dataset statistics from built SQLite databases as JSON.
+ *
+ * Used twice: once against the pre-refactor databases to capture a baseline,
+ * and again after the schema unification. Comparing the two outputs is what
+ * proves no score data was lost or invented.
+ *
+ * Schema-agnostic on purpose — columns come from PRAGMA table_info, so the same
+ * script runs against the old 18/20-column tables and the new 22-column one.
+ *
+ * Usage:
+ *   node db-stats.js <label>=<db-path> [<label>=<db-path> ...] > stats.json
+ */
+
+import { DatabaseSync } from "node:sqlite";
+import { statSync } from "node:fs";
+
+// Deterministic value-level sample: every SBD ending in these digits.
+// Re-running against a rebuilt DB compares the same students.
+const SAMPLE_SUFFIX = "0000";
+
+function collect(dbPath) {
+  const db = new DatabaseSync(dbPath, { readOnly: true });
+
+  const columns = db
+    .prepare("PRAGMA table_info(student)")
+    .all()
+    .map((c) => c.name);
+
+  const rowCount = db.prepare("SELECT COUNT(*) AS c FROM student").get().c;
+
+  // One pass over the table counting non-NULLs for every column at once.
+  const sums = columns
+    .map((c) => `SUM(CASE WHEN "${c}" IS NOT NULL THEN 1 ELSE 0 END) AS "${c}"`)
+    .join(", ");
+  const nonNull = db.prepare(`SELECT ${sums} FROM student`).get();
+
+  const sample = db
+    .prepare(
+      `SELECT * FROM student WHERE so_bao_danh LIKE '%${SAMPLE_SUFFIX}'
+       ORDER BY so_bao_danh`,
+    )
+    .all();
+
+  db.close();
+
+  return {
+    rowCount,
+    columns,
+    nonNull: Object.fromEntries(columns.map((c) => [c, Number(nonNull[c])])),
+    sizeBytes: statSync(dbPath).size,
+    sampleCount: sample.length,
+    // Store the sample keyed by SBD so a later diff can report which student
+    // and which field changed, not just that something did.
+    sample: Object.fromEntries(sample.map((r) => [r.so_bao_danh, r])),
+  };
+}
+
+const args = process.argv.slice(2);
+if (args.length === 0) {
+  console.error("usage: db-stats.js <label>=<db-path> [...]");
+  process.exit(2);
+}
+
+const out = {};
+for (const arg of args) {
+  const idx = arg.indexOf("=");
+  if (idx === -1) {
+    console.error(`bad argument (expected label=path): ${arg}`);
+    process.exit(2);
+  }
+  const label = arg.slice(0, idx);
+  const path = arg.slice(idx + 1);
+  process.stderr.write(`collecting ${label} from ${path}\n`);
+  out[label] = collect(path);
+}
+
+process.stdout.write(JSON.stringify(out, null, 2) + "\n");
@@ -0,0 +1,120 @@
+// Compare two builds of the 2017 database, column by column.
+//
+// BROKEN as committed, for two reasons that both predate the repo unification:
+//   - `better-sqlite3` is imported but declared in no package.json
+//   - the public/ and backup/ paths it reads no longer exist
+//
+// Left unchanged rather than half-fixed. For a working comparison of two
+// builds use db-stats.js + verify-parity.js, which need no dependencies.
+import Database from "better-sqlite3";
+
+const NEW = new Database("public/thptqg2017.db", { readonly: true });
+const OLD = new Database("backup/thptqg2017.old.db", { readonly: true });
+
+const SCORE_COLS = [
+  "toan", "ngu_van", "vat_ly", "hoa_hoc", "sinh_hoc", "khtn",
+  "lich_su", "dia_ly", "gdcd", "khxh",
+  "tieng_anh", "tieng_phap", "tieng_nga", "tieng_trung",
+];
+
+// Old schema may lack some newer columns — detect and project a subset
+const oldCols = new Set(
+  OLD.prepare("PRAGMA table_info(student)").all().map((c) => c.name),
+);
+const commonCols = SCORE_COLS.filter((c) => oldCols.has(c));
+
+const newCount = NEW.prepare("SELECT COUNT(*) c FROM student").get().c;
+const oldCount = OLD.prepare("SELECT COUNT(*) c FROM student").get().c;
+console.log(`\n=== Row counts ===`);
+console.log(`  new: ${newCount}`);
+console.log(`  old: ${oldCount}`);
+console.log(`  diff: ${newCount - oldCount >= 0 ? "+" : ""}${newCount - oldCount}`);
+
+// Build SBD sets (memory: 861k strings ~50 MB, fine)
+const newSbd = new Set(
+  NEW.prepare("SELECT so_bao_danh FROM student").all().map((r) => r.so_bao_danh),
+);
+const oldSbd = new Set(
+  OLD.prepare("SELECT so_bao_danh FROM student").all().map((r) => r.so_bao_danh),
+);
+
+let newOnly = 0, oldOnly = 0, common = 0;
+for (const s of newSbd) (oldSbd.has(s) ? common++ : newOnly++);
+for (const s of oldSbd) if (!newSbd.has(s)) oldOnly++;
+
+console.log(`\n=== SBD membership ===`);
+console.log(`  new-only (added):     ${newOnly}`);
+console.log(`  old-only (removed):   ${oldOnly}`);
+console.log(`  common:               ${common}`);
+
+// Sample 5 new-only SBDs to peek
+const sampleNewOnly = [];
+for (const s of newSbd) {
+  if (!oldSbd.has(s)) sampleNewOnly.push(s);
+  if (sampleNewOnly.length >= 5) break;
+}
+if (sampleNewOnly.length) {
+  const rows = NEW.prepare(
+    `SELECT so_bao_danh, ho_ten FROM student WHERE so_bao_danh IN (${sampleNewOnly.map(() => "?").join(",")})`,
+  ).all(...sampleNewOnly);
+  console.log(`  sample new-only rows:`);
+  rows.forEach((r) => console.log(`    ${r.so_bao_danh}  ${r.ho_ten}`));
+}
+
+// Score comparison on common SBDs
+console.log(`\n=== Score comparison (common SBDs) ===`);
+console.log(`  columns compared: ${commonCols.join(", ")}`);
+
+const newStmt = NEW.prepare(
+  `SELECT so_bao_danh, ${commonCols.join(",")} FROM student`,
+);
+const oldStmt = OLD.prepare(
+  `SELECT so_bao_danh, ${commonCols.join(",")} FROM student`,
+);
+
+// Build old map
+const oldMap = new Map();
+for (const r of oldStmt.iterate()) oldMap.set(r.so_bao_danh, r);
+
+let identical = 0,
+  differ = 0;
+const colChanges = Object.fromEntries(commonCols.map((c) => [c, { changed: 0, newNull: 0, oldNull: 0 }]));
+const sampleDiffs = [];
+
+for (const nr of newStmt.iterate()) {
+  const or = oldMap.get(nr.so_bao_danh);
+  if (!or) continue;
+  let rowDiffers = false;
+  const rowChanges = [];
+  for (const c of commonCols) {
+    const a = nr[c], b = or[c];
+    if (a === b) continue;
+    // treat very-close floats as equal (xlsx may emit 5.5 vs 5.50)
+    if (a !== null && b !== null && Math.abs(a - b) < 1e-9) continue;
+    rowDiffers = true;
+    colChanges[c].changed++;
+    if (a === null) colChanges[c].newNull++;
+    if (b === null) colChanges[c].oldNull++;
+    rowChanges.push(`${c}: ${b} → ${a}`);
+  }
+  if (rowDiffers) {
+    differ++;
+    if (sampleDiffs.length < 5) sampleDiffs.push({ sbd: nr.so_bao_danh, changes: rowChanges });
+  } else {
+    identical++;
+  }
+}
+
+console.log(`  identical:  ${identical}`);
+console.log(`  differing:  ${differ}`);
+console.log(`  per-column changes:`);
+for (const [c, info] of Object.entries(colChanges)) {
+  if (info.changed === 0) continue;
+  console.log(`    ${c.padEnd(14)} changed=${info.changed}  (new-null=${info.newNull}, old-null=${info.oldNull})`);
+}
+if (sampleDiffs.length) {
+  console.log(`  sample diffs:`);
+  sampleDiffs.forEach((d) => {
+    console.log(`    SBD ${d.sbd}: ${d.changes.slice(0, 4).join("; ")}${d.changes.length > 4 ? " ..." : ""}`);
+  });
+}
@@ -0,0 +1,157 @@
+#!/usr/bin/env node
+/**
+ * Compare rebuilt databases against the pre-refactor parity baseline.
+ *
+ * The schema deliberately changed shape, so a whole-file hash is meaningless.
+ * What must hold instead:
+ *
+ *   1. Row count per dataset is unchanged.
+ *   2. Every column that existed before has the same non-NULL count.
+ *   3. Every column newly added to a dataset has a non-NULL count of exactly 0.
+ *      This is the check that catches the union-regex risk — if the 16-pattern
+ *      map starts matching text the narrower per-year map ignored, it shows up
+ *      here rather than silently corrupting the dataset.
+ *   4. A deterministic sample of students is identical field by field.
+ *
+ * Exits non-zero on any mismatch.
+ *
+ * Usage:
+ *   node verify-parity.js <baseline.json> <current.json>
+ */
+
+import { readFileSync } from "node:fs";
+
+/**
+ * Foreign-language scores the pre-refactor configs silently discarded.
+ *
+ * The 2016 config listed 12 subject regexes and the 2017 configs listed 14;
+ * neither list was complete. Candidates could sit German, Japanese and Russian
+ * in both exam years, so every one of these students previously ended up with
+ * no foreign-language score at all.
+ *
+ * Unifying to the canonical 16 patterns recovers them. Verified real, not
+ * spurious matches: across all four datasets every student holds either zero
+ * or exactly one foreign language — never two — and each affected student had
+ * all language columns NULL beforehand.
+ *
+ * These exact counts are approved. Any other newly-populated column, or any
+ * drift in these numbers, still fails the gate.
+ */
+const APPROVED_RECOVERY = {
+  "2016": { tieng_nga: 182 },
+  "2017": { tieng_duc: 93, tieng_nhat: 512 },
+  "2017-old": { tieng_duc: 85, tieng_nhat: 484 },
+  "2017-old2": { tieng_duc: 22, tieng_nhat: 313 },
+};
+
+const [baselinePath, currentPath] = process.argv.slice(2);
+if (!baselinePath || !currentPath) {
+  console.error("usage: verify-parity.js <baseline.json> <current.json>");
+  process.exit(2);
+}
+
+const baseline = JSON.parse(readFileSync(baselinePath, "utf8"));
+const current = JSON.parse(readFileSync(currentPath, "utf8"));
+
+const failures = [];
+const notes = [];
+
+for (const dataset of Object.keys(baseline)) {
+  const b = baseline[dataset];
+  const c = current[dataset];
+
+  if (!c) {
+    failures.push(`${dataset}: missing from current stats`);
+    continue;
+  }
+
+  // 1. Row count
+  if (b.rowCount !== c.rowCount) {
+    failures.push(
+      `${dataset}: row count ${b.rowCount} → ${c.rowCount} (${c.rowCount - b.rowCount >= 0 ? "+" : ""}${c.rowCount - b.rowCount})`,
+    );
+  }
+
+  // 2. Pre-existing columns keep their non-NULL counts
+  for (const col of b.columns) {
+    if (!(col in c.nonNull)) {
+      failures.push(`${dataset}.${col}: column disappeared from schema`);
+      continue;
+    }
+    if (b.nonNull[col] !== c.nonNull[col]) {
+      failures.push(
+        `${dataset}.${col}: non-NULL ${b.nonNull[col]} → ${c.nonNull[col]} (${c.nonNull[col] - b.nonNull[col] >= 0 ? "+" : ""}${c.nonNull[col] - b.nonNull[col]})`,
+      );
+    }
+  }
+
+  // 3. Newly added columns must be NULL — except the approved recoveries,
+  //    which must match their approved count exactly.
+  const approved = APPROVED_RECOVERY[dataset] ?? {};
+  const added = c.columns.filter((col) => !b.columns.includes(col));
+  const recovered = [];
+  for (const col of added) {
+    const expected = approved[col] ?? 0;
+    if (c.nonNull[col] !== expected) {
+      failures.push(
+        expected === 0
+          ? `${dataset}.${col}: new column has ${c.nonNull[col]} non-NULL values, expected 0`
+          : `${dataset}.${col}: recovered ${c.nonNull[col]} values, approved count is ${expected}`,
+      );
+    } else if (expected > 0) {
+      recovered.push(`${col}=${expected}`);
+    }
+  }
+  // An approved recovery that vanished means the union patterns regressed.
+  for (const [col, expected] of Object.entries(approved)) {
+    if (!added.includes(col)) {
+      failures.push(
+        `${dataset}.${col}: expected ${expected} recovered values but column is not new`,
+      );
+    }
+  }
+  if (added.length) {
+    const nulls = added.length - recovered.length;
+    notes.push(
+      `${dataset}: +${added.length} new columns (${nulls} all-NULL as expected` +
+        (recovered.length ? `, recovered ${recovered.join(", ")}` : "") +
+        ")",
+    );
+  }
+
+  // 4. Deterministic sample compared field by field
+  if (b.sampleCount !== c.sampleCount) {
+    failures.push(
+      `${dataset}: sample size ${b.sampleCount} → ${c.sampleCount}`,
+    );
+  }
+  for (const [sbd, bRow] of Object.entries(b.sample)) {
+    const cRow = c.sample[sbd];
+    if (!cRow) {
+      failures.push(`${dataset}: sampled student ${sbd} missing after rebuild`);
+      continue;
+    }
+    for (const [field, bVal] of Object.entries(bRow)) {
+      if (cRow[field] !== bVal) {
+        failures.push(
+          `${dataset}: student ${sbd} field ${field}: ${JSON.stringify(bVal)} → ${JSON.stringify(cRow[field])}`,
+        );
+      }
+    }
+  }
+
+  const sizeDelta = ((c.sizeBytes - b.sizeBytes) / b.sizeBytes) * 100;
+  notes.push(
+    `${dataset}: ${c.rowCount} rows, ${b.columns.length} → ${c.columns.length} cols, size ${sizeDelta >= 0 ? "+" : ""}${sizeDelta.toFixed(1)}%`,
+  );
+}
+
+for (const n of notes) console.log(`  ${n}`);
+
+if (failures.length) {
+  console.error(`\nPARITY FAILED — ${failures.length} mismatch(es):\n`);
+  for (const f of failures) console.error(`  ✗ ${f}`);
+  process.exit(1);
+}
+
+console.log("\nPARITY OK — row counts, per-column non-NULL counts and samples all match.");