mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-13 12:15:09 +00:00
refactor: collapse the two projects into one tree and move to npm
The repo held two near-duplicate projects. 2016/ and 2017/ each carried their
own React frontend, their own copy of the same Rust crate, and their own
package manager setup. 2016/tools/sync-from-thptqg2017.sh existed purely to
copy the parser source between them.
New layout:
index.html + src/ the 2017 frontend, now the only one
data/<id>/ 2016, 2017, 2017-old, 2017-old2
parser/ the single Rust crate, configs renamed to <id>.toml
docs/ both projects' docs, 2016 copies suffixed -2016-legacy
pending the merge pass
<id> is now one identifier end to end: data/<id>/ feeds parser/configs/<id>.toml
and produces db/<id>.db.gz.
pnpm gives way to npm. pnpm-workspace.yaml existed only to whitelist
better-sqlite3's native build, which npm permits by default, so it has no
equivalent and is simply gone. Lockfiles cannot be converted; package-lock.json
is generated fresh. The migration direction is safe — pnpm's strict layout
forbids phantom dependencies, so anything that resolved under pnpm resolves
under npm's flat tree.
Adds parser/scripts/build-db.js and src/datasets.js: the four dataset IDs are
declared once and read by both the build tooling and (from the next phase) the
frontend.
Follow-on fixes the move made necessary:
- eslint's Node-globals override pointed at scripts/, now parser/scripts/
- crawl-baotintuc.js wrote to <root>/data, now data/2017
- golden tests loaded configs by their old thptqg*-data.toml names
Drops the #[ignore]d Rust-vs-Node golden test. It shelled out to pnpm to run
scripts/build-database.js, a file removed when the parser was ported to Rust,
so it could never pass. check-duplicates.js and diff-datasets.js were already
broken before this change and are annotated as such rather than half-fixed.
63 Rust tests pass and clippy is clean from the new location.
This commit is contained in:
tiennm99 committed
1 parent
82db8295bb
commit
6ff2ed99ec
396 files changed
+2753
-8720
No files matched your search
Generated
+1159
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,30 @@
|
||||
[package]
|
||||
name = "xlsxread"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Rust CLI replacing SheetJS xlsx build scripts for thptqg2017/thptqg2016"
|
||||
|
||||
[dependencies]
|
||||
calamine = "0.26"
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
toml = "0.8"
|
||||
regex = "1"
|
||||
unicode-normalization = "0.1"
|
||||
thiserror = "1"
|
||||
anyhow = "1"
|
||||
glob = "0.3"
|
||||
|
||||
# zip is already a transitive dep of calamine; pin explicitly so tests can use it
|
||||
[dev-dependencies]
|
||||
zip = "2"
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
|
||||
[[bin]]
|
||||
name = "xlsxread"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[test]]
|
||||
name = "golden"
|
||||
path = "tests/golden.rs"
|
||||
@@ -0,0 +1,32 @@
|
||||
# thptqg2016 data/ — 4 .xls + 115 .xlsx mixed files.
|
||||
#
|
||||
# Three column layouts exist across the 119 files; the binary selects the
|
||||
# right one per-file at runtime via format_detection = "thptqg2016":
|
||||
#
|
||||
# separate-scores header SBD(0)/HOTEN(1)/TOAN(2)... — dhhanghai files
|
||||
# mapped header SOBAODANH|SBD + DIEM_THI — most provinces
|
||||
# default no header; positional 6-col layout — remaining files
|
||||
#
|
||||
# This is the only dataset that populates ten_cum_thi and gioi_tinh, and the
|
||||
# only one whose files carry Tiếng Đức / Tiếng Nhật scores.
|
||||
#
|
||||
# sheet_mode = "all": several provinces overflow into Sheet2 (65k Excel row cap).
|
||||
# strip_blank_rows = false: no blank-row anomaly observed in this dataset.
|
||||
#
|
||||
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
|
||||
|
||||
format_detection = "thptqg2016"
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
# Tokens that identify a header row by first-cell content (uppercased).
|
||||
# Covers both SOBAODANH-style and SBD-style headers.
|
||||
tokens = ["SOBAODANH", "SBD", "HO_TEN", "HOTEN", "HỌ TÊN", "STT"]
|
||||
@@ -0,0 +1,25 @@
|
||||
# thptqg2017 data-old/ — 63 .xlsx files (pre-baotintuc refresh).
|
||||
#
|
||||
# sheet_mode = "first": single-sheet workbooks, never hit the 65k row cap.
|
||||
# SBD validation: require ^\d+$ (build-database-old.js:55 guard).
|
||||
# strip_blank_rows = false: no explicit blank-skip in build-database-old.js.
|
||||
#
|
||||
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
|
||||
|
||||
[reader]
|
||||
sheet_mode = "first"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = true
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
@@ -0,0 +1,26 @@
|
||||
# thptqg2017 data-old2/ — 54 .xlsx files (corrected-export set).
|
||||
#
|
||||
# sheet_mode = "all": 24.HCM_UTLQ.xlsx overflows into Sheet2 (+6,446 rows).
|
||||
# SBD validation: require ^\d+$ (build-database-old2.js:57 guard).
|
||||
# strip_blank_rows = true: skip fully blank rows BEFORE counting sourceRows
|
||||
# (build-database-old2.js:50-51 checks blank before sourceRows++).
|
||||
#
|
||||
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = true
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = true
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
@@ -0,0 +1,25 @@
|
||||
# thptqg2017 data/ — 63 .xls files from baotintuc.vn (current generation).
|
||||
#
|
||||
# sheet_mode = "all": Hà Nội and HCM overflow into Sheet2 (65k Excel row cap).
|
||||
# SBD validation: no numeric guard — build-database.js did not apply ^\d+$.
|
||||
# strip_blank_rows = false: no blank-row anomaly in this dataset.
|
||||
#
|
||||
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build the SQLite database for one or all datasets, then gzip it.
|
||||
*
|
||||
* Replaces the six per-dataset npm scripts the two old projects carried. The
|
||||
* dataset list comes from src/datasets.js so it is written in exactly one place.
|
||||
*
|
||||
* Output goes to .build/public/db/ — the directory Vite copies as its publicDir.
|
||||
* Only the .gz survives: shipping a 100+ MB uncompressed database is made
|
||||
* structurally impossible rather than left to a cleanup step.
|
||||
*
|
||||
* Usage:
|
||||
* node parser/scripts/build-db.js # all four datasets
|
||||
* node parser/scripts/build-db.js 2017-old # just one
|
||||
*/
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, rmSync, existsSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { DATASET_IDS } from "../../src/datasets.js";
|
||||
|
||||
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const BIN = resolve(ROOT, "parser/target/release/xlsxread");
|
||||
const OUT_DIR = resolve(ROOT, ".build/public/db");
|
||||
|
||||
const requested = process.argv.slice(2);
|
||||
const unknown = requested.filter((id) => !DATASET_IDS.includes(id));
|
||||
if (unknown.length) {
|
||||
console.error(`unknown dataset(s): ${unknown.join(", ")}`);
|
||||
console.error(`known: ${DATASET_IDS.join(", ")}`);
|
||||
process.exit(2);
|
||||
}
|
||||
const targets = requested.length ? requested : DATASET_IDS;
|
||||
|
||||
if (!existsSync(BIN)) {
|
||||
console.error(`parser binary not found at ${BIN}`);
|
||||
console.error("run: npm run build:rust");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
mkdirSync(OUT_DIR, { recursive: true });
|
||||
|
||||
for (const id of targets) {
|
||||
const db = resolve(OUT_DIR, `${id}.db`);
|
||||
|
||||
execFileSync(
|
||||
BIN,
|
||||
[
|
||||
"build",
|
||||
"--schema",
|
||||
resolve(ROOT, `parser/configs/${id}.toml`),
|
||||
"--input",
|
||||
resolve(ROOT, `data/${id}`),
|
||||
"--output",
|
||||
db,
|
||||
],
|
||||
{ stdio: "inherit" },
|
||||
);
|
||||
|
||||
// -9 without -k: the raw .db must not reach the published artifact.
|
||||
rmSync(`${db}.gz`, { force: true });
|
||||
execFileSync("gzip", ["-9", db], { stdio: "inherit" });
|
||||
console.log(` → db/${id}.db.gz\n`);
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
// One-off audit: detect content-identical Excel files via md5.
|
||||
//
|
||||
// BROKEN as committed — `dirs` below is a hardcoded Windows path from the
|
||||
// original author's machine. Predates the repo unification; left unchanged
|
||||
// rather than half-fixed. Point `dirs` at data/<dataset> to use it.
|
||||
import crypto from "crypto";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
const dirs = ["D:/tiennm99/thptqg2017/data"];
|
||||
|
||||
const byHash = {};
|
||||
for (const d of dirs) {
|
||||
for (const f of fs.readdirSync(d)) {
|
||||
const full = path.join(d, f);
|
||||
if (!fs.statSync(full).isFile() || !f.endsWith(".xlsx")) continue;
|
||||
const h = crypto.createHash("md5").update(fs.readFileSync(full)).digest("hex");
|
||||
(byHash[h] ||= []).push(full);
|
||||
}
|
||||
}
|
||||
|
||||
const total = Object.values(byHash).reduce((s, a) => s + a.length, 0);
|
||||
const dupes = Object.entries(byHash).filter(([, a]) => a.length > 1);
|
||||
console.log(`Total files: ${total}`);
|
||||
console.log(`Unique by md5: ${Object.keys(byHash).length}`);
|
||||
console.log(`Duplicate groups: ${dupes.length}`);
|
||||
for (const [h, a] of dupes) {
|
||||
console.log(` ${h.slice(0, 12)}:`);
|
||||
a.forEach((p) => console.log(` ${p}`));
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
// Crawl Excel score files for all 63 provinces from baotintuc.vn article:
|
||||
// https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm
|
||||
// Saves to data/2017/<ascii-kebab-name>.xls (63 files). Idempotent: skips already-downloaded files.
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const OUT_DIR = path.join(__dirname, "..", "..", "data", "2017");
|
||||
|
||||
const LINKS = [
|
||||
["An Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/17/57/Angiang.xls"],
|
||||
["Bac Lieu", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Baclieu.xls"],
|
||||
["Ba Ria - Vung Tau", "https://cdnmedia.baotintuc.vn/2017/07/06/08/17/1BaRiaVungTau.xls"],
|
||||
["Bac Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacGiang.xls"],
|
||||
["Bac Kan", "https://cdnmedia.baotintuc.vn/2017/07/06/08/27/BacKan.xls"],
|
||||
["Bac Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacNinh.xls"],
|
||||
["Ben Tre", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BenTre.xls"],
|
||||
["Binh Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BinhDuong.xls"],
|
||||
["Binh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/BinhThuan.xls"],
|
||||
["Binh Phuoc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/BinhPhuoc.xls"],
|
||||
["Binh Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Binhdinh.xls"],
|
||||
["Ca Mau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Camau.xls"],
|
||||
["Cao Bang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Caobang.xls"],
|
||||
["Can Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/13/07/Cantho.xls"],
|
||||
["Da Nang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/03/Danang.xls"],
|
||||
["Dak Nong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/DakNong.xls"],
|
||||
["Dak Lak", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Daklak.xls"],
|
||||
["Dong Nai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/08/dongnai.xls"],
|
||||
["Dong Thap", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Dongthap.xls"],
|
||||
["Dien Bien", "https://cdnmedia.baotintuc.vn/2017/07/06/09/08/DienBien.xls"],
|
||||
["Gia Lai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/09/Gia-Lai.xls"],
|
||||
["Ha Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/18/Hagiang.xls"],
|
||||
["Ha Noi", "https://cdnmedia.baotintuc.vn/2017/07/07/08/16/HaNoi.xls"],
|
||||
["Ha Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/07/Hanam.xls"],
|
||||
["Ha Tinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaTinh.xls"],
|
||||
["Hai Phong", "https://cdnmedia.baotintuc.vn/2017/07/06/08/18/23HaiPhong.xls"],
|
||||
["Hai Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaiDuong.xls"],
|
||||
["Hau Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Haugiang.xls"],
|
||||
["Ho Chi Minh", "https://cdnmedia.baotintuc.vn/2017/07/06/08/26/HCM.xls"],
|
||||
["Hoa Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/06/HoaBinh.xls"],
|
||||
["Hung Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/08/25/HungYen.xls"],
|
||||
["Khanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/18/04/Khanhhoa.xls"],
|
||||
["Kien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/08/28/KienGiang.xls"],
|
||||
["Kon Tum", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/KonTum.xls"],
|
||||
["Nam Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/13NamDinh.xls"],
|
||||
["Nghe An", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/14NgheAn.xls"],
|
||||
["Ninh Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/37/NinhBinh.xls"],
|
||||
["Ninh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Ninhthuan.xls"],
|
||||
["Lao Cai", "https://cdnmedia.baotintuc.vn/2017/07/06/08/52/11LaoCai.xls"],
|
||||
["Lai Chau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/33/LaiChau.xls"],
|
||||
["Lang Son", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/Langson.xls"],
|
||||
["Lam Dong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/00/LamDong.xls"],
|
||||
["Long An", "https://cdnmedia.baotintuc.vn/2017/07/06/08/53/12LongAn.xls"],
|
||||
["Quang Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/QuangBinh.xls"],
|
||||
["Quang Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNam.xls"],
|
||||
["Quang Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Quangninh.xls"],
|
||||
["Quang Ngai", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNgai.xls"],
|
||||
["Quang Tri", "https://cdnmedia.baotintuc.vn/2017/07/06/09/25/QuangTri.xls"],
|
||||
["Phu Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/PhuTho.xls"],
|
||||
["Phu Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/PhuYen.xls"],
|
||||
["Son La", "https://cdnmedia.baotintuc.vn/2017/07/06/18/01/Sonla.xls"],
|
||||
["Soc Trang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Soctrang.xls"],
|
||||
["Tay Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/26/TayNinh.xls"],
|
||||
["Thai Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/ThaiBinh.xls"],
|
||||
["Thai Nguyen", "https://cdnmedia.baotintuc.vn/2017/07/06/13/35/ThaiNguyen.xls"],
|
||||
["Thanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thanhoa.xls"],
|
||||
["Tra Vinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/TraVinh.xls"],
|
||||
["Thua Thien Hue", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thuathienhue.xls"],
|
||||
["Tien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/13/11/tiengiang.xls"],
|
||||
["Tuyen Quang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/39/TuyenQuang.xls"],
|
||||
["Vinh Phuc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/40/VinhPhuc.xls"],
|
||||
["Vinh Long", "https://cdnmedia.baotintuc.vn/2017/07/06/13/13/vinhlong.xls"],
|
||||
["Yen Bai", "https://cdnmedia.baotintuc.vn/2017/07/06/18/07/Yenbai.xls"],
|
||||
];
|
||||
|
||||
function slug(name) {
|
||||
return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
|
||||
}
|
||||
|
||||
async function downloadOne(name, url) {
|
||||
const outPath = path.join(OUT_DIR, `${slug(name)}.xls`);
|
||||
if (fs.existsSync(outPath) && fs.statSync(outPath).size > 0) {
|
||||
return { name, url, outPath, status: "skip", size: fs.statSync(outPath).size };
|
||||
}
|
||||
const res = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
|
||||
Referer:
|
||||
"https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm",
|
||||
},
|
||||
});
|
||||
if (!res.ok) {
|
||||
return { name, url, outPath, status: "fail", httpStatus: res.status };
|
||||
}
|
||||
const buf = Buffer.from(await res.arrayBuffer());
|
||||
fs.writeFileSync(outPath, buf);
|
||||
return { name, url, outPath, status: "ok", size: buf.length };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
console.log(`Downloading ${LINKS.length} files to ${OUT_DIR}...`);
|
||||
|
||||
const CONCURRENCY = 6;
|
||||
const results = [];
|
||||
let i = 0;
|
||||
async function worker() {
|
||||
while (i < LINKS.length) {
|
||||
const idx = i++;
|
||||
const [name, url] = LINKS[idx];
|
||||
try {
|
||||
const r = await downloadOne(name, url);
|
||||
results.push(r);
|
||||
const tag = r.status === "ok" ? "✓" : r.status === "skip" ? "·" : "✗";
|
||||
const sz = r.size ? `${(r.size / 1024).toFixed(0)} KB` : "";
|
||||
console.log(` ${tag} [${idx + 1}/${LINKS.length}] ${name.padEnd(20)} ${sz} ${r.status === "fail" ? "HTTP " + r.httpStatus : ""}`);
|
||||
} catch (err) {
|
||||
console.log(` ✗ [${idx + 1}/${LINKS.length}] ${name}: ${err.message}`);
|
||||
results.push({ name, url, status: "error", message: err.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: CONCURRENCY }, worker));
|
||||
|
||||
const ok = results.filter((r) => r.status === "ok").length;
|
||||
const skip = results.filter((r) => r.status === "skip").length;
|
||||
const fail = results.filter((r) => r.status === "fail" || r.status === "error");
|
||||
console.log(`\nDone. ok=${ok} skip=${skip} fail=${fail.length}`);
|
||||
if (fail.length) {
|
||||
console.log("Failed:", fail.map((f) => f.name).join(", "));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Dump per-dataset statistics from built SQLite databases as JSON.
|
||||
*
|
||||
* Used twice: once against the pre-refactor databases to capture a baseline,
|
||||
* and again after the schema unification. Comparing the two outputs is what
|
||||
* proves no score data was lost or invented.
|
||||
*
|
||||
* Schema-agnostic on purpose — columns come from PRAGMA table_info, so the same
|
||||
* script runs against the old 18/20-column tables and the new 22-column one.
|
||||
*
|
||||
* Usage:
|
||||
* node db-stats.js <label>=<db-path> [<label>=<db-path> ...] > stats.json
|
||||
*/
|
||||
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { statSync } from "node:fs";
|
||||
|
||||
// Deterministic value-level sample: every SBD ending in these digits.
|
||||
// Re-running against a rebuilt DB compares the same students.
|
||||
const SAMPLE_SUFFIX = "0000";
|
||||
|
||||
function collect(dbPath) {
|
||||
const db = new DatabaseSync(dbPath, { readOnly: true });
|
||||
|
||||
const columns = db
|
||||
.prepare("PRAGMA table_info(student)")
|
||||
.all()
|
||||
.map((c) => c.name);
|
||||
|
||||
const rowCount = db.prepare("SELECT COUNT(*) AS c FROM student").get().c;
|
||||
|
||||
// One pass over the table counting non-NULLs for every column at once.
|
||||
const sums = columns
|
||||
.map((c) => `SUM(CASE WHEN "${c}" IS NOT NULL THEN 1 ELSE 0 END) AS "${c}"`)
|
||||
.join(", ");
|
||||
const nonNull = db.prepare(`SELECT ${sums} FROM student`).get();
|
||||
|
||||
const sample = db
|
||||
.prepare(
|
||||
`SELECT * FROM student WHERE so_bao_danh LIKE '%${SAMPLE_SUFFIX}'
|
||||
ORDER BY so_bao_danh`,
|
||||
)
|
||||
.all();
|
||||
|
||||
db.close();
|
||||
|
||||
return {
|
||||
rowCount,
|
||||
columns,
|
||||
nonNull: Object.fromEntries(columns.map((c) => [c, Number(nonNull[c])])),
|
||||
sizeBytes: statSync(dbPath).size,
|
||||
sampleCount: sample.length,
|
||||
// Store the sample keyed by SBD so a later diff can report which student
|
||||
// and which field changed, not just that something did.
|
||||
sample: Object.fromEntries(sample.map((r) => [r.so_bao_danh, r])),
|
||||
};
|
||||
}
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
if (args.length === 0) {
|
||||
console.error("usage: db-stats.js <label>=<db-path> [...]");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const out = {};
|
||||
for (const arg of args) {
|
||||
const idx = arg.indexOf("=");
|
||||
if (idx === -1) {
|
||||
console.error(`bad argument (expected label=path): ${arg}`);
|
||||
process.exit(2);
|
||||
}
|
||||
const label = arg.slice(0, idx);
|
||||
const path = arg.slice(idx + 1);
|
||||
process.stderr.write(`collecting ${label} from ${path}\n`);
|
||||
out[label] = collect(path);
|
||||
}
|
||||
|
||||
process.stdout.write(JSON.stringify(out, null, 2) + "\n");
|
||||
@@ -0,0 +1,120 @@
|
||||
// Compare two builds of the 2017 database, column by column.
|
||||
//
|
||||
// BROKEN as committed, for two reasons that both predate the repo unification:
|
||||
// - `better-sqlite3` is imported but declared in no package.json
|
||||
// - the public/ and backup/ paths it reads no longer exist
|
||||
//
|
||||
// Left unchanged rather than half-fixed. For a working comparison of two
|
||||
// builds use db-stats.js + verify-parity.js, which need no dependencies.
|
||||
import Database from "better-sqlite3";
|
||||
|
||||
const NEW = new Database("public/thptqg2017.db", { readonly: true });
|
||||
const OLD = new Database("backup/thptqg2017.old.db", { readonly: true });
|
||||
|
||||
const SCORE_COLS = [
|
||||
"toan", "ngu_van", "vat_ly", "hoa_hoc", "sinh_hoc", "khtn",
|
||||
"lich_su", "dia_ly", "gdcd", "khxh",
|
||||
"tieng_anh", "tieng_phap", "tieng_nga", "tieng_trung",
|
||||
];
|
||||
|
||||
// Old schema may lack some newer columns — detect and project a subset
|
||||
const oldCols = new Set(
|
||||
OLD.prepare("PRAGMA table_info(student)").all().map((c) => c.name),
|
||||
);
|
||||
const commonCols = SCORE_COLS.filter((c) => oldCols.has(c));
|
||||
|
||||
const newCount = NEW.prepare("SELECT COUNT(*) c FROM student").get().c;
|
||||
const oldCount = OLD.prepare("SELECT COUNT(*) c FROM student").get().c;
|
||||
console.log(`\n=== Row counts ===`);
|
||||
console.log(` new: ${newCount}`);
|
||||
console.log(` old: ${oldCount}`);
|
||||
console.log(` diff: ${newCount - oldCount >= 0 ? "+" : ""}${newCount - oldCount}`);
|
||||
|
||||
// Build SBD sets (memory: 861k strings ~50 MB, fine)
|
||||
const newSbd = new Set(
|
||||
NEW.prepare("SELECT so_bao_danh FROM student").all().map((r) => r.so_bao_danh),
|
||||
);
|
||||
const oldSbd = new Set(
|
||||
OLD.prepare("SELECT so_bao_danh FROM student").all().map((r) => r.so_bao_danh),
|
||||
);
|
||||
|
||||
let newOnly = 0, oldOnly = 0, common = 0;
|
||||
for (const s of newSbd) (oldSbd.has(s) ? common++ : newOnly++);
|
||||
for (const s of oldSbd) if (!newSbd.has(s)) oldOnly++;
|
||||
|
||||
console.log(`\n=== SBD membership ===`);
|
||||
console.log(` new-only (added): ${newOnly}`);
|
||||
console.log(` old-only (removed): ${oldOnly}`);
|
||||
console.log(` common: ${common}`);
|
||||
|
||||
// Sample 5 new-only SBDs to peek
|
||||
const sampleNewOnly = [];
|
||||
for (const s of newSbd) {
|
||||
if (!oldSbd.has(s)) sampleNewOnly.push(s);
|
||||
if (sampleNewOnly.length >= 5) break;
|
||||
}
|
||||
if (sampleNewOnly.length) {
|
||||
const rows = NEW.prepare(
|
||||
`SELECT so_bao_danh, ho_ten FROM student WHERE so_bao_danh IN (${sampleNewOnly.map(() => "?").join(",")})`,
|
||||
).all(...sampleNewOnly);
|
||||
console.log(` sample new-only rows:`);
|
||||
rows.forEach((r) => console.log(` ${r.so_bao_danh} ${r.ho_ten}`));
|
||||
}
|
||||
|
||||
// Score comparison on common SBDs
|
||||
console.log(`\n=== Score comparison (common SBDs) ===`);
|
||||
console.log(` columns compared: ${commonCols.join(", ")}`);
|
||||
|
||||
const newStmt = NEW.prepare(
|
||||
`SELECT so_bao_danh, ${commonCols.join(",")} FROM student`,
|
||||
);
|
||||
const oldStmt = OLD.prepare(
|
||||
`SELECT so_bao_danh, ${commonCols.join(",")} FROM student`,
|
||||
);
|
||||
|
||||
// Build old map
|
||||
const oldMap = new Map();
|
||||
for (const r of oldStmt.iterate()) oldMap.set(r.so_bao_danh, r);
|
||||
|
||||
let identical = 0,
|
||||
differ = 0;
|
||||
const colChanges = Object.fromEntries(commonCols.map((c) => [c, { changed: 0, newNull: 0, oldNull: 0 }]));
|
||||
const sampleDiffs = [];
|
||||
|
||||
for (const nr of newStmt.iterate()) {
|
||||
const or = oldMap.get(nr.so_bao_danh);
|
||||
if (!or) continue;
|
||||
let rowDiffers = false;
|
||||
const rowChanges = [];
|
||||
for (const c of commonCols) {
|
||||
const a = nr[c], b = or[c];
|
||||
if (a === b) continue;
|
||||
// treat very-close floats as equal (xlsx may emit 5.5 vs 5.50)
|
||||
if (a !== null && b !== null && Math.abs(a - b) < 1e-9) continue;
|
||||
rowDiffers = true;
|
||||
colChanges[c].changed++;
|
||||
if (a === null) colChanges[c].newNull++;
|
||||
if (b === null) colChanges[c].oldNull++;
|
||||
rowChanges.push(`${c}: ${b} → ${a}`);
|
||||
}
|
||||
if (rowDiffers) {
|
||||
differ++;
|
||||
if (sampleDiffs.length < 5) sampleDiffs.push({ sbd: nr.so_bao_danh, changes: rowChanges });
|
||||
} else {
|
||||
identical++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` identical: ${identical}`);
|
||||
console.log(` differing: ${differ}`);
|
||||
console.log(` per-column changes:`);
|
||||
for (const [c, info] of Object.entries(colChanges)) {
|
||||
if (info.changed === 0) continue;
|
||||
console.log(` ${c.padEnd(14)} changed=${info.changed} (new-null=${info.newNull}, old-null=${info.oldNull})`);
|
||||
}
|
||||
if (sampleDiffs.length) {
|
||||
console.log(` sample diffs:`);
|
||||
sampleDiffs.forEach((d) => {
|
||||
console.log(` SBD ${d.sbd}: ${d.changes.slice(0, 4).join("; ")}${d.changes.length > 4 ? " ..." : ""}`);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Compare rebuilt databases against the pre-refactor parity baseline.
|
||||
*
|
||||
* The schema deliberately changed shape, so a whole-file hash is meaningless.
|
||||
* What must hold instead:
|
||||
*
|
||||
* 1. Row count per dataset is unchanged.
|
||||
* 2. Every column that existed before has the same non-NULL count.
|
||||
* 3. Every column newly added to a dataset has a non-NULL count of exactly 0.
|
||||
* This is the check that catches the union-regex risk — if the 16-pattern
|
||||
* map starts matching text the narrower per-year map ignored, it shows up
|
||||
* here rather than silently corrupting the dataset.
|
||||
* 4. A deterministic sample of students is identical field by field.
|
||||
*
|
||||
* Exits non-zero on any mismatch.
|
||||
*
|
||||
* Usage:
|
||||
* node verify-parity.js <baseline.json> <current.json>
|
||||
*/
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
/**
|
||||
* Foreign-language scores the pre-refactor configs silently discarded.
|
||||
*
|
||||
* The 2016 config listed 12 subject regexes and the 2017 configs listed 14;
|
||||
* neither list was complete. Candidates could sit German, Japanese and Russian
|
||||
* in both exam years, so every one of these students previously ended up with
|
||||
* no foreign-language score at all.
|
||||
*
|
||||
* Unifying to the canonical 16 patterns recovers them. Verified real, not
|
||||
* spurious matches: across all four datasets every student holds either zero
|
||||
* or exactly one foreign language — never two — and each affected student had
|
||||
* all language columns NULL beforehand.
|
||||
*
|
||||
* These exact counts are approved. Any other newly-populated column, or any
|
||||
* drift in these numbers, still fails the gate.
|
||||
*/
|
||||
const APPROVED_RECOVERY = {
|
||||
"2016": { tieng_nga: 182 },
|
||||
"2017": { tieng_duc: 93, tieng_nhat: 512 },
|
||||
"2017-old": { tieng_duc: 85, tieng_nhat: 484 },
|
||||
"2017-old2": { tieng_duc: 22, tieng_nhat: 313 },
|
||||
};
|
||||
|
||||
const [baselinePath, currentPath] = process.argv.slice(2);
|
||||
if (!baselinePath || !currentPath) {
|
||||
console.error("usage: verify-parity.js <baseline.json> <current.json>");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const baseline = JSON.parse(readFileSync(baselinePath, "utf8"));
|
||||
const current = JSON.parse(readFileSync(currentPath, "utf8"));
|
||||
|
||||
const failures = [];
|
||||
const notes = [];
|
||||
|
||||
for (const dataset of Object.keys(baseline)) {
|
||||
const b = baseline[dataset];
|
||||
const c = current[dataset];
|
||||
|
||||
if (!c) {
|
||||
failures.push(`${dataset}: missing from current stats`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 1. Row count
|
||||
if (b.rowCount !== c.rowCount) {
|
||||
failures.push(
|
||||
`${dataset}: row count ${b.rowCount} → ${c.rowCount} (${c.rowCount - b.rowCount >= 0 ? "+" : ""}${c.rowCount - b.rowCount})`,
|
||||
);
|
||||
}
|
||||
|
||||
// 2. Pre-existing columns keep their non-NULL counts
|
||||
for (const col of b.columns) {
|
||||
if (!(col in c.nonNull)) {
|
||||
failures.push(`${dataset}.${col}: column disappeared from schema`);
|
||||
continue;
|
||||
}
|
||||
if (b.nonNull[col] !== c.nonNull[col]) {
|
||||
failures.push(
|
||||
`${dataset}.${col}: non-NULL ${b.nonNull[col]} → ${c.nonNull[col]} (${c.nonNull[col] - b.nonNull[col] >= 0 ? "+" : ""}${c.nonNull[col] - b.nonNull[col]})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Newly added columns must be NULL — except the approved recoveries,
|
||||
// which must match their approved count exactly.
|
||||
const approved = APPROVED_RECOVERY[dataset] ?? {};
|
||||
const added = c.columns.filter((col) => !b.columns.includes(col));
|
||||
const recovered = [];
|
||||
for (const col of added) {
|
||||
const expected = approved[col] ?? 0;
|
||||
if (c.nonNull[col] !== expected) {
|
||||
failures.push(
|
||||
expected === 0
|
||||
? `${dataset}.${col}: new column has ${c.nonNull[col]} non-NULL values, expected 0`
|
||||
: `${dataset}.${col}: recovered ${c.nonNull[col]} values, approved count is ${expected}`,
|
||||
);
|
||||
} else if (expected > 0) {
|
||||
recovered.push(`${col}=${expected}`);
|
||||
}
|
||||
}
|
||||
// An approved recovery that vanished means the union patterns regressed.
|
||||
for (const [col, expected] of Object.entries(approved)) {
|
||||
if (!added.includes(col)) {
|
||||
failures.push(
|
||||
`${dataset}.${col}: expected ${expected} recovered values but column is not new`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (added.length) {
|
||||
const nulls = added.length - recovered.length;
|
||||
notes.push(
|
||||
`${dataset}: +${added.length} new columns (${nulls} all-NULL as expected` +
|
||||
(recovered.length ? `, recovered ${recovered.join(", ")}` : "") +
|
||||
")",
|
||||
);
|
||||
}
|
||||
|
||||
// 4. Deterministic sample compared field by field
|
||||
if (b.sampleCount !== c.sampleCount) {
|
||||
failures.push(
|
||||
`${dataset}: sample size ${b.sampleCount} → ${c.sampleCount}`,
|
||||
);
|
||||
}
|
||||
for (const [sbd, bRow] of Object.entries(b.sample)) {
|
||||
const cRow = c.sample[sbd];
|
||||
if (!cRow) {
|
||||
failures.push(`${dataset}: sampled student ${sbd} missing after rebuild`);
|
||||
continue;
|
||||
}
|
||||
for (const [field, bVal] of Object.entries(bRow)) {
|
||||
if (cRow[field] !== bVal) {
|
||||
failures.push(
|
||||
`${dataset}: student ${sbd} field ${field}: ${JSON.stringify(bVal)} → ${JSON.stringify(cRow[field])}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const sizeDelta = ((c.sizeBytes - b.sizeBytes) / b.sizeBytes) * 100;
|
||||
notes.push(
|
||||
`${dataset}: ${c.rowCount} rows, ${b.columns.length} → ${c.columns.length} cols, size ${sizeDelta >= 0 ? "+" : ""}${sizeDelta.toFixed(1)}%`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const n of notes) console.log(` ${n}`);
|
||||
|
||||
if (failures.length) {
|
||||
console.error(`\nPARITY FAILED — ${failures.length} mismatch(es):\n`);
|
||||
for (const f of failures) console.error(` ✗ ${f}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log("\nPARITY OK — row counts, per-column non-NULL counts and samples all match.");
|
||||
@@ -0,0 +1,182 @@
|
||||
/// Audit subcommand: replicates audit-row-counts.js exactly.
|
||||
///
|
||||
/// Reads all .xlsx files from the input directory (sheet 0 only, matching the
|
||||
/// JS script's behaviour at audit-row-counts.js:33), collects distinct SBDs
|
||||
/// into a HashSet, then queries `SELECT COUNT(*) FROM student` from the DB.
|
||||
/// Prints the same lines as audit-row-counts.js:54-62 and exits 0 on match,
|
||||
/// 1 on mismatch.
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
|
||||
use calamine::{open_workbook_auto, Data, Reader};
|
||||
|
||||
use crate::config::DatasetConfig;
|
||||
use crate::error::BuildError;
|
||||
use crate::reader::is_header_row;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Audit result
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct AuditResult {
|
||||
pub total_data_rows: u64,
|
||||
pub both_empty: u64,
|
||||
pub empty_name: u64,
|
||||
pub empty_sbd: u64,
|
||||
pub distinct_sbds: usize,
|
||||
pub db_count: i64,
|
||||
pub matched: bool,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main audit logic
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Collect distinct SBDs from all xlsx files in `input_dir`, query `db_path`,
|
||||
/// print the audit report and return the result.
|
||||
///
|
||||
/// The JS script reads only sheet 0 for every file (audit-row-counts.js:33).
|
||||
/// Unlike build-database.js, the audit script does NOT iterate all sheets.
|
||||
pub fn run_audit(
|
||||
input_dir: &Path,
|
||||
db_path: &Path,
|
||||
cfg: &DatasetConfig,
|
||||
) -> Result<AuditResult, BuildError> {
|
||||
// Collect .xlsx files (audit-row-counts.js only checks .xlsx — line 15)
|
||||
let mut files: Vec<std::path::PathBuf> = std::fs::read_dir(input_dir)
|
||||
.map_err(|e| BuildError::Io {
|
||||
path: input_dir.display().to_string(),
|
||||
source: e,
|
||||
})?
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.is_file()
|
||||
&& p.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e.eq_ignore_ascii_case("xlsx"))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
|
||||
let mut all_sbd: HashSet<String> = HashSet::new();
|
||||
let mut total_data_rows: u64 = 0;
|
||||
let mut empty_name: u64 = 0;
|
||||
let mut empty_sbd: u64 = 0;
|
||||
let mut both_empty: u64 = 0;
|
||||
|
||||
for file in &files {
|
||||
let path_str = file.display().to_string();
|
||||
let mut workbook = open_workbook_auto(file).map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let sheet_names = workbook.sheet_names().to_vec();
|
||||
if sheet_names.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// audit-row-counts.js reads only sheet 0 (line 33: wb.SheetNames[0])
|
||||
let range =
|
||||
workbook
|
||||
.worksheet_range(&sheet_names[0])
|
||||
.map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let mut first_row = true;
|
||||
for raw in range.rows() {
|
||||
let row: Vec<Data> = raw.to_vec();
|
||||
|
||||
// Skip header row on first row only
|
||||
if first_row {
|
||||
first_row = false;
|
||||
if is_header_row(&row, &cfg.header) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
total_data_rows += 1;
|
||||
|
||||
// For format_detection configs the audit uses positional defaults (col 0 = SBD,
|
||||
// col 1 = HO_TEN) because the audit is best-effort and mirrors the JS script
|
||||
// which also uses a fixed column assumption (audit-row-counts.js:33–36).
|
||||
let (ho_ten_col, sbd_col) = cfg
|
||||
.columns
|
||||
.as_ref()
|
||||
.map(|c| (c.ho_ten, c.so_bao_danh))
|
||||
.unwrap_or((1, 0));
|
||||
let ho_ten = row
|
||||
.get(ho_ten_col)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
let sbd = row
|
||||
.get(sbd_col)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
|
||||
if ho_ten.is_empty() && sbd.is_empty() {
|
||||
both_empty += 1;
|
||||
continue;
|
||||
}
|
||||
if ho_ten.is_empty() {
|
||||
empty_name += 1;
|
||||
}
|
||||
if sbd.is_empty() {
|
||||
empty_sbd += 1;
|
||||
}
|
||||
if !sbd.is_empty() {
|
||||
all_sbd.insert(sbd);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Query DB count
|
||||
let conn =
|
||||
rusqlite::Connection::open_with_flags(db_path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?;
|
||||
let db_count: i64 = conn.query_row("SELECT COUNT(*) FROM student", [], |row| row.get(0))?;
|
||||
|
||||
let distinct_sbds = all_sbd.len();
|
||||
let matched = distinct_sbds as i64 == db_count;
|
||||
|
||||
Ok(AuditResult {
|
||||
total_data_rows,
|
||||
both_empty,
|
||||
empty_name,
|
||||
empty_sbd,
|
||||
distinct_sbds,
|
||||
db_count,
|
||||
matched,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Print audit report — mirrors audit-row-counts.js:54-62 exactly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn print_audit_report(r: &AuditResult) {
|
||||
println!("=== Source vs DB ===");
|
||||
println!(
|
||||
"Source: total data rows across all files: {}",
|
||||
r.total_data_rows
|
||||
);
|
||||
println!(
|
||||
"Source: rows with empty name AND sbd (skipped): {}",
|
||||
r.both_empty
|
||||
);
|
||||
println!("Source: rows with missing name only: {}", r.empty_name);
|
||||
println!("Source: rows with missing sbd only: {}", r.empty_sbd);
|
||||
println!("Source: distinct SBDs: {}", r.distinct_sbds);
|
||||
println!("DB: row count: {}", r.db_count);
|
||||
println!(
|
||||
"Match: {}",
|
||||
if r.matched {
|
||||
"YES — all unique SBDs accounted for".to_string()
|
||||
} else {
|
||||
format!("NO — gap of {}", r.distinct_sbds as i64 - r.db_count)
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/// CLI argument structs via clap derive.
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "xlsxread",
|
||||
version,
|
||||
about = "Read .xls/.xlsx files and build SQLite databases for thptqg datasets"
|
||||
)]
|
||||
pub struct Cli {
|
||||
#[command(subcommand)]
|
||||
pub cmd: Cmd,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
pub enum Cmd {
|
||||
/// Read input spreadsheets and write a SQLite database
|
||||
Build {
|
||||
/// Path to the dataset TOML config file
|
||||
#[arg(long)]
|
||||
schema: PathBuf,
|
||||
|
||||
/// Directory containing the .xls / .xlsx source files
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
|
||||
/// Output SQLite database path
|
||||
#[arg(long)]
|
||||
output: PathBuf,
|
||||
},
|
||||
|
||||
/// Audit: compare distinct SBD count from xlsx files vs DB row count
|
||||
Audit {
|
||||
/// Path to the dataset TOML config file
|
||||
#[arg(long)]
|
||||
schema: PathBuf,
|
||||
|
||||
/// Directory containing the .xlsx source files
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
|
||||
/// SQLite database to compare against
|
||||
#[arg(long)]
|
||||
db: PathBuf,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BuildError;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Top-level dataset configuration loaded from a .toml file
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Per-dataset parse rules.
|
||||
///
|
||||
/// Deliberately carries no SQL. The table shape, the INSERT and the subject
|
||||
/// regexes are identical for every dataset and live in `crate::schema` — keeping
|
||||
/// them here meant four copies of the same DDL, which is how the 2016 and 2017
|
||||
/// schemas drifted apart.
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct DatasetConfig {
|
||||
pub reader: ReaderCfg,
|
||||
/// Fixed column indices. Optional when format_detection handles per-file mapping.
|
||||
#[serde(default)]
|
||||
pub columns: Option<ColumnMap>,
|
||||
pub validation: ValidationCfg,
|
||||
pub header: HeaderCfg,
|
||||
/// When set to "thptqg2016", enables per-file format auto-detection.
|
||||
/// Each file's header row is inspected at runtime to choose the right
|
||||
/// column layout (separate-scores / mapped / default-positional).
|
||||
#[serde(default)]
|
||||
pub format_detection: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct ReaderCfg {
|
||||
/// "all" → iterate every sheet (handles HCM/HN overflow); "first" → sheet 0 only
|
||||
pub sheet_mode: SheetMode,
|
||||
/// If true, skip rows where every cell is empty/null before counting (data-old2 quirk)
|
||||
pub strip_blank_rows: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum SheetMode {
|
||||
All,
|
||||
First,
|
||||
}
|
||||
|
||||
/// Zero-indexed column positions in the source spreadsheet row.
|
||||
/// Used by thptqg2017 configs. thptqg2016 uses runtime format detection instead.
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct ColumnMap {
|
||||
pub ho_ten: usize,
|
||||
pub ngay_sinh: usize,
|
||||
pub so_bao_danh: usize,
|
||||
pub diem_thi: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct ValidationCfg {
|
||||
/// build-database-old.js / -old2.js require soBaoDanh to match ^\d+$
|
||||
pub require_numeric_sbd: bool,
|
||||
pub require_nonempty_name: bool,
|
||||
pub require_nonempty_sbd: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct HeaderCfg {
|
||||
/// Tokens to match against row[0].to_uppercase() to detect a header row
|
||||
pub tokens: Vec<String>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Loader
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn load_config(path: &Path) -> Result<DatasetConfig, BuildError> {
|
||||
let text = fs::read_to_string(path).map_err(|e| BuildError::Io {
|
||||
path: path.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
let cfg: DatasetConfig = toml::from_str(&text)?;
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SAMPLE_TOML: &str = r#"
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
"#;
|
||||
|
||||
#[test]
|
||||
fn config_round_trip() {
|
||||
let cfg: DatasetConfig = toml::from_str(SAMPLE_TOML).expect("parse failed");
|
||||
assert_eq!(cfg.reader.sheet_mode, SheetMode::All);
|
||||
assert!(!cfg.reader.strip_blank_rows);
|
||||
let cols = cfg.columns.as_ref().unwrap();
|
||||
assert_eq!(cols.ho_ten, 0);
|
||||
assert_eq!(cols.diem_thi, 3);
|
||||
assert!(!cfg.validation.require_numeric_sbd);
|
||||
assert!(cfg.validation.require_nonempty_name);
|
||||
assert_eq!(cfg.header.tokens.len(), 3);
|
||||
assert!(cfg.format_detection.is_none());
|
||||
}
|
||||
|
||||
/// A config carrying leftover SQL sections must be rejected rather than
|
||||
/// silently ignored — otherwise a stale [schema] block would look effective
|
||||
/// while `crate::schema` was actually driving the build.
|
||||
#[test]
|
||||
fn config_rejects_leftover_sql_sections() {
|
||||
let with_ddl = format!(
|
||||
"{SAMPLE_TOML}\n[schema]\nddl = \"CREATE TABLE student (so_bao_danh TEXT);\"\n"
|
||||
);
|
||||
assert!(toml::from_str::<DatasetConfig>(&with_ddl).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_first_sheet_mode() {
|
||||
let toml_str = SAMPLE_TOML.replace(r#"sheet_mode = "all""#, r#"sheet_mode = "first""#);
|
||||
let cfg: DatasetConfig = toml::from_str(&toml_str).expect("parse failed");
|
||||
assert_eq!(cfg.reader.sheet_mode, SheetMode::First);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_format_detection_field() {
|
||||
// Configs without [columns] and with format_detection = "thptqg2016" parse correctly
|
||||
let toml_str = r#"
|
||||
format_detection = "thptqg2016"
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["SBD", "SOBAODANH", "STT"]
|
||||
"#;
|
||||
let cfg: DatasetConfig = toml::from_str(toml_str).expect("parse failed");
|
||||
assert_eq!(cfg.format_detection.as_deref(), Some("thptqg2016"));
|
||||
assert!(cfg.columns.is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum BuildError {
|
||||
#[error("I/O error for {path}: {source}")]
|
||||
Io {
|
||||
path: String,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
|
||||
#[error("Calamine error for {path}: {source}")]
|
||||
Calamine {
|
||||
path: String,
|
||||
#[source]
|
||||
source: calamine::Error,
|
||||
},
|
||||
|
||||
#[error("SQLite error: {0}")]
|
||||
Sqlite(#[from] rusqlite::Error),
|
||||
|
||||
#[error("Config parse error: {0}")]
|
||||
Config(#[from] toml::de::Error),
|
||||
|
||||
#[error("Regex compile error for pattern '{pattern}': {source}")]
|
||||
Regex {
|
||||
pattern: String,
|
||||
#[source]
|
||||
source: regex::Error,
|
||||
},
|
||||
|
||||
#[error("Schema has no sheets in file: {0}")]
|
||||
NoSheets(String),
|
||||
}
|
||||
@@ -0,0 +1,545 @@
|
||||
/// Per-file format auto-detection for the thptqg2016 dataset.
|
||||
///
|
||||
/// Translates `detectFormat` from scripts/build-database.js (lines 63–87) and the
|
||||
/// three row-processing functions (lines 90–146) into Rust.
|
||||
///
|
||||
/// The JS source has three formats:
|
||||
///
|
||||
/// 1. `separate-scores` — header row[0]=="SBD" && row[2]=="TOAN"
|
||||
/// Columns: SBD(0) HOTEN(1) TOAN(2) VAN(3) LY(4) HOA(5) SINH(6) SU(7) DIA(8)
|
||||
/// NGOAINGUTN(9) NGOAINGUTL(10) NGOAINGU-total(11)
|
||||
/// → maps col 11 → tieng_anh; no ngay_sinh / ten_cum_thi / gioi_tinh / DIEM_THI
|
||||
/// → JS: build-database.js:90–116 (processSeparateScoresRow)
|
||||
///
|
||||
/// 2. `mapped` — header row has SOBAODANH|SBD and DIEM_THI columns
|
||||
/// → dynamic column indices built from header names
|
||||
/// → JS: build-database.js:119–146 (processMappedRow with map from detectFormat)
|
||||
///
|
||||
/// 3. `default` — no recognised header; positional 6-col layout
|
||||
/// SBD(0) HO_TEN(1) NGAY_SINH(2) TEN_CUMTHI(3) GIOI_TINH(4) DIEM_THI(5)
|
||||
/// → JS: build-database.js:149–151 DEFAULT_MAP + processMappedRow
|
||||
///
|
||||
/// JS citations are line numbers in /config/workspace/tiennm99/thptqg2016/scripts/build-database.js.
|
||||
use calamine::Data;
|
||||
|
||||
use crate::transform::{parse_scores, to_ascii, CompiledPatterns, ParsedRow};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Known header tokens (mirrors JS KNOWN_HEADERS set, build-database.js:50–54)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Upper-cased strings that identify a header row's first cell.
|
||||
/// Mirrors `KNOWN_HEADERS` in build-database.js:50–54.
|
||||
const KNOWN_HEADERS: &[&str] = &[
|
||||
"SOBAODANH",
|
||||
"SBD",
|
||||
"HO_TEN",
|
||||
"HOTEN",
|
||||
"HỌ TÊN",
|
||||
"NGAY_SINH",
|
||||
"TEN_CUMTHI",
|
||||
"GIOI_TINH",
|
||||
"DIEM_THI",
|
||||
"STT",
|
||||
"TOAN",
|
||||
"VAN",
|
||||
"LY",
|
||||
"HOA",
|
||||
"SINH ",
|
||||
"SU",
|
||||
"DIA",
|
||||
];
|
||||
|
||||
/// Returns true when `row[0]` (uppercased, trimmed) is in the known-headers set.
|
||||
/// Mirrors `isHeaderRow` at build-database.js:56–60.
|
||||
pub fn is_header_row_2016(row: &[Data]) -> bool {
|
||||
if row.len() < 2 {
|
||||
return false;
|
||||
}
|
||||
let first = row[0].to_string().trim().to_uppercase();
|
||||
KNOWN_HEADERS.contains(&first.as_str())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Detected per-file format
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The three layouts the thptqg2016 dataset uses, detected per file.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum DetectedFormat {
|
||||
/// SBD/HOTEN/TOAN/VAN/LY/HOA/SINH/SU/DIA/NGOAINGUTN/NGOAINGUTL/NGOAINGU columns.
|
||||
/// Corresponds to the dhhanghai-style files. build-database.js:67–68.
|
||||
SeparateScores,
|
||||
/// Header present with SOBAODANH|SBD and DIEM_THI; dynamic column indices.
|
||||
/// build-database.js:70–86.
|
||||
Mapped {
|
||||
sbd: usize,
|
||||
ho_ten: usize,
|
||||
ngay_sinh: Option<usize>,
|
||||
ten_cum_thi: Option<usize>,
|
||||
gioi_tinh: Option<usize>,
|
||||
diem_thi: usize,
|
||||
},
|
||||
/// No recognised header; standard 6-column positional layout.
|
||||
/// build-database.js:149–151 DEFAULT_MAP.
|
||||
Default,
|
||||
}
|
||||
|
||||
/// Inspect a header row and decide which format applies.
|
||||
/// Returns `None` when the header is present but unrecognised (treated as Default).
|
||||
///
|
||||
/// Mirrors `detectFormat` at build-database.js:63–87.
|
||||
pub fn detect_format(header_row: &[Data]) -> DetectedFormat {
|
||||
let cols: Vec<String> = header_row
|
||||
.iter()
|
||||
.map(|c| c.to_string().trim().to_uppercase())
|
||||
.collect();
|
||||
|
||||
// Format 1: SBD in col 0 AND TOAN in col 2 → separate-scores
|
||||
// build-database.js:68: if (cols[0] === "SBD" && cols[2] === "TOAN")
|
||||
if cols.first().map(|s| s.as_str()) == Some("SBD")
|
||||
&& cols.get(2).map(|s| s.as_str()) == Some("TOAN")
|
||||
{
|
||||
return DetectedFormat::SeparateScores;
|
||||
}
|
||||
|
||||
// Format 2: build column index map — check for SOBAODANH|SBD and DIEM_THI
|
||||
// build-database.js:70–86
|
||||
let mut sbd_idx: Option<usize> = None;
|
||||
let mut ho_ten_idx: Option<usize> = None;
|
||||
let mut ngay_sinh_idx: Option<usize> = None;
|
||||
let mut ten_cum_thi_idx: Option<usize> = None;
|
||||
let mut gioi_tinh_idx: Option<usize> = None;
|
||||
let mut diem_thi_idx: Option<usize> = None;
|
||||
|
||||
for (i, c) in cols.iter().enumerate() {
|
||||
match c.as_str() {
|
||||
"SOBAODANH" | "SBD" => sbd_idx = Some(i),
|
||||
"HO_TEN" | "HOTEN" | "HỌ TÊN" => ho_ten_idx = Some(i),
|
||||
"NGAY_SINH" => ngay_sinh_idx = Some(i),
|
||||
"TEN_CUMTHI" => ten_cum_thi_idx = Some(i),
|
||||
"GIOI_TINH" => gioi_tinh_idx = Some(i),
|
||||
"DIEM_THI" => diem_thi_idx = Some(i),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// build-database.js:82–84: if (map.sbd !== undefined && map.diem_thi !== undefined)
|
||||
if let (Some(sbd), Some(diem_thi)) = (sbd_idx, diem_thi_idx) {
|
||||
let ho_ten = ho_ten_idx.unwrap_or(1); // fallback: col 1 (present in all known files)
|
||||
return DetectedFormat::Mapped {
|
||||
sbd,
|
||||
ho_ten,
|
||||
ngay_sinh: ngay_sinh_idx,
|
||||
ten_cum_thi: ten_cum_thi_idx,
|
||||
gioi_tinh: gioi_tinh_idx,
|
||||
diem_thi,
|
||||
};
|
||||
}
|
||||
|
||||
// Unrecognised header (or no header) → positional default
|
||||
DetectedFormat::Default
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Row processors
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Cell accessor helper.
|
||||
fn cell_str(row: &[Data], idx: usize) -> String {
|
||||
row.get(idx)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Parse a cell that should hold a float score; returns None for blank/non-numeric.
|
||||
/// Mirrors `parseFloat(row[N]) || null` in JS.
|
||||
fn parse_float_cell(row: &[Data], idx: usize) -> Option<f64> {
|
||||
let s = cell_str(row, idx);
|
||||
if s.is_empty() {
|
||||
return None;
|
||||
}
|
||||
s.parse::<f64>().ok().filter(|v| v.is_finite() && *v != 0.0)
|
||||
}
|
||||
|
||||
/// Process a row in the `separate-scores` format.
|
||||
///
|
||||
/// Column layout (build-database.js:90–116 `processSeparateScoresRow`):
|
||||
/// 0=SBD 1=HOTEN 2=TOAN 3=VAN 4=LY 5=HOA 6=SINH 7=SU 8=DIA
|
||||
/// 9=NGOAINGUTN 10=NGOAINGUTL 11=NGOAINGU(total→tieng_anh)
|
||||
///
|
||||
/// tieng_phap / tieng_duc / tieng_nhat / tieng_trung all → None
|
||||
/// ngay_sinh / ten_cum_thi / gioi_tinh all → None (not in this format)
|
||||
pub fn process_separate_scores_row(row: &[Data], patterns: &CompiledPatterns) -> Option<ParsedRow> {
|
||||
let sbd = cell_str(row, 0);
|
||||
let ho_ten = cell_str(row, 1);
|
||||
if sbd.is_empty() || ho_ten.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let ho_ten_ascii = to_ascii(&ho_ten);
|
||||
|
||||
// build-database.js:102–114: explicit per-column score mapping
|
||||
let mut scores = std::collections::HashMap::new();
|
||||
macro_rules! add_score {
|
||||
($field:expr, $idx:expr) => {
|
||||
if let Some(v) = parse_float_cell(row, $idx) {
|
||||
scores.insert($field.to_string(), v);
|
||||
}
|
||||
};
|
||||
}
|
||||
add_score!("toan", 2);
|
||||
add_score!("ngu_van", 3);
|
||||
add_score!("vat_ly", 4);
|
||||
add_score!("hoa_hoc", 5);
|
||||
add_score!("sinh_hoc", 6);
|
||||
add_score!("lich_su", 7);
|
||||
add_score!("dia_ly", 8);
|
||||
// col 11 = NGOAINGU total → tieng_anh (build-database.js:110–111)
|
||||
add_score!("tieng_anh", 11);
|
||||
|
||||
// Suppress unused-variable warning; patterns not used in this path (no DIEM_THI string)
|
||||
let _ = patterns;
|
||||
|
||||
Some(ParsedRow {
|
||||
so_bao_danh: sbd,
|
||||
ho_ten,
|
||||
ho_ten_ascii,
|
||||
ngay_sinh: None,
|
||||
ten_cum_thi: None,
|
||||
gioi_tinh: None,
|
||||
scores,
|
||||
})
|
||||
}
|
||||
|
||||
/// Process a row in the `mapped` format (header-derived column indices).
|
||||
///
|
||||
/// Mirrors `processMappedRow` at build-database.js:119–146.
|
||||
/// Gender is normalised: only "Nam" or "Nữ" are kept; everything else → None.
|
||||
/// (build-database.js:132: `(rawGioiTinh === "Nam" || rawGioiTinh === "Nữ") ? rawGioiTinh : null`)
|
||||
// Mirrors the JS column map one-for-one; grouping the indices into a struct
|
||||
// would obscure that correspondence for no benefit.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn process_mapped_row(
|
||||
row: &[Data],
|
||||
sbd_idx: usize,
|
||||
ho_ten_idx: usize,
|
||||
ngay_sinh_idx: Option<usize>,
|
||||
ten_cum_thi_idx: Option<usize>,
|
||||
gioi_tinh_idx: Option<usize>,
|
||||
diem_thi_idx: usize,
|
||||
patterns: &CompiledPatterns,
|
||||
) -> Option<ParsedRow> {
|
||||
let sbd = cell_str(row, sbd_idx);
|
||||
let ho_ten = cell_str(row, ho_ten_idx);
|
||||
if sbd.is_empty() || ho_ten.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Skip rows where SBD or HO_TEN are themselves header tokens (leaked header rows).
|
||||
// build-database.js:125–126: KNOWN_HEADERS.has(sbdUpper) || KNOWN_HEADERS.has(hoTenUpper)
|
||||
let sbd_upper = sbd.to_uppercase();
|
||||
let ho_ten_upper = ho_ten.to_uppercase();
|
||||
if KNOWN_HEADERS.contains(&sbd_upper.as_str())
|
||||
|| KNOWN_HEADERS.contains(&ho_ten_upper.as_str())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
let ho_ten_ascii = to_ascii(&ho_ten);
|
||||
|
||||
let ngay_sinh = ngay_sinh_idx
|
||||
.map(|i| cell_str(row, i))
|
||||
.filter(|s| !s.is_empty());
|
||||
|
||||
let ten_cum_thi = ten_cum_thi_idx
|
||||
.map(|i| cell_str(row, i))
|
||||
.filter(|s| !s.is_empty());
|
||||
|
||||
// build-database.js:130–132: normalise gender
|
||||
let gioi_tinh = gioi_tinh_idx
|
||||
.map(|i| cell_str(row, i))
|
||||
.and_then(|s| {
|
||||
if s == "Nam" || s == "Nữ" {
|
||||
Some(s)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
let diem_thi = row
|
||||
.get(diem_thi_idx)
|
||||
.map(|c| c.to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let scores = parse_scores(&diem_thi, patterns);
|
||||
|
||||
Some(ParsedRow {
|
||||
so_bao_danh: sbd,
|
||||
ho_ten,
|
||||
ho_ten_ascii,
|
||||
ngay_sinh,
|
||||
ten_cum_thi,
|
||||
gioi_tinh,
|
||||
scores,
|
||||
})
|
||||
}
|
||||
|
||||
/// Process a row using the default positional 6-column layout.
|
||||
///
|
||||
/// Column order: SBD(0) HO_TEN(1) NGAY_SINH(2) TEN_CUMTHI(3) GIOI_TINH(4) DIEM_THI(5)
|
||||
/// Mirrors `processMappedRow(row, DEFAULT_MAP)` at build-database.js:233–236.
|
||||
pub fn process_default_row(row: &[Data], patterns: &CompiledPatterns) -> Option<ParsedRow> {
|
||||
process_mapped_row(
|
||||
row,
|
||||
0, // sbd
|
||||
1, // ho_ten
|
||||
Some(2),
|
||||
Some(3),
|
||||
Some(4),
|
||||
5, // diem_thi
|
||||
patterns,
|
||||
)
|
||||
}
|
||||
|
||||
/// Dispatch a data row through the correct processor for the detected format.
|
||||
///
|
||||
/// Returns `None` when the row is empty/invalid and should be skipped.
|
||||
pub fn process_row_2016(
|
||||
row: &[Data],
|
||||
fmt: &DetectedFormat,
|
||||
patterns: &CompiledPatterns,
|
||||
) -> Option<ParsedRow> {
|
||||
match fmt {
|
||||
DetectedFormat::SeparateScores => process_separate_scores_row(row, patterns),
|
||||
DetectedFormat::Mapped {
|
||||
sbd,
|
||||
ho_ten,
|
||||
ngay_sinh,
|
||||
ten_cum_thi,
|
||||
gioi_tinh,
|
||||
diem_thi,
|
||||
} => process_mapped_row(
|
||||
row,
|
||||
*sbd,
|
||||
*ho_ten,
|
||||
*ngay_sinh,
|
||||
*ten_cum_thi,
|
||||
*gioi_tinh,
|
||||
*diem_thi,
|
||||
patterns,
|
||||
),
|
||||
DetectedFormat::Default => process_default_row(row, patterns),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests — 3 detection branches + key processing cases
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn s(v: &str) -> Data {
|
||||
Data::String(v.to_string())
|
||||
}
|
||||
|
||||
fn make_patterns() -> CompiledPatterns {
|
||||
CompiledPatterns::new().unwrap()
|
||||
}
|
||||
|
||||
// --- detect_format: branch 1 — separate-scores ---
|
||||
|
||||
#[test]
|
||||
fn detect_separate_scores() {
|
||||
let header = vec![s("SBD"), s("HOTEN"), s("TOAN"), s("VAN")];
|
||||
match detect_format(&header) {
|
||||
DetectedFormat::SeparateScores => {}
|
||||
other => panic!("expected SeparateScores, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// --- detect_format: branch 2 — mapped ---
|
||||
|
||||
#[test]
|
||||
fn detect_mapped_with_named_cols() {
|
||||
let header = vec![
|
||||
s("SOBAODANH"),
|
||||
s("HO_TEN"),
|
||||
s("NGAY_SINH"),
|
||||
s("TEN_CUMTHI"),
|
||||
s("GIOI_TINH"),
|
||||
s("DIEM_THI"),
|
||||
];
|
||||
match detect_format(&header) {
|
||||
DetectedFormat::Mapped {
|
||||
sbd,
|
||||
ho_ten,
|
||||
ngay_sinh,
|
||||
ten_cum_thi,
|
||||
gioi_tinh,
|
||||
diem_thi,
|
||||
} => {
|
||||
assert_eq!(sbd, 0);
|
||||
assert_eq!(ho_ten, 1);
|
||||
assert_eq!(ngay_sinh, Some(2));
|
||||
assert_eq!(ten_cum_thi, Some(3));
|
||||
assert_eq!(gioi_tinh, Some(4));
|
||||
assert_eq!(diem_thi, 5);
|
||||
}
|
||||
other => panic!("expected Mapped, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_mapped_sbd_variant() {
|
||||
// "SBD" (not "SOBAODANH") + DIEM_THI at different positions
|
||||
let header = vec![s("STT"), s("SBD"), s("HOTEN"), s("DIEM_THI")];
|
||||
match detect_format(&header) {
|
||||
DetectedFormat::Mapped { sbd, diem_thi, .. } => {
|
||||
assert_eq!(sbd, 1);
|
||||
assert_eq!(diem_thi, 3);
|
||||
}
|
||||
other => panic!("expected Mapped, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// --- detect_format: branch 3 — default ---
|
||||
|
||||
#[test]
|
||||
fn detect_default_when_no_header() {
|
||||
// A data row: positional default used
|
||||
let data_row = vec![
|
||||
s("12345678"),
|
||||
s("Nguyễn Văn A"),
|
||||
s("01/01/2000"),
|
||||
s("TP HCM"),
|
||||
s("Nam"),
|
||||
s("Toán: 8.5"),
|
||||
];
|
||||
// Default is returned when there is no recognised header
|
||||
match detect_format(&data_row) {
|
||||
DetectedFormat::Default => {}
|
||||
other => panic!("expected Default, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
// --- process_separate_scores_row ---
|
||||
|
||||
#[test]
|
||||
fn separate_scores_basic() {
|
||||
let p = make_patterns();
|
||||
// 12 columns: SBD HOTEN TOAN VAN LY HOA SINH SU DIA NGUTN NGUTL NGUTOTAL
|
||||
let row = vec![
|
||||
s("TP001"),
|
||||
s("Nguyễn Thị Lan"),
|
||||
Data::Float(8.0),
|
||||
Data::Float(7.5),
|
||||
Data::Float(9.0),
|
||||
Data::Float(6.5),
|
||||
Data::Float(5.0),
|
||||
Data::Float(4.5),
|
||||
Data::Float(8.0),
|
||||
Data::Empty,
|
||||
Data::Empty,
|
||||
Data::Float(7.0), // col 11 → tieng_anh
|
||||
];
|
||||
let row = process_separate_scores_row(&row, &p).expect("should parse");
|
||||
assert_eq!(row.so_bao_danh, "TP001");
|
||||
assert_eq!(row.ho_ten, "Nguyễn Thị Lan");
|
||||
assert_eq!(row.ho_ten_ascii, "nguyen thi lan");
|
||||
assert_eq!(row.scores.get("toan"), Some(&8.0));
|
||||
assert_eq!(row.scores.get("tieng_anh"), Some(&7.0));
|
||||
assert!(row.ngay_sinh.is_none());
|
||||
assert!(row.ten_cum_thi.is_none());
|
||||
assert!(row.gioi_tinh.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn separate_scores_skips_empty_sbd() {
|
||||
let p = make_patterns();
|
||||
let row = vec![s(""), s("Nguyễn Văn A"), Data::Float(5.0)];
|
||||
assert!(process_separate_scores_row(&row, &p).is_none());
|
||||
}
|
||||
|
||||
// --- process_mapped_row ---
|
||||
|
||||
#[test]
|
||||
fn mapped_row_full_fields() {
|
||||
let p = make_patterns();
|
||||
let row = vec![
|
||||
s("HCM001"),
|
||||
s("Trần Thị Bích"),
|
||||
s("15/3/1999"),
|
||||
s("Cụm thi HCM"),
|
||||
s("Nữ"),
|
||||
s("Toán: 9.0 Ngữ văn: 8.5 Tiếng Anh: 7.75"),
|
||||
];
|
||||
let parsed = process_mapped_row(&row, 0, 1, Some(2), Some(3), Some(4), 5, &p)
|
||||
.expect("should parse");
|
||||
assert_eq!(parsed.so_bao_danh, "HCM001");
|
||||
assert_eq!(parsed.ngay_sinh.as_deref(), Some("15/3/1999"));
|
||||
assert_eq!(parsed.ten_cum_thi.as_deref(), Some("Cụm thi HCM"));
|
||||
assert_eq!(parsed.gioi_tinh.as_deref(), Some("Nữ"));
|
||||
assert_eq!(parsed.scores.get("toan"), Some(&9.0));
|
||||
assert_eq!(parsed.scores.get("ngu_van"), Some(&8.5));
|
||||
assert_eq!(parsed.scores.get("tieng_anh"), Some(&7.75));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mapped_row_gender_normalisation() {
|
||||
let p = make_patterns();
|
||||
// Gender "Unknown" → None
|
||||
let row = vec![s("ABC"), s("Nguyen Van A"), s(""), s(""), s("Unknown"), s("")];
|
||||
let parsed =
|
||||
process_mapped_row(&row, 0, 1, Some(2), Some(3), Some(4), 5, &p).expect("should parse");
|
||||
assert!(parsed.gioi_tinh.is_none());
|
||||
|
||||
// "Nam" passes through
|
||||
let row2 = vec![s("ABC"), s("Nguyen Van A"), s(""), s(""), s("Nam"), s("")];
|
||||
let parsed2 =
|
||||
process_mapped_row(&row2, 0, 1, Some(2), Some(3), Some(4), 5, &p).expect("should parse");
|
||||
assert_eq!(parsed2.gioi_tinh.as_deref(), Some("Nam"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mapped_row_skips_leaked_header() {
|
||||
let p = make_patterns();
|
||||
// A leaked header row — SBD cell contains "SOBAODANH"
|
||||
let row = vec![s("SOBAODANH"), s("HO_TEN"), s("NGAY_SINH"), s(""), s(""), s("")];
|
||||
assert!(process_mapped_row(&row, 0, 1, Some(2), Some(3), Some(4), 5, &p).is_none());
|
||||
}
|
||||
|
||||
// --- process_default_row ---
|
||||
|
||||
#[test]
|
||||
fn default_row_positional() {
|
||||
let p = make_patterns();
|
||||
let row = vec![
|
||||
s("DN001"),
|
||||
s("Lê Văn Long"),
|
||||
s("10/5/1998"),
|
||||
s("Cụm Đà Nẵng"),
|
||||
s("Nam"),
|
||||
s("Tiếng Đức: 6.25"),
|
||||
];
|
||||
let parsed = process_default_row(&row, &p).expect("should parse");
|
||||
assert_eq!(parsed.so_bao_danh, "DN001");
|
||||
assert_eq!(parsed.ho_ten_ascii, "le van long");
|
||||
assert_eq!(parsed.ngay_sinh.as_deref(), Some("10/5/1998"));
|
||||
assert_eq!(parsed.ten_cum_thi.as_deref(), Some("Cụm Đà Nẵng"));
|
||||
assert_eq!(parsed.gioi_tinh.as_deref(), Some("Nam"));
|
||||
assert_eq!(parsed.scores.get("tieng_duc"), Some(&6.25));
|
||||
}
|
||||
|
||||
// --- is_header_row_2016 ---
|
||||
|
||||
#[test]
|
||||
fn header_row_detection_2016() {
|
||||
assert!(is_header_row_2016(&[s("SOBAODANH"), s("HO_TEN")]));
|
||||
assert!(is_header_row_2016(&[s("SBD"), s("HOTEN"), s("TOAN")]));
|
||||
assert!(!is_header_row_2016(&[s("12345678"), s("Nguyen Van A")]));
|
||||
assert!(!is_header_row_2016(&[s("SBD")])); // too short (< 2 cells)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
/// Public library interface for integration tests.
|
||||
/// The binary entry point is src/main.rs; this file re-exports the internal
|
||||
/// modules so tests/golden.rs can call them without going through the CLI.
|
||||
pub mod audit;
|
||||
pub mod config;
|
||||
pub mod error;
|
||||
pub mod format_detect_2016;
|
||||
pub mod reader;
|
||||
pub mod schema;
|
||||
pub mod transform;
|
||||
pub mod writer;
|
||||
@@ -0,0 +1,377 @@
|
||||
/// xlsxread — Rust CLI replacing the SheetJS xlsx build scripts.
|
||||
///
|
||||
/// Subcommands:
|
||||
/// build — read .xls/.xlsx files → write SQLite DB
|
||||
/// audit — compare distinct SBD count from xlsx vs DB row count
|
||||
///
|
||||
/// Library modules are declared in lib.rs; main.rs only adds the CLI layer.
|
||||
///
|
||||
/// When config contains `format_detection = "thptqg2016"` the build subcommand
|
||||
/// uses per-file header inspection to pick the right column layout, replicating
|
||||
/// the `detectFormat` logic from scripts/build-database.js (lines 63–87).
|
||||
mod cli;
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use calamine::Data;
|
||||
use clap::Parser;
|
||||
|
||||
use cli::{Cli, Cmd};
|
||||
use xlsxread::audit;
|
||||
use xlsxread::config::load_config;
|
||||
use xlsxread::format_detect_2016::{
|
||||
detect_format, is_header_row_2016, process_row_2016, DetectedFormat,
|
||||
};
|
||||
use xlsxread::reader::{is_all_blank, process_file};
|
||||
use xlsxread::transform::{validate_row, CompiledPatterns, SkipReason};
|
||||
use xlsxread::writer::{finish_db, insert_row, open_db};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
match cli.cmd {
|
||||
Cmd::Build {
|
||||
schema,
|
||||
input,
|
||||
output,
|
||||
} => {
|
||||
run_build(&schema, &input, &output)?;
|
||||
}
|
||||
Cmd::Audit { schema, input, db } => {
|
||||
let cfg = load_config(&schema)
|
||||
.with_context(|| format!("Failed to load config: {}", schema.display()))?;
|
||||
let result = audit::run_audit(&input, &db, &cfg).with_context(|| "Audit failed")?;
|
||||
audit::print_audit_report(&result);
|
||||
if !result.matched {
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Build subcommand — dispatches to thptqg2016 or standard path
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn run_build(schema_path: &Path, input_dir: &Path, output_path: &Path) -> Result<()> {
|
||||
let cfg = load_config(schema_path)
|
||||
.with_context(|| format!("Failed to load config: {}", schema_path.display()))?;
|
||||
|
||||
if cfg.format_detection.as_deref() == Some("thptqg2016") {
|
||||
run_build_2016(&cfg, input_dir, output_path)
|
||||
} else {
|
||||
run_build_standard(&cfg, input_dir, output_path)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Standard build path (thptqg2017 and similar fixed-column configs)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn run_build_standard(
|
||||
cfg: &xlsxread::config::DatasetConfig,
|
||||
input_dir: &Path,
|
||||
output_path: &Path,
|
||||
) -> Result<()> {
|
||||
let patterns =
|
||||
CompiledPatterns::new().with_context(|| "Failed to compile score regexes")?;
|
||||
|
||||
let mut files: Vec<std::path::PathBuf> = std::fs::read_dir(input_dir)
|
||||
.with_context(|| format!("Cannot read input dir: {}", input_dir.display()))?
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.is_file()
|
||||
&& p.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| {
|
||||
let lower = e.to_lowercase();
|
||||
lower == "xls" || lower == "xlsx"
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
|
||||
let dataset_label = input_dir
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("data");
|
||||
|
||||
println!(
|
||||
"[build] {dataset_label}/ → {} ({} files)",
|
||||
output_path.display(),
|
||||
files.len()
|
||||
);
|
||||
|
||||
let conn = open_db(output_path)
|
||||
.with_context(|| format!("Failed to open DB: {}", output_path.display()))?;
|
||||
|
||||
let mut total_source_rows: u64 = 0;
|
||||
let mut total_skipped: u64 = 0;
|
||||
let mut total_errors: u64 = 0;
|
||||
|
||||
let is_old2 = dataset_label.contains("old2");
|
||||
let strip_blank = cfg.reader.strip_blank_rows;
|
||||
|
||||
conn.execute_batch("BEGIN")?;
|
||||
|
||||
for file in &files {
|
||||
let base = file
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("?")
|
||||
.to_owned();
|
||||
let mut file_rows: u64 = 0;
|
||||
let mut file_skipped: u64 = 0;
|
||||
let mut file_errors: u64 = 0;
|
||||
|
||||
let process_result = process_file(file, cfg, |_sheet_idx, raw| {
|
||||
let all_blank = is_all_blank(raw);
|
||||
if strip_blank && all_blank {
|
||||
return;
|
||||
}
|
||||
|
||||
total_source_rows += 1;
|
||||
|
||||
let ho_ten = raw
|
||||
.get(cfg.columns.as_ref().unwrap().ho_ten)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
let so_bao_danh = raw
|
||||
.get(cfg.columns.as_ref().unwrap().so_bao_danh)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
|
||||
match validate_row(&ho_ten, &so_bao_danh, &cfg.validation, strip_blank, all_blank) {
|
||||
Err(SkipReason::BlankRow) => {}
|
||||
Err(_) => {
|
||||
file_skipped += 1;
|
||||
return;
|
||||
}
|
||||
Ok(()) => {}
|
||||
}
|
||||
|
||||
let parsed = xlsxread::transform::transform_row(raw, cfg, &patterns);
|
||||
|
||||
match insert_row(&conn, &parsed) {
|
||||
Ok(()) => file_rows += 1,
|
||||
Err(e) => {
|
||||
file_errors += 1;
|
||||
if total_errors + file_errors <= 5 {
|
||||
eprintln!(" [warn] {base}: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
match process_result {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
eprintln!(" [error] {base}: {e}");
|
||||
file_errors += 1;
|
||||
}
|
||||
}
|
||||
|
||||
total_skipped += file_skipped;
|
||||
total_errors += file_errors;
|
||||
println!(" {base}: {file_rows} rows");
|
||||
}
|
||||
|
||||
conn.execute_batch("COMMIT")?;
|
||||
|
||||
finish_db(
|
||||
&conn,
|
||||
output_path,
|
||||
total_source_rows,
|
||||
total_skipped,
|
||||
total_errors,
|
||||
dataset_label,
|
||||
files.len(),
|
||||
is_old2,
|
||||
)
|
||||
.with_context(|| "Failed to finalise DB")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// thptqg2016 build path — per-file format detection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Build the thptqg2016 database.
|
||||
///
|
||||
/// Each file is processed independently: the first row is inspected to determine
|
||||
/// which of the three column layouts applies (separate-scores / mapped / default).
|
||||
/// This mirrors `detectFormat` in scripts/build-database.js lines 63–87, called
|
||||
/// once per file inside the file loop at build-database.js:218–219.
|
||||
fn run_build_2016(
|
||||
cfg: &xlsxread::config::DatasetConfig,
|
||||
input_dir: &Path,
|
||||
output_path: &Path,
|
||||
) -> Result<()> {
|
||||
let patterns =
|
||||
CompiledPatterns::new().with_context(|| "Failed to compile score regexes")?;
|
||||
|
||||
let mut files: Vec<std::path::PathBuf> = std::fs::read_dir(input_dir)
|
||||
.with_context(|| format!("Cannot read input dir: {}", input_dir.display()))?
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.is_file()
|
||||
&& p.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| {
|
||||
let lower = e.to_lowercase();
|
||||
lower == "xls" || lower == "xlsx"
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
|
||||
let dataset_label = input_dir
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("data");
|
||||
|
||||
println!(
|
||||
"[build:2016] {dataset_label}/ → {} ({} files)",
|
||||
output_path.display(),
|
||||
files.len()
|
||||
);
|
||||
|
||||
let conn = open_db(output_path)
|
||||
.with_context(|| format!("Failed to open DB: {}", output_path.display()))?;
|
||||
|
||||
let mut total_source_rows: u64 = 0;
|
||||
let total_skipped: u64 = 0;
|
||||
let mut total_errors: u64 = 0;
|
||||
|
||||
conn.execute_batch("BEGIN")?;
|
||||
|
||||
for file in &files {
|
||||
let base = file
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("?")
|
||||
.to_owned();
|
||||
|
||||
match process_file_2016(
|
||||
file,
|
||||
cfg,
|
||||
&patterns,
|
||||
&conn,
|
||||
&base,
|
||||
&mut total_source_rows,
|
||||
&mut total_errors,
|
||||
) {
|
||||
Ok(file_rows) => {
|
||||
println!(" {base}: {file_rows} rows");
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" [error] {base}: {e}");
|
||||
total_errors += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
conn.execute_batch("COMMIT")?;
|
||||
|
||||
finish_db(
|
||||
&conn,
|
||||
output_path,
|
||||
total_source_rows,
|
||||
total_skipped,
|
||||
total_errors,
|
||||
dataset_label,
|
||||
files.len(),
|
||||
false,
|
||||
)
|
||||
.with_context(|| "Failed to finalise DB")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Process one file in the thptqg2016 format-detection path.
|
||||
///
|
||||
/// Reads the file, uses the first row to detect the column layout, then processes
|
||||
/// all subsequent data rows. Returns the count of successfully inserted rows.
|
||||
fn process_file_2016(
|
||||
file: &Path,
|
||||
cfg: &xlsxread::config::DatasetConfig,
|
||||
patterns: &CompiledPatterns,
|
||||
conn: &rusqlite::Connection,
|
||||
base: &str,
|
||||
total_source_rows: &mut u64,
|
||||
total_errors: &mut u64,
|
||||
) -> Result<u64> {
|
||||
use calamine::{open_workbook_auto, Reader, Sheets};
|
||||
|
||||
let path_str = file.display().to_string();
|
||||
let mut workbook: Sheets<_> =
|
||||
open_workbook_auto(file).with_context(|| format!("Cannot open {path_str}"))?;
|
||||
|
||||
let sheet_names: Vec<String> = workbook.sheet_names().to_vec();
|
||||
if sheet_names.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// Sheet selection: thptqg2016 data/ has all-sheets mode to handle
|
||||
// HCM/HN overflow (same reason as thptqg2017 data/).
|
||||
let sheets_to_read: Vec<String> = match cfg.reader.sheet_mode {
|
||||
xlsxread::config::SheetMode::All => sheet_names.clone(),
|
||||
xlsxread::config::SheetMode::First => vec![sheet_names[0].clone()],
|
||||
};
|
||||
|
||||
let mut file_rows: u64 = 0;
|
||||
|
||||
for sheet_name in &sheets_to_read {
|
||||
let range = workbook
|
||||
.worksheet_range(sheet_name)
|
||||
.with_context(|| format!("Cannot read sheet {sheet_name} in {path_str}"))?;
|
||||
|
||||
let rows: Vec<Vec<Data>> = range.rows().map(|r| r.to_vec()).collect();
|
||||
if rows.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Detect format from first row, then determine start index.
|
||||
// Mirrors build-database.js:215–220: isHeaderRow check + detectFormat.
|
||||
let (fmt, start_idx) = if is_header_row_2016(&rows[0]) {
|
||||
(detect_format(&rows[0]), 1)
|
||||
} else {
|
||||
(DetectedFormat::Default, 0)
|
||||
};
|
||||
|
||||
for row in rows.iter().skip(start_idx) {
|
||||
if row.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
|
||||
*total_source_rows += 1;
|
||||
|
||||
match process_row_2016(row, &fmt, patterns) {
|
||||
None => {
|
||||
// Row was empty/invalid — skipped (mirrors JS `if (!record) continue`)
|
||||
}
|
||||
Some(parsed) => {
|
||||
match insert_row(conn, &parsed) {
|
||||
Ok(()) => file_rows += 1,
|
||||
Err(e) => {
|
||||
*total_errors += 1;
|
||||
if *total_errors <= 5 {
|
||||
eprintln!(" [warn] {base}: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(file_rows)
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
/// Spreadsheet reader: wraps calamine to iterate rows across sheets.
|
||||
///
|
||||
/// Sheet selection mirrors the JS scripts:
|
||||
/// - sheet_mode = "all" → iterate every sheet (handles HCM/HN 65k overflow in data/)
|
||||
/// - sheet_mode = "first" → sheet 0 only (data-old/)
|
||||
///
|
||||
/// Header detection mirrors build-lib.js isHeaderRow:
|
||||
/// row[0].toUpperCase() in {"HO_TEN", "HỌ TÊN", "STT"}
|
||||
use std::path::Path;
|
||||
|
||||
use calamine::{open_workbook_auto, Data, Reader, Sheets};
|
||||
|
||||
use crate::config::{DatasetConfig, HeaderCfg, SheetMode};
|
||||
use crate::error::BuildError;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public row representation from calamine
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub type RawRow = Vec<Data>;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Header detection — mirrors build-lib.js isHeaderRow
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Returns true when the first cell (uppercased) matches one of the configured
|
||||
/// header tokens. Used to skip the header row on the first row of each sheet.
|
||||
pub fn is_header_row(row: &[Data], header_cfg: &HeaderCfg) -> bool {
|
||||
if row.len() < 3 {
|
||||
return false;
|
||||
}
|
||||
let first = row[0].to_string().trim().to_uppercase();
|
||||
header_cfg.tokens.iter().any(|t| t.to_uppercase() == first)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// All-blank row check (data-old2: strip_blank_rows)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn is_all_blank(row: &[Data]) -> bool {
|
||||
row.iter()
|
||||
.all(|c| matches!(c, Data::Empty) || c.to_string().trim().is_empty())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// File processor — yields all data rows from the file
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Process one spreadsheet file, calling `on_row` for each data row.
|
||||
///
|
||||
/// `on_row` receives `(sheet_index, row_index_in_sheet, raw_row)` where
|
||||
/// `row_index_in_sheet` is 0-based AFTER the header has been consumed.
|
||||
/// Returns `(sheets_seen, total_rows_yielded)`.
|
||||
pub fn process_file<F>(
|
||||
path: &Path,
|
||||
cfg: &DatasetConfig,
|
||||
mut on_row: F,
|
||||
) -> Result<(usize, usize), BuildError>
|
||||
where
|
||||
F: FnMut(usize, &RawRow),
|
||||
{
|
||||
let path_str = path.display().to_string();
|
||||
|
||||
// calamine::open_workbook_auto dispatches on file extension
|
||||
let mut workbook: Sheets<_> = open_workbook_auto(path).map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let sheet_names: Vec<String> = workbook.sheet_names().to_vec();
|
||||
if sheet_names.is_empty() {
|
||||
return Err(BuildError::NoSheets(path_str.clone()));
|
||||
}
|
||||
|
||||
// Sheet selection per config
|
||||
let sheets_to_read: Vec<String> = match cfg.reader.sheet_mode {
|
||||
SheetMode::All => sheet_names.clone(),
|
||||
SheetMode::First => vec![sheet_names[0].clone()],
|
||||
};
|
||||
|
||||
let mut total_rows = 0usize;
|
||||
|
||||
for (sheet_idx, sheet_name) in sheets_to_read.iter().enumerate() {
|
||||
let range = workbook
|
||||
.worksheet_range(sheet_name)
|
||||
.map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let mut first_row = true;
|
||||
|
||||
for raw in range.rows() {
|
||||
let row: RawRow = raw.to_vec();
|
||||
|
||||
// Skip header row on first row of each sheet (matches JS: `if (i === 0 && isHeaderRow(...))`)
|
||||
if first_row {
|
||||
first_row = false;
|
||||
if is_header_row(&row, &cfg.header) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
on_row(sheet_idx, &row);
|
||||
total_rows += 1;
|
||||
}
|
||||
}
|
||||
|
||||
Ok((sheets_to_read.len(), total_rows))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::HeaderCfg;
|
||||
|
||||
fn hdr(tokens: &[&str]) -> HeaderCfg {
|
||||
HeaderCfg {
|
||||
tokens: tokens.iter().map(|s| s.to_string()).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_detects_ho_ten() {
|
||||
let row = vec![
|
||||
Data::String("HO_TEN".into()),
|
||||
Data::String("NGAY_SINH".into()),
|
||||
Data::String("SBD".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_detects_stt() {
|
||||
let row = vec![
|
||||
Data::String("STT".into()),
|
||||
Data::String("B".into()),
|
||||
Data::String("C".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_detects_ho_ten_unicode() {
|
||||
let row = vec![
|
||||
Data::String("HỌ TÊN".into()),
|
||||
Data::String("B".into()),
|
||||
Data::String("C".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_rejects_data_row() {
|
||||
let row = vec![
|
||||
Data::String("Nguyen Van A".into()),
|
||||
Data::String("01/01/2000".into()),
|
||||
Data::String("12345678".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(!is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_rejects_short_row() {
|
||||
let row = vec![Data::String("HO_TEN".into()), Data::Empty];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(!is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_case_insensitive() {
|
||||
let row = vec![
|
||||
Data::String("ho_ten".into()),
|
||||
Data::String("B".into()),
|
||||
Data::String("C".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blank_row_detection() {
|
||||
let row = vec![Data::Empty, Data::Empty, Data::String("".into())];
|
||||
assert!(is_all_blank(&row));
|
||||
|
||||
let row2 = vec![Data::String("Nguyen".into()), Data::Empty, Data::Empty];
|
||||
assert!(!is_all_blank(&row2));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
//! Canonical `student` table definition — the single source of truth for the
|
||||
//! SQL shape of every dataset.
|
||||
//!
|
||||
//! Every dataset (2016, 2017, 2017-old, 2017-old2) is written into this same
|
||||
//! 22-column table. Columns a dataset has no data for bind NULL, which costs
|
||||
//! ~1 byte per row in SQLite's record header.
|
||||
//!
|
||||
//! Column provenance:
|
||||
//! ten_cum_thi, gioi_tinh, tieng_duc, tieng_nhat → 2016 only
|
||||
//! khtn, khxh, gdcd, tieng_nga → 2017 datasets only
|
||||
//! everything else → both
|
||||
//!
|
||||
//! Before this module existed the DDL, the INSERT statement and the subject
|
||||
//! regex table were duplicated across four TOML configs, which is how the 2016
|
||||
//! and 2017 schemas drifted apart in the first place. The configs now carry only
|
||||
//! per-dataset parse rules.
|
||||
// ---------------------------------------------------------------------------
|
||||
// DDL
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Executed verbatim after the output database is (re)created.
|
||||
///
|
||||
/// `idx_ten_cum_thi` is partial so it holds zero entries on the three 2017
|
||||
/// datasets — where the column is always NULL — while staying fully useful for
|
||||
/// the 2016 cluster-grouping queries.
|
||||
pub const DDL: &str = "
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
ten_cum_thi TEXT,
|
||||
gioi_tinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
khtn REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
gdcd REAL,
|
||||
khxh REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_nga REAL,
|
||||
tieng_duc REAL,
|
||||
tieng_nhat REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi) WHERE ten_cum_thi IS NOT NULL;
|
||||
";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Column order
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Identity columns, in INSERT parameter order.
|
||||
pub const IDENTITY_FIELDS: &[&str] = &[
|
||||
"so_bao_danh",
|
||||
"ho_ten",
|
||||
"ho_ten_ascii",
|
||||
"ngay_sinh",
|
||||
"ten_cum_thi",
|
||||
"gioi_tinh",
|
||||
];
|
||||
|
||||
/// Subject columns, in INSERT parameter order. Bound as NULL when a row has no
|
||||
/// score for that subject.
|
||||
pub const SCORE_FIELDS: &[&str] = &[
|
||||
"toan",
|
||||
"ngu_van",
|
||||
"vat_ly",
|
||||
"hoa_hoc",
|
||||
"sinh_hoc",
|
||||
"khtn",
|
||||
"lich_su",
|
||||
"dia_ly",
|
||||
"gdcd",
|
||||
"khxh",
|
||||
"tieng_anh",
|
||||
"tieng_phap",
|
||||
"tieng_nga",
|
||||
"tieng_duc",
|
||||
"tieng_nhat",
|
||||
"tieng_trung",
|
||||
];
|
||||
|
||||
/// Total bound parameters per row.
|
||||
pub const PARAM_COUNT: usize = IDENTITY_FIELDS.len() + SCORE_FIELDS.len();
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// INSERT
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Positional INSERT matching IDENTITY_FIELDS followed by SCORE_FIELDS.
|
||||
///
|
||||
/// `OR REPLACE` preserves the pre-existing behaviour where a repeated SBD
|
||||
/// overwrites the earlier row rather than aborting the transaction.
|
||||
pub const INSERT_SQL: &str = "
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, ten_cum_thi, gioi_tinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_duc, tieng_nhat, tieng_trung)
|
||||
VALUES
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Subject score patterns
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Regex per subject, applied to the DIEM_THI cell text.
|
||||
///
|
||||
/// Every pattern runs against every dataset. A subject absent from a given exam
|
||||
/// year simply never matches and stays NULL — 2016 source files contain no
|
||||
/// "KHTN:" or "Tiếng Nga:" tokens, and 2017 files contain no "Tiếng Đức:" or
|
||||
/// "Tiếng Nhật:". The parity check asserts those counts are exactly zero rather
|
||||
/// than assuming it.
|
||||
///
|
||||
/// Order here is irrelevant (matching is by name); SCORE_FIELDS fixes the
|
||||
/// INSERT order.
|
||||
pub const SCORE_PATTERNS: &[(&str, &str)] = &[
|
||||
("toan", r"Toán:\s*(\d+(?:\.\d+)?)"),
|
||||
("ngu_van", r"Ngữ văn:\s*(\d+(?:\.\d+)?)"),
|
||||
("vat_ly", r"Vật lí:\s*(\d+(?:\.\d+)?)"),
|
||||
("hoa_hoc", r"Hóa học:\s*(\d+(?:\.\d+)?)"),
|
||||
("sinh_hoc", r"Sinh học:\s*(\d+(?:\.\d+)?)"),
|
||||
("khtn", r"KHTN:\s*(\d+(?:\.\d+)?)"),
|
||||
("lich_su", r"Lịch sử:\s*(\d+(?:\.\d+)?)"),
|
||||
("dia_ly", r"Địa lí:\s*(\d+(?:\.\d+)?)"),
|
||||
("gdcd", r"GDCD:\s*(\d+(?:\.\d+)?)"),
|
||||
("khxh", r"KHXH:\s*(\d+(?:\.\d+)?)"),
|
||||
("tieng_anh", r"Tiếng Anh:\s*(\d+(?:\.\d+)?)"),
|
||||
("tieng_phap", r"Tiếng Pháp:\s*(\d+(?:\.\d+)?)"),
|
||||
("tieng_nga", r"Tiếng Nga:\s*(\d+(?:\.\d+)?)"),
|
||||
("tieng_duc", r"Tiếng Đức:\s*(\d+(?:\.\d+)?)"),
|
||||
("tieng_nhat", r"Tiếng Nhật:\s*(\d+(?:\.\d+)?)"),
|
||||
("tieng_trung", r"Tiếng Trung:\s*(\d+(?:\.\d+)?)"),
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The INSERT placeholder count, the named column list and the field
|
||||
/// constants must agree, or rows silently land in the wrong columns.
|
||||
#[test]
|
||||
fn insert_matches_field_order() {
|
||||
assert_eq!(PARAM_COUNT, 22);
|
||||
assert_eq!(INSERT_SQL.matches('?').count(), PARAM_COUNT);
|
||||
|
||||
let named = INSERT_SQL
|
||||
.split_once('(')
|
||||
.and_then(|(_, rest)| rest.split_once(')'))
|
||||
.map(|(cols, _)| cols)
|
||||
.expect("INSERT must contain a column list");
|
||||
|
||||
let listed: Vec<&str> = named
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect();
|
||||
|
||||
let expected: Vec<&str> = IDENTITY_FIELDS
|
||||
.iter()
|
||||
.chain(SCORE_FIELDS.iter())
|
||||
.copied()
|
||||
.collect();
|
||||
|
||||
assert_eq!(listed, expected);
|
||||
}
|
||||
|
||||
/// Every subject column must have a pattern and vice versa.
|
||||
#[test]
|
||||
fn score_patterns_cover_score_fields() {
|
||||
assert_eq!(SCORE_PATTERNS.len(), SCORE_FIELDS.len());
|
||||
for (field, _) in SCORE_PATTERNS {
|
||||
assert!(
|
||||
SCORE_FIELDS.contains(field),
|
||||
"pattern {field} has no column"
|
||||
);
|
||||
}
|
||||
for field in SCORE_FIELDS {
|
||||
assert!(
|
||||
SCORE_PATTERNS.iter().any(|(f, _)| f == field),
|
||||
"column {field} has no pattern"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Every column named in the DDL must be bound by the INSERT.
|
||||
#[test]
|
||||
fn ddl_columns_match_insert() {
|
||||
for field in IDENTITY_FIELDS.iter().chain(SCORE_FIELDS.iter()) {
|
||||
assert!(DDL.contains(field), "DDL missing column {field}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_patterns_compile() {
|
||||
for (field, src) in SCORE_PATTERNS {
|
||||
regex::Regex::new(src).unwrap_or_else(|e| panic!("{field}: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,409 @@
|
||||
/// Row transformation: ascii normalisation, score regex parsing, validation.
|
||||
///
|
||||
/// `to_ascii` replicates build-lib.js `toAscii` exactly:
|
||||
/// str.normalize("NFD").replace(/[̀-ͯ]/g,"").replace(/đ/gi,"d").toLowerCase()
|
||||
use std::collections::HashMap;
|
||||
|
||||
use regex::Regex;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
use crate::config::{DatasetConfig, ValidationCfg};
|
||||
use crate::error::BuildError;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Compiled score patterns (built once at startup from config)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct CompiledPatterns {
|
||||
/// One compiled regex per subject column in `schema::SCORE_PATTERNS`.
|
||||
pub patterns: Vec<(String, Regex)>,
|
||||
}
|
||||
|
||||
impl CompiledPatterns {
|
||||
/// Compile the canonical subject patterns once at startup.
|
||||
///
|
||||
/// All 16 patterns run against every dataset. A subject that did not exist
|
||||
/// in a given exam year simply never matches and stays NULL — the parity
|
||||
/// check asserts those counts are exactly zero rather than assuming it.
|
||||
pub fn new() -> Result<Self, BuildError> {
|
||||
let mut patterns = Vec::with_capacity(crate::schema::SCORE_PATTERNS.len());
|
||||
for (field, src) in crate::schema::SCORE_PATTERNS {
|
||||
let re = Regex::new(src).map_err(|e| BuildError::Regex {
|
||||
pattern: (*src).to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
patterns.push(((*field).to_string(), re));
|
||||
}
|
||||
Ok(Self { patterns })
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// to_ascii — must be byte-for-byte equivalent to build-lib.js toAscii
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Normalise a Vietnamese name to an ASCII slug.
|
||||
///
|
||||
/// Algorithm mirrors the JavaScript `toAscii` in build-lib.js:
|
||||
/// 1. NFD decompose (splits base + combining diacritics)
|
||||
/// 2. Drop all Unicode combining marks (U+0300–U+036F)
|
||||
/// 3. Replace đ/Đ with d (NFD does not decompose đ)
|
||||
/// 4. Lowercase
|
||||
pub fn to_ascii(s: &str) -> String {
|
||||
// Step 1 + 2: NFD then filter out combining marks (Unicode category M)
|
||||
let decomposed: String = s
|
||||
.nfd()
|
||||
.filter(|c| !('\u{0300}'..='\u{036f}').contains(c))
|
||||
.collect();
|
||||
|
||||
// Step 3: đ/Đ are not decomposed by NFD — replace explicitly
|
||||
let replaced = decomposed.replace(['đ', 'Đ'], "d");
|
||||
|
||||
// Step 4: lowercase
|
||||
replaced.to_lowercase()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parsed row ready for DB insert
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct ParsedRow {
|
||||
pub so_bao_danh: String,
|
||||
pub ho_ten: String,
|
||||
pub ho_ten_ascii: String,
|
||||
pub ngay_sinh: Option<String>,
|
||||
/// thptqg2016 only: examination cluster name (TEN_CUMTHI column)
|
||||
pub ten_cum_thi: Option<String>,
|
||||
/// thptqg2016 only: gender (GIOI_TINH column), normalised to "Nam"/"Nữ" or None
|
||||
pub gioi_tinh: Option<String>,
|
||||
/// Subject field → float value; absent subjects not in map → NULL
|
||||
pub scores: HashMap<String, f64>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Row validation — mirrors the per-script skip logic
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Returns `None` when the row should be skipped entirely (before sourceRows counter).
|
||||
/// Returns `Some(reason)` when the row should be counted as sourceRows but skipped.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum SkipReason {
|
||||
/// Row is fully blank (data-old2 only, before sourceRows counter)
|
||||
BlankRow,
|
||||
/// soBaoDanh or hoTen empty/missing
|
||||
EmptyField,
|
||||
/// soBaoDanh contains non-digit characters (data-old / data-old2 guard)
|
||||
NonNumericSbd,
|
||||
}
|
||||
|
||||
/// Validates a raw cell slice against the dataset's `ValidationCfg`.
|
||||
/// Returns `Ok(())` on pass, `Err(SkipReason)` on fail.
|
||||
pub fn validate_row(
|
||||
ho_ten: &str,
|
||||
so_bao_danh: &str,
|
||||
cfg: &ValidationCfg,
|
||||
strip_blank_rows: bool,
|
||||
all_blank: bool,
|
||||
) -> Result<(), SkipReason> {
|
||||
// data-old2: skip fully blank rows BEFORE counting sourceRows
|
||||
if strip_blank_rows && all_blank {
|
||||
return Err(SkipReason::BlankRow);
|
||||
}
|
||||
|
||||
if cfg.require_nonempty_sbd && so_bao_danh.is_empty() {
|
||||
return Err(SkipReason::EmptyField);
|
||||
}
|
||||
if cfg.require_nonempty_name && ho_ten.is_empty() {
|
||||
return Err(SkipReason::EmptyField);
|
||||
}
|
||||
if cfg.require_numeric_sbd && !so_bao_danh.chars().all(|c| c.is_ascii_digit()) {
|
||||
return Err(SkipReason::NonNumericSbd);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Score parsing — mirrors build-lib.js parseScores
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Parse a DIEM_THI cell string and extract matching subject scores.
|
||||
pub fn parse_scores(diem_thi: &str, patterns: &CompiledPatterns) -> HashMap<String, f64> {
|
||||
let mut out = HashMap::new();
|
||||
for (field, re) in &patterns.patterns {
|
||||
if let Some(caps) = re.captures(diem_thi) {
|
||||
if let Some(m) = caps.get(1) {
|
||||
if let Ok(v) = m.as_str().parse::<f64>() {
|
||||
if v.is_finite() {
|
||||
out.insert(field.clone(), v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Full row transform (thptqg2017 fixed-column path)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Extract and transform one spreadsheet row into a `ParsedRow` using fixed column indices.
|
||||
/// `raw` is the full cell slice; column indices come from `cfg.columns`.
|
||||
/// Used for thptqg2017 configs that have a static [columns] table.
|
||||
pub fn transform_row(
|
||||
raw: &[calamine::Data],
|
||||
cfg: &DatasetConfig,
|
||||
patterns: &CompiledPatterns,
|
||||
) -> ParsedRow {
|
||||
let cols = cfg
|
||||
.columns
|
||||
.as_ref()
|
||||
.expect("transform_row requires [columns] section in config");
|
||||
|
||||
let get = |idx: usize| -> String {
|
||||
raw.get(idx)
|
||||
.map(|cell| cell.to_string().trim().to_owned())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
|
||||
let ho_ten = get(cols.ho_ten);
|
||||
let ngay_sinh = get(cols.ngay_sinh);
|
||||
let so_bao_danh = get(cols.so_bao_danh);
|
||||
let diem_thi = raw
|
||||
.get(cols.diem_thi)
|
||||
.map(|c| c.to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let ho_ten_ascii = to_ascii(&ho_ten);
|
||||
let scores = parse_scores(&diem_thi, patterns);
|
||||
let ngay_sinh_opt = if ngay_sinh.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(ngay_sinh)
|
||||
};
|
||||
|
||||
ParsedRow {
|
||||
so_bao_danh,
|
||||
ho_ten,
|
||||
ho_ten_ascii,
|
||||
ngay_sinh: ngay_sinh_opt,
|
||||
ten_cum_thi: None,
|
||||
gioi_tinh: None,
|
||||
scores,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests — 20 cases for to_ascii (real Vietnamese names)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// Helper: assert to_ascii(input) == expected
|
||||
fn check(input: &str, expected: &str) {
|
||||
assert_eq!(
|
||||
to_ascii(input),
|
||||
expected,
|
||||
"to_ascii({input:?}) expected {expected:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_plain_latin() {
|
||||
check("Nguyen Van A", "nguyen van a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_thi_hoa() {
|
||||
check("Nguyễn Thị Hoa", "nguyen thi hoa");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_tran_van_duc() {
|
||||
// đ/Đ replacement
|
||||
check("Trần Văn Đức", "tran van duc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_le_thi_my_duyen() {
|
||||
check("Lê Thị Mỹ Duyên", "le thi my duyen");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_pham_thi_lan() {
|
||||
check("Phạm Thị Lan", "pham thi lan");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_bui_thi_thu() {
|
||||
check("Bùi Thị Thu", "bui thi thu");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_hoang_van_truong() {
|
||||
check("Hoàng Văn Trường", "hoang van truong");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_do_thi_ngan() {
|
||||
// Đ uppercase at start
|
||||
check("Đỗ Thị Ngân", "do thi ngan");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_van_khanh() {
|
||||
check("Nguyễn Văn Khánh", "nguyen van khanh");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_trinh_thi_bich_ngoc() {
|
||||
check("Trịnh Thị Bích Ngọc", "trinh thi bich ngoc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_vu_thi_dieu() {
|
||||
// ề = e + combining grave + combining circumflex (after NFD)
|
||||
check("Vũ Thị Diệu", "vu thi dieu");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_thi_tuong_vi() {
|
||||
check("Nguyễn Thị Tường Vi", "nguyen thi tuong vi");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_lowercase_d_stroke() {
|
||||
// Lowercase đ → d
|
||||
check("đặng thị hằng", "dang thi hang");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_uppercase_d_stroke() {
|
||||
check("ĐẶNG THỊ HẰNG", "dang thi hang");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_mixed_case() {
|
||||
check("NGUYỄN VĂN AN", "nguyen van an");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_tran_thi_kim_anh() {
|
||||
check("Trần Thị Kim Anh", "tran thi kim anh");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_thi_phuong_thao() {
|
||||
check("Nguyễn Thị Phương Thảo", "nguyen thi phuong thao");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_le_van_long() {
|
||||
check("Lê Văn Long", "le van long");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_vo_thi_xuan_mai() {
|
||||
check("Võ Thị Xuân Mai", "vo thi xuan mai");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_empty_string() {
|
||||
check("", "");
|
||||
}
|
||||
|
||||
// --- Score parsing tests ---
|
||||
|
||||
fn make_patterns() -> CompiledPatterns {
|
||||
CompiledPatterns::new().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_scores_single() {
|
||||
let p = make_patterns();
|
||||
let s = "Toán: 8.5";
|
||||
let scores = parse_scores(s, &p);
|
||||
assert_eq!(scores.get("toan"), Some(&8.5));
|
||||
assert!(!scores.contains_key("ngu_van"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_scores_multiple() {
|
||||
let p = make_patterns();
|
||||
let s = "Toán: 7.25 Ngữ văn: 6.0 Vật lí: 9";
|
||||
let scores = parse_scores(s, &p);
|
||||
assert_eq!(scores.get("toan"), Some(&7.25));
|
||||
assert_eq!(scores.get("ngu_van"), Some(&6.0));
|
||||
assert_eq!(scores.get("vat_ly"), Some(&9.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_scores_empty_cell() {
|
||||
let p = make_patterns();
|
||||
let scores = parse_scores("", &p);
|
||||
assert!(scores.is_empty());
|
||||
}
|
||||
|
||||
// --- Validation tests ---
|
||||
|
||||
fn default_validation() -> ValidationCfg {
|
||||
ValidationCfg {
|
||||
require_numeric_sbd: false,
|
||||
require_nonempty_name: true,
|
||||
require_nonempty_sbd: true,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_ok() {
|
||||
let v = default_validation();
|
||||
assert!(validate_row("Nguyen Van A", "12345678", &v, false, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_empty_sbd() {
|
||||
let v = default_validation();
|
||||
assert_eq!(
|
||||
validate_row("Nguyen Van A", "", &v, false, false),
|
||||
Err(SkipReason::EmptyField)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_empty_name() {
|
||||
let v = default_validation();
|
||||
assert_eq!(
|
||||
validate_row("", "12345678", &v, false, false),
|
||||
Err(SkipReason::EmptyField)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_non_numeric_sbd_rejected() {
|
||||
let mut v = default_validation();
|
||||
v.require_numeric_sbd = true;
|
||||
assert_eq!(
|
||||
validate_row("Nguyen Van A", "12AB5678", &v, false, false),
|
||||
Err(SkipReason::NonNumericSbd)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_numeric_sbd_accepted() {
|
||||
let mut v = default_validation();
|
||||
v.require_numeric_sbd = true;
|
||||
assert!(validate_row("Nguyen Van A", "12345678", &v, false, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_blank_row_skipped() {
|
||||
let v = default_validation();
|
||||
// strip_blank_rows=true AND all_blank=true → BlankRow
|
||||
assert_eq!(
|
||||
validate_row("", "", &v, true, true),
|
||||
Err(SkipReason::BlankRow)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/// SQLite writer: DDL setup, batched INSERT OR REPLACE, VACUUM, stats output.
|
||||
///
|
||||
/// Mirrors build-lib.js createDb + the transaction loop in each build-database*.js.
|
||||
/// Stats output lines match the JS stdout exactly so existing CI log-greps still work.
|
||||
///
|
||||
/// Every dataset writes the same canonical table (see `crate::schema`), so there
|
||||
/// is exactly one insert path. Columns a dataset carries no data for bind NULL.
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use rusqlite::{params_from_iter, Connection, ToSql};
|
||||
|
||||
use crate::error::BuildError;
|
||||
use crate::schema;
|
||||
use crate::transform::ParsedRow;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DB initialisation — mirrors build-lib.js createDb (delete + recreate)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Open (or recreate) the output SQLite database, execute the canonical DDL,
|
||||
/// and return the open connection ready for inserts.
|
||||
pub fn open_db(db_path: &Path) -> Result<Connection, BuildError> {
|
||||
// Mirror Node behaviour: delete existing file before creating (build-lib.js:54)
|
||||
if db_path.exists() {
|
||||
fs::remove_file(db_path).map_err(|e| BuildError::Io {
|
||||
path: db_path.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
if let Some(parent) = db_path.parent() {
|
||||
if !parent.as_os_str().is_empty() {
|
||||
fs::create_dir_all(parent).map_err(|e| BuildError::Io {
|
||||
path: parent.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
}
|
||||
}
|
||||
|
||||
let conn = Connection::open(db_path)?;
|
||||
conn.execute_batch(schema::DDL)?;
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Insert a single parsed row
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Bind every field from `row` into the canonical INSERT and execute it.
|
||||
///
|
||||
/// Parameter order is `schema::IDENTITY_FIELDS` followed by
|
||||
/// `schema::SCORE_FIELDS`. Subjects absent from `row.scores` — and the two
|
||||
/// identity columns only the thptqg2016 layouts populate — bind NULL.
|
||||
pub fn insert_row(conn: &Connection, row: &ParsedRow) -> Result<(), BuildError> {
|
||||
let mut params: Vec<Box<dyn ToSql>> = Vec::with_capacity(schema::PARAM_COUNT);
|
||||
|
||||
params.push(Box::new(row.so_bao_danh.clone()));
|
||||
params.push(Box::new(row.ho_ten.clone()));
|
||||
params.push(Box::new(row.ho_ten_ascii.clone()));
|
||||
params.push(Box::new(row.ngay_sinh.clone()));
|
||||
params.push(Box::new(row.ten_cum_thi.clone()));
|
||||
params.push(Box::new(row.gioi_tinh.clone()));
|
||||
|
||||
for field in schema::SCORE_FIELDS {
|
||||
let val: Option<f64> = row.scores.get(*field).copied();
|
||||
params.push(Box::new(val));
|
||||
}
|
||||
|
||||
debug_assert_eq!(params.len(), schema::PARAM_COUNT);
|
||||
|
||||
conn.execute(
|
||||
schema::INSERT_SQL,
|
||||
params_from_iter(params.iter().map(|p| p.as_ref())),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Post-build: VACUUM + stats output
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Run VACUUM and print statistics lines that mirror the Node scripts' stdout.
|
||||
/// The exact prefix tokens ("Source data rows", "DB rows", "Size:") are preserved
|
||||
/// so any log-grep in the deploy pipeline keeps working.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn finish_db(
|
||||
conn: &Connection,
|
||||
db_path: &Path,
|
||||
source_rows: u64,
|
||||
skipped: u64,
|
||||
errors: u64,
|
||||
dataset_label: &str,
|
||||
_file_count: usize,
|
||||
is_old2: bool,
|
||||
) -> Result<(), BuildError> {
|
||||
conn.execute_batch("VACUUM")?;
|
||||
|
||||
let db_count: i64 = conn.query_row("SELECT COUNT(*) FROM student", [], |row| row.get(0))?;
|
||||
|
||||
let insertable = source_rows - skipped;
|
||||
|
||||
println!();
|
||||
if is_old2 {
|
||||
println!("Source non-blank data rows: {source_rows}");
|
||||
println!(" skipped (empty/non-numeric SBD): {skipped}");
|
||||
} else {
|
||||
println!("Source data rows (post-header): {source_rows}");
|
||||
if dataset_label.contains("old") {
|
||||
println!(" skipped (empty/non-numeric SBD): {skipped}");
|
||||
} else {
|
||||
println!(" skipped (empty/invalid): {skipped}");
|
||||
}
|
||||
}
|
||||
println!(" insertable: {insertable}");
|
||||
println!(" insert errors: {errors}");
|
||||
println!("DB rows (distinct SBD): {db_count}");
|
||||
|
||||
if !dataset_label.contains("old") && errors == 0 {
|
||||
let gap = insertable as i64 - db_count;
|
||||
if gap == 0 {
|
||||
println!("Audit: OK — every source row made it in.");
|
||||
} else {
|
||||
println!("Audit: {gap} row(s) collapsed (duplicate SBDs overwriting).");
|
||||
}
|
||||
}
|
||||
|
||||
let sz = fs::metadata(db_path).map(|m| m.len()).unwrap_or(0);
|
||||
println!("Size: {:.1} MB", sz as f64 / 1024.0 / 1024.0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
# Test Fixtures
|
||||
|
||||
Anonymised `.xlsx` files for integration testing. All student PII has been replaced:
|
||||
|
||||
- `ho_ten` replaced with `Nguyen Van Test NNN` / `Tran Thi Test NNN` patterns
|
||||
- `so_bao_danh` replaced with sequential synthetic numbers (e.g. `10000001`)
|
||||
- `ngay_sinh` replaced with fixed synthetic dates
|
||||
- Scores are realistic random values in the 0–10 range
|
||||
|
||||
Files:
|
||||
|
||||
- `province-100.xlsx` — 100-row single-sheet file (simulates a normal province)
|
||||
- `hcm-overflow.xlsx` — 2-sheet file (200 rows Sheet1 + 200 rows Sheet2, simulating HCM overflow)
|
||||
- `province-numeric-sbd.xlsx` — 100 rows with strictly numeric SBDs (for data-old variant)
|
||||
|
||||
These files are generated by `tests/golden.rs` `generate_fixtures()` if they do not already exist
|
||||
on disk. The generator is pure Rust (uses the `zip` crate already pulled in via calamine).
|
||||
No external Python or Node tooling required for unit/integration tests.
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,591 @@
|
||||
/// Golden tests — integration tests using anonymised fixture files.
|
||||
///
|
||||
/// Fixture files are generated in-process via raw OOXML + zip if they do not
|
||||
/// already exist on disk. No external tooling is required.
|
||||
use std::io::Write as IoWrite;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Minimal OOXML xlsx generator
|
||||
//
|
||||
// Produces a valid .xlsx that calamine can read. Only uses the `zip` crate
|
||||
// which is already pulled in as a transitive dependency of calamine.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One row of cell data for a fixture sheet.
|
||||
struct XlsxRow {
|
||||
values: Vec<String>,
|
||||
}
|
||||
|
||||
/// Write a minimal .xlsx to `path` with the given sheets.
|
||||
/// `sheets`: Vec<(sheet_name, rows)> where rows[0] is the header.
|
||||
fn write_xlsx(path: &Path, sheets: &[(String, Vec<XlsxRow>)]) {
|
||||
use zip::{write::SimpleFileOptions, ZipWriter};
|
||||
|
||||
let file = std::fs::File::create(path).expect("create fixture xlsx");
|
||||
let mut zip = ZipWriter::new(file);
|
||||
let opts = SimpleFileOptions::default();
|
||||
|
||||
// [Content_Types].xml
|
||||
let mut content_types = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
|
||||
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
|
||||
<Default Extension="xml" ContentType="application/xml"/>
|
||||
<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>
|
||||
"#,
|
||||
);
|
||||
for (i, _) in sheets.iter().enumerate() {
|
||||
content_types.push_str(&format!(
|
||||
r#" <Override PartName="/xl/worksheets/sheet{}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>
|
||||
"#,
|
||||
i + 1
|
||||
));
|
||||
}
|
||||
content_types.push_str("</Types>");
|
||||
zip.start_file("[Content_Types].xml", opts).unwrap();
|
||||
zip.write_all(content_types.as_bytes()).unwrap();
|
||||
|
||||
// _rels/.rels
|
||||
zip.start_file("_rels/.rels", opts).unwrap();
|
||||
zip.write_all(
|
||||
br#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>
|
||||
</Relationships>"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// xl/_rels/workbook.xml.rels
|
||||
let mut wb_rels = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
"#,
|
||||
);
|
||||
for (i, _) in sheets.iter().enumerate() {
|
||||
wb_rels.push_str(&format!(
|
||||
r#" <Relationship Id="rId{}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet{}.xml"/>
|
||||
"#,
|
||||
i + 1,
|
||||
i + 1
|
||||
));
|
||||
}
|
||||
wb_rels.push_str("</Relationships>");
|
||||
zip.start_file("xl/_rels/workbook.xml.rels", opts).unwrap();
|
||||
zip.write_all(wb_rels.as_bytes()).unwrap();
|
||||
|
||||
// xl/workbook.xml
|
||||
let mut wb = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
|
||||
xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
|
||||
<sheets>
|
||||
"#,
|
||||
);
|
||||
for (i, (name, _)) in sheets.iter().enumerate() {
|
||||
let escaped = xml_escape(name);
|
||||
wb.push_str(&format!(
|
||||
r#" <sheet name="{}" sheetId="{}" r:id="rId{}"/>
|
||||
"#,
|
||||
escaped,
|
||||
i + 1,
|
||||
i + 1
|
||||
));
|
||||
}
|
||||
wb.push_str(" </sheets>\n</workbook>");
|
||||
zip.start_file("xl/workbook.xml", opts).unwrap();
|
||||
zip.write_all(wb.as_bytes()).unwrap();
|
||||
|
||||
// xl/worksheets/sheetN.xml
|
||||
for (i, (_, rows)) in sheets.iter().enumerate() {
|
||||
let mut ws = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<sheetData>
|
||||
"#,
|
||||
);
|
||||
for (row_idx, row) in rows.iter().enumerate() {
|
||||
ws.push_str(&format!(
|
||||
r#" <row r="{}">
|
||||
"#,
|
||||
row_idx + 1
|
||||
));
|
||||
for (col_idx, val) in row.values.iter().enumerate() {
|
||||
let col_letter = col_letter(col_idx);
|
||||
let cell_ref = format!("{}{}", col_letter, row_idx + 1);
|
||||
let escaped = xml_escape(val);
|
||||
ws.push_str(&format!(
|
||||
r#" <c r="{}" t="inlineStr"><is><t>{}</t></is></c>
|
||||
"#,
|
||||
cell_ref, escaped
|
||||
));
|
||||
}
|
||||
ws.push_str(" </row>\n");
|
||||
}
|
||||
ws.push_str(" </sheetData>\n</worksheet>");
|
||||
zip.start_file(format!("xl/worksheets/sheet{}.xml", i + 1), opts)
|
||||
.unwrap();
|
||||
zip.write_all(ws.as_bytes()).unwrap();
|
||||
}
|
||||
|
||||
zip.finish().unwrap();
|
||||
}
|
||||
|
||||
fn col_letter(idx: usize) -> &'static str {
|
||||
const LETTERS: &[&str] = &[
|
||||
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R",
|
||||
"S", "T", "U", "V", "W", "X", "Y", "Z",
|
||||
];
|
||||
LETTERS[idx % 26]
|
||||
}
|
||||
|
||||
fn xml_escape(s: &str) -> String {
|
||||
s.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('>', ">")
|
||||
.replace('"', """)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture data builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn header_row() -> XlsxRow {
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
"HO_TEN".into(),
|
||||
"NGAY_SINH".into(),
|
||||
"SO_BAO_DANH".into(),
|
||||
"DIEM_THI".into(),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn data_row(idx: usize, scores: &str) -> XlsxRow {
|
||||
// Anonymised: name uses sequential pattern, SBD is purely synthetic
|
||||
let name = if idx.is_multiple_of(2) {
|
||||
format!("Nguyen Van Test {:03}", idx)
|
||||
} else {
|
||||
format!("Tran Thi Test {:03}", idx)
|
||||
};
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
name,
|
||||
format!("15/0{}/{}", (idx % 9) + 1, 1999 + (idx % 5)),
|
||||
format!("1000{:04}", idx),
|
||||
scores.to_owned(),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_scores(idx: usize) -> String {
|
||||
// Realistic scores in 0–10 range, varies by idx
|
||||
let toan = 4.0 + (idx % 60) as f64 / 10.0;
|
||||
let van = 3.5 + (idx % 65) as f64 / 10.0;
|
||||
format!("Toán: {toan:.1} Ngữ văn: {van:.1} Tiếng Anh: 7.5")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture file paths
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
// tests/fixtures/ relative to the crate root
|
||||
let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
p.push("tests");
|
||||
p.push("fixtures");
|
||||
p
|
||||
}
|
||||
|
||||
fn province_fixture_path() -> PathBuf {
|
||||
fixtures_dir().join("province-100.xlsx")
|
||||
}
|
||||
|
||||
fn hcm_overflow_fixture_path() -> PathBuf {
|
||||
fixtures_dir().join("hcm-overflow.xlsx")
|
||||
}
|
||||
|
||||
fn numeric_sbd_fixture_path() -> PathBuf {
|
||||
fixtures_dir().join("province-numeric-sbd.xlsx")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture generation — called once per test run if files missing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn ensure_fixtures() {
|
||||
let dir = fixtures_dir();
|
||||
std::fs::create_dir_all(&dir).expect("create fixtures dir");
|
||||
|
||||
// province-100.xlsx — 100 data rows, single sheet, with header
|
||||
if !province_fixture_path().exists() {
|
||||
let mut rows = vec![header_row()];
|
||||
for i in 0..100 {
|
||||
rows.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(&province_fixture_path(), &[("Sheet1".to_owned(), rows)]);
|
||||
}
|
||||
|
||||
// hcm-overflow.xlsx — 2 sheets × 200 rows each (no header on sheet 2)
|
||||
if !hcm_overflow_fixture_path().exists() {
|
||||
let mut sheet1 = vec![header_row()];
|
||||
for i in 0..200 {
|
||||
sheet1.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
// Sheet2: continuation rows, no header row (as in real HCM overflow)
|
||||
let mut sheet2 = Vec::new();
|
||||
for i in 200..400 {
|
||||
sheet2.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(
|
||||
&hcm_overflow_fixture_path(),
|
||||
&[("Sheet1".to_owned(), sheet1), ("Sheet2".to_owned(), sheet2)],
|
||||
);
|
||||
}
|
||||
|
||||
// province-numeric-sbd.xlsx — strictly numeric SBDs for data-old config
|
||||
if !numeric_sbd_fixture_path().exists() {
|
||||
let mut rows = vec![header_row()];
|
||||
for i in 0..100 {
|
||||
rows.push(XlsxRow {
|
||||
values: vec![
|
||||
format!("Nguyen Van Test {:03}", i),
|
||||
"01/01/2000".to_owned(),
|
||||
format!("{:08}", 20000000 + i), // pure digits
|
||||
sample_scores(i),
|
||||
],
|
||||
});
|
||||
}
|
||||
write_xlsx(&numeric_sbd_fixture_path(), &[("Sheet1".to_owned(), rows)]);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn make_data_config() -> xlsxread::config::DatasetConfig {
|
||||
let cfg_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("configs")
|
||||
.join("2017.toml");
|
||||
xlsxread::config::load_config(&cfg_path).expect("load data config")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Integration tests — pure Rust, no Node dependency
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn province_100_builds_100_rows() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "2017.toml");
|
||||
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(count, 100, "expected 100 rows from province-100 fixture");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hcm_overflow_builds_400_rows() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
std::fs::copy(hcm_overflow_fixture_path(), fixture_dir.join("hcm.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "2017.toml");
|
||||
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(
|
||||
count, 400,
|
||||
"expected 400 rows (200 × 2 sheets) from hcm-overflow fixture"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_old_first_sheet_only_100_rows() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
// Use the overflow file but with data-old config (first sheet only → 200 rows)
|
||||
std::fs::copy(hcm_overflow_fixture_path(), fixture_dir.join("hcm.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "2017-old.toml");
|
||||
|
||||
// data-old: sheet_mode=first → only 200 rows from sheet1; but SBDs "1000NNNN" are
|
||||
// all digits so all pass the numeric guard
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(
|
||||
count, 200,
|
||||
"data-old config should read only first sheet (200 rows)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numeric_sbd_guard_rejects_non_numeric() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
// Write a fixture with one non-numeric SBD mixed in
|
||||
let mut rows = vec![header_row()];
|
||||
for i in 0..10 {
|
||||
rows.push(XlsxRow {
|
||||
values: vec![
|
||||
format!("Test {:03}", i),
|
||||
"01/01/2000".to_owned(),
|
||||
if i == 5 {
|
||||
"ABC123".to_owned()
|
||||
} else {
|
||||
format!("{:08}", 20000000 + i)
|
||||
},
|
||||
sample_scores(i),
|
||||
],
|
||||
});
|
||||
}
|
||||
let mixed_path = fixture_dir.join("mixed.xlsx");
|
||||
write_xlsx(&mixed_path, &[("Sheet1".to_owned(), rows)]);
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "2017-old.toml");
|
||||
|
||||
// Row i=5 has non-numeric SBD → rejected by data-old config
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(
|
||||
count, 9,
|
||||
"non-numeric SBD row should be skipped by data-old config"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scores_parsed_correctly_into_db() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
let rows = vec![
|
||||
header_row(),
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
"Nguyen Van Test 001".to_owned(),
|
||||
"01/01/2000".to_owned(),
|
||||
"10000001".to_owned(),
|
||||
"Toán: 8.5 Ngữ văn: 7.0 Tiếng Anh: 9.25".to_owned(),
|
||||
],
|
||||
},
|
||||
];
|
||||
write_xlsx(
|
||||
&fixture_dir.join("one.xlsx"),
|
||||
&[("Sheet1".to_owned(), rows)],
|
||||
);
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "2017.toml");
|
||||
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
let (toan, van, anh): (f64, f64, f64) = conn
|
||||
.query_row(
|
||||
"SELECT toan, ngu_van, tieng_anh FROM student WHERE so_bao_danh = '10000001'",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
)
|
||||
.expect("row not found");
|
||||
assert!((toan - 8.5).abs() < 1e-9);
|
||||
assert!((van - 7.0).abs() < 1e-9);
|
||||
assert!((anh - 9.25).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_ascii_stored_correctly() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
let rows = vec![
|
||||
header_row(),
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
"Nguyễn Văn Đức".to_owned(),
|
||||
"".to_owned(),
|
||||
"20000001".to_owned(),
|
||||
"".to_owned(),
|
||||
],
|
||||
},
|
||||
];
|
||||
write_xlsx(
|
||||
&fixture_dir.join("one.xlsx"),
|
||||
&[("Sheet1".to_owned(), rows)],
|
||||
);
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "2017.toml");
|
||||
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
let ascii: String = conn
|
||||
.query_row(
|
||||
"SELECT ho_ten_ascii FROM student WHERE so_bao_danh = '20000001'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.expect("row not found");
|
||||
assert_eq!(ascii, "nguyen van duc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audit_subcommand_matches_after_build() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "2017.toml");
|
||||
|
||||
// audit should match (100 distinct SBDs in xlsx == 100 rows in DB)
|
||||
let cfg = make_data_config();
|
||||
let result = xlsxread::audit::run_audit(&fixture_dir, &db_path, &cfg).expect("audit failed");
|
||||
assert!(result.matched, "audit should match after build");
|
||||
assert_eq!(result.distinct_sbds, 100);
|
||||
assert_eq!(result.db_count, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audit_subcommand_mismatch_detected() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
// Write 10 rows to xlsx but build DB from only 5 rows
|
||||
let mut all_rows = vec![header_row()];
|
||||
for i in 0..10 {
|
||||
all_rows.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(
|
||||
&fixture_dir.join("all.xlsx"),
|
||||
&[("Sheet1".to_owned(), all_rows)],
|
||||
);
|
||||
|
||||
// Build DB with only first 5 rows in a different file
|
||||
let build_dir = dir.join("build_input");
|
||||
std::fs::create_dir_all(&build_dir).unwrap();
|
||||
let mut five_rows = vec![header_row()];
|
||||
for i in 0..5 {
|
||||
five_rows.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(
|
||||
&build_dir.join("five.xlsx"),
|
||||
&[("Sheet1".to_owned(), five_rows)],
|
||||
);
|
||||
|
||||
run_build_cmd(&build_dir, &db_path, "2017.toml");
|
||||
|
||||
// audit against fixture_dir (10 xlsx rows) but DB has 5 rows → mismatch
|
||||
let cfg = make_data_config();
|
||||
let result = xlsxread::audit::run_audit(&fixture_dir, &db_path, &cfg).expect("audit failed");
|
||||
assert!(!result.matched, "audit should not match (10 xlsx vs 5 db)");
|
||||
assert_eq!(result.distinct_sbds, 10);
|
||||
assert_eq!(result.db_count, 5);
|
||||
}
|
||||
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn tempdir() -> PathBuf {
|
||||
let base = std::env::temp_dir().join(format!(
|
||||
"xlsxread-test-{}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.subsec_nanos()
|
||||
));
|
||||
std::fs::create_dir_all(&base).unwrap();
|
||||
base
|
||||
}
|
||||
|
||||
fn run_build_cmd(input_dir: &Path, db_path: &Path, config_name: &str) {
|
||||
let cfg_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("configs")
|
||||
.join(config_name);
|
||||
|
||||
let cfg = xlsxread::config::load_config(&cfg_path)
|
||||
.unwrap_or_else(|e| panic!("load config {config_name}: {e}"));
|
||||
let patterns = xlsxread::transform::CompiledPatterns::new().expect("compile patterns");
|
||||
|
||||
// Collect files
|
||||
let mut files: Vec<PathBuf> = std::fs::read_dir(input_dir)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.is_file()
|
||||
&& p.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| {
|
||||
let l = e.to_lowercase();
|
||||
l == "xls" || l == "xlsx"
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
|
||||
let conn = xlsxread::writer::open_db(db_path).expect("open db");
|
||||
conn.execute_batch("BEGIN").unwrap();
|
||||
|
||||
for file in &files {
|
||||
xlsxread::reader::process_file(file, &cfg, |_, raw| {
|
||||
let all_blank = xlsxread::reader::is_all_blank(raw);
|
||||
if cfg.reader.strip_blank_rows && all_blank {
|
||||
return;
|
||||
}
|
||||
let cols = cfg.columns.as_ref().expect("golden test requires [columns]");
|
||||
let ho_ten = raw
|
||||
.get(cols.ho_ten)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
let so_bao_danh = raw
|
||||
.get(cols.so_bao_danh)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
if xlsxread::transform::validate_row(
|
||||
&ho_ten,
|
||||
&so_bao_danh,
|
||||
&cfg.validation,
|
||||
cfg.reader.strip_blank_rows,
|
||||
all_blank,
|
||||
)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
let row = xlsxread::transform::transform_row(raw, &cfg, &patterns);
|
||||
let _ = xlsxread::writer::insert_row(&conn, &row);
|
||||
})
|
||||
.expect("process file");
|
||||
}
|
||||
|
||||
conn.execute_batch("COMMIT").unwrap();
|
||||
conn.execute_batch("VACUUM").unwrap();
|
||||
}
|
||||
|
||||
fn query_count(db_path: &Path) -> i64 {
|
||||
let conn = rusqlite::Connection::open(db_path).expect("open db for count");
|
||||
conn.query_row("SELECT COUNT(*) FROM student", [], |r| r.get(0))
|
||||
.expect("count query")
|
||||
}
|
||||
Reference in new issue
Block a user