From 6b21c625ac0764ab0a7b080317c970bf00253e3d Mon Sep 17 00:00:00 2001 From: Tien Nguyen Minh Date: Tue, 19 May 2026 16:33:20 +0700 Subject: [PATCH] feat: replace xlsx (SheetJS) build pipeline with Rust xlsxread CLI (#1) * feat(xlsxread): vendor Rust binary cloned from thptqg2017 Copies the xlsxread Rust crate from thptqg2017@8b4a755 (chore/xlsxread-rust). Adds format_detect_2016 module with per-file column-layout auto-detection mirroring detectFormat() in scripts/build-database.js (lines 63-87): - separate-scores: SBD/HOTEN/TOAN... fixed columns (dhhanghai files) - mapped: header-derived SOBAODANH|SBD + DIEM_THI dynamic indices - default: positional 6-col layout (no header) Extends ParsedRow with ten_cum_thi and gioi_tinh fields. Adds SCORE_FIELDS_2016 (12 cols: tieng_duc/tieng_nhat; no khtn/khxh/tieng_nga). Adds thptqg2016-data.toml config with 18-column schema and format_detection flag. 58 tests pass (50 unit + 8 integration), 0 failures. * feat(build): wire build:db to xlsxread CLI Replaces the Node.js build:db script with the xlsxread Rust binary. Adds build:rust script for the cargo compile step in isolation. * chore: remove deprecated build-database.js Superseded by the xlsxread Rust binary. All 119 source files (4 .xls + 115 .xlsx) are now processed by xlsxread with per-file format detection. * ci: build xlsxread before running database build job Adds dtolnay/rust-toolchain@stable and Swatinem/rust-cache@v2 steps before the xlsxread build and database generation steps. Node/pnpm steps now follow the Rust build rather than preceding it. * chore: add Rust build artifacts to .gitignore * docs: update README build instructions for xlsxread pipeline * chore(deps): drop xlsx and better-sqlite3 from package.json and lockfile --- 2016/.github/workflows/deploy.yml | 19 +- 2016/.gitignore | 3 + 2016/README.md | 29 +- 2016/package.json | 7 +- 2016/pnpm-lock.yaml | 30 - 2016/scripts/build-database.js | 271 ---- 2016/tools/sync-from-thptqg2017.sh | 44 + 2016/tools/xlsxread/Cargo.lock | 1159 +++++++++++++++++ 2016/tools/xlsxread/Cargo.toml | 30 + .../xlsxread/configs/thptqg2016-data.toml | 86 ++ .../xlsxread/configs/thptqg2017-data-old.toml | 76 ++ .../xlsxread/configs/thptqg2017-data.toml | 76 ++ 2016/tools/xlsxread/src/audit.rs | 182 +++ 2016/tools/xlsxread/src/cli.rs | 48 + 2016/tools/xlsxread/src/config.rs | 188 +++ 2016/tools/xlsxread/src/error.rs | 34 + 2016/tools/xlsxread/src/format_detect_2016.rs | 549 ++++++++ 2016/tools/xlsxread/src/lib.rs | 10 + 2016/tools/xlsxread/src/main.rs | 379 ++++++ 2016/tools/xlsxread/src/reader.rs | 197 +++ 2016/tools/xlsxread/src/transform.rs | 410 ++++++ 2016/tools/xlsxread/src/writer.rs | 201 +++ 2016/tools/xlsxread/tests/fixtures/README.md | 18 + .../xlsxread/tests/fixtures/hcm-overflow.xlsx | Bin 0 -> 12658 bytes .../xlsxread/tests/fixtures/province-100.xlsx | Bin 0 -> 4460 bytes .../tests/fixtures/province-numeric-sbd.xlsx | Bin 0 -> 4165 bytes 2016/tools/xlsxread/tests/golden.rs | 691 ++++++++++ 27 files changed, 4422 insertions(+), 315 deletions(-) delete mode 100644 2016/scripts/build-database.js create mode 100755 2016/tools/sync-from-thptqg2017.sh create mode 100644 2016/tools/xlsxread/Cargo.lock create mode 100644 2016/tools/xlsxread/Cargo.toml create mode 100644 2016/tools/xlsxread/configs/thptqg2016-data.toml create mode 100644 2016/tools/xlsxread/configs/thptqg2017-data-old.toml create mode 100644 2016/tools/xlsxread/configs/thptqg2017-data.toml create mode 100644 2016/tools/xlsxread/src/audit.rs create mode 100644 2016/tools/xlsxread/src/cli.rs create mode 100644 2016/tools/xlsxread/src/config.rs create mode 100644 2016/tools/xlsxread/src/error.rs create mode 100644 2016/tools/xlsxread/src/format_detect_2016.rs create mode 100644 2016/tools/xlsxread/src/lib.rs create mode 100644 2016/tools/xlsxread/src/main.rs create mode 100644 2016/tools/xlsxread/src/reader.rs create mode 100644 2016/tools/xlsxread/src/transform.rs create mode 100644 2016/tools/xlsxread/src/writer.rs create mode 100644 2016/tools/xlsxread/tests/fixtures/README.md create mode 100644 2016/tools/xlsxread/tests/fixtures/hcm-overflow.xlsx create mode 100644 2016/tools/xlsxread/tests/fixtures/province-100.xlsx create mode 100644 2016/tools/xlsxread/tests/fixtures/province-numeric-sbd.xlsx create mode 100644 2016/tools/xlsxread/tests/golden.rs diff --git a/2016/.github/workflows/deploy.yml b/2016/.github/workflows/deploy.yml index 9ccebcd..9d9fb6d 100644 --- a/2016/.github/workflows/deploy.yml +++ b/2016/.github/workflows/deploy.yml @@ -20,6 +20,22 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: tools/xlsxread + + - name: Build xlsxread binary + run: cargo build --release --manifest-path tools/xlsxread/Cargo.toml + + - name: Build database + run: > + ./tools/xlsxread/target/release/xlsxread build + --schema tools/xlsxread/configs/thptqg2016-data.toml + --input data + --output public/thptqg2016.db + - uses: pnpm/action-setup@v4 - uses: actions/setup-node@v4 @@ -30,9 +46,6 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Build database - run: pnpm build:db - - name: Compress database run: gzip -k -9 public/thptqg2016.db diff --git a/2016/.gitignore b/2016/.gitignore index e0e0037..051b283 100644 --- a/2016/.gitignore +++ b/2016/.gitignore @@ -141,3 +141,6 @@ vite.config.ts.timestamp-* # Generated database files public/*.db public/*.db.gz + +# Rust build artifacts +tools/xlsxread/target/ diff --git a/2016/README.md b/2016/README.md index 4c04d76..7ba96f6 100644 --- a/2016/README.md +++ b/2016/README.md @@ -19,18 +19,33 @@ Fully static app running entirely in the browser (SQLite via `sql.js`). No backe ## Development ```bash -npm install -npm run build:db # Parse data/*.xlsx → public/thptqg2016.db -npm run dev # Vite dev server -npm run build # Production bundle → dist/ -npm run lint # ESLint +# Build the SQLite database from source Excel files (requires Rust stable) +pnpm run build:db + +# Or build in two steps: +pnpm run build:rust # compile the xlsxread binary +./tools/xlsxread/target/release/xlsxread build \ + --schema tools/xlsxread/configs/thptqg2016-data.toml \ + --input data \ + --output public/thptqg2016.db + +pnpm run dev # Vite dev server +pnpm run build # Production bundle → dist/ +pnpm run lint # ESLint ``` -The GitHub Actions workflow (`.github/workflows/deploy.yml`) builds the DB, gzips it, and deploys to GitHub Pages on every push to `main`. +The database is built by the `xlsxread` Rust binary (`tools/xlsxread/`), which +reads the 119 mixed `.xls`/`.xlsx` files and auto-detects the column layout per +file (`separate-scores`, `mapped`, or positional default). No Node.js Excel +library is required at build time. + +The GitHub Actions workflow (`.github/workflows/deploy.yml`) compiles +`xlsxread`, builds the DB, gzips it, and deploys to GitHub Pages on every push +to `main`. ## Tech stack -React 19 · Vite · sql.js (WASM) · better-sqlite3 (build-time only) · GitHub Pages +React 19 · Vite · sql.js (WASM) · xlsxread (Rust, build-time) · GitHub Pages ## Documentation diff --git a/2016/package.json b/2016/package.json index b7a9052..44080af 100644 --- a/2016/package.json +++ b/2016/package.json @@ -6,7 +6,8 @@ "type": "module", "description": "Tra cứu điểm thi THPT QG 2016 - Hỗ trợ truy vấn SQL tùy chỉnh", "scripts": { - "build:db": "node scripts/build-database.js", + "build:rust": "cargo build --release --manifest-path tools/xlsxread/Cargo.toml", + "build:db": "cargo build --release --manifest-path tools/xlsxread/Cargo.toml && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2016-data.toml --input data --output public/thptqg2016.db", "dev": "vite", "build": "vite build", "preview": "vite preview", @@ -27,12 +28,10 @@ "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^6.0.1", - "better-sqlite3": "^12.8.0", "eslint": "^9.39.4", "eslint-plugin-react-hooks": "^7.0.1", "eslint-plugin-react-refresh": "^0.5.2", "globals": "^17.4.0", - "vite": "^8.0.4", - "xlsx": "^0.18.5" + "vite": "^8.0.4" } } diff --git a/2016/pnpm-lock.yaml b/2016/pnpm-lock.yaml index ec7a8b8..ce60a15 100644 --- a/2016/pnpm-lock.yaml +++ b/2016/pnpm-lock.yaml @@ -30,9 +30,6 @@ importers: '@vitejs/plugin-react': specifier: ^6.0.1 version: 6.0.1(vite@8.0.12) - better-sqlite3: - specifier: ^12.8.0 - version: 12.9.0 eslint: specifier: ^9.39.4 version: 9.39.4 @@ -48,9 +45,6 @@ importers: vite: specifier: ^8.0.4 version: 8.0.12 - xlsx: - specifier: ^0.18.5 - version: 0.18.5 packages: @@ -379,10 +373,6 @@ packages: engines: {node: '>=6.0.0'} hasBin: true - better-sqlite3@12.9.0: - resolution: {integrity: sha512-wqUv4Gm3toFpHDQmaKD4QhZm3g1DjUBI0yzS4UBl6lElUmXFYdTQmmEDpAFa5o8FiFiymURypEnfVHzILKaxqQ==} - engines: {node: 20.x || 22.x || 23.x || 24.x || 25.x} - bindings@1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} @@ -1034,11 +1024,6 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - xlsx@0.18.5: - resolution: {integrity: sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==} - engines: {node: '>=0.8'} - hasBin: true - yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} @@ -1365,11 +1350,6 @@ snapshots: baseline-browser-mapping@2.10.29: {} - better-sqlite3@12.9.0: - dependencies: - bindings: 1.5.0 - prebuild-install: 7.1.3 - bindings@1.5.0: dependencies: file-uri-to-path: 1.0.0 @@ -1942,16 +1922,6 @@ snapshots: wrappy@1.0.2: {} - xlsx@0.18.5: - dependencies: - adler-32: 1.3.1 - cfb: 1.2.2 - codepage: 1.15.0 - crc-32: 1.2.2 - ssf: 0.11.2 - wmf: 1.0.2 - word: 0.3.0 - yallist@3.1.1: {} yocto-queue@0.1.0: {} diff --git a/2016/scripts/build-database.js b/2016/scripts/build-database.js deleted file mode 100644 index 34bb87d..0000000 --- a/2016/scripts/build-database.js +++ /dev/null @@ -1,271 +0,0 @@ -import XLSX from "xlsx"; -import Database from "better-sqlite3"; -import fs from "fs"; -import path from "path"; -import { fileURLToPath } from "url"; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const DATA_DIR = path.join(__dirname, "..", "data"); -const DB_PATH = path.join(__dirname, "..", "public", "thptqg2016.db"); - -// Score patterns for the DIEM_THI string format -const SCORE_PATTERNS = { - toan: /Toán:\s*([\d.]+)/, - ngu_van: /Ngữ văn:\s*([\d.]+)/, - vat_ly: /Vật lí:\s*([\d.]+)/, - hoa_hoc: /Hóa học:\s*([\d.]+)/, - sinh_hoc: /Sinh học:\s*([\d.]+)/, - lich_su: /Lịch sử:\s*([\d.]+)/, - dia_ly: /Địa lí:\s*([\d.]+)/, - tieng_anh: /Tiếng Anh:\s*([\d.]+)/, - tieng_phap: /Tiếng Pháp:\s*([\d.]+)/, - tieng_duc: /Tiếng Đức:\s*([\d.]+)/, - tieng_nhat: /Tiếng Nhật:\s*([\d.]+)/, - tieng_trung: /Tiếng Trung:\s*([\d.]+)/, -}; - -const ALL_SCORE_FIELDS = Object.keys(SCORE_PATTERNS); - -// Strip Vietnamese diacritics: "NGUYỄN BŨU LỘC" → "nguyen buu loc" -function toAscii(str) { - return str - .normalize("NFD") - .replace(/[\u0300-\u036f]/g, "") - .replace(/đ/g, "d") - .replace(/Đ/g, "D") - .toLowerCase(); -} - -// Parse score text "Toán: 3.75 Ngữ văn: 5.00 ..." into { toan: 3.75, ... } -function parseScoreString(diemThi) { - const scores = {}; - for (const [field, pattern] of Object.entries(SCORE_PATTERNS)) { - const match = diemThi.match(pattern); - if (match) scores[field] = parseFloat(match[1]); - } - return scores; -} - -// Detect header row by checking for known column names -const KNOWN_HEADERS = new Set([ - "SOBAODANH", "SBD", "HO_TEN", "HOTEN", "HỌ TÊN", - "NGAY_SINH", "TEN_CUMTHI", "GIOI_TINH", "DIEM_THI", "STT", - "TOAN", "VAN", "LY", "HOA", "SINH ", "SU", "DIA", -]); - -function isHeaderRow(row) { - if (!row || row.length < 2) return false; - const first = String(row[0] || "").trim().toUpperCase(); - return KNOWN_HEADERS.has(first); -} - -// Detect which format a file uses based on its header row -function detectFormat(headerRow) { - if (!headerRow) return null; - const cols = headerRow.map((c) => String(c || "").trim().toUpperCase()); - - // Format: SBD, HOTEN, TOAN, VAN, LY, HOA, SINH, SU, DIA, ... - if (cols[0] === "SBD" && cols[2] === "TOAN") return "separate-scores"; - - // Build a column index map for flexible column ordering - const map = {}; - for (let i = 0; i < cols.length; i++) { - const c = cols[i]; - if (c === "SOBAODANH" || c === "SBD") map.sbd = i; - else if (c === "HO_TEN" || c === "HOTEN" || c === "HỌ TÊN") map.ho_ten = i; - else if (c === "NGAY_SINH") map.ngay_sinh = i; - else if (c === "TEN_CUMTHI") map.ten_cum_thi = i; - else if (c === "GIOI_TINH") map.gioi_tinh = i; - else if (c === "DIEM_THI") map.diem_thi = i; - } - - if (map.sbd !== undefined && map.diem_thi !== undefined) { - return { type: "mapped", map }; - } - - return null; -} - -// Process a file with separate score columns (dhhanghai format) -function processSeparateScoresRow(row) { - const sbd = String(row[0] || "").trim(); - const hoTen = String(row[1] || "").trim(); - if (!sbd || !hoTen) return null; - - return { - so_bao_danh: sbd, - ho_ten: hoTen, - ho_ten_ascii: toAscii(hoTen), - ngay_sinh: null, - ten_cum_thi: null, - gioi_tinh: null, - toan: parseFloat(row[2]) || null, - ngu_van: parseFloat(row[3]) || null, - vat_ly: parseFloat(row[4]) || null, - hoa_hoc: parseFloat(row[5]) || null, - sinh_hoc: parseFloat(row[6]) || null, - lich_su: parseFloat(row[7]) || null, - dia_ly: parseFloat(row[8]) || null, - // row[9]=NGOAINGUTN, row[10]=NGOAINGUTL, row[11]=NGOAINGU (total) - tieng_anh: parseFloat(row[11]) || null, - tieng_phap: null, - tieng_duc: null, - tieng_nhat: null, - tieng_trung: null, - }; -} - -// Process a row using the column map -function processMappedRow(row, map) { - const sbd = String(row[map.sbd] || "").trim(); - const hoTen = String(row[map.ho_ten] || "").trim(); - if (!sbd || !hoTen) return null; - - // Skip leaked header rows - const sbdUpper = sbd.toUpperCase(); - if (KNOWN_HEADERS.has(sbdUpper) || KNOWN_HEADERS.has(hoTen.toUpperCase())) return null; - - const ngaySinh = map.ngay_sinh !== undefined ? String(row[map.ngay_sinh] || "").trim() : null; - const tenCumThi = map.ten_cum_thi !== undefined ? String(row[map.ten_cum_thi] || "").trim() : null; - const rawGioiTinh = map.gioi_tinh !== undefined ? String(row[map.gioi_tinh] || "").trim() : null; - // Normalize gender: only accept "Nam" or "Nữ" - const gioiTinh = (rawGioiTinh === "Nam" || rawGioiTinh === "Nữ") ? rawGioiTinh : null; - const diemThi = map.diem_thi !== undefined ? String(row[map.diem_thi] || "") : ""; - - const scores = parseScoreString(diemThi); - - return { - so_bao_danh: sbd, - ho_ten: hoTen, - ho_ten_ascii: toAscii(hoTen), - ngay_sinh: ngaySinh || null, - ten_cum_thi: tenCumThi || null, - gioi_tinh: gioiTinh || null, - ...Object.fromEntries(ALL_SCORE_FIELDS.map((f) => [f, scores[f] ?? null])), - }; -} - -// Standard 6-column format without header: SBD, HO_TEN, NGAY_SINH, TEN_CUMTHI, GIOI_TINH, DIEM_THI -const DEFAULT_MAP = { - sbd: 0, ho_ten: 1, ngay_sinh: 2, ten_cum_thi: 3, gioi_tinh: 4, diem_thi: 5, -}; - -function main() { - fs.mkdirSync(path.dirname(DB_PATH), { recursive: true }); - if (fs.existsSync(DB_PATH)) fs.unlinkSync(DB_PATH); - - const db = new Database(DB_PATH); - - db.exec(` - CREATE TABLE student ( - so_bao_danh TEXT PRIMARY KEY, - ho_ten TEXT NOT NULL, - ho_ten_ascii TEXT NOT NULL, - ngay_sinh TEXT, - ten_cum_thi TEXT, - gioi_tinh TEXT, - toan REAL, - ngu_van REAL, - vat_ly REAL, - hoa_hoc REAL, - sinh_hoc REAL, - lich_su REAL, - dia_ly REAL, - tieng_anh REAL, - tieng_phap REAL, - tieng_duc REAL, - tieng_nhat REAL, - tieng_trung REAL - ); - CREATE INDEX idx_ho_ten ON student(ho_ten); - CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii); - CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi); - `); - - const insert = db.prepare(` - INSERT OR REPLACE INTO student - (so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, ten_cum_thi, gioi_tinh, - toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, lich_su, dia_ly, - tieng_anh, tieng_phap, tieng_duc, tieng_nhat, tieng_trung) - VALUES - (@so_bao_danh, @ho_ten, @ho_ten_ascii, @ngay_sinh, @ten_cum_thi, @gioi_tinh, - @toan, @ngu_van, @vat_ly, @hoa_hoc, @sinh_hoc, @lich_su, @dia_ly, - @tieng_anh, @tieng_phap, @tieng_duc, @tieng_nhat, @tieng_trung) - `); - - // Collect all Excel files (.xlsx and .xls) - const files = fs.readdirSync(DATA_DIR) - .filter((f) => f.endsWith(".xlsx") || f.endsWith(".xls")) - .map((f) => path.join(DATA_DIR, f)); - - let totalRows = 0; - let errorCount = 0; - - const insertAll = db.transaction((files) => { - for (const file of files) { - const basename = path.basename(file); - let fileRows = 0; - - try { - const wb = XLSX.readFile(file); - const ws = wb.Sheets[wb.SheetNames[0]]; - const rows = XLSX.utils.sheet_to_json(ws, { header: 1 }); - if (rows.length === 0) continue; - - let startRow = 0; - let format = null; - - if (isHeaderRow(rows[0])) { - format = detectFormat(rows[0]); - startRow = 1; - } - - for (let i = startRow; i < rows.length; i++) { - const row = rows[i]; - if (!row || row.length < 2) continue; - - try { - let record; - - if (format === "separate-scores") { - record = processSeparateScoresRow(row); - } else if (format && format.type === "mapped") { - record = processMappedRow(row, format.map); - } else { - // No header or unrecognized: assume standard 6-column order - record = processMappedRow(row, DEFAULT_MAP); - } - - if (!record) continue; - insert.run(record); - fileRows++; - } catch { - errorCount++; - } - } - } catch (err) { - console.error(`Failed to read ${basename}: ${err.message}`); - } - - totalRows += fileRows; - console.log(` ${basename}: ${fileRows} rows`); - } - }); - - console.log(`Processing ${files.length} Excel files...\n`); - insertAll(files); - - db.exec("VACUUM"); - - const count = db.prepare("SELECT COUNT(*) as cnt FROM student").get(); - console.log(`\nDone! ${count.cnt} students in database.`); - console.log(`Errors skipped: ${errorCount}`); - console.log(`Output: ${DB_PATH}`); - - const stat = fs.statSync(DB_PATH); - console.log(`Size: ${(stat.size / 1024 / 1024).toFixed(1)} MB`); - - db.close(); -} - -main(); diff --git a/2016/tools/sync-from-thptqg2017.sh b/2016/tools/sync-from-thptqg2017.sh new file mode 100755 index 0000000..fd86408 --- /dev/null +++ b/2016/tools/sync-from-thptqg2017.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Re-sync xlsxread Rust source from thptqg2017. +# +# Source SHA: 8b4a755c115595bf1b937d749eb1133efb3a6e22 (chore/xlsxread-rust) +# +# Re-run this when xlsxread is updated upstream. The configs/ directory is +# NOT synced — it contains thptqg2016-specific configs and test stubs that +# must be maintained here independently. +# +# Usage: +# ./tools/sync-from-thptqg2017.sh /path/to/thptqg2017 +# +set -euo pipefail + +SRC="${1:?Usage: $0 /path/to/thptqg2017}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEST="$SCRIPT_DIR/xlsxread" + +if [ ! -d "$SRC/tools/xlsxread" ]; then + echo "Error: $SRC/tools/xlsxread not found" >&2 + exit 1 +fi + +echo "Syncing from: $SRC/tools/xlsxread" +echo "Syncing to: $DEST" +echo "" + +# Sync source, tests, and Cargo manifests — exclude build artifacts and dataset configs +for item in src tests Cargo.toml Cargo.lock; do + if [ -e "$SRC/tools/xlsxread/$item" ]; then + cp -r "$SRC/tools/xlsxread/$item" "$DEST/" + echo " synced: $item" + fi +done + +echo "" +echo "Sync complete." +echo "Next steps:" +echo " 1. Review src/ for breaking changes to config.rs / writer.rs that" +echo " may affect format_detect_2016.rs or the thptqg2016-data.toml config." +echo " 2. Rebuild: cargo build --release --manifest-path $DEST/Cargo.toml" +echo " 3. Test: cargo test --manifest-path $DEST/Cargo.toml" +echo " 4. Commit on a chore/xlsxread-sync-... branch." diff --git a/2016/tools/xlsxread/Cargo.lock b/2016/tools/xlsxread/Cargo.lock new file mode 100644 index 0000000..e57f452 --- /dev/null +++ b/2016/tools/xlsxread/Cargo.lock @@ -0,0 +1,1159 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "calamine" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138646b9af2c5d7f1804ea4bf93afc597737d2bd4f7341d67c48b03316976eb1" +dependencies = [ + "byteorder", + "codepage", + "encoding_rs", + "log", + "quick-xml", + "serde", + "zip", +] + +[[package]] +name = "cc" +version = "1.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "codepage" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deflate64" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "encoding_rs", + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "xlsxread" +version = "0.1.0" +dependencies = [ + "anyhow", + "calamine", + "clap", + "glob", + "regex", + "rusqlite", + "serde", + "thiserror 1.0.69", + "toml", + "unicode-normalization", + "zip", +] + +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "getrandom", + "hmac", + "indexmap", + "lzma-rs", + "memchr", + "pbkdf2", + "sha1", + "thiserror 2.0.18", + "time", + "xz2", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/2016/tools/xlsxread/Cargo.toml b/2016/tools/xlsxread/Cargo.toml new file mode 100644 index 0000000..d23dcca --- /dev/null +++ b/2016/tools/xlsxread/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "xlsxread" +version = "0.1.0" +edition = "2021" +description = "Rust CLI replacing SheetJS xlsx build scripts for thptqg2017/thptqg2016" + +[dependencies] +calamine = "0.26" +rusqlite = { version = "0.32", features = ["bundled"] } +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +toml = "0.8" +regex = "1" +unicode-normalization = "0.1" +thiserror = "1" +anyhow = "1" +glob = "0.3" + +# zip is already a transitive dep of calamine; pin explicitly so tests can use it +[dev-dependencies] +zip = "2" +rusqlite = { version = "0.32", features = ["bundled"] } + +[[bin]] +name = "xlsxread" +path = "src/main.rs" + +[[test]] +name = "golden" +path = "tests/golden.rs" diff --git a/2016/tools/xlsxread/configs/thptqg2016-data.toml b/2016/tools/xlsxread/configs/thptqg2016-data.toml new file mode 100644 index 0000000..646ad68 --- /dev/null +++ b/2016/tools/xlsxread/configs/thptqg2016-data.toml @@ -0,0 +1,86 @@ +# Config for thptqg2016 data/ — 4 .xls + 115 .xlsx mixed files. +# +# Three column layouts exist across the 119 files; the binary selects the +# right one per-file at runtime via format_detection = "thptqg2016": +# +# separate-scores header SBD(0)/HOTEN(1)/TOAN(2)... — dhhanghai files +# mapped header SOBAODANH|SBD + DIEM_THI — most provinces +# default no header; positional 6-col layout — remaining files +# +# Schema differences from thptqg2017: +# + ten_cum_thi TEXT (exam-cluster name, TEN_CUMTHI column) +# + gioi_tinh TEXT (gender: "Nam"/"Nữ", GIOI_TINH column) +# + tieng_duc REAL (German) +# + tieng_nhat REAL (Japanese) +# - khtn, khxh, tieng_nga (not in 2016 dataset) +# +# sheet_mode = "all": several provinces overflow into Sheet2 (65k Excel row cap). +# strip_blank_rows = false: no blank-row anomaly observed in this dataset. + +format_detection = "thptqg2016" + +[reader] +sheet_mode = "all" +strip_blank_rows = false + +[validation] +require_numeric_sbd = false +require_nonempty_name = true +require_nonempty_sbd = true + +[header] +# Tokens that identify a header row by first-cell content (uppercased). +# Covers both SOBAODANH-style and SBD-style headers. +tokens = ["SOBAODANH", "SBD", "HO_TEN", "HOTEN", "HỌ TÊN", "STT"] + +[schema] +ddl = """ +CREATE TABLE student ( + so_bao_danh TEXT PRIMARY KEY, + ho_ten TEXT NOT NULL, + ho_ten_ascii TEXT NOT NULL, + ngay_sinh TEXT, + ten_cum_thi TEXT, + gioi_tinh TEXT, + toan REAL, + ngu_van REAL, + vat_ly REAL, + hoa_hoc REAL, + sinh_hoc REAL, + lich_su REAL, + dia_ly REAL, + tieng_anh REAL, + tieng_phap REAL, + tieng_duc REAL, + tieng_nhat REAL, + tieng_trung REAL +); +CREATE INDEX idx_ho_ten ON student(ho_ten); +CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii); +CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi); +""" + +[scores] +toan = 'Toán:\s*(\d+(?:\.\d+)?)' +ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)' +vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)' +hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)' +sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)' +lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)' +dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)' +tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)' +tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)' +tieng_duc = 'Tiếng Đức:\s*(\d+(?:\.\d+)?)' +tieng_nhat = 'Tiếng Nhật:\s*(\d+(?:\.\d+)?)' +tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)' + +[insert] +sql = """ +INSERT OR REPLACE INTO student + (so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, ten_cum_thi, gioi_tinh, + toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, + lich_su, dia_ly, + tieng_anh, tieng_phap, tieng_duc, tieng_nhat, tieng_trung) +VALUES + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +""" diff --git a/2016/tools/xlsxread/configs/thptqg2017-data-old.toml b/2016/tools/xlsxread/configs/thptqg2017-data-old.toml new file mode 100644 index 0000000..34d59c0 --- /dev/null +++ b/2016/tools/xlsxread/configs/thptqg2017-data-old.toml @@ -0,0 +1,76 @@ +# Test-only config used by golden integration tests (data-old variant). +# Same column layout as thptqg2017-data.toml but with: +# - sheet_mode = "first" (reads only first sheet) +# - require_numeric_sbd = true (rejects non-digit SBDs) +# Schema and INSERT match SCORE_FIELDS_2017 (14 score cols). + +[reader] +sheet_mode = "first" +strip_blank_rows = false + +[columns] +ho_ten = 0 +ngay_sinh = 1 +so_bao_danh = 2 +diem_thi = 3 + +[validation] +require_numeric_sbd = true +require_nonempty_name = true +require_nonempty_sbd = true + +[header] +tokens = ["HO_TEN", "HỌ TÊN", "STT"] + +[schema] +ddl = """ +CREATE TABLE student ( + so_bao_danh TEXT PRIMARY KEY, + ho_ten TEXT NOT NULL, + ho_ten_ascii TEXT NOT NULL, + ngay_sinh TEXT, + toan REAL, + ngu_van REAL, + vat_ly REAL, + hoa_hoc REAL, + sinh_hoc REAL, + khtn REAL, + lich_su REAL, + dia_ly REAL, + gdcd REAL, + khxh REAL, + tieng_anh REAL, + tieng_phap REAL, + tieng_nga REAL, + tieng_trung REAL +); +CREATE INDEX idx_ho_ten ON student(ho_ten); +CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii); +""" + +[scores] +toan = 'Toán:\s*(\d+(?:\.\d+)?)' +ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)' +vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)' +hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)' +sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)' +khtn = 'KHTN:\s*(\d+(?:\.\d+)?)' +lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)' +dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)' +gdcd = 'GDCD:\s*(\d+(?:\.\d+)?)' +khxh = 'KHXH:\s*(\d+(?:\.\d+)?)' +tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)' +tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)' +tieng_nga = 'Tiếng Nga:\s*(\d+(?:\.\d+)?)' +tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)' + +[insert] +sql = """ +INSERT OR REPLACE INTO student + (so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, + toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn, + lich_su, dia_ly, gdcd, khxh, + tieng_anh, tieng_phap, tieng_nga, tieng_trung) +VALUES + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +""" diff --git a/2016/tools/xlsxread/configs/thptqg2017-data.toml b/2016/tools/xlsxread/configs/thptqg2017-data.toml new file mode 100644 index 0000000..892fc6f --- /dev/null +++ b/2016/tools/xlsxread/configs/thptqg2017-data.toml @@ -0,0 +1,76 @@ +# Test-only config used by golden integration tests. +# Matches the fixture row layout produced by tests/golden.rs: write_xlsx() +# header: HO_TEN(0) NGAY_SINH(1) SO_BAO_DANH(2) DIEM_THI(3) +# Schema and INSERT match SCORE_FIELDS_2017 (14 score cols) so run_build_cmd +# in golden.rs can call insert_row(..., SCORE_FIELDS) without modification. + +[reader] +sheet_mode = "all" +strip_blank_rows = false + +[columns] +ho_ten = 0 +ngay_sinh = 1 +so_bao_danh = 2 +diem_thi = 3 + +[validation] +require_numeric_sbd = false +require_nonempty_name = true +require_nonempty_sbd = true + +[header] +tokens = ["HO_TEN", "HỌ TÊN", "STT"] + +[schema] +ddl = """ +CREATE TABLE student ( + so_bao_danh TEXT PRIMARY KEY, + ho_ten TEXT NOT NULL, + ho_ten_ascii TEXT NOT NULL, + ngay_sinh TEXT, + toan REAL, + ngu_van REAL, + vat_ly REAL, + hoa_hoc REAL, + sinh_hoc REAL, + khtn REAL, + lich_su REAL, + dia_ly REAL, + gdcd REAL, + khxh REAL, + tieng_anh REAL, + tieng_phap REAL, + tieng_nga REAL, + tieng_trung REAL +); +CREATE INDEX idx_ho_ten ON student(ho_ten); +CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii); +""" + +[scores] +toan = 'Toán:\s*(\d+(?:\.\d+)?)' +ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)' +vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)' +hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)' +sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)' +khtn = 'KHTN:\s*(\d+(?:\.\d+)?)' +lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)' +dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)' +gdcd = 'GDCD:\s*(\d+(?:\.\d+)?)' +khxh = 'KHXH:\s*(\d+(?:\.\d+)?)' +tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)' +tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)' +tieng_nga = 'Tiếng Nga:\s*(\d+(?:\.\d+)?)' +tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)' + +[insert] +sql = """ +INSERT OR REPLACE INTO student + (so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, + toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn, + lich_su, dia_ly, gdcd, khxh, + tieng_anh, tieng_phap, tieng_nga, tieng_trung) +VALUES + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +""" diff --git a/2016/tools/xlsxread/src/audit.rs b/2016/tools/xlsxread/src/audit.rs new file mode 100644 index 0000000..c0bd9b4 --- /dev/null +++ b/2016/tools/xlsxread/src/audit.rs @@ -0,0 +1,182 @@ +/// Audit subcommand: replicates audit-row-counts.js exactly. +/// +/// Reads all .xlsx files from the input directory (sheet 0 only, matching the +/// JS script's behaviour at audit-row-counts.js:33), collects distinct SBDs +/// into a HashSet, then queries `SELECT COUNT(*) FROM student` from the DB. +/// Prints the same lines as audit-row-counts.js:54-62 and exits 0 on match, +/// 1 on mismatch. +use std::collections::HashSet; +use std::path::Path; + +use calamine::{open_workbook_auto, Data, Reader}; + +use crate::config::DatasetConfig; +use crate::error::BuildError; +use crate::reader::is_header_row; + +// --------------------------------------------------------------------------- +// Audit result +// --------------------------------------------------------------------------- + +pub struct AuditResult { + pub total_data_rows: u64, + pub both_empty: u64, + pub empty_name: u64, + pub empty_sbd: u64, + pub distinct_sbds: usize, + pub db_count: i64, + pub matched: bool, +} + +// --------------------------------------------------------------------------- +// Main audit logic +// --------------------------------------------------------------------------- + +/// Collect distinct SBDs from all xlsx files in `input_dir`, query `db_path`, +/// print the audit report and return the result. +/// +/// The JS script reads only sheet 0 for every file (audit-row-counts.js:33). +/// Unlike build-database.js, the audit script does NOT iterate all sheets. +pub fn run_audit( + input_dir: &Path, + db_path: &Path, + cfg: &DatasetConfig, +) -> Result { + // Collect .xlsx files (audit-row-counts.js only checks .xlsx — line 15) + let mut files: Vec = std::fs::read_dir(input_dir) + .map_err(|e| BuildError::Io { + path: input_dir.display().to_string(), + source: e, + })? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + p.is_file() + && p.extension() + .and_then(|e| e.to_str()) + .map(|e| e.eq_ignore_ascii_case("xlsx")) + .unwrap_or(false) + }) + .collect(); + files.sort(); + + let mut all_sbd: HashSet = HashSet::new(); + let mut total_data_rows: u64 = 0; + let mut empty_name: u64 = 0; + let mut empty_sbd: u64 = 0; + let mut both_empty: u64 = 0; + + for file in &files { + let path_str = file.display().to_string(); + let mut workbook = open_workbook_auto(file).map_err(|e| BuildError::Calamine { + path: path_str.clone(), + source: e, + })?; + + let sheet_names = workbook.sheet_names().to_vec(); + if sheet_names.is_empty() { + continue; + } + + // audit-row-counts.js reads only sheet 0 (line 33: wb.SheetNames[0]) + let range = + workbook + .worksheet_range(&sheet_names[0]) + .map_err(|e| BuildError::Calamine { + path: path_str.clone(), + source: e, + })?; + + let mut first_row = true; + for raw in range.rows() { + let row: Vec = raw.to_vec(); + + // Skip header row on first row only + if first_row { + first_row = false; + if is_header_row(&row, &cfg.header) { + continue; + } + } + + total_data_rows += 1; + + // For format_detection configs the audit uses positional defaults (col 0 = SBD, + // col 1 = HO_TEN) because the audit is best-effort and mirrors the JS script + // which also uses a fixed column assumption (audit-row-counts.js:33–36). + let (ho_ten_col, sbd_col) = cfg + .columns + .as_ref() + .map(|c| (c.ho_ten, c.so_bao_danh)) + .unwrap_or((1, 0)); + let ho_ten = row + .get(ho_ten_col) + .map(|c| c.to_string().trim().to_owned()) + .unwrap_or_default(); + let sbd = row + .get(sbd_col) + .map(|c| c.to_string().trim().to_owned()) + .unwrap_or_default(); + + if ho_ten.is_empty() && sbd.is_empty() { + both_empty += 1; + continue; + } + if ho_ten.is_empty() { + empty_name += 1; + } + if sbd.is_empty() { + empty_sbd += 1; + } + if !sbd.is_empty() { + all_sbd.insert(sbd); + } + } + } + + // Query DB count + let conn = + rusqlite::Connection::open_with_flags(db_path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?; + let db_count: i64 = conn.query_row("SELECT COUNT(*) FROM student", [], |row| row.get(0))?; + + let distinct_sbds = all_sbd.len(); + let matched = distinct_sbds as i64 == db_count; + + Ok(AuditResult { + total_data_rows, + both_empty, + empty_name, + empty_sbd, + distinct_sbds, + db_count, + matched, + }) +} + +// --------------------------------------------------------------------------- +// Print audit report — mirrors audit-row-counts.js:54-62 exactly +// --------------------------------------------------------------------------- + +pub fn print_audit_report(r: &AuditResult) { + println!("=== Source vs DB ==="); + println!( + "Source: total data rows across all files: {}", + r.total_data_rows + ); + println!( + "Source: rows with empty name AND sbd (skipped): {}", + r.both_empty + ); + println!("Source: rows with missing name only: {}", r.empty_name); + println!("Source: rows with missing sbd only: {}", r.empty_sbd); + println!("Source: distinct SBDs: {}", r.distinct_sbds); + println!("DB: row count: {}", r.db_count); + println!( + "Match: {}", + if r.matched { + "YES — all unique SBDs accounted for".to_string() + } else { + format!("NO — gap of {}", r.distinct_sbds as i64 - r.db_count) + } + ); +} diff --git a/2016/tools/xlsxread/src/cli.rs b/2016/tools/xlsxread/src/cli.rs new file mode 100644 index 0000000..bce8285 --- /dev/null +++ b/2016/tools/xlsxread/src/cli.rs @@ -0,0 +1,48 @@ +/// CLI argument structs via clap derive. +use std::path::PathBuf; + +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command( + name = "xlsxread", + version, + about = "Read .xls/.xlsx files and build SQLite databases for thptqg datasets" +)] +pub struct Cli { + #[command(subcommand)] + pub cmd: Cmd, +} + +#[derive(Subcommand)] +pub enum Cmd { + /// Read input spreadsheets and write a SQLite database + Build { + /// Path to the dataset TOML config file + #[arg(long)] + schema: PathBuf, + + /// Directory containing the .xls / .xlsx source files + #[arg(long)] + input: PathBuf, + + /// Output SQLite database path + #[arg(long)] + output: PathBuf, + }, + + /// Audit: compare distinct SBD count from xlsx files vs DB row count + Audit { + /// Path to the dataset TOML config file + #[arg(long)] + schema: PathBuf, + + /// Directory containing the .xlsx source files + #[arg(long)] + input: PathBuf, + + /// SQLite database to compare against + #[arg(long)] + db: PathBuf, + }, +} diff --git a/2016/tools/xlsxread/src/config.rs b/2016/tools/xlsxread/src/config.rs new file mode 100644 index 0000000..e2aaa8e --- /dev/null +++ b/2016/tools/xlsxread/src/config.rs @@ -0,0 +1,188 @@ +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +use serde::Deserialize; + +use crate::error::BuildError; + +// --------------------------------------------------------------------------- +// Top-level dataset configuration loaded from a .toml file +// --------------------------------------------------------------------------- + +#[derive(Debug, Deserialize, Clone)] +pub struct DatasetConfig { + pub reader: ReaderCfg, + /// Fixed column indices. Optional when format_detection handles per-file mapping. + #[serde(default)] + pub columns: Option, + pub validation: ValidationCfg, + pub header: HeaderCfg, + pub schema: SchemaCfg, + /// field name → regex source string (one entry per scoreable subject) + pub scores: HashMap, + pub insert: InsertCfg, + /// When set to "thptqg2016", enables per-file format auto-detection. + /// Each file's header row is inspected at runtime to choose the right + /// column layout (separate-scores / mapped / default-positional). + #[serde(default)] + pub format_detection: Option, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct ReaderCfg { + /// "all" → iterate every sheet (handles HCM/HN overflow); "first" → sheet 0 only + pub sheet_mode: SheetMode, + /// If true, skip rows where every cell is empty/null before counting (data-old2 quirk) + pub strip_blank_rows: bool, +} + +#[derive(Debug, Deserialize, Clone, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SheetMode { + All, + First, +} + +/// Zero-indexed column positions in the source spreadsheet row. +/// Used by thptqg2017 configs. thptqg2016 uses runtime format detection instead. +#[derive(Debug, Deserialize, Clone)] +pub struct ColumnMap { + pub ho_ten: usize, + pub ngay_sinh: usize, + pub so_bao_danh: usize, + pub diem_thi: usize, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct ValidationCfg { + /// build-database-old.js / -old2.js require soBaoDanh to match ^\d+$ + pub require_numeric_sbd: bool, + pub require_nonempty_name: bool, + pub require_nonempty_sbd: bool, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct HeaderCfg { + /// Tokens to match against row[0].to_uppercase() to detect a header row + pub tokens: Vec, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct SchemaCfg { + /// DDL executed verbatim before inserts (CREATE TABLE + CREATE INDEX) + pub ddl: String, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct InsertCfg { + /// Parameterised INSERT OR REPLACE SQL using :named_param style + pub sql: String, +} + +// --------------------------------------------------------------------------- +// Loader +// --------------------------------------------------------------------------- + +pub fn load_config(path: &Path) -> Result { + let text = fs::read_to_string(path).map_err(|e| BuildError::Io { + path: path.display().to_string(), + source: e, + })?; + let cfg: DatasetConfig = toml::from_str(&text)?; + Ok(cfg) +} + +// --------------------------------------------------------------------------- +// Unit tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE_TOML: &str = r#" +[reader] +sheet_mode = "all" +strip_blank_rows = false + +[columns] +ho_ten = 0 +ngay_sinh = 1 +so_bao_danh = 2 +diem_thi = 3 + +[validation] +require_numeric_sbd = false +require_nonempty_name = true +require_nonempty_sbd = true + +[header] +tokens = ["HO_TEN", "HỌ TÊN", "STT"] + +[schema] +ddl = "CREATE TABLE student (so_bao_danh TEXT PRIMARY KEY);" + +[scores] +toan = 'Toán:\s*(\d+(?:\.\d+)?)' +ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)' + +[insert] +sql = "INSERT OR REPLACE INTO student (so_bao_danh) VALUES (:so_bao_danh)" +"#; + + #[test] + fn config_round_trip() { + let cfg: DatasetConfig = toml::from_str(SAMPLE_TOML).expect("parse failed"); + assert_eq!(cfg.reader.sheet_mode, SheetMode::All); + assert!(!cfg.reader.strip_blank_rows); + let cols = cfg.columns.as_ref().unwrap(); + assert_eq!(cols.ho_ten, 0); + assert_eq!(cols.diem_thi, 3); + assert!(!cfg.validation.require_numeric_sbd); + assert!(cfg.validation.require_nonempty_name); + assert_eq!(cfg.header.tokens.len(), 3); + assert!(cfg.scores.contains_key("toan")); + assert!(cfg.scores.contains_key("ngu_van")); + assert!(cfg.format_detection.is_none()); + } + + #[test] + fn config_first_sheet_mode() { + let toml_str = SAMPLE_TOML.replace(r#"sheet_mode = "all""#, r#"sheet_mode = "first""#); + let cfg: DatasetConfig = toml::from_str(&toml_str).expect("parse failed"); + assert_eq!(cfg.reader.sheet_mode, SheetMode::First); + } + + #[test] + fn config_format_detection_field() { + // Configs without [columns] and with format_detection = "thptqg2016" parse correctly + let toml_str = r#" +format_detection = "thptqg2016" + +[reader] +sheet_mode = "all" +strip_blank_rows = false + +[validation] +require_numeric_sbd = false +require_nonempty_name = true +require_nonempty_sbd = true + +[header] +tokens = ["SBD", "SOBAODANH", "STT"] + +[schema] +ddl = "CREATE TABLE student (so_bao_danh TEXT PRIMARY KEY);" + +[scores] +toan = 'Toán:\s*(\d+(?:\.\d+)?)' + +[insert] +sql = "INSERT OR REPLACE INTO student (so_bao_danh) VALUES (?)" +"#; + let cfg: DatasetConfig = toml::from_str(toml_str).expect("parse failed"); + assert_eq!(cfg.format_detection.as_deref(), Some("thptqg2016")); + assert!(cfg.columns.is_none()); + } +} diff --git a/2016/tools/xlsxread/src/error.rs b/2016/tools/xlsxread/src/error.rs new file mode 100644 index 0000000..e15edc9 --- /dev/null +++ b/2016/tools/xlsxread/src/error.rs @@ -0,0 +1,34 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum BuildError { + #[error("I/O error for {path}: {source}")] + Io { + path: String, + #[source] + source: std::io::Error, + }, + + #[error("Calamine error for {path}: {source}")] + Calamine { + path: String, + #[source] + source: calamine::Error, + }, + + #[error("SQLite error: {0}")] + Sqlite(#[from] rusqlite::Error), + + #[error("Config parse error: {0}")] + Config(#[from] toml::de::Error), + + #[error("Regex compile error for pattern '{pattern}': {source}")] + Regex { + pattern: String, + #[source] + source: regex::Error, + }, + + #[error("Schema has no sheets in file: {0}")] + NoSheets(String), +} diff --git a/2016/tools/xlsxread/src/format_detect_2016.rs b/2016/tools/xlsxread/src/format_detect_2016.rs new file mode 100644 index 0000000..36c2bfd --- /dev/null +++ b/2016/tools/xlsxread/src/format_detect_2016.rs @@ -0,0 +1,549 @@ +/// Per-file format auto-detection for the thptqg2016 dataset. +/// +/// Translates `detectFormat` from scripts/build-database.js (lines 63–87) and the +/// three row-processing functions (lines 90–146) into Rust. +/// +/// The JS source has three formats: +/// +/// 1. `separate-scores` — header row[0]=="SBD" && row[2]=="TOAN" +/// Columns: SBD(0) HOTEN(1) TOAN(2) VAN(3) LY(4) HOA(5) SINH(6) SU(7) DIA(8) +/// NGOAINGUTN(9) NGOAINGUTL(10) NGOAINGU-total(11) +/// → maps col 11 → tieng_anh; no ngay_sinh / ten_cum_thi / gioi_tinh / DIEM_THI +/// → JS: build-database.js:90–116 (processSeparateScoresRow) +/// +/// 2. `mapped` — header row has SOBAODANH|SBD and DIEM_THI columns +/// → dynamic column indices built from header names +/// → JS: build-database.js:119–146 (processMappedRow with map from detectFormat) +/// +/// 3. `default` — no recognised header; positional 6-col layout +/// SBD(0) HO_TEN(1) NGAY_SINH(2) TEN_CUMTHI(3) GIOI_TINH(4) DIEM_THI(5) +/// → JS: build-database.js:149–151 DEFAULT_MAP + processMappedRow +/// +/// JS citations are line numbers in /config/workspace/tiennm99/thptqg2016/scripts/build-database.js. +use calamine::Data; + +use crate::transform::{parse_scores, to_ascii, CompiledPatterns, ParsedRow}; + +// --------------------------------------------------------------------------- +// Known header tokens (mirrors JS KNOWN_HEADERS set, build-database.js:50–54) +// --------------------------------------------------------------------------- + +/// Upper-cased strings that identify a header row's first cell. +/// Mirrors `KNOWN_HEADERS` in build-database.js:50–54. +const KNOWN_HEADERS: &[&str] = &[ + "SOBAODANH", + "SBD", + "HO_TEN", + "HOTEN", + "HỌ TÊN", + "NGAY_SINH", + "TEN_CUMTHI", + "GIOI_TINH", + "DIEM_THI", + "STT", + "TOAN", + "VAN", + "LY", + "HOA", + "SINH ", + "SU", + "DIA", +]; + +/// Returns true when `row[0]` (uppercased, trimmed) is in the known-headers set. +/// Mirrors `isHeaderRow` at build-database.js:56–60. +pub fn is_header_row_2016(row: &[Data]) -> bool { + if row.len() < 2 { + return false; + } + let first = row[0].to_string().trim().to_uppercase(); + KNOWN_HEADERS.iter().any(|h| *h == first.as_str()) +} + +// --------------------------------------------------------------------------- +// Detected per-file format +// --------------------------------------------------------------------------- + +/// The three layouts the thptqg2016 dataset uses, detected per file. +#[derive(Debug, Clone)] +pub enum DetectedFormat { + /// SBD/HOTEN/TOAN/VAN/LY/HOA/SINH/SU/DIA/NGOAINGUTN/NGOAINGUTL/NGOAINGU columns. + /// Corresponds to the dhhanghai-style files. build-database.js:67–68. + SeparateScores, + /// Header present with SOBAODANH|SBD and DIEM_THI; dynamic column indices. + /// build-database.js:70–86. + Mapped { + sbd: usize, + ho_ten: usize, + ngay_sinh: Option, + ten_cum_thi: Option, + gioi_tinh: Option, + diem_thi: usize, + }, + /// No recognised header; standard 6-column positional layout. + /// build-database.js:149–151 DEFAULT_MAP. + Default, +} + +/// Inspect a header row and decide which format applies. +/// Returns `None` when the header is present but unrecognised (treated as Default). +/// +/// Mirrors `detectFormat` at build-database.js:63–87. +pub fn detect_format(header_row: &[Data]) -> DetectedFormat { + let cols: Vec = header_row + .iter() + .map(|c| c.to_string().trim().to_uppercase()) + .collect(); + + // Format 1: SBD in col 0 AND TOAN in col 2 → separate-scores + // build-database.js:68: if (cols[0] === "SBD" && cols[2] === "TOAN") + if cols.first().map(|s| s.as_str()) == Some("SBD") + && cols.get(2).map(|s| s.as_str()) == Some("TOAN") + { + return DetectedFormat::SeparateScores; + } + + // Format 2: build column index map — check for SOBAODANH|SBD and DIEM_THI + // build-database.js:70–86 + let mut sbd_idx: Option = None; + let mut ho_ten_idx: Option = None; + let mut ngay_sinh_idx: Option = None; + let mut ten_cum_thi_idx: Option = None; + let mut gioi_tinh_idx: Option = None; + let mut diem_thi_idx: Option = None; + + for (i, c) in cols.iter().enumerate() { + match c.as_str() { + "SOBAODANH" | "SBD" => sbd_idx = Some(i), + "HO_TEN" | "HOTEN" | "HỌ TÊN" => ho_ten_idx = Some(i), + "NGAY_SINH" => ngay_sinh_idx = Some(i), + "TEN_CUMTHI" => ten_cum_thi_idx = Some(i), + "GIOI_TINH" => gioi_tinh_idx = Some(i), + "DIEM_THI" => diem_thi_idx = Some(i), + _ => {} + } + } + + // build-database.js:82–84: if (map.sbd !== undefined && map.diem_thi !== undefined) + if let (Some(sbd), Some(diem_thi)) = (sbd_idx, diem_thi_idx) { + let ho_ten = ho_ten_idx.unwrap_or(1); // fallback: col 1 (present in all known files) + return DetectedFormat::Mapped { + sbd, + ho_ten, + ngay_sinh: ngay_sinh_idx, + ten_cum_thi: ten_cum_thi_idx, + gioi_tinh: gioi_tinh_idx, + diem_thi, + }; + } + + // Unrecognised header (or no header) → positional default + DetectedFormat::Default +} + +// --------------------------------------------------------------------------- +// Row processors +// --------------------------------------------------------------------------- + +/// Cell accessor helper. +fn cell_str(row: &[Data], idx: usize) -> String { + row.get(idx) + .map(|c| c.to_string().trim().to_owned()) + .unwrap_or_default() +} + +/// Parse a cell that should hold a float score; returns None for blank/non-numeric. +/// Mirrors `parseFloat(row[N]) || null` in JS. +fn parse_float_cell(row: &[Data], idx: usize) -> Option { + let s = cell_str(row, idx); + if s.is_empty() { + return None; + } + s.parse::().ok().filter(|v| v.is_finite() && *v != 0.0) +} + +/// Process a row in the `separate-scores` format. +/// +/// Column layout (build-database.js:90–116 `processSeparateScoresRow`): +/// 0=SBD 1=HOTEN 2=TOAN 3=VAN 4=LY 5=HOA 6=SINH 7=SU 8=DIA +/// 9=NGOAINGUTN 10=NGOAINGUTL 11=NGOAINGU(total→tieng_anh) +/// +/// tieng_phap / tieng_duc / tieng_nhat / tieng_trung all → None +/// ngay_sinh / ten_cum_thi / gioi_tinh all → None (not in this format) +pub fn process_separate_scores_row(row: &[Data], patterns: &CompiledPatterns) -> Option { + let sbd = cell_str(row, 0); + let ho_ten = cell_str(row, 1); + if sbd.is_empty() || ho_ten.is_empty() { + return None; + } + + let ho_ten_ascii = to_ascii(&ho_ten); + + // build-database.js:102–114: explicit per-column score mapping + let mut scores = std::collections::HashMap::new(); + macro_rules! add_score { + ($field:expr, $idx:expr) => { + if let Some(v) = parse_float_cell(row, $idx) { + scores.insert($field.to_string(), v); + } + }; + } + add_score!("toan", 2); + add_score!("ngu_van", 3); + add_score!("vat_ly", 4); + add_score!("hoa_hoc", 5); + add_score!("sinh_hoc", 6); + add_score!("lich_su", 7); + add_score!("dia_ly", 8); + // col 11 = NGOAINGU total → tieng_anh (build-database.js:110–111) + add_score!("tieng_anh", 11); + + // Suppress unused-variable warning; patterns not used in this path (no DIEM_THI string) + let _ = patterns; + + Some(ParsedRow { + so_bao_danh: sbd, + ho_ten, + ho_ten_ascii, + ngay_sinh: None, + ten_cum_thi: None, + gioi_tinh: None, + scores, + }) +} + +/// Process a row in the `mapped` format (header-derived column indices). +/// +/// Mirrors `processMappedRow` at build-database.js:119–146. +/// Gender is normalised: only "Nam" or "Nữ" are kept; everything else → None. +/// (build-database.js:132: `(rawGioiTinh === "Nam" || rawGioiTinh === "Nữ") ? rawGioiTinh : null`) +pub fn process_mapped_row( + row: &[Data], + sbd_idx: usize, + ho_ten_idx: usize, + ngay_sinh_idx: Option, + ten_cum_thi_idx: Option, + gioi_tinh_idx: Option, + diem_thi_idx: usize, + patterns: &CompiledPatterns, +) -> Option { + let sbd = cell_str(row, sbd_idx); + let ho_ten = cell_str(row, ho_ten_idx); + if sbd.is_empty() || ho_ten.is_empty() { + return None; + } + + // Skip rows where SBD or HO_TEN are themselves header tokens (leaked header rows). + // build-database.js:125–126: KNOWN_HEADERS.has(sbdUpper) || KNOWN_HEADERS.has(hoTenUpper) + let sbd_upper = sbd.to_uppercase(); + let ho_ten_upper = ho_ten.to_uppercase(); + if KNOWN_HEADERS.iter().any(|h| *h == sbd_upper.as_str()) + || KNOWN_HEADERS.iter().any(|h| *h == ho_ten_upper.as_str()) + { + return None; + } + + let ho_ten_ascii = to_ascii(&ho_ten); + + let ngay_sinh = ngay_sinh_idx + .map(|i| cell_str(row, i)) + .filter(|s| !s.is_empty()); + + let ten_cum_thi = ten_cum_thi_idx + .map(|i| cell_str(row, i)) + .filter(|s| !s.is_empty()); + + // build-database.js:130–132: normalise gender + let gioi_tinh = gioi_tinh_idx + .map(|i| cell_str(row, i)) + .and_then(|s| { + if s == "Nam" || s == "Nữ" { + Some(s) + } else { + None + } + }); + + let diem_thi = row + .get(diem_thi_idx) + .map(|c| c.to_string()) + .unwrap_or_default(); + + let scores = parse_scores(&diem_thi, patterns); + + Some(ParsedRow { + so_bao_danh: sbd, + ho_ten, + ho_ten_ascii, + ngay_sinh, + ten_cum_thi, + gioi_tinh, + scores, + }) +} + +/// Process a row using the default positional 6-column layout. +/// +/// Column order: SBD(0) HO_TEN(1) NGAY_SINH(2) TEN_CUMTHI(3) GIOI_TINH(4) DIEM_THI(5) +/// Mirrors `processMappedRow(row, DEFAULT_MAP)` at build-database.js:233–236. +pub fn process_default_row(row: &[Data], patterns: &CompiledPatterns) -> Option { + process_mapped_row( + row, + 0, // sbd + 1, // ho_ten + Some(2), + Some(3), + Some(4), + 5, // diem_thi + patterns, + ) +} + +/// Dispatch a data row through the correct processor for the detected format. +/// +/// Returns `None` when the row is empty/invalid and should be skipped. +pub fn process_row_2016( + row: &[Data], + fmt: &DetectedFormat, + patterns: &CompiledPatterns, +) -> Option { + match fmt { + DetectedFormat::SeparateScores => process_separate_scores_row(row, patterns), + DetectedFormat::Mapped { + sbd, + ho_ten, + ngay_sinh, + ten_cum_thi, + gioi_tinh, + diem_thi, + } => process_mapped_row( + row, + *sbd, + *ho_ten, + *ngay_sinh, + *ten_cum_thi, + *gioi_tinh, + *diem_thi, + patterns, + ), + DetectedFormat::Default => process_default_row(row, patterns), + } +} + +// --------------------------------------------------------------------------- +// Unit tests — 3 detection branches + key processing cases +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + fn s(v: &str) -> Data { + Data::String(v.to_string()) + } + + fn make_patterns() -> CompiledPatterns { + let mut map = HashMap::new(); + map.insert("toan".into(), r"Toán:\s*(\d+(?:\.\d+)?)".into()); + map.insert("ngu_van".into(), r"Ngữ văn:\s*(\d+(?:\.\d+)?)".into()); + map.insert("tieng_anh".into(), r"Tiếng Anh:\s*(\d+(?:\.\d+)?)".into()); + map.insert("tieng_duc".into(), r"Tiếng Đức:\s*(\d+(?:\.\d+)?)".into()); + map.insert("tieng_nhat".into(), r"Tiếng Nhật:\s*(\d+(?:\.\d+)?)".into()); + CompiledPatterns::new(&map).unwrap() + } + + // --- detect_format: branch 1 — separate-scores --- + + #[test] + fn detect_separate_scores() { + let header = vec![s("SBD"), s("HOTEN"), s("TOAN"), s("VAN")]; + match detect_format(&header) { + DetectedFormat::SeparateScores => {} + other => panic!("expected SeparateScores, got {other:?}"), + } + } + + // --- detect_format: branch 2 — mapped --- + + #[test] + fn detect_mapped_with_named_cols() { + let header = vec![ + s("SOBAODANH"), + s("HO_TEN"), + s("NGAY_SINH"), + s("TEN_CUMTHI"), + s("GIOI_TINH"), + s("DIEM_THI"), + ]; + match detect_format(&header) { + DetectedFormat::Mapped { + sbd, + ho_ten, + ngay_sinh, + ten_cum_thi, + gioi_tinh, + diem_thi, + } => { + assert_eq!(sbd, 0); + assert_eq!(ho_ten, 1); + assert_eq!(ngay_sinh, Some(2)); + assert_eq!(ten_cum_thi, Some(3)); + assert_eq!(gioi_tinh, Some(4)); + assert_eq!(diem_thi, 5); + } + other => panic!("expected Mapped, got {other:?}"), + } + } + + #[test] + fn detect_mapped_sbd_variant() { + // "SBD" (not "SOBAODANH") + DIEM_THI at different positions + let header = vec![s("STT"), s("SBD"), s("HOTEN"), s("DIEM_THI")]; + match detect_format(&header) { + DetectedFormat::Mapped { sbd, diem_thi, .. } => { + assert_eq!(sbd, 1); + assert_eq!(diem_thi, 3); + } + other => panic!("expected Mapped, got {other:?}"), + } + } + + // --- detect_format: branch 3 — default --- + + #[test] + fn detect_default_when_no_header() { + // A data row: positional default used + let data_row = vec![ + s("12345678"), + s("Nguyễn Văn A"), + s("01/01/2000"), + s("TP HCM"), + s("Nam"), + s("Toán: 8.5"), + ]; + // Default is returned when there is no recognised header + match detect_format(&data_row) { + DetectedFormat::Default => {} + other => panic!("expected Default, got {other:?}"), + } + } + + // --- process_separate_scores_row --- + + #[test] + fn separate_scores_basic() { + let p = make_patterns(); + // 12 columns: SBD HOTEN TOAN VAN LY HOA SINH SU DIA NGUTN NGUTL NGUTOTAL + let row = vec![ + s("TP001"), + s("Nguyễn Thị Lan"), + Data::Float(8.0), + Data::Float(7.5), + Data::Float(9.0), + Data::Float(6.5), + Data::Float(5.0), + Data::Float(4.5), + Data::Float(8.0), + Data::Empty, + Data::Empty, + Data::Float(7.0), // col 11 → tieng_anh + ]; + let row = process_separate_scores_row(&row, &p).expect("should parse"); + assert_eq!(row.so_bao_danh, "TP001"); + assert_eq!(row.ho_ten, "Nguyễn Thị Lan"); + assert_eq!(row.ho_ten_ascii, "nguyen thi lan"); + assert_eq!(row.scores.get("toan"), Some(&8.0)); + assert_eq!(row.scores.get("tieng_anh"), Some(&7.0)); + assert!(row.ngay_sinh.is_none()); + assert!(row.ten_cum_thi.is_none()); + assert!(row.gioi_tinh.is_none()); + } + + #[test] + fn separate_scores_skips_empty_sbd() { + let p = make_patterns(); + let row = vec![s(""), s("Nguyễn Văn A"), Data::Float(5.0)]; + assert!(process_separate_scores_row(&row, &p).is_none()); + } + + // --- process_mapped_row --- + + #[test] + fn mapped_row_full_fields() { + let p = make_patterns(); + let row = vec![ + s("HCM001"), + s("Trần Thị Bích"), + s("15/3/1999"), + s("Cụm thi HCM"), + s("Nữ"), + s("Toán: 9.0 Ngữ văn: 8.5 Tiếng Anh: 7.75"), + ]; + let parsed = process_mapped_row(&row, 0, 1, Some(2), Some(3), Some(4), 5, &p) + .expect("should parse"); + assert_eq!(parsed.so_bao_danh, "HCM001"); + assert_eq!(parsed.ngay_sinh.as_deref(), Some("15/3/1999")); + assert_eq!(parsed.ten_cum_thi.as_deref(), Some("Cụm thi HCM")); + assert_eq!(parsed.gioi_tinh.as_deref(), Some("Nữ")); + assert_eq!(parsed.scores.get("toan"), Some(&9.0)); + assert_eq!(parsed.scores.get("ngu_van"), Some(&8.5)); + assert_eq!(parsed.scores.get("tieng_anh"), Some(&7.75)); + } + + #[test] + fn mapped_row_gender_normalisation() { + let p = make_patterns(); + // Gender "Unknown" → None + let row = vec![s("ABC"), s("Nguyen Van A"), s(""), s(""), s("Unknown"), s("")]; + let parsed = + process_mapped_row(&row, 0, 1, Some(2), Some(3), Some(4), 5, &p).expect("should parse"); + assert!(parsed.gioi_tinh.is_none()); + + // "Nam" passes through + let row2 = vec![s("ABC"), s("Nguyen Van A"), s(""), s(""), s("Nam"), s("")]; + let parsed2 = + process_mapped_row(&row2, 0, 1, Some(2), Some(3), Some(4), 5, &p).expect("should parse"); + assert_eq!(parsed2.gioi_tinh.as_deref(), Some("Nam")); + } + + #[test] + fn mapped_row_skips_leaked_header() { + let p = make_patterns(); + // A leaked header row — SBD cell contains "SOBAODANH" + let row = vec![s("SOBAODANH"), s("HO_TEN"), s("NGAY_SINH"), s(""), s(""), s("")]; + assert!(process_mapped_row(&row, 0, 1, Some(2), Some(3), Some(4), 5, &p).is_none()); + } + + // --- process_default_row --- + + #[test] + fn default_row_positional() { + let p = make_patterns(); + let row = vec![ + s("DN001"), + s("Lê Văn Long"), + s("10/5/1998"), + s("Cụm Đà Nẵng"), + s("Nam"), + s("Tiếng Đức: 6.25"), + ]; + let parsed = process_default_row(&row, &p).expect("should parse"); + assert_eq!(parsed.so_bao_danh, "DN001"); + assert_eq!(parsed.ho_ten_ascii, "le van long"); + assert_eq!(parsed.ngay_sinh.as_deref(), Some("10/5/1998")); + assert_eq!(parsed.ten_cum_thi.as_deref(), Some("Cụm Đà Nẵng")); + assert_eq!(parsed.gioi_tinh.as_deref(), Some("Nam")); + assert_eq!(parsed.scores.get("tieng_duc"), Some(&6.25)); + } + + // --- is_header_row_2016 --- + + #[test] + fn header_row_detection_2016() { + assert!(is_header_row_2016(&[s("SOBAODANH"), s("HO_TEN")])); + assert!(is_header_row_2016(&[s("SBD"), s("HOTEN"), s("TOAN")])); + assert!(!is_header_row_2016(&[s("12345678"), s("Nguyen Van A")])); + assert!(!is_header_row_2016(&[s("SBD")])); // too short (< 2 cells) + } +} diff --git a/2016/tools/xlsxread/src/lib.rs b/2016/tools/xlsxread/src/lib.rs new file mode 100644 index 0000000..b61560c --- /dev/null +++ b/2016/tools/xlsxread/src/lib.rs @@ -0,0 +1,10 @@ +/// Public library interface for integration tests. +/// The binary entry point is src/main.rs; this file re-exports the internal +/// modules so tests/golden.rs can call them without going through the CLI. +pub mod audit; +pub mod config; +pub mod error; +pub mod format_detect_2016; +pub mod reader; +pub mod transform; +pub mod writer; diff --git a/2016/tools/xlsxread/src/main.rs b/2016/tools/xlsxread/src/main.rs new file mode 100644 index 0000000..0736ed7 --- /dev/null +++ b/2016/tools/xlsxread/src/main.rs @@ -0,0 +1,379 @@ +/// xlsxread — Rust CLI replacing the SheetJS xlsx build scripts. +/// +/// Subcommands: +/// build — read .xls/.xlsx files → write SQLite DB +/// audit — compare distinct SBD count from xlsx vs DB row count +/// +/// Library modules are declared in lib.rs; main.rs only adds the CLI layer. +/// +/// When config contains `format_detection = "thptqg2016"` the build subcommand +/// uses per-file header inspection to pick the right column layout, replicating +/// the `detectFormat` logic from scripts/build-database.js (lines 63–87). +mod cli; + +use std::path::Path; + +use anyhow::{Context, Result}; +use calamine::Data; +use clap::Parser; + +use cli::{Cli, Cmd}; +use xlsxread::audit; +use xlsxread::config::load_config; +use xlsxread::format_detect_2016::{ + detect_format, is_header_row_2016, process_row_2016, DetectedFormat, +}; +use xlsxread::reader::{is_all_blank, process_file}; +use xlsxread::transform::{validate_row, CompiledPatterns, SkipReason}; +use xlsxread::writer::{ + finish_db, insert_row, insert_row_2016, open_db, SCORE_FIELDS, SCORE_FIELDS_2016, +}; + +fn main() -> Result<()> { + let cli = Cli::parse(); + + match cli.cmd { + Cmd::Build { + schema, + input, + output, + } => { + run_build(&schema, &input, &output)?; + } + Cmd::Audit { schema, input, db } => { + let cfg = load_config(&schema) + .with_context(|| format!("Failed to load config: {}", schema.display()))?; + let result = audit::run_audit(&input, &db, &cfg).with_context(|| "Audit failed")?; + audit::print_audit_report(&result); + if !result.matched { + std::process::exit(1); + } + } + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Build subcommand — dispatches to thptqg2016 or standard path +// --------------------------------------------------------------------------- + +fn run_build(schema_path: &Path, input_dir: &Path, output_path: &Path) -> Result<()> { + let cfg = load_config(schema_path) + .with_context(|| format!("Failed to load config: {}", schema_path.display()))?; + + if cfg.format_detection.as_deref() == Some("thptqg2016") { + run_build_2016(&cfg, input_dir, output_path) + } else { + run_build_standard(&cfg, input_dir, output_path) + } +} + +// --------------------------------------------------------------------------- +// Standard build path (thptqg2017 and similar fixed-column configs) +// --------------------------------------------------------------------------- + +fn run_build_standard( + cfg: &xlsxread::config::DatasetConfig, + input_dir: &Path, + output_path: &Path, +) -> Result<()> { + let patterns = + CompiledPatterns::new(&cfg.scores).with_context(|| "Failed to compile score regexes")?; + + let mut files: Vec = std::fs::read_dir(input_dir) + .with_context(|| format!("Cannot read input dir: {}", input_dir.display()))? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + p.is_file() + && p.extension() + .and_then(|e| e.to_str()) + .map(|e| { + let lower = e.to_lowercase(); + lower == "xls" || lower == "xlsx" + }) + .unwrap_or(false) + }) + .collect(); + files.sort(); + + let dataset_label = input_dir + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("data"); + + println!( + "[build] {dataset_label}/ → {} ({} files)", + output_path.display(), + files.len() + ); + + let conn = open_db(output_path, cfg) + .with_context(|| format!("Failed to open DB: {}", output_path.display()))?; + + let mut total_source_rows: u64 = 0; + let mut total_skipped: u64 = 0; + let mut total_errors: u64 = 0; + + let is_old2 = dataset_label.contains("old2"); + let strip_blank = cfg.reader.strip_blank_rows; + + conn.execute_batch("BEGIN")?; + + for file in &files { + let base = file + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("?") + .to_owned(); + let mut file_rows: u64 = 0; + let mut file_skipped: u64 = 0; + let mut file_errors: u64 = 0; + + let process_result = process_file(file, cfg, |_sheet_idx, raw| { + let all_blank = is_all_blank(raw); + if strip_blank && all_blank { + return; + } + + total_source_rows += 1; + + let ho_ten = raw + .get(cfg.columns.as_ref().unwrap().ho_ten) + .map(|c| c.to_string().trim().to_owned()) + .unwrap_or_default(); + let so_bao_danh = raw + .get(cfg.columns.as_ref().unwrap().so_bao_danh) + .map(|c| c.to_string().trim().to_owned()) + .unwrap_or_default(); + + match validate_row(&ho_ten, &so_bao_danh, &cfg.validation, strip_blank, all_blank) { + Err(SkipReason::BlankRow) => {} + Err(_) => { + file_skipped += 1; + return; + } + Ok(()) => {} + } + + let parsed = xlsxread::transform::transform_row(raw, cfg, &patterns); + + match insert_row(&conn, &cfg.insert.sql, &parsed, SCORE_FIELDS) { + Ok(()) => file_rows += 1, + Err(e) => { + file_errors += 1; + if total_errors + file_errors <= 5 { + eprintln!(" [warn] {base}: {e}"); + } + } + } + }); + + match process_result { + Ok(_) => {} + Err(e) => { + eprintln!(" [error] {base}: {e}"); + file_errors += 1; + } + } + + total_skipped += file_skipped; + total_errors += file_errors; + println!(" {base}: {file_rows} rows"); + } + + conn.execute_batch("COMMIT")?; + + finish_db( + &conn, + output_path, + total_source_rows, + total_skipped, + total_errors, + dataset_label, + files.len(), + is_old2, + ) + .with_context(|| "Failed to finalise DB")?; + + Ok(()) +} + +// --------------------------------------------------------------------------- +// thptqg2016 build path — per-file format detection +// --------------------------------------------------------------------------- + +/// Build the thptqg2016 database. +/// +/// Each file is processed independently: the first row is inspected to determine +/// which of the three column layouts applies (separate-scores / mapped / default). +/// This mirrors `detectFormat` in scripts/build-database.js lines 63–87, called +/// once per file inside the file loop at build-database.js:218–219. +fn run_build_2016( + cfg: &xlsxread::config::DatasetConfig, + input_dir: &Path, + output_path: &Path, +) -> Result<()> { + let patterns = + CompiledPatterns::new(&cfg.scores).with_context(|| "Failed to compile score regexes")?; + + let mut files: Vec = std::fs::read_dir(input_dir) + .with_context(|| format!("Cannot read input dir: {}", input_dir.display()))? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + p.is_file() + && p.extension() + .and_then(|e| e.to_str()) + .map(|e| { + let lower = e.to_lowercase(); + lower == "xls" || lower == "xlsx" + }) + .unwrap_or(false) + }) + .collect(); + files.sort(); + + let dataset_label = input_dir + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("data"); + + println!( + "[build:2016] {dataset_label}/ → {} ({} files)", + output_path.display(), + files.len() + ); + + let conn = open_db(output_path, cfg) + .with_context(|| format!("Failed to open DB: {}", output_path.display()))?; + + let mut total_source_rows: u64 = 0; + let total_skipped: u64 = 0; + let mut total_errors: u64 = 0; + + conn.execute_batch("BEGIN")?; + + for file in &files { + let base = file + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("?") + .to_owned(); + + match process_file_2016( + file, + cfg, + &patterns, + &conn, + &base, + &mut total_source_rows, + &mut total_errors, + ) { + Ok(file_rows) => { + println!(" {base}: {file_rows} rows"); + } + Err(e) => { + eprintln!(" [error] {base}: {e}"); + total_errors += 1; + } + } + } + + conn.execute_batch("COMMIT")?; + + finish_db( + &conn, + output_path, + total_source_rows, + total_skipped, + total_errors, + dataset_label, + files.len(), + false, + ) + .with_context(|| "Failed to finalise DB")?; + + Ok(()) +} + +/// Process one file in the thptqg2016 format-detection path. +/// +/// Reads the file, uses the first row to detect the column layout, then processes +/// all subsequent data rows. Returns the count of successfully inserted rows. +fn process_file_2016( + file: &Path, + cfg: &xlsxread::config::DatasetConfig, + patterns: &CompiledPatterns, + conn: &rusqlite::Connection, + base: &str, + total_source_rows: &mut u64, + total_errors: &mut u64, +) -> Result { + use calamine::{open_workbook_auto, Reader, Sheets}; + + let path_str = file.display().to_string(); + let mut workbook: Sheets<_> = + open_workbook_auto(file).with_context(|| format!("Cannot open {path_str}"))?; + + let sheet_names: Vec = workbook.sheet_names().to_vec(); + if sheet_names.is_empty() { + return Ok(0); + } + + // Sheet selection: thptqg2016 data/ has all-sheets mode to handle + // HCM/HN overflow (same reason as thptqg2017 data/). + let sheets_to_read: Vec = match cfg.reader.sheet_mode { + xlsxread::config::SheetMode::All => sheet_names.clone(), + xlsxread::config::SheetMode::First => vec![sheet_names[0].clone()], + }; + + let mut file_rows: u64 = 0; + + for sheet_name in &sheets_to_read { + let range = workbook + .worksheet_range(sheet_name) + .with_context(|| format!("Cannot read sheet {sheet_name} in {path_str}"))?; + + let rows: Vec> = range.rows().map(|r| r.to_vec()).collect(); + if rows.is_empty() { + continue; + } + + // Detect format from first row, then determine start index. + // Mirrors build-database.js:215–220: isHeaderRow check + detectFormat. + let (fmt, start_idx) = if is_header_row_2016(&rows[0]) { + (detect_format(&rows[0]), 1) + } else { + (DetectedFormat::Default, 0) + }; + + for row in rows.iter().skip(start_idx) { + if row.len() < 2 { + continue; + } + + *total_source_rows += 1; + + match process_row_2016(row, &fmt, patterns) { + None => { + // Row was empty/invalid — skipped (mirrors JS `if (!record) continue`) + } + Some(parsed) => { + match insert_row_2016(conn, &cfg.insert.sql, &parsed, SCORE_FIELDS_2016) { + Ok(()) => file_rows += 1, + Err(e) => { + *total_errors += 1; + if *total_errors <= 5 { + eprintln!(" [warn] {base}: {e}"); + } + } + } + } + } + } + } + + Ok(file_rows) +} diff --git a/2016/tools/xlsxread/src/reader.rs b/2016/tools/xlsxread/src/reader.rs new file mode 100644 index 0000000..ac71da1 --- /dev/null +++ b/2016/tools/xlsxread/src/reader.rs @@ -0,0 +1,197 @@ +/// Spreadsheet reader: wraps calamine to iterate rows across sheets. +/// +/// Sheet selection mirrors the JS scripts: +/// - sheet_mode = "all" → iterate every sheet (handles HCM/HN 65k overflow in data/) +/// - sheet_mode = "first" → sheet 0 only (data-old/) +/// +/// Header detection mirrors build-lib.js isHeaderRow: +/// row[0].toUpperCase() in {"HO_TEN", "HỌ TÊN", "STT"} +use std::path::Path; + +use calamine::{open_workbook_auto, Data, Reader, Sheets}; + +use crate::config::{DatasetConfig, HeaderCfg, SheetMode}; +use crate::error::BuildError; + +// --------------------------------------------------------------------------- +// Public row representation from calamine +// --------------------------------------------------------------------------- + +pub type RawRow = Vec; + +// --------------------------------------------------------------------------- +// Header detection — mirrors build-lib.js isHeaderRow +// --------------------------------------------------------------------------- + +/// Returns true when the first cell (uppercased) matches one of the configured +/// header tokens. Used to skip the header row on the first row of each sheet. +pub fn is_header_row(row: &[Data], header_cfg: &HeaderCfg) -> bool { + if row.len() < 3 { + return false; + } + let first = row[0].to_string().trim().to_uppercase(); + header_cfg.tokens.iter().any(|t| t.to_uppercase() == first) +} + +// --------------------------------------------------------------------------- +// All-blank row check (data-old2: strip_blank_rows) +// --------------------------------------------------------------------------- + +pub fn is_all_blank(row: &[Data]) -> bool { + row.iter() + .all(|c| matches!(c, Data::Empty) || c.to_string().trim().is_empty()) +} + +// --------------------------------------------------------------------------- +// File processor — yields all data rows from the file +// --------------------------------------------------------------------------- + +/// Process one spreadsheet file, calling `on_row` for each data row. +/// +/// `on_row` receives `(sheet_index, row_index_in_sheet, raw_row)` where +/// `row_index_in_sheet` is 0-based AFTER the header has been consumed. +/// Returns `(sheets_seen, total_rows_yielded)`. +pub fn process_file( + path: &Path, + cfg: &DatasetConfig, + mut on_row: F, +) -> Result<(usize, usize), BuildError> +where + F: FnMut(usize, &RawRow), +{ + let path_str = path.display().to_string(); + + // calamine::open_workbook_auto dispatches on file extension + let mut workbook: Sheets<_> = open_workbook_auto(path).map_err(|e| BuildError::Calamine { + path: path_str.clone(), + source: e, + })?; + + let sheet_names: Vec = workbook.sheet_names().to_vec(); + if sheet_names.is_empty() { + return Err(BuildError::NoSheets(path_str.clone())); + } + + // Sheet selection per config + let sheets_to_read: Vec = match cfg.reader.sheet_mode { + SheetMode::All => sheet_names.clone(), + SheetMode::First => vec![sheet_names[0].clone()], + }; + + let mut total_rows = 0usize; + + for (sheet_idx, sheet_name) in sheets_to_read.iter().enumerate() { + let range = workbook + .worksheet_range(sheet_name) + .map_err(|e| BuildError::Calamine { + path: path_str.clone(), + source: e, + })?; + + let mut first_row = true; + + for raw in range.rows() { + let row: RawRow = raw.to_vec(); + + // Skip header row on first row of each sheet (matches JS: `if (i === 0 && isHeaderRow(...))`) + if first_row { + first_row = false; + if is_header_row(&row, &cfg.header) { + continue; + } + } + + on_row(sheet_idx, &row); + total_rows += 1; + } + } + + Ok((sheets_to_read.len(), total_rows)) +} + +// --------------------------------------------------------------------------- +// Unit tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::HeaderCfg; + + fn hdr(tokens: &[&str]) -> HeaderCfg { + HeaderCfg { + tokens: tokens.iter().map(|s| s.to_string()).collect(), + } + } + + #[test] + fn header_detects_ho_ten() { + let row = vec![ + Data::String("HO_TEN".into()), + Data::String("NGAY_SINH".into()), + Data::String("SBD".into()), + ]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(is_header_row(&row, &cfg)); + } + + #[test] + fn header_detects_stt() { + let row = vec![ + Data::String("STT".into()), + Data::String("B".into()), + Data::String("C".into()), + ]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(is_header_row(&row, &cfg)); + } + + #[test] + fn header_detects_ho_ten_unicode() { + let row = vec![ + Data::String("HỌ TÊN".into()), + Data::String("B".into()), + Data::String("C".into()), + ]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(is_header_row(&row, &cfg)); + } + + #[test] + fn header_rejects_data_row() { + let row = vec![ + Data::String("Nguyen Van A".into()), + Data::String("01/01/2000".into()), + Data::String("12345678".into()), + ]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(!is_header_row(&row, &cfg)); + } + + #[test] + fn header_rejects_short_row() { + let row = vec![Data::String("HO_TEN".into()), Data::Empty]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(!is_header_row(&row, &cfg)); + } + + #[test] + fn header_case_insensitive() { + let row = vec![ + Data::String("ho_ten".into()), + Data::String("B".into()), + Data::String("C".into()), + ]; + let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]); + assert!(is_header_row(&row, &cfg)); + } + + #[test] + fn blank_row_detection() { + let row = vec![Data::Empty, Data::Empty, Data::String("".into())]; + assert!(is_all_blank(&row)); + + let row2 = vec![Data::String("Nguyen".into()), Data::Empty, Data::Empty]; + assert!(!is_all_blank(&row2)); + } +} diff --git a/2016/tools/xlsxread/src/transform.rs b/2016/tools/xlsxread/src/transform.rs new file mode 100644 index 0000000..c688be4 --- /dev/null +++ b/2016/tools/xlsxread/src/transform.rs @@ -0,0 +1,410 @@ +/// Row transformation: ascii normalisation, score regex parsing, validation. +/// +/// `to_ascii` replicates build-lib.js `toAscii` exactly: +/// str.normalize("NFD").replace(/[̀-ͯ]/g,"").replace(/đ/gi,"d").toLowerCase() +use std::collections::HashMap; + +use regex::Regex; +use unicode_normalization::UnicodeNormalization; + +use crate::config::{DatasetConfig, ValidationCfg}; +use crate::error::BuildError; + +// --------------------------------------------------------------------------- +// Compiled score patterns (built once at startup from config) +// --------------------------------------------------------------------------- + +pub struct CompiledPatterns { + /// Ordered list so INSERT column order is deterministic + pub patterns: Vec<(String, Regex)>, +} + +impl CompiledPatterns { + pub fn new(scores: &HashMap) -> Result { + let mut patterns = Vec::with_capacity(scores.len()); + for (field, src) in scores { + let re = Regex::new(src).map_err(|e| BuildError::Regex { + pattern: src.clone(), + source: e, + })?; + patterns.push((field.clone(), re)); + } + // Sort for deterministic order across HashMap iteration + patterns.sort_by(|a, b| a.0.cmp(&b.0)); + Ok(Self { patterns }) + } +} + +// --------------------------------------------------------------------------- +// to_ascii — must be byte-for-byte equivalent to build-lib.js toAscii +// --------------------------------------------------------------------------- + +/// Normalise a Vietnamese name to an ASCII slug. +/// +/// Algorithm mirrors the JavaScript `toAscii` in build-lib.js: +/// 1. NFD decompose (splits base + combining diacritics) +/// 2. Drop all Unicode combining marks (U+0300–U+036F) +/// 3. Replace đ/Đ with d (NFD does not decompose đ) +/// 4. Lowercase +pub fn to_ascii(s: &str) -> String { + // Step 1 + 2: NFD then filter out combining marks (Unicode category M) + let decomposed: String = s + .nfd() + .filter(|c| !('\u{0300}'..='\u{036f}').contains(c)) + .collect(); + + // Step 3: đ/Đ are not decomposed by NFD — replace explicitly + let replaced = decomposed.replace(['đ', 'Đ'], "d"); + + // Step 4: lowercase + replaced.to_lowercase() +} + +// --------------------------------------------------------------------------- +// Parsed row ready for DB insert +// --------------------------------------------------------------------------- + +pub struct ParsedRow { + pub so_bao_danh: String, + pub ho_ten: String, + pub ho_ten_ascii: String, + pub ngay_sinh: Option, + /// thptqg2016 only: examination cluster name (TEN_CUMTHI column) + pub ten_cum_thi: Option, + /// thptqg2016 only: gender (GIOI_TINH column), normalised to "Nam"/"Nữ" or None + pub gioi_tinh: Option, + /// Subject field → float value; absent subjects not in map → NULL + pub scores: HashMap, +} + +// --------------------------------------------------------------------------- +// Row validation — mirrors the per-script skip logic +// --------------------------------------------------------------------------- + +/// Returns `None` when the row should be skipped entirely (before sourceRows counter). +/// Returns `Some(reason)` when the row should be counted as sourceRows but skipped. +#[derive(Debug, PartialEq, Eq)] +pub enum SkipReason { + /// Row is fully blank (data-old2 only, before sourceRows counter) + BlankRow, + /// soBaoDanh or hoTen empty/missing + EmptyField, + /// soBaoDanh contains non-digit characters (data-old / data-old2 guard) + NonNumericSbd, +} + +/// Validates a raw cell slice against the dataset's `ValidationCfg`. +/// Returns `Ok(())` on pass, `Err(SkipReason)` on fail. +pub fn validate_row( + ho_ten: &str, + so_bao_danh: &str, + cfg: &ValidationCfg, + strip_blank_rows: bool, + all_blank: bool, +) -> Result<(), SkipReason> { + // data-old2: skip fully blank rows BEFORE counting sourceRows + if strip_blank_rows && all_blank { + return Err(SkipReason::BlankRow); + } + + if cfg.require_nonempty_sbd && so_bao_danh.is_empty() { + return Err(SkipReason::EmptyField); + } + if cfg.require_nonempty_name && ho_ten.is_empty() { + return Err(SkipReason::EmptyField); + } + if cfg.require_numeric_sbd && !so_bao_danh.chars().all(|c| c.is_ascii_digit()) { + return Err(SkipReason::NonNumericSbd); + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Score parsing — mirrors build-lib.js parseScores +// --------------------------------------------------------------------------- + +/// Parse a DIEM_THI cell string and extract matching subject scores. +pub fn parse_scores(diem_thi: &str, patterns: &CompiledPatterns) -> HashMap { + let mut out = HashMap::new(); + for (field, re) in &patterns.patterns { + if let Some(caps) = re.captures(diem_thi) { + if let Some(m) = caps.get(1) { + if let Ok(v) = m.as_str().parse::() { + if v.is_finite() { + out.insert(field.clone(), v); + } + } + } + } + } + out +} + +// --------------------------------------------------------------------------- +// Full row transform (thptqg2017 fixed-column path) +// --------------------------------------------------------------------------- + +/// Extract and transform one spreadsheet row into a `ParsedRow` using fixed column indices. +/// `raw` is the full cell slice; column indices come from `cfg.columns`. +/// Used for thptqg2017 configs that have a static [columns] table. +pub fn transform_row( + raw: &[calamine::Data], + cfg: &DatasetConfig, + patterns: &CompiledPatterns, +) -> ParsedRow { + let cols = cfg + .columns + .as_ref() + .expect("transform_row requires [columns] section in config"); + + let get = |idx: usize| -> String { + raw.get(idx) + .map(|cell| cell.to_string().trim().to_owned()) + .unwrap_or_default() + }; + + let ho_ten = get(cols.ho_ten); + let ngay_sinh = get(cols.ngay_sinh); + let so_bao_danh = get(cols.so_bao_danh); + let diem_thi = raw + .get(cols.diem_thi) + .map(|c| c.to_string()) + .unwrap_or_default(); + + let ho_ten_ascii = to_ascii(&ho_ten); + let scores = parse_scores(&diem_thi, patterns); + let ngay_sinh_opt = if ngay_sinh.is_empty() { + None + } else { + Some(ngay_sinh) + }; + + ParsedRow { + so_bao_danh, + ho_ten, + ho_ten_ascii, + ngay_sinh: ngay_sinh_opt, + ten_cum_thi: None, + gioi_tinh: None, + scores, + } +} + +// --------------------------------------------------------------------------- +// Unit tests — 20 cases for to_ascii (real Vietnamese names) +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // Helper: assert to_ascii(input) == expected + fn check(input: &str, expected: &str) { + assert_eq!( + to_ascii(input), + expected, + "to_ascii({input:?}) expected {expected:?}" + ); + } + + #[test] + fn ascii_plain_latin() { + check("Nguyen Van A", "nguyen van a"); + } + + #[test] + fn ascii_nguyen_thi_hoa() { + check("Nguyễn Thị Hoa", "nguyen thi hoa"); + } + + #[test] + fn ascii_tran_van_duc() { + // đ/Đ replacement + check("Trần Văn Đức", "tran van duc"); + } + + #[test] + fn ascii_le_thi_my_duyen() { + check("Lê Thị Mỹ Duyên", "le thi my duyen"); + } + + #[test] + fn ascii_pham_thi_lan() { + check("Phạm Thị Lan", "pham thi lan"); + } + + #[test] + fn ascii_bui_thi_thu() { + check("Bùi Thị Thu", "bui thi thu"); + } + + #[test] + fn ascii_hoang_van_truong() { + check("Hoàng Văn Trường", "hoang van truong"); + } + + #[test] + fn ascii_do_thi_ngan() { + // Đ uppercase at start + check("Đỗ Thị Ngân", "do thi ngan"); + } + + #[test] + fn ascii_nguyen_van_khanh() { + check("Nguyễn Văn Khánh", "nguyen van khanh"); + } + + #[test] + fn ascii_trinh_thi_bich_ngoc() { + check("Trịnh Thị Bích Ngọc", "trinh thi bich ngoc"); + } + + #[test] + fn ascii_vu_thi_dieu() { + // ề = e + combining grave + combining circumflex (after NFD) + check("Vũ Thị Diệu", "vu thi dieu"); + } + + #[test] + fn ascii_nguyen_thi_tuong_vi() { + check("Nguyễn Thị Tường Vi", "nguyen thi tuong vi"); + } + + #[test] + fn ascii_lowercase_d_stroke() { + // Lowercase đ → d + check("đặng thị hằng", "dang thi hang"); + } + + #[test] + fn ascii_uppercase_d_stroke() { + check("ĐẶNG THỊ HẰNG", "dang thi hang"); + } + + #[test] + fn ascii_mixed_case() { + check("NGUYỄN VĂN AN", "nguyen van an"); + } + + #[test] + fn ascii_tran_thi_kim_anh() { + check("Trần Thị Kim Anh", "tran thi kim anh"); + } + + #[test] + fn ascii_nguyen_thi_phuong_thao() { + check("Nguyễn Thị Phương Thảo", "nguyen thi phuong thao"); + } + + #[test] + fn ascii_le_van_long() { + check("Lê Văn Long", "le van long"); + } + + #[test] + fn ascii_vo_thi_xuan_mai() { + check("Võ Thị Xuân Mai", "vo thi xuan mai"); + } + + #[test] + fn ascii_empty_string() { + check("", ""); + } + + // --- Score parsing tests --- + + fn make_patterns() -> CompiledPatterns { + let mut map = HashMap::new(); + map.insert("toan".into(), r"Toán:\s*(\d+(?:\.\d+)?)".into()); + map.insert("ngu_van".into(), r"Ngữ văn:\s*(\d+(?:\.\d+)?)".into()); + map.insert("vat_ly".into(), r"Vật lí:\s*(\d+(?:\.\d+)?)".into()); + CompiledPatterns::new(&map).unwrap() + } + + #[test] + fn parse_scores_single() { + let p = make_patterns(); + let s = "Toán: 8.5"; + let scores = parse_scores(s, &p); + assert_eq!(scores.get("toan"), Some(&8.5)); + assert!(scores.get("ngu_van").is_none()); + } + + #[test] + fn parse_scores_multiple() { + let p = make_patterns(); + let s = "Toán: 7.25 Ngữ văn: 6.0 Vật lí: 9"; + let scores = parse_scores(s, &p); + assert_eq!(scores.get("toan"), Some(&7.25)); + assert_eq!(scores.get("ngu_van"), Some(&6.0)); + assert_eq!(scores.get("vat_ly"), Some(&9.0)); + } + + #[test] + fn parse_scores_empty_cell() { + let p = make_patterns(); + let scores = parse_scores("", &p); + assert!(scores.is_empty()); + } + + // --- Validation tests --- + + fn default_validation() -> ValidationCfg { + ValidationCfg { + require_numeric_sbd: false, + require_nonempty_name: true, + require_nonempty_sbd: true, + } + } + + #[test] + fn validate_ok() { + let v = default_validation(); + assert!(validate_row("Nguyen Van A", "12345678", &v, false, false).is_ok()); + } + + #[test] + fn validate_empty_sbd() { + let v = default_validation(); + assert_eq!( + validate_row("Nguyen Van A", "", &v, false, false), + Err(SkipReason::EmptyField) + ); + } + + #[test] + fn validate_empty_name() { + let v = default_validation(); + assert_eq!( + validate_row("", "12345678", &v, false, false), + Err(SkipReason::EmptyField) + ); + } + + #[test] + fn validate_non_numeric_sbd_rejected() { + let mut v = default_validation(); + v.require_numeric_sbd = true; + assert_eq!( + validate_row("Nguyen Van A", "12AB5678", &v, false, false), + Err(SkipReason::NonNumericSbd) + ); + } + + #[test] + fn validate_numeric_sbd_accepted() { + let mut v = default_validation(); + v.require_numeric_sbd = true; + assert!(validate_row("Nguyen Van A", "12345678", &v, false, false).is_ok()); + } + + #[test] + fn validate_blank_row_skipped() { + let v = default_validation(); + // strip_blank_rows=true AND all_blank=true → BlankRow + assert_eq!( + validate_row("", "", &v, true, true), + Err(SkipReason::BlankRow) + ); + } +} diff --git a/2016/tools/xlsxread/src/writer.rs b/2016/tools/xlsxread/src/writer.rs new file mode 100644 index 0000000..07a83f0 --- /dev/null +++ b/2016/tools/xlsxread/src/writer.rs @@ -0,0 +1,201 @@ +/// SQLite writer: DDL setup, batched INSERT OR REPLACE, VACUUM, stats output. +/// +/// Mirrors build-lib.js createDb + the transaction loop in each build-database*.js. +/// Stats output lines match the JS stdout exactly so existing CI log-greps still work. +/// +/// thptqg2016 differences vs thptqg2017: +/// - INSERT includes ten_cum_thi and gioi_tinh after ngay_sinh +/// - Score columns are toan/ngu_van/vat_ly/hoa_hoc/sinh_hoc/lich_su/dia_ly/ +/// tieng_anh/tieng_phap/tieng_duc/tieng_nhat/tieng_trung +/// (no khtn, khxh, tieng_nga) +use std::fs; +use std::path::Path; + +use rusqlite::{params_from_iter, Connection, ToSql}; + +use crate::config::DatasetConfig; +use crate::error::BuildError; +use crate::transform::ParsedRow; + +// --------------------------------------------------------------------------- +// DB initialisation — mirrors build-lib.js createDb (delete + recreate) +// --------------------------------------------------------------------------- + +/// Open (or recreate) the output SQLite database, execute the DDL from config, +/// and return the open connection ready for inserts. +pub fn open_db(db_path: &Path, cfg: &DatasetConfig) -> Result { + // Mirror Node behaviour: delete existing file before creating (build-lib.js:54) + if db_path.exists() { + fs::remove_file(db_path).map_err(|e| BuildError::Io { + path: db_path.display().to_string(), + source: e, + })?; + } + + // Ensure parent directory exists + if let Some(parent) = db_path.parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent).map_err(|e| BuildError::Io { + path: parent.display().to_string(), + source: e, + })?; + } + } + + let conn = Connection::open(db_path)?; + conn.execute_batch(&cfg.schema.ddl)?; + Ok(conn) +} + +// --------------------------------------------------------------------------- +// Score field lists — one per dataset schema +// --------------------------------------------------------------------------- + +/// thptqg2017 score columns (14 fields including khtn/khxh/tieng_nga). +pub const SCORE_FIELDS_2017: &[&str] = &[ + "toan", + "ngu_van", + "vat_ly", + "hoa_hoc", + "sinh_hoc", + "khtn", + "lich_su", + "dia_ly", + "gdcd", + "khxh", + "tieng_anh", + "tieng_phap", + "tieng_nga", + "tieng_trung", +]; + +/// thptqg2016 score columns (12 fields: tieng_duc/tieng_nhat present; no khtn/khxh/tieng_nga). +pub const SCORE_FIELDS_2016: &[&str] = &[ + "toan", + "ngu_van", + "vat_ly", + "hoa_hoc", + "sinh_hoc", + "lich_su", + "dia_ly", + "tieng_anh", + "tieng_phap", + "tieng_duc", + "tieng_nhat", + "tieng_trung", +]; + +/// Alias kept so thptqg2017 callers that import SCORE_FIELDS continue to compile. +pub const SCORE_FIELDS: &[&str] = SCORE_FIELDS_2017; + +// --------------------------------------------------------------------------- +// Insert a single parsed row (thptqg2017 schema — no ten_cum_thi / gioi_tinh) +// --------------------------------------------------------------------------- + +/// Bind all fields from `row` into the prepared statement and execute it. +/// Positional params: so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, +pub fn insert_row( + conn: &Connection, + sql: &str, + row: &ParsedRow, + score_fields: &[&str], +) -> Result<(), BuildError> { + let mut params: Vec> = Vec::with_capacity(4 + score_fields.len()); + params.push(Box::new(row.so_bao_danh.clone())); + params.push(Box::new(row.ho_ten.clone())); + params.push(Box::new(row.ho_ten_ascii.clone())); + params.push(Box::new(row.ngay_sinh.clone())); + + for field in score_fields { + let val: Option = row.scores.get(*field).copied(); + params.push(Box::new(val)); + } + + conn.execute(sql, params_from_iter(params.iter().map(|p| p.as_ref())))?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Insert a single parsed row (thptqg2016 schema — includes ten_cum_thi + gioi_tinh) +// --------------------------------------------------------------------------- + +/// Bind all fields from `row` into the prepared statement for the thptqg2016 schema. +/// Positional params: so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, ten_cum_thi, +/// gioi_tinh, +pub fn insert_row_2016( + conn: &Connection, + sql: &str, + row: &ParsedRow, + score_fields: &[&str], +) -> Result<(), BuildError> { + let mut params: Vec> = Vec::with_capacity(6 + score_fields.len()); + params.push(Box::new(row.so_bao_danh.clone())); + params.push(Box::new(row.ho_ten.clone())); + params.push(Box::new(row.ho_ten_ascii.clone())); + params.push(Box::new(row.ngay_sinh.clone())); + params.push(Box::new(row.ten_cum_thi.clone())); + params.push(Box::new(row.gioi_tinh.clone())); + + for field in score_fields { + let val: Option = row.scores.get(*field).copied(); + params.push(Box::new(val)); + } + + conn.execute(sql, params_from_iter(params.iter().map(|p| p.as_ref())))?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Post-build: VACUUM + stats output +// --------------------------------------------------------------------------- + +/// Run VACUUM and print statistics lines that mirror the Node scripts' stdout. +/// The exact prefix tokens ("Source data rows", "DB rows", "Size:") are preserved +/// so any log-grep in the deploy pipeline keeps working. +#[allow(clippy::too_many_arguments)] +pub fn finish_db( + conn: &Connection, + db_path: &Path, + source_rows: u64, + skipped: u64, + errors: u64, + dataset_label: &str, + _file_count: usize, + is_old2: bool, +) -> Result<(), BuildError> { + conn.execute_batch("VACUUM")?; + + let db_count: i64 = conn.query_row("SELECT COUNT(*) FROM student", [], |row| row.get(0))?; + + let insertable = source_rows - skipped; + + println!(); + if is_old2 { + println!("Source non-blank data rows: {source_rows}"); + println!(" skipped (empty/non-numeric SBD): {skipped}"); + } else { + println!("Source data rows (post-header): {source_rows}"); + if dataset_label.contains("old") { + println!(" skipped (empty/non-numeric SBD): {skipped}"); + } else { + println!(" skipped (empty/invalid): {skipped}"); + } + } + println!(" insertable: {insertable}"); + println!(" insert errors: {errors}"); + println!("DB rows (distinct SBD): {db_count}"); + + if !dataset_label.contains("old") && errors == 0 { + let gap = insertable as i64 - db_count; + if gap == 0 { + println!("Audit: OK — every source row made it in."); + } else { + println!("Audit: {gap} row(s) collapsed (duplicate SBDs overwriting)."); + } + } + + let sz = fs::metadata(db_path).map(|m| m.len()).unwrap_or(0); + println!("Size: {:.1} MB", sz as f64 / 1024.0 / 1024.0); + + Ok(()) +} diff --git a/2016/tools/xlsxread/tests/fixtures/README.md b/2016/tools/xlsxread/tests/fixtures/README.md new file mode 100644 index 0000000..c0885f3 --- /dev/null +++ b/2016/tools/xlsxread/tests/fixtures/README.md @@ -0,0 +1,18 @@ +# Test Fixtures + +Anonymised `.xlsx` files for integration testing. All student PII has been replaced: + +- `ho_ten` replaced with `Nguyen Van Test NNN` / `Tran Thi Test NNN` patterns +- `so_bao_danh` replaced with sequential synthetic numbers (e.g. `10000001`) +- `ngay_sinh` replaced with fixed synthetic dates +- Scores are realistic random values in the 0–10 range + +Files: + +- `province-100.xlsx` — 100-row single-sheet file (simulates a normal province) +- `hcm-overflow.xlsx` — 2-sheet file (200 rows Sheet1 + 200 rows Sheet2, simulating HCM overflow) +- `province-numeric-sbd.xlsx` — 100 rows with strictly numeric SBDs (for data-old variant) + +These files are generated by `tests/golden.rs` `generate_fixtures()` if they do not already exist +on disk. The generator is pure Rust (uses the `zip` crate already pulled in via calamine). +No external Python or Node tooling required for unit/integration tests. diff --git a/2016/tools/xlsxread/tests/fixtures/hcm-overflow.xlsx b/2016/tools/xlsxread/tests/fixtures/hcm-overflow.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..c92e498e9f9213a1744cc3f22d8338a98d7b89d3 GIT binary patch literal 12658 zcmZ{L2{@GN|9+=bM0U+oBqRtnTDlS^UKwImPdD6ST%*({s$U@Ik837I_gwlHT zwT7y+=3p>r+?@f%G{$NKI4)giqd}G0nw{K?gPAfIdwBtq& zRYhD{n;7I(Y7-Xs>dh18tPXp2((v&%|C-sNwQtf1Yp6d*e1hg>jsNAGdGJ@nTj1Cs z;2tOVXz5~m-Blg|{TWtY%zX#RyX|x0tE>3fI{^wobSuLPJ5QXHt4KQOeFh`c+Z%2r zEG+N&?Z>Q8>$58h+M_y(n`f1_1fkGs4=vuOVaFFO{CXx7I28^J38-{}v~;6wve@ zh$;l)_JEH!uFJ3M^K^1?uyS&8fP%r&TKM-FPCAk+Xl6AulE%mlxSKjBe#fj))VN@B z3pH11-g>Q9T$)?(_{|-9l~KXe3w=Qr*h<$cVZ=jvr~Yot9XFI1xY1D;yI)!Quh+eM zB9Z(h1&@Th$IstA*5eU=nh1|+2ZDAg7blF+quDzIZ|TVj(`EAPi^1t-_) zZn+#RWs~)YW-CrnqKM<^NC(5Gy z$*3bwf0@4Z9)l0FtA#5q#Bhsk&8PVXzxn#O1ZOBE@YL|ft5UuR!JpsOdj8k;4EeL3 zJeQx}*x_eK=Pp9`H_Y@%jn3_z>y$p*^ueUM>5^;-v5N5+cA${-C`!KIV^p*xnMYA9 zMydN{^0eb@#N{h~4z2cc{w+6zhi=_J{!Z$7R+a4|-$r4@veQq@aMUinVz2A>#FCUM z=4{iCbiWV{Ky8oerf8iA+B0dX>l&bROgCGs&O#)yu!yqupKbTB4 zzva7LwY7C~m1liWfOh^xVlCcW$LsW{-;O(TcdxYj?~dGKe_m=~|H6CyTO>BR-GQfa z)x7Z@UXP@z1Z*C2De780?|Uw}SN-#!F0E$g)g|`EcK76+Y*ig(PK?d0uB_aL`4=qS z_%>ZKm2IG|f{@4Ki`PDv(7KP+Bi6=R`_|0!-{sMYi&h4fDtM^S7kykk7ipuG$5$3h zDl~cv3-j}fzkl6=pLuLvR9L(=#z;w_&GhxYD=zY$9e9<{eRi$B*zf!I)uq+uSkXQ# zbLH!ok4wRp7GGB@WIct)##iQP-HcI7EcW=yNcYsz>e8H9Qqk)7fzf;SlU4%|{wqt( zo<(cR^PJ`uJsLD0&mTjy?u40X4T}^0OViVf%|AvheSa*qiWc4Ua$j2>YaN}z`mS}u zR#!7&esetXt0VKCmi}|CLm9q{9S+4m7UpLxeSQ=aJ~5+KSTKuLOEil7=3CoW43aE- zm*)L3L^asJz8}85%iT#%c=^@ErG>%PTINcbvv5PTwZ`c$;|Gu1S4epJ~Nkv3rM)Y9yZ-?P{`e9Kd3b#eKqpuqVoE z`O4p?EVm)JT`+K=(YmscvcbXpysJLNHz{(Ojsvcapq2z`bCr{8lN@C4<-BlptXlTt zG{)W;92_0&r~1d|DOWmPGr=mu%wYEzqk~yrR(3tj<;r={nE7}9(MrdgCYWVM$Kpo< z2078Lj%CY!2FBRigR`G5OJjwHa}{L}iT9eVoaeFFBSn3REv8R%6;*f1v~SB#3%6&C zcJ){}sumn6uifJLLgK;g7H#nSHVG zbR72%1x;g@C{g!UuAV7ePr1cyUb6F@mL^S?xLMp)xf7W{r>J}4orDgwL=Z{RVwxOf zjnVLW&L9qbd?eQ);vDg=)Pv7<$5j>Lq^0x!4C~??T`-N%6#1)?T}Yf_ zT+aCN>D}ejC&Mp~k588hImgSk+zza<5fkT(ZWP&lmdlP)A1^~*Xv!}oiPWdoSdAXeu7k0N#sQ-F z%HwrOG>jxyJ73UYfglMw;o4VrAP*vnSoee_fq;CPli9vQfLCW=j&7e zQE@Ia6xHUq0t-^nUOusV1Ck$tba&G!pSuGwNg+?qLi#?7;BK(W{8_%-fZDD8^ks&< zWG_bvAMQrj;{(k%2jT(En_&ehEI&Po1GU6~5ahcO%xr(@XG8ddp54;A&KJbPGUhQvhiAhWh>isAhB%k#FJl&e z8q>!YbW9*fivIc9*8>~IocL)BU=s$h@%e4c9AwO}?S?`CY5B{T#h=DNko*wjZ(}kx zjA0kSu^k3XNCcKKY>+Vn8^)kMu}i=?N(69hEMxc}v$t*-BjHBmt0D5;Mc!cq^;)u!x5<@pX5KmRfm7+aYp`bya@erskS*q%W6%uFc!g;O5LYYd{ zhd-4dT|PiMN#s-PjnEn03i3()TZfs*PCg_6LB?@FkctJ8Lm@242WaoHp5Apx7EcP` z`4|Ri&=hfxt!Dxzlk5@?p|VnIK(qdn0%&)yf>P*$cZx<@>A)Lx;kVf(?7tp}ue(Ea7cI%`ND{NA;@Q<%!sjfccF&Q z*6$S}pBnwH9sRC0T!FO_+R#QQkh)w)7h*ITnEubYb>Vy(O(Gk45%;60g%`|05rsf& zAyBJ5>U!k@0g%~-nu>PyP9a99(sfag z20gGv%4~GV7uID9e0eQFvN=f7CPoOee6a$)#MlKt8&||V4@X8uAo=-Fv#hkR5FeV~ zOJ=a}?C84M@IP6e9DqEz=AoF35I~KyOYG*5;KM!t9hBN>RO>vG;-_3~237m~MX%cL&s=OkU-{ZxSIZN6yp2Ub+Vnhhi1%aBMdcztLyqL|Oz=k-Nq2 zoy7e*&VwY9Kn_H@n??E{$&eVq_!L8u+QuS4lgueP5eL9f5b6Bz>B<`x(5YkT><)M# zL9&8M(BX}8@kS(Qv(swudTALvG2zHi(1s7y2(};*76?Too*8+mZ{HyioZw-cpg3vp zbIfsNg&j~IMIiK@eJ>wMfgdGCm-Yi{F1kWHZR=-XAC1)4ftMM=1KG*Mf3yLv@uSAr z$#4!bFYdvw2&C2gT!>>WD0xM0h=H2u!y(ECNSZ|S?`uL4QmKByYiCeAa)%ePdR;#1 zlt4MO&r6qoHBtR}A+ba{@w9~_lR*g|N*&y{stPiY6jhdG7u4t!6kS>yejc0xHwzO; zi@S$`y|4Eh4!H#~5ZP0xMJIFy~T;XjTCQoJt184EpZVx+wSB)*YO*Rzr9+ zI~jj%ejT1AB1eK`e}JS1W=N63g%8>HMAFXEH9?m$$L1+ z{J0i5hJ`JNqA7@C8+E(GFYMO~|D^8S{WtqgcGX zvC?z$&&BXkp=VJuaY+f6+Vb`lM$c51f7@@r7_Khg7mAy?yUY|HH&5mqg`Sv=D^deF2BI^tsH3xcBe*^!|z`rOlIb{uXb~bvD@E z4>^*P0%{-1j0m;&(Q=2YwwMygyRfNS+hcOh9@+-G_u)vfZcfbs9?FMWJRx&07BhY# zQv`c`d;9XCR>i2}{o9Z2pqJetI4=Gu4JBlXW9Pa}7>2XA4)k2-fMMu4l3xV8>1B@z zwLB&e_)7v=2AjI0J*MjHAx;>BeuQ=|r^ueN-EF|mGOq)cE|-|p0Y@ASa_~sD_w6y} zQ7#EXE+wg$IPt`y zMG-B*49Cuu(#_5_c$aL(`3ZyIaiKv)wbDRt=X|-cqz>`$&NFzwBKvqfulsoU(p^F4E#4BKr*Xsz7XatM!V{mFr{PSa>42mF3rfejax@Z9*N{{VGLCzw#JK>M@C$R_Q+OtNP*Ls}MA4GOS#tjwW;&cxJt z^V9Qmm-w&|O>5A>*m&{~)IlvK=R^jj_Y#7fi>1h!b!6=bEgOkr3fjT4+hDs|Qy8`$ zv7F429T@vlnT0?Dee!zSQo}PZ|9sMB*Qf8XIEK8mX`HpkUvYhPP>!q+*=5+P3T7RV zT%qb80no?oZ=oE;GboC%!JF`sl?1ztP*={tZN zy~-z@bA8%^b!MnYw><>#kNTP+`0I%5PVCjAW7bO`Hga7sG&5d@Ujeo55EWJVFcl*Qjt(3Nc6uYeK9dfNv{oFk*57`$fYD#U1cx=k_a>t?t3ou4$f z6g0&=F`hgLwm_+v2JV}}t>&nm1;zpCZ*Bh+yw?EWYiIBPemwp_d_0@^r~7Nd-8upS zhdH_$ME$>5(=ZbyVdK7^6n8WRYP!9UvwV6X1aQ6P;pyN@Ll+8~+sPI*gr>ihloRxB zgutfpj)uw$c5b^{KWQET&GU^9=|IhCv>b*0j)_Lw1v2N#Cw* zXk}^u_}O(jPR6n5rw9{@tkZ7pGJbdPd^QcUcT0F7%G~aLK?51!s|Y5vK=|u=5$?Nf z$2H>%W@9FdQ~v^dj<4(GY_&a_%o#RbjiGC0{`HRr=VQ?+h?Znor;9vgR#1a=LMg4b z9x0qm@cg(|raAQdx?Vr`-CHL#KLGqSv|X5ch65~t`tN6ZEhx~BX~YbJM%vD{JLJ0$L{D!O-WjHi~{0`BbuZopx>xIKzodW>*|x z(sxR6M|U8d1uxqT!2=h#loy&j;?t%y9cbf^4@VSE#W78;cv6oZ6!vzdk;Xlq7MQ!& z$5+k*vB?{57Ma`m(;E6%>;oY7oy@c_j*8v^1i92)+S!chdN8XTm@p1@W$6}qLVdb! z(i!H}!^|8pbenXpnEV(HD&{$K+P!-c8;1~I^JvzL_)i@};hnW2phxzW@Ut1yy}dm> z)QKMxU}##n1Wip>XTwJQxBh$n&1Ut9N&*95iKYKJ09ITd02ktIE20Bk)wh1|Db2QT z&v=h7Y2GWKy1RCmSUQHkaAwDWiY#l5kfitdBFb+pOP#FF1#L+>lirnMu=~%y4X_0# zTRkgjMH80G3k)7ualY@3Wgq`<)BKjAu`sGW8lPs9L^b?B(qixXhcNEF@{CQI@%s+OzkO$UVrMcmG>VG-yw+0gudUI# zY4_F&^J(7~7QT!PE_Pd5suva)c`de^^{xEyT&$-SaaOmAnZd2!t(1xOE)NbZiatzn z@*Sgo^Is`p9)rzK3=IyNiLR}VYFI3PbSm~=)%IH~r6#SdzAEnVefNEY`N|2=)n4Ye zHjy$~uv(bshCmbwt$aS$`Ko>`dFS(7ZF`3Nnod>ys@Rt<-Sn3i9hKXIE<3uUpXte< zYKID(!`}QeeEF*=5vJpE-mdJ0tMi{1uqR-#i5bVla%|6Gd|!txPl_sLgbmw2Zap~K zb>IRVq&S8Sb?6zwSl|^73v!Q}LhtiYoP1LnxBy!x(?X|1W9L<>&57b$sNoce))TRkrgS_Mr$@03hmK6FM*`&iBllu2XFbA@DAa;5GW~0ze zWcJ}s$GF5@{lXt2g>JSbKBC>b!^WleK8*je^leS~vTT8X5;ggv~~e zHw|%^c*#FbgguhJkoBytKz4NcneHDYf&{ItaFZvs=@nl~`!4`E$2JRfDr{s2{LQVX zL+>j$`+8`-9Nl__GbZ&@_HcA1g7W1zHf3=+?e$ZqIV5(| zb&cqoBuH*G`}nix#GUDvs~(15^c?Cje}Ie>BMl!Um9k6R?v0d^LJbl>L>@WAPEO(= zm-`>Xu}i3MNHq2t0XO1Tz3vF0o)stQW=|8Q5&G~dUX+Uf%8D3a<6&U8QCXE*=!TbH zU*m_?W+%gI>h6_t1V}nmXHO6Rt7CL$v$dBM0b)~~TIlnyI+;R=`Z`V{_@Rq`CAymd z`&laRbX^kiTmqP6gnWP??Xwrg^9!YM2ejZ-dhifF)Ofa_279^Z4RWW=Y9-or^SW4 zn}8(p1>J&t34&P42+-kI=vQ%cUM=|G0WKU-?3#y;-Dn_djTMr>HMZ*2oex0WD&K<| zeAU;CC(1bXmBZ5UIk9zMCm_%$2y__unFyl;4JpGHJzXF>t=P4Ch+-t<^BOQj76=g7 zW?=ef_QEVA@2(h^Gk*}0{2W%k@1Xz6wKbT!y@){+3Ubd)B!QFU#@lwVaRXcEoq zRb%T}`CX;(t7>=l!V=WWMho7l2dA@3h_XDXe1yD^_0hVGsSJ5}on3;)A#s>q`Gh)L z6%wHWe6AIsS6-oi!qF>(kXcbkMG4X)J5L)A3meEx*N5)GuAXStS^~6o2!vBjK@r-7 zi*{2lT)hW8FLh8aDRW3Pf4c;kQq&1T_Hrh-aw9BRVx-A~q#AaK*v=@Mjj%8_!qUl$ z^593=5hG-Og~jz}SoYE@@$^a((&cwp@Eg9YD~#tuP5%l@?a#3M68JkTH#fq9=E4a| z;cQUwkvMg|mop7R;H2aTXHSYD0vplvY>Fr^P*G)%Gm`_3PkiH zA{vf+phT0%o_hwxg!=*VgNf|E`-&Ra5bV z`Dsyob*Kr*9li#h@qi78HA%Ct#oTUc#MK*3mkvUPMj>}dkj{;X-1M+uH5^Z+v0_%n zi*gn~;fUz2$r!`4T+l8VB;=Su_XHVEmo}nHlaTsJV7h#hb;5oo=9j?T>(>j!)4I(< zg9=h%&rVk5AS3CNib*|1HF+6mcjq^b5qS@tf~QkRNSRM^H;-;*8Dq}88~Hf|y%T35 zg)>3HkJL_APQIf+L2GM$D$S7uYvo0`3!rR?Xj|6JG6FfBo_<>IdnqLdnH+_bksv+1 z0)@;3qCYWi^PsezQw!dD1|H3ak^{K~mnN%b8l|^f`lbYJM5nl^BsZzePQL3WqQ^X1 zmTs7Q4KX=FL^Fx#-ME%tho`NeXupu}!Sg@4<dWZQUhjoxb!NdG`Cgmby)r>EqJXS zJd6)z0HVm;I}G(n!|Rp7wkL>@J{}~Mvy-bPV`_{@)7NK#0WWXDcc;UFu}uljoEfbTS{F1`jb`fy060Lq%!2nL)R0fJ;cKo*FR zrVl`r*Kdwbkj#a7x!2M-H!V2x3_Ot!wVic3>L4+eKuiXY2UyXAR?f1-s6uUg*Tw9_ zxgEo~q2T6rU_5&L|Ej6~pEZTQ4Jcw6Fd2j_h(fALkSu;1(9Sa8XPd%Wa9yv*Cc{Y& z90ttJAW$)X`75}U9>X3XRgd6OSqppVpYZfgB;-=uukS;aeUYL3?0c)*Tt;9>f<7ehiLkE0Z}c}_d(e^a*Q!ZR_V4|W(l;GSTry?Y|o)9yPN z|BX9z4?U4tF`jJV6U?iVCA4%^i<19iYG$x`MkEBLyW(9j{?^1NlsEXuXOTg_o#(dp zWsdf%5zTZZeJaL_Onf4EgIhm~bT00BzOTsoY^kS>#r`8-L{cRFa@1nloBQZE7jJt_ zn(4^?A`)n(3x>JlO(s4Gyum`VBCVhX1!}51O)U1i&x%aA{)G#fz@_=UKft^H6~`98vm%aFzf?<`gcyPFZ0B4p|<>eZS|g9-yHtlu@tVZ z)u(VH(4^5bHREhx{IJl@yyh*4o-N^WXGHTjuz4K969OiaQQG=x2JE3 z4dM2250)vy-Zdlk8%g95B$RWBzWqjo0V6_cIJ&HY^Gm2uPuf;k`qqk=93Cte0?QTN zdOG=gkeCHJK#kc?XZClT`*yqgLtyuZOjQG+B!yC0KLT|>HgCT$X7vQ&bROZfvMKfT zphM7&)BMwKiJVlTwZY)_X%5&lN5#^~^Kp6>3W2oq1N8F)9g2&$8T3E~J+tVXkX@vi z+klZ}9>G#M*UhiLoG?&MNIiQV5_TmIvj>6Q6E62kH18-j@2K#E_j%mG%t3;3?SN+O zfG);LgW#!LYx;b?F&P8)`3Rd_UV`Y0f4ElP3gXCTv@by;Rto(HPgDq|#gv0AJK{!oL{!-R*Gfe$Oi zFwagBU@D-&IJ>Ej5?>pNn;MR#gbQccaT`mx&48O4ho!^`XN~M+GmC->@K$AJg1C`kLRVE_S5*Xc_;$aa%fqzFl#&Zy#7X3o*(FueIK9-D!bWHFjnAsg#-Mn6OaRrJzT&bq2ineaQ&X{& zRNvU$cdUq5Xj%Ghx@BqydMO%9eUr_-CY zh((6HU0{4^GGo9kmeV~}!s+$a4^tybm&Hrr-zh>M zP&V8;mHNF)JhfEV=w80@JwV8Yx5pYH{|?Vlc)QVPYHp&UO!5Y&_K2yfJhJ zK_9Of_!`eSA5XqMw(UdMMjm{(5&na@;t zWas+S&h>%Zm;HI&z>4Z63SV zcokqp)`<*Y#e_;Q#tmWO7M?XC#%7iX2wm*!09HgNmRwVV*{kVvM)KYHQFkr(St_?< zS@3RL*eF4^K2W9}ol7G2H@QAclTLBKeDOtHm{NJv>-w}85RwziTZ9OW51aZt<7@Nf zrY>VC%fhydGwkUaQ0D@gkfl9Q=t;@7lQ8>}I!u1SJ8BJ}Mfx)YGfYQ#{}@zb!>4OdTSr-Qkvyyg^M5qE|e zd%8AgQcy&LcKUR1b10%EG`v7kj_unsAkVyy3FI9im!zR!$tWGBBJ2*M5$ZjI2Rl99 zIPjIsIZr0z$2dRi90S{62Bu)!gwrHM@OlNtdaC0wCA`=uV5_@3G<|CfZ5mK|!Fl8b z86U#+!TcWuaw2&10%LPfP|J&r`&}T@JmB_{)BPpc@z0$f(m+9@>3WA_B6#NlV`tT_ zn08)lA}Db5g_M~-=6KS`1O^lov6Q7?f}5#` z7HW9W3oW)|=|Gt!OPPCmiJUo1#$2b*Mf{FKJ5ZLvQWn}ipp?ORB!lb}!uNqW0hG0~ zlub<#CfWnX+Z(Bqcl$ScfWE94W$y4$7Mau(nfwICx&%(OTGOwsU~>6~&IB$PQ;IMt z4KEteV>8Qy`m!dbZxub19Vay%K|z)_LG*WlgH14|3Sm+eUSw;?7Mu0E;F5=O;iP8a zBpi%h@peF1CS+azRpumt(HU6Xsj%=U1X~Zb*m^xgJGusx-f|v!YdxtVkWWsulPwKxYhQql)8GOUhc5dn$bIKbL zYcQ*go>IanK-oGdoc314$bn|;pjHlMyWZW`a;;K-Ej{R7&XUIo3EE$ z08VT4W-WOruS{vKOtAzIAOT=!{PJu?rU?c;f%-ix0Yi`gkkO?ZvKQ}ryLj~kdi6Ae z;jUp1SUDmB%y!_Rrf2NaVD_FEt8K3Rk&&L^SK$903@mS~@faKYu~~4-|9+((yvNJ> z_53#U;s1ES|GxnLeY1VNeAA}qw^hMw(d+Mlv)=vwZ{dI68vk7g1OL6?f4@WiU!4EG zh5Z|cXA2AGpYLS0BF|rIPWJ@uYY%@q=Op~n;%2u+Rn4x8GhM`h8#S{_Q zi{_}0#&VKF$S$IevbC8G;s4&D<@EnQ*X6s${oT*=+|PSI?{z(6zfW+rw19wsh=90B zncKaLYrp;UlYoH!Dgl9Y@PA$R1_p$91%yx>PX&1elhsci^S@1OvaQ2QaCqBPDFxj) zVVZr4{DGtg*CHQ0x0c#AIC^~F98VW_fu=n2;HY*)f%mOR&!6=VVJFEgI@d?VGT+?} zi)zUZ=RI2Q!+KD0=zd*>0`pU)i&Krq?!ztZtm+Og&OFHZt&`}K-mO2oE_lDK0JD~4jv z^CXyQ$b>a=M=}v^S;)%|-^)_k8zQggxq8}3JQeJ=W~DNVjgmA5lSjg{6{_coo{G8_ z&+go}ntt}L_Fpd*O8zO0DtBGVhajbUj27eYe?Bbbk($PMEgS9s)iH9y2laBeQ>t*+ zknodo!plEcLmm80Y^_}zi$=qgx_)pl>F}ma z8sWV8ck4T^`^=iYGS^!5PFpaZK-AB0e{lQWMAC!E-)^LBmzBPAJ>i!1`E5_M>J^L` zU1ev{8izz4X6f>LcdJENp=|m?M$5&HhZt6oY)Z}lXe|{C&tAhqcx+7Xye;5drFX}> z;*+=?Sx3bnHR6c&uzB+{GZy=erpN*B#mzbc=Wb|Uf8b&M2hQb>pZ+cG;~~M=Dp-^n zEN%n*Jn65otS>z9gx}%7KtEuJ6|F6z5ME4prpAtVBD}}oY-2UqwA<;WAWMDzkgIFP zTuD{cNeN+j13Gq8X7SWNSCx({wZ`p`D#Z$!5c<6FnKZwsLHXT5_BV*BJFwJW;!C?n zWilMk>hy~znVE|1`+IMYxoMYS%lUP08tZNR77F+HMN%A|V&@82kC6YC z8>S6%Ki`qj6sMB;+B#Yd$; z1+Ix!0`ipN)LKf-d~4afSzCRF3Fr2MUw+=SPHxwyxyYCZp_^Om#kQ{t3)({`*B)Iu zkTmqz+$Xwy?F~{o_q5#B$0k+x6?=R1_X)@Yy$%&dHZA zXFtzOeSXnNUGkiNy68PWIj*B*>`q=9e_0nDQ*&=#c&(0Ir9wcUvEkrkz+iMYC2Po_ zdam5Kf1yR8xJaWlaN6Cyc4#27dN8|oKC*4BA(N7Rzs8_e*Wa$T$KTdy>2CH=)=+Kr z>8PlWFJ8_~8J91-o10P!oTeCu_O(o=PCgob@rb%KHyd#};>7Xbh!dyhK6hRzo2Fz< z1lZYy@r-*o3v#8SZmvFS~deGavM}wtG`ON^xh*Qd>N4Q8mclzDgLkQ-_zzDG~KO zLBmG6CwZ-#g@r>;vtr+S1U*8#ZHU{~-0J#kw^Ti9+9M989*3z)xJo>Ce`3j)W@X{x zeKQgDy+PxAsurN8$6oLVYC*fn#%+VCvE8Ee>Fi2--JN-T7o=V1c2%}ULDH^r9tE#k zUE`mttai@j+Ux1&^%Y3F$;SnApZB8O9HIwB>$?;yRVwJ*5|e?!-0o)d6&>!&^rI={ zj_kWu!IP_ccI4|AM1H(Dqta4wf#oju6xsE`TUd9jp@7an znQ7HPHg|%2U363E1?>!mT--6Zh5}N=DeKnrNS$#AcWU^oOp)bwty$!nHBM7SmOW{j z`UUvUSg~SMJcjzoEF&e~<0J}(qM7Xla3<;}OSC&q%p97rt#+_pfXeWBCwR8aMTtEw z^kF6H$qKx|Vi@#-rMeIiOCNsATNjIer_As~#b?uo)1Vowm_n}Xjd$# z*{+S zloAi1IN8NYBMEjBD1!~kKwV|B`ED{q;c86KovL3+B~}hvVPZg5iPiX+reOgdD3&H* zsr6f18oM>Xc{FVGa^_AHLKdm`l!Is{jfpyH#r63Bpqpk^#UK-ND;3Mnq&IG>_3=VI zY+OS`1t!=A}nO)V#aDzp(89Gz)_#=*W=VF}^%b-rgw zvvHSKSo~IOH~!1exb3zy0Wz;nUG@bwGEd*KhYser9m~i?5jInERR=7&Nnm~qy>J}e z8_Q*|KpBofg!DNfcocY!xf5%s+GXEJDmG2alXX*teGxbiC?=uL(U=)u9GI2CBFh2? z+Ch!JRIC)iJr~sH4OGeCIJ-cmEE+QnWSYp;Z@Mc&17{UR3OfeQ$^d~WpffGeJ@ADQ z!gvhoD<(^KK86}o@uej6Wg0V18{<&h2ps*s_ZYE7^(#dWKB>?z6xr`7D{X))cs7;j zJIf)yA5pHrYXu}VFHZ-9QIpf9p6u&Q9koQe;273WDVO0)5zh#c&fn{pYz0H6W@p0E{{t<8^Rn6EW` z(@8Yvd2Da&_Rr*UB)9q9>fTD#X5&>5Im?&8H^dMi75EK^9J$a-?jG>040cVY zl#1rqKu4^o+EN7l*QT81*Wqf7b{&->I3RPF_+Dbe>m8s^$N1`cv5Xti1Prx2=Si0z zFkIgpnY5o;Zi%*0V-TPt=5lwQflI0CV@+-$bR-p@K|&|e+LCs86_>yV6bKF|T)gtz z>lMqdSJdQA^a=Mx_GXdLKPUI;Pu({}5$XhaT~jHg)AIM{;ZP?Mu==p6ke{;Mh_vcr*ir9tK>b89L z(NC+S1^@q70`QW+|H(ZA9{=MDf*%9?xFIjo1q5=>fIaKGefdY?kK5-rqB#8Z#Q)z# ye~k0vTK6r^;&Pntm%JazKQ2h$$ZrMtMF8@DtxNX%gw}vJh2UQ^Sf!){`1Bu}sM*8- literal 0 HcmV?d00001 diff --git a/2016/tools/xlsxread/tests/fixtures/province-numeric-sbd.xlsx b/2016/tools/xlsxread/tests/fixtures/province-numeric-sbd.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..5eb518c1c8d934f6e8446d483965365379ce6472 GIT binary patch literal 4165 zcmai13p7;uAD36&k0_c_GFqh?k(*W`i(ywSBd_rmgWP0_DDQ-kT(W5`ZQ4+Y@o2nh zkSUKOtIQ=~cJf>zDtV0Ie=qxAR_)oDbLO7;p5OQTd3?Y3p8GX7<>Qy-;o%YDk<>48 zs7SkWZQVK^p2NI6JTges-oW>aKk1CWv*krH$Eb%IFun^&u}H zwB>f-^wW)=$+aaej~Hp&lV(bZR#k3CPZHl{4;uWyw)???&rXreR~hfGYiDv1qKo~8 zA0{{@58Sov-Rb|pq-edbb-Yzs=i^@G04-8~`_7CVFV6hf{cZlBRAifoMg73z5>{l~ zv{cd!QHr2~*DYa(bkWquA2&O{YzWE7ws$v^dd1gyEctp;fo_6YZ{pj)+uJMW@?VMA z7koZw${!v3`^$)5sM5`{n~LohAJo;URK&OF1r6`IW%Zc4lgDhub&u|AxP3JExc$c7-h`7| zx2gqBb3SZjW_W%!95UA6{h-MggC`tLbE?g&c$ZKca_!!Y1Dj%;GrBxVn|b7kFDRS=epM&&)g?o3d{U zJxXRI%^bBbz9aR^y^;l5S>bY`HGo#kcxOz9Sdfh+8O}X$zmLa0{ zA>uH|^}=bjWqpCZ=fIP`z92dfe`#$IsoO8Mexk~Za4zt*PHbbPqhY7jAYU?$<6>{0 zHutcs?1I$#Z937){c;NzXMa^WtI~QI*g#Vj(8s^=*ncYu4DH=^lx%*Fa2-%~-4R3U z?3YWkjMeIvOfWJOGaWV{8ymt$TH<6z8=o8pxzz7Yhf;q0lW;z&O0y?(@!)TrLJvr>O`NYE_Hi_iiosTb*3Wwz{!x5y;oFS#IEEfKNbC_ z+|V0MG4+T{4MeWwvfR}|p+x!_#(uG=X`Sd=dQ#7wU1D#DalEN-bR)dm9!g!+G?`NI z-CnP+K@q*EI4drskRiI$Q-sRpmQ{G+j(DAb_M*GmIpA=|` zzzlQD_cP48z&Y?w$$UF;Mhp=io|mCKJn}1&{k%vde?PUQi#pomyFK(XzjjF1Pxjh8 z?9}a=4!JFDf+=peVri)Qgx|zS>Sc_Q5>DWpTf_N$Wt(fdyDgNp-X0dd%dhWWAM)bS zfru?HZI8e;>fy?Pz1IsC{W-25xeE*I!Eth>c@9lt1(XBWZ%c0arPL8JH})bRV8v5 zeKoV(9^~KTB!e^B!)V}enN_W7mE6Ls%@%=WHd>Ui*ETm>nug3iGABn~87FRMvABg5 z;q#JR3qySkt&g~$CNFlLNx1*!CWCU5vCuLZ794QyBRSxlPVnU1hqqNWDph^2o5o*x z2Kd&5u(izS6U>4a;eoS*1G#c$HJ{(ycj63A4QLG&sLgJrr`-+$YUal~QsmT}o~5L} z+qO98924snJJZe@noHChm%PbxQaC8bq4aqWH<=GS&?5;~HH0gLOqQ9^XI?dLnz0BQ zxhogAIN!{k{YKCJ;7ywjJubK4=KIB|p$4-oPf}4u`Rt#*i-#P1Ur4^c?b}t9A(B#0 z{$2qWL2i=4vB@qeul72~GpyXobz+)F$kI{G>Ez86zN%k6*>C0ZH#2UiI}j+QiVl-e zdk$^w9MBL&vU}w76WBT04ub3)6^EznT3iPa$;P^sAB+J<$O2JdI$0bk(e1&0mcrIu zWYx*$C$cNG9R%2wDh>_E`DtRgc8q8}Su+NEzFe>_No6Za&o9$;2IS2L+SHy4n~9Cf3+=cyc#YM&BfislEc>%dy7Js)YaRkI%zHsmi+;*vQX@X$D}I}>X&~0Hj7B#4;OhW3H``lqQd*ZkbFtW zg@ByDXswnoMa>eblhudfBFv>7a)f@_=#^@a$~1L5lDj7|!pOW?Ns-7ew~x0}z;4jo zbLxX_Yur^uf9du|La}m>2d+k3Ldq8t>H6|d_r@#6O1IYwHOfDZj*qy3#6J<~+HjEu zR>+ed(BzH8lfTEhcq6cyxgGUE)}Wy)lwFEH3diC=!_(r$&(ES-kK_Uwg?LMQtRiTb z0w|RV+6~HkOnFf~OYpLAtT$*lBO}U6+*=4J&8I$nf;nG|SGUKCfQBW25)JQ9#GV9y zBPE=&R*Rpa!m)K2f)5l+m3D+0yPpePhJns@{=O3X`&0)Fbk2otdt)TBb-ca8g3fG#h{dr@Hb>=p}e zPm&8QI)MM0XV&!V7v37|p42N1DtNE=_sfS=khEvKQq)FdX{ z*ob*eg5ptw?xW*;payGDz0<%45@TIw2cHlu#mBsV)>ssD78S4wZuAD--Jy3hd_5|l znd-D;X*59V!fYmDtEfp25le@(b5l@%++87RDIO0u;y`!QpNf_dad_C{%XsSub45VX z-vaIt=?4)3nL?oKl7O?sOcz7|4uRc{_(O;S;!xPkaw-AsWXk?rW7MCxLOj7fQxTL+ z0Yp%LG#&~R+!)z) z7%Lwrj!JNZv(fu;v$7s<;v*}r!=J^Ag2&-WuK`8TBZQ1s5zCNyJZoE(LhcY zW+O4Pl1e~3*ioW&zcI}3hi$hk>@(|fPgED3=)`R7y=51CA@2EC07p-Jeidb9o0*eG zNCI-2F&j}I9fKpEh21U`0WC`aRT}<0F%x}?l{B zFiMUvs96GFXm~$jrYiz?X7s%|vBrQI1*COh#EF>|)HZZl%BBLdsjNHRb(rHmPz<%r z0ro=8doy>~5`J$ETpY0WfdpH-=f=BFf|?hG15kcb%luX|p&O8DoI2OmE(y$j*CAR} zh_|uN+yVNh0NSY6)t@D9aD!as;Nd0O<1~B&O1q=?4$WEZT^+{J2TGu}Il{vz5e`CR zkTd+1s5R*C3gwsLjp1P&=#QEg%2iO!1uB=|uq$v25;vF_N88NYytX8uvKga*cEvfI z^2{5;QQMgCa3iLN1SOz87cA=-a0A-m;fV@N^?J|+^q+BuRx4RmdP&I+VencPMuB+Q z0aiq56#r*5imHzTr4TPX>`)rw8(_r}Kxm}^!g%)5u{umvso)OqS~CWHaHpbewYwYz znD~f)2P>ukLWqn5dB_Nq2rd!$ucE71tvVV^c@{>$*6%mp4lB+m>aniR7Xm`M&wQ8r zTsZ%nG2h;$IIpQU)Hh$KC8XEtJEpG65oQ!Ns?6>4J|`8D#d;fjj}_UbKXE6lx$CMW z_+CE!-NO4Tf2ve19!cQ7W0{+-, +} + +/// Write a minimal .xlsx to `path` with the given sheets. +/// `sheets`: Vec<(sheet_name, rows)> where rows[0] is the header. +fn write_xlsx(path: &Path, sheets: &[(String, Vec)]) { + use zip::{write::SimpleFileOptions, ZipWriter}; + + let file = std::fs::File::create(path).expect("create fixture xlsx"); + let mut zip = ZipWriter::new(file); + let opts = SimpleFileOptions::default(); + + // [Content_Types].xml + let mut content_types = String::from( + r#" + + + + +"#, + ); + for (i, _) in sheets.iter().enumerate() { + content_types.push_str(&format!( + r#" +"#, + i + 1 + )); + } + content_types.push_str(""); + zip.start_file("[Content_Types].xml", opts).unwrap(); + zip.write_all(content_types.as_bytes()).unwrap(); + + // _rels/.rels + zip.start_file("_rels/.rels", opts).unwrap(); + zip.write_all( + br#" + + +"#, + ) + .unwrap(); + + // xl/_rels/workbook.xml.rels + let mut wb_rels = String::from( + r#" + +"#, + ); + for (i, _) in sheets.iter().enumerate() { + wb_rels.push_str(&format!( + r#" +"#, + i + 1, + i + 1 + )); + } + wb_rels.push_str(""); + zip.start_file("xl/_rels/workbook.xml.rels", opts).unwrap(); + zip.write_all(wb_rels.as_bytes()).unwrap(); + + // xl/workbook.xml + let mut wb = String::from( + r#" + + +"#, + ); + for (i, (name, _)) in sheets.iter().enumerate() { + let escaped = xml_escape(name); + wb.push_str(&format!( + r#" +"#, + escaped, + i + 1, + i + 1 + )); + } + wb.push_str(" \n"); + zip.start_file("xl/workbook.xml", opts).unwrap(); + zip.write_all(wb.as_bytes()).unwrap(); + + // xl/worksheets/sheetN.xml + for (i, (_, rows)) in sheets.iter().enumerate() { + let mut ws = String::from( + r#" + + +"#, + ); + for (row_idx, row) in rows.iter().enumerate() { + ws.push_str(&format!( + r#" +"#, + row_idx + 1 + )); + for (col_idx, val) in row.values.iter().enumerate() { + let col_letter = col_letter(col_idx); + let cell_ref = format!("{}{}", col_letter, row_idx + 1); + let escaped = xml_escape(val); + ws.push_str(&format!( + r#" {} +"#, + cell_ref, escaped + )); + } + ws.push_str(" \n"); + } + ws.push_str(" \n"); + zip.start_file(&format!("xl/worksheets/sheet{}.xml", i + 1), opts) + .unwrap(); + zip.write_all(ws.as_bytes()).unwrap(); + } + + zip.finish().unwrap(); +} + +fn col_letter(idx: usize) -> &'static str { + const LETTERS: &[&str] = &[ + "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", + "S", "T", "U", "V", "W", "X", "Y", "Z", + ]; + LETTERS[idx % 26] +} + +fn xml_escape(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +// --------------------------------------------------------------------------- +// Fixture data builders +// --------------------------------------------------------------------------- + +fn header_row() -> XlsxRow { + XlsxRow { + values: vec![ + "HO_TEN".into(), + "NGAY_SINH".into(), + "SO_BAO_DANH".into(), + "DIEM_THI".into(), + ], + } +} + +fn data_row(idx: usize, scores: &str) -> XlsxRow { + // Anonymised: name uses sequential pattern, SBD is purely synthetic + let name = if idx % 2 == 0 { + format!("Nguyen Van Test {:03}", idx) + } else { + format!("Tran Thi Test {:03}", idx) + }; + XlsxRow { + values: vec![ + name, + format!("15/0{}/{}", (idx % 9) + 1, 1999 + (idx % 5)), + format!("1000{:04}", idx), + scores.to_owned(), + ], + } +} + +fn sample_scores(idx: usize) -> String { + // Realistic scores in 0–10 range, varies by idx + let toan = 4.0 + (idx % 60) as f64 / 10.0; + let van = 3.5 + (idx % 65) as f64 / 10.0; + format!("Toán: {toan:.1} Ngữ văn: {van:.1} Tiếng Anh: 7.5") +} + +// --------------------------------------------------------------------------- +// Fixture file paths +// --------------------------------------------------------------------------- + +fn fixtures_dir() -> PathBuf { + // tests/fixtures/ relative to the crate root + let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + p.push("tests"); + p.push("fixtures"); + p +} + +fn province_fixture_path() -> PathBuf { + fixtures_dir().join("province-100.xlsx") +} + +fn hcm_overflow_fixture_path() -> PathBuf { + fixtures_dir().join("hcm-overflow.xlsx") +} + +fn numeric_sbd_fixture_path() -> PathBuf { + fixtures_dir().join("province-numeric-sbd.xlsx") +} + +// --------------------------------------------------------------------------- +// Fixture generation — called once per test run if files missing +// --------------------------------------------------------------------------- + +fn ensure_fixtures() { + let dir = fixtures_dir(); + std::fs::create_dir_all(&dir).expect("create fixtures dir"); + + // province-100.xlsx — 100 data rows, single sheet, with header + if !province_fixture_path().exists() { + let mut rows = vec![header_row()]; + for i in 0..100 { + rows.push(data_row(i, &sample_scores(i))); + } + write_xlsx(&province_fixture_path(), &[("Sheet1".to_owned(), rows)]); + } + + // hcm-overflow.xlsx — 2 sheets × 200 rows each (no header on sheet 2) + if !hcm_overflow_fixture_path().exists() { + let mut sheet1 = vec![header_row()]; + for i in 0..200 { + sheet1.push(data_row(i, &sample_scores(i))); + } + // Sheet2: continuation rows, no header row (as in real HCM overflow) + let mut sheet2 = Vec::new(); + for i in 200..400 { + sheet2.push(data_row(i, &sample_scores(i))); + } + write_xlsx( + &hcm_overflow_fixture_path(), + &[("Sheet1".to_owned(), sheet1), ("Sheet2".to_owned(), sheet2)], + ); + } + + // province-numeric-sbd.xlsx — strictly numeric SBDs for data-old config + if !numeric_sbd_fixture_path().exists() { + let mut rows = vec![header_row()]; + for i in 0..100 { + rows.push(XlsxRow { + values: vec![ + format!("Nguyen Van Test {:03}", i), + "01/01/2000".to_owned(), + format!("{:08}", 20000000 + i), // pure digits + sample_scores(i), + ], + }); + } + write_xlsx(&numeric_sbd_fixture_path(), &[("Sheet1".to_owned(), rows)]); + } +} + +// --------------------------------------------------------------------------- +// Config helpers +// --------------------------------------------------------------------------- + +fn make_data_config() -> xlsxread::config::DatasetConfig { + let cfg_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("configs") + .join("thptqg2017-data.toml"); + xlsxread::config::load_config(&cfg_path).expect("load data config") +} + +// --------------------------------------------------------------------------- +// Integration tests — pure Rust, no Node dependency +// --------------------------------------------------------------------------- + +#[test] +fn province_100_builds_100_rows() { + ensure_fixtures(); + let dir = tempdir(); + let db_path = dir.join("test.db"); + let fixture_dir = dir.join("input"); + std::fs::create_dir_all(&fixture_dir).unwrap(); + std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap(); + + run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml"); + + let count = query_count(&db_path); + assert_eq!(count, 100, "expected 100 rows from province-100 fixture"); +} + +#[test] +fn hcm_overflow_builds_400_rows() { + ensure_fixtures(); + let dir = tempdir(); + let db_path = dir.join("test.db"); + let fixture_dir = dir.join("input"); + std::fs::create_dir_all(&fixture_dir).unwrap(); + std::fs::copy(hcm_overflow_fixture_path(), fixture_dir.join("hcm.xlsx")).unwrap(); + + run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml"); + + let count = query_count(&db_path); + assert_eq!( + count, 400, + "expected 400 rows (200 × 2 sheets) from hcm-overflow fixture" + ); +} + +#[test] +fn data_old_first_sheet_only_100_rows() { + ensure_fixtures(); + let dir = tempdir(); + let db_path = dir.join("test.db"); + let fixture_dir = dir.join("input"); + std::fs::create_dir_all(&fixture_dir).unwrap(); + // Use the overflow file but with data-old config (first sheet only → 200 rows) + std::fs::copy(hcm_overflow_fixture_path(), fixture_dir.join("hcm.xlsx")).unwrap(); + + run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data-old.toml"); + + // data-old: sheet_mode=first → only 200 rows from sheet1; but SBDs "1000NNNN" are + // all digits so all pass the numeric guard + let count = query_count(&db_path); + assert_eq!( + count, 200, + "data-old config should read only first sheet (200 rows)" + ); +} + +#[test] +fn numeric_sbd_guard_rejects_non_numeric() { + ensure_fixtures(); + let dir = tempdir(); + let db_path = dir.join("test.db"); + let fixture_dir = dir.join("input"); + std::fs::create_dir_all(&fixture_dir).unwrap(); + + // Write a fixture with one non-numeric SBD mixed in + let mut rows = vec![header_row()]; + for i in 0..10 { + rows.push(XlsxRow { + values: vec![ + format!("Test {:03}", i), + "01/01/2000".to_owned(), + if i == 5 { + "ABC123".to_owned() + } else { + format!("{:08}", 20000000 + i) + }, + sample_scores(i), + ], + }); + } + let mixed_path = fixture_dir.join("mixed.xlsx"); + write_xlsx(&mixed_path, &[("Sheet1".to_owned(), rows)]); + + run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data-old.toml"); + + // Row i=5 has non-numeric SBD → rejected by data-old config + let count = query_count(&db_path); + assert_eq!( + count, 9, + "non-numeric SBD row should be skipped by data-old config" + ); +} + +#[test] +fn scores_parsed_correctly_into_db() { + ensure_fixtures(); + let dir = tempdir(); + let db_path = dir.join("test.db"); + let fixture_dir = dir.join("input"); + std::fs::create_dir_all(&fixture_dir).unwrap(); + + let rows = vec![ + header_row(), + XlsxRow { + values: vec![ + "Nguyen Van Test 001".to_owned(), + "01/01/2000".to_owned(), + "10000001".to_owned(), + "Toán: 8.5 Ngữ văn: 7.0 Tiếng Anh: 9.25".to_owned(), + ], + }, + ]; + write_xlsx( + &fixture_dir.join("one.xlsx"), + &[("Sheet1".to_owned(), rows)], + ); + + run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml"); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + let (toan, van, anh): (f64, f64, f64) = conn + .query_row( + "SELECT toan, ngu_van, tieng_anh FROM student WHERE so_bao_danh = '10000001'", + [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .expect("row not found"); + assert!((toan - 8.5).abs() < 1e-9); + assert!((van - 7.0).abs() < 1e-9); + assert!((anh - 9.25).abs() < 1e-9); +} + +#[test] +fn to_ascii_stored_correctly() { + ensure_fixtures(); + let dir = tempdir(); + let db_path = dir.join("test.db"); + let fixture_dir = dir.join("input"); + std::fs::create_dir_all(&fixture_dir).unwrap(); + + let rows = vec![ + header_row(), + XlsxRow { + values: vec![ + "Nguyễn Văn Đức".to_owned(), + "".to_owned(), + "20000001".to_owned(), + "".to_owned(), + ], + }, + ]; + write_xlsx( + &fixture_dir.join("one.xlsx"), + &[("Sheet1".to_owned(), rows)], + ); + + run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml"); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + let ascii: String = conn + .query_row( + "SELECT ho_ten_ascii FROM student WHERE so_bao_danh = '20000001'", + [], + |r| r.get(0), + ) + .expect("row not found"); + assert_eq!(ascii, "nguyen van duc"); +} + +#[test] +fn audit_subcommand_matches_after_build() { + ensure_fixtures(); + let dir = tempdir(); + let db_path = dir.join("test.db"); + let fixture_dir = dir.join("input"); + std::fs::create_dir_all(&fixture_dir).unwrap(); + std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap(); + + run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml"); + + // audit should match (100 distinct SBDs in xlsx == 100 rows in DB) + let cfg = make_data_config(); + let result = xlsxread::audit::run_audit(&fixture_dir, &db_path, &cfg).expect("audit failed"); + assert!(result.matched, "audit should match after build"); + assert_eq!(result.distinct_sbds, 100); + assert_eq!(result.db_count, 100); +} + +#[test] +fn audit_subcommand_mismatch_detected() { + ensure_fixtures(); + let dir = tempdir(); + let db_path = dir.join("test.db"); + let fixture_dir = dir.join("input"); + std::fs::create_dir_all(&fixture_dir).unwrap(); + + // Write 10 rows to xlsx but build DB from only 5 rows + let mut all_rows = vec![header_row()]; + for i in 0..10 { + all_rows.push(data_row(i, &sample_scores(i))); + } + write_xlsx( + &fixture_dir.join("all.xlsx"), + &[("Sheet1".to_owned(), all_rows)], + ); + + // Build DB with only first 5 rows in a different file + let build_dir = dir.join("build_input"); + std::fs::create_dir_all(&build_dir).unwrap(); + let mut five_rows = vec![header_row()]; + for i in 0..5 { + five_rows.push(data_row(i, &sample_scores(i))); + } + write_xlsx( + &build_dir.join("five.xlsx"), + &[("Sheet1".to_owned(), five_rows)], + ); + + run_build_cmd(&build_dir, &db_path, "thptqg2017-data.toml"); + + // audit against fixture_dir (10 xlsx rows) but DB has 5 rows → mismatch + let cfg = make_data_config(); + let result = xlsxread::audit::run_audit(&fixture_dir, &db_path, &cfg).expect("audit failed"); + assert!(!result.matched, "audit should not match (10 xlsx vs 5 db)"); + assert_eq!(result.distinct_sbds, 10); + assert_eq!(result.db_count, 5); +} + +// --------------------------------------------------------------------------- +// Golden test: compare Rust DB vs Node DB on identical fixture +// Marked #[ignore] when pnpm / node is not in PATH — CI installs them first. +// --------------------------------------------------------------------------- + +#[test] +#[ignore] +fn golden_rust_matches_node_db() { + // This test requires: pnpm, node, and the thptqg2017 package to be installed + // Run with: cargo test -- --ignored golden_rust_matches_node_db + let which_pnpm = std::process::Command::new("which") + .arg("pnpm") + .output() + .map(|o| o.status.success()) + .unwrap_or(false); + if !which_pnpm { + eprintln!("pnpm not in PATH — skipping golden test"); + return; + } + + let dir = tempdir(); + let fixture_dir = dir.join("input"); + std::fs::create_dir_all(&fixture_dir).unwrap(); + std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap(); + + // Build with Rust + let rust_db = dir.join("rust.db"); + run_build_cmd(&fixture_dir, &rust_db, "thptqg2017-data.toml"); + + // Build with Node (run build-database.js with DATA_DIR / DB_PATH overrides) + // Node script reads env-vars via a thin wrapper — see scripts/build-database.js + // For now: diff via SELECT * ORDER BY so_bao_danh + let node_db = dir.join("node.db"); + let status = std::process::Command::new("pnpm") + .args(["exec", "node", "scripts/build-database.js"]) + .env("OVERRIDE_SRC_DIR", fixture_dir.to_str().unwrap()) + .env("OVERRIDE_DB_PATH", node_db.to_str().unwrap()) + .current_dir( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap(), + ) + .status() + .expect("failed to run node build script"); + + if !status.success() { + panic!("Node build script failed with: {status}"); + } + + // Row-by-row comparison + diff_dbs(&rust_db, &node_db); +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn tempdir() -> PathBuf { + let base = std::env::temp_dir().join(format!( + "xlsxread-test-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .subsec_nanos() + )); + std::fs::create_dir_all(&base).unwrap(); + base +} + +fn run_build_cmd(input_dir: &Path, db_path: &Path, config_name: &str) { + let cfg_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("configs") + .join(config_name); + + let cfg = xlsxread::config::load_config(&cfg_path) + .unwrap_or_else(|e| panic!("load config {config_name}: {e}")); + let patterns = + xlsxread::transform::CompiledPatterns::new(&cfg.scores).expect("compile patterns"); + + // Collect files + let mut files: Vec = std::fs::read_dir(input_dir) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + p.is_file() + && p.extension() + .and_then(|e| e.to_str()) + .map(|e| { + let l = e.to_lowercase(); + l == "xls" || l == "xlsx" + }) + .unwrap_or(false) + }) + .collect(); + files.sort(); + + let conn = xlsxread::writer::open_db(db_path, &cfg).expect("open db"); + conn.execute_batch("BEGIN").unwrap(); + + for file in &files { + xlsxread::reader::process_file(file, &cfg, |_, raw| { + let all_blank = xlsxread::reader::is_all_blank(raw); + if cfg.reader.strip_blank_rows && all_blank { + return; + } + let cols = cfg.columns.as_ref().expect("golden test requires [columns]"); + let ho_ten = raw + .get(cols.ho_ten) + .map(|c| c.to_string().trim().to_owned()) + .unwrap_or_default(); + let so_bao_danh = raw + .get(cols.so_bao_danh) + .map(|c| c.to_string().trim().to_owned()) + .unwrap_or_default(); + if xlsxread::transform::validate_row( + &ho_ten, + &so_bao_danh, + &cfg.validation, + cfg.reader.strip_blank_rows, + all_blank, + ) + .is_err() + { + return; + } + let row = xlsxread::transform::transform_row(raw, &cfg, &patterns); + let _ = xlsxread::writer::insert_row( + &conn, + &cfg.insert.sql, + &row, + xlsxread::writer::SCORE_FIELDS, + ); + }) + .expect("process file"); + } + + conn.execute_batch("COMMIT").unwrap(); + conn.execute_batch("VACUUM").unwrap(); +} + +fn query_count(db_path: &Path) -> i64 { + let conn = rusqlite::Connection::open(db_path).expect("open db for count"); + conn.query_row("SELECT COUNT(*) FROM student", [], |r| r.get(0)) + .expect("count query") +} + +fn diff_dbs(a: &Path, b: &Path) { + let conn_a = rusqlite::Connection::open(a).unwrap(); + + // Attach b as "other" + conn_a + .execute_batch(&format!("ATTACH DATABASE '{}' AS other", b.display())) + .unwrap(); + + // Rows in a not in b + let missing_in_b: i64 = conn_a + .query_row( + "SELECT COUNT(*) FROM main.student s + WHERE NOT EXISTS (SELECT 1 FROM other.student o WHERE o.so_bao_danh = s.so_bao_danh)", + [], + |r| r.get(0), + ) + .unwrap(); + + // Rows in b not in a + let missing_in_a: i64 = conn_a + .query_row( + "SELECT COUNT(*) FROM other.student o + WHERE NOT EXISTS (SELECT 1 FROM main.student s WHERE s.so_bao_danh = o.so_bao_danh)", + [], + |r| r.get(0), + ) + .unwrap(); + + assert_eq!( + missing_in_b, 0, + "{missing_in_b} rows in Rust DB missing from Node DB" + ); + assert_eq!( + missing_in_a, 0, + "{missing_in_a} rows in Node DB missing from Rust DB" + ); +}