mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-11 03:13:48 +00:00
tiennm99
99465fda59
* feat(xlsxread): stage 0 scaffold with pinned deps and clap CLI skeleton * feat(xlsxread): stage 1 reader — calamine sheet enumeration and header skip * feat(xlsxread): stage 2 transform — to_ascii, score regex, validation with 38 unit tests * feat(xlsxread): stage 3 writer — SQLite DDL, INSERT OR REPLACE, VACUUM, stats output * feat(xlsxread): stage 4 audit — distinct SBD scan vs DB count, mirrors audit-row-counts.js output * feat(xlsxread): stage 5 golden tests — in-process xlsx fixtures, 8 integration tests pass * chore(xlsxread): commit Cargo.lock for reproducible Rust builds * feat(build): wire root build:db scripts to xlsxread CLI Replace node scripts/build-database*.js invocations with the Rust xlsxread binary. Each build:db* script now calls `pnpm build:rust` (cargo build --release) before invoking the xlsxread build subcommand with the matching per-dataset config. Drop xlsx and better-sqlite3 from devDependencies — no Node script consumes them anymore. sql.js (runtime DB reader in the SPA) is unaffected and remains in dependencies. * ci: build xlsxread before running database build jobs Add dtolnay/rust-toolchain@stable and Swatinem/rust-cache@v2 (workspaces: tools/xlsxread) for warm incremental Rust builds. Replace the single `pnpm build:db:all` step with explicit xlsxread invocations so CI doesn't call pnpm build:rust redundantly three times. The binary is built once, then each of the three datasets is processed in sequence. * chore: remove deprecated xlsx-based build scripts Delete scripts/build-database.js, build-database-old.js, build-database-old2.js, build-lib.js, and audit-row-counts.js. Functionality replaced by the xlsxread Rust CLI configured via tools/xlsxread/configs/*.toml. History preserved in git; one-click revert available via the chore/migration-backup-260519 branch. * docs: update README build instructions for xlsxread pipeline Replace Node.js + xlsx references with Rust + xlsxread workflow. Update requirements (Node 24+, pnpm, Rust stable), quickstart, scripts table, and project layout tree to reflect the current state after the xlsx-based build scripts were removed. * chore(deps): drop xlsx and better-sqlite3 from package.json and lockfile Remove xlsx (SheetJS, vulnerable: GHSA-4r6h-8v6p-xvw6, GHSA-5pgg-2g8v-p4x9) and better-sqlite3 from devDependencies. Both were only used by the now-deleted Node build scripts. The Rust xlsxread CLI vendors SQLite via rusqlite-bundled; no Node-side SQLite dependency is needed. `pnpm audit` returns clean.
77 lines
2.1 KiB
TOML
77 lines
2.1 KiB
TOML
# Config for data-old2/ — 54 .xlsx files (corrected-export set)
|
|
# Sheet mode: "all" — HCM (24.HCM_UTLQ.xlsx) overflows into Sheet2 (+6,446 rows)
|
|
# SBD validation: require ^\d+$ (build-database-old2.js:57 guard)
|
|
# Blank row strip: true — skip fully blank rows BEFORE counting sourceRows
|
|
# (build-database-old2.js:50-51: blank row check before sourceRows++)
|
|
|
|
[reader]
|
|
sheet_mode = "all"
|
|
strip_blank_rows = true
|
|
|
|
[columns]
|
|
ho_ten = 0
|
|
ngay_sinh = 1
|
|
so_bao_danh = 2
|
|
diem_thi = 3
|
|
|
|
[validation]
|
|
require_numeric_sbd = true
|
|
require_nonempty_name = true
|
|
require_nonempty_sbd = true
|
|
|
|
[header]
|
|
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
|
|
|
[schema]
|
|
ddl = """
|
|
CREATE TABLE student (
|
|
so_bao_danh TEXT PRIMARY KEY,
|
|
ho_ten TEXT NOT NULL,
|
|
ho_ten_ascii TEXT NOT NULL,
|
|
ngay_sinh TEXT,
|
|
toan REAL,
|
|
ngu_van REAL,
|
|
vat_ly REAL,
|
|
hoa_hoc REAL,
|
|
sinh_hoc REAL,
|
|
khtn REAL,
|
|
lich_su REAL,
|
|
dia_ly REAL,
|
|
gdcd REAL,
|
|
khxh REAL,
|
|
tieng_anh REAL,
|
|
tieng_phap REAL,
|
|
tieng_nga REAL,
|
|
tieng_trung REAL
|
|
);
|
|
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
|
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
|
"""
|
|
|
|
[scores]
|
|
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
|
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
|
vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)'
|
|
hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)'
|
|
sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)'
|
|
khtn = 'KHTN:\s*(\d+(?:\.\d+)?)'
|
|
lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)'
|
|
dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)'
|
|
gdcd = 'GDCD:\s*(\d+(?:\.\d+)?)'
|
|
khxh = 'KHXH:\s*(\d+(?:\.\d+)?)'
|
|
tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)'
|
|
tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)'
|
|
tieng_nga = 'Tiếng Nga:\s*(\d+(?:\.\d+)?)'
|
|
tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)'
|
|
|
|
[insert]
|
|
sql = """
|
|
INSERT OR REPLACE INTO student
|
|
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh,
|
|
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
|
lich_su, dia_ly, gdcd, khxh,
|
|
tieng_anh, tieng_phap, tieng_nga, tieng_trung)
|
|
VALUES
|
|
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
"""
|