mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-14 18:19:21 +00:00
refactor: collapse the two projects into one tree and move to npm
The repo held two near-duplicate projects. 2016/ and 2017/ each carried their
own React frontend, their own copy of the same Rust crate, and their own
package manager setup. 2016/tools/sync-from-thptqg2017.sh existed purely to
copy the parser source between them.
New layout:
index.html + src/ the 2017 frontend, now the only one
data/<id>/ 2016, 2017, 2017-old, 2017-old2
parser/ the single Rust crate, configs renamed to <id>.toml
docs/ both projects' docs, 2016 copies suffixed -2016-legacy
pending the merge pass
<id> is now one identifier end to end: data/<id>/ feeds parser/configs/<id>.toml
and produces db/<id>.db.gz.
pnpm gives way to npm. pnpm-workspace.yaml existed only to whitelist
better-sqlite3's native build, which npm permits by default, so it has no
equivalent and is simply gone. Lockfiles cannot be converted; package-lock.json
is generated fresh. The migration direction is safe — pnpm's strict layout
forbids phantom dependencies, so anything that resolved under pnpm resolves
under npm's flat tree.
Adds parser/scripts/build-db.js and src/datasets.js: the four dataset IDs are
declared once and read by both the build tooling and (from the next phase) the
frontend.
Follow-on fixes the move made necessary:
- eslint's Node-globals override pointed at scripts/, now parser/scripts/
- crawl-baotintuc.js wrote to <root>/data, now data/2017
- golden tests loaded configs by their old thptqg*-data.toml names
Drops the #[ignore]d Rust-vs-Node golden test. It shelled out to pnpm to run
scripts/build-database.js, a file removed when the parser was ported to Rust,
so it could never pass. check-duplicates.js and diff-datasets.js were already
broken before this change and are annotated as such rather than half-fixed.
63 Rust tests pass and clippy is clean from the new location.
This commit is contained in:
1 parent
82db8295bb
commit
6ff2ed99ec
396 files changed
+2753
-8720
No files matched your search
@@ -0,0 +1,32 @@
|
||||
# thptqg2016 data/ — 4 .xls + 115 .xlsx mixed files.
|
||||
#
|
||||
# Three column layouts exist across the 119 files; the binary selects the
|
||||
# right one per-file at runtime via format_detection = "thptqg2016":
|
||||
#
|
||||
# separate-scores header SBD(0)/HOTEN(1)/TOAN(2)... — dhhanghai files
|
||||
# mapped header SOBAODANH|SBD + DIEM_THI — most provinces
|
||||
# default no header; positional 6-col layout — remaining files
|
||||
#
|
||||
# This is the only dataset that populates ten_cum_thi and gioi_tinh, and the
|
||||
# only one whose files carry Tiếng Đức / Tiếng Nhật scores.
|
||||
#
|
||||
# sheet_mode = "all": several provinces overflow into Sheet2 (65k Excel row cap).
|
||||
# strip_blank_rows = false: no blank-row anomaly observed in this dataset.
|
||||
#
|
||||
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
|
||||
|
||||
format_detection = "thptqg2016"
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
# Tokens that identify a header row by first-cell content (uppercased).
|
||||
# Covers both SOBAODANH-style and SBD-style headers.
|
||||
tokens = ["SOBAODANH", "SBD", "HO_TEN", "HOTEN", "HỌ TÊN", "STT"]
|
||||
@@ -0,0 +1,25 @@
|
||||
# thptqg2017 data-old/ — 63 .xlsx files (pre-baotintuc refresh).
|
||||
#
|
||||
# sheet_mode = "first": single-sheet workbooks, never hit the 65k row cap.
|
||||
# SBD validation: require ^\d+$ (build-database-old.js:55 guard).
|
||||
# strip_blank_rows = false: no explicit blank-skip in build-database-old.js.
|
||||
#
|
||||
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
|
||||
|
||||
[reader]
|
||||
sheet_mode = "first"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = true
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
@@ -0,0 +1,26 @@
|
||||
# thptqg2017 data-old2/ — 54 .xlsx files (corrected-export set).
|
||||
#
|
||||
# sheet_mode = "all": 24.HCM_UTLQ.xlsx overflows into Sheet2 (+6,446 rows).
|
||||
# SBD validation: require ^\d+$ (build-database-old2.js:57 guard).
|
||||
# strip_blank_rows = true: skip fully blank rows BEFORE counting sourceRows
|
||||
# (build-database-old2.js:50-51 checks blank before sourceRows++).
|
||||
#
|
||||
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = true
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = true
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
@@ -0,0 +1,25 @@
|
||||
# thptqg2017 data/ — 63 .xls files from baotintuc.vn (current generation).
|
||||
#
|
||||
# sheet_mode = "all": Hà Nội and HCM overflow into Sheet2 (65k Excel row cap).
|
||||
# SBD validation: no numeric guard — build-database.js did not apply ^\d+$.
|
||||
# strip_blank_rows = false: no blank-row anomaly in this dataset.
|
||||
#
|
||||
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
Reference in new issue
Block a user