refactor(parser): reimplement the parser in Go alongside the Rust crate

Adds go-parser/, a Go reimplementation of the xlsxread parser, verified
byte-for-byte against the Rust original before any cutover.

Reader fidelity is exact across all 299 input files: the canonical cell dump
of every sheet matches calamine's, locked in as a test against a committed
hash oracle. Reaching that required replacing extrame/xls, which corrupted
69% of cells and dropped a further 28% on the BIFF corpus, with pbnjay/grate;
correcting excelize's number-format application and trailing-cell trimming;
restoring carriage returns that XML line-ending normalisation strips from
2,233 ten_cum_thi values; and gating numeric re-rendering on cell type so
shared strings that merely look numeric keep their leading zeros.

The differential gate compares both parsers over all four datasets:
3,265,641 rows with identical full-table SHA-256, identical per-column
non-NULL counts, identical schema metadata and identical stdout.

Config moves from TOML to YAML for both parsers, so they keep reading the
same files and the gate stays meaningful. Verified by rebuilding 2016 and
2017-old2 with Rust under the new configs and matching the recorded counts.

build-db.js now refuses to publish a database whose row count does not match
the known figure, closing a path where an under-producing parser could ship a
truncated public dataset with green CI. The deploy workflow gains a
pull_request trigger and guards deploy to main, so branch verification can no
longer publish to production.
This commit is contained in:
tiennm99 committed 2026-08-13 20:27:44 +07:00
1 parent 8902232747
commit 0eb174721d
60 files changed
+6050 -189

No files matched your search

+26 -54
View File
@@ -489,6 +489,12 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jobserver"
version = "0.1.34"
@@ -693,6 +699,12 @@ version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "ryu"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "serde"
version = "1.0.228"
@@ -724,12 +736,16 @@ dependencies = [
]
[[package]]
name = "serde_spanned"
version = "0.6.9"
name = "serde_yaml"
version = "0.9.34+deprecated"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
dependencies = [
"indexmap",
"itoa",
"ryu",
"serde",
"unsafe-libyaml",
]
[[package]]
@@ -858,47 +874,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "toml"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit",
]
[[package]]
name = "toml_datetime"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
dependencies = [
"serde",
]
[[package]]
name = "toml_edit"
version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
"indexmap",
"serde",
"serde_spanned",
"toml_datetime",
"toml_write",
"winnow",
]
[[package]]
name = "toml_write"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
[[package]]
name = "typenum"
version = "1.20.0"
@@ -920,6 +895,12 @@ dependencies = [
"tinyvec",
]
[[package]]
name = "unsafe-libyaml"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
[[package]]
name = "utf8parse"
version = "0.2.2"
@@ -1007,15 +988,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "winnow"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
dependencies = [
"memchr",
]
[[package]]
name = "wit-bindgen"
version = "0.57.1"
@@ -1033,8 +1005,8 @@ dependencies = [
"regex",
"rusqlite",
"serde",
"serde_yaml",
"thiserror 1.0.69",
"toml",
"unicode-normalization",
"zip",
]
+1 -1
View File
@@ -9,7 +9,7 @@ calamine = "0.26"
rusqlite = { version = "0.32", features = ["bundled"] }
clap = { version = "4", features = ["derive"] }
serde = { version = "1", features = ["derive"] }
toml = "0.8"
serde_yaml = "0.9" # deprecated upstream but stable; this crate is deleted at cutover
regex = "1"
unicode-normalization = "0.1"
thiserror = "1"
@@ -15,18 +15,18 @@
#
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
format_detection = "thptqg2016"
format_detection: thptqg2016
[reader]
sheet_mode = "all"
strip_blank_rows = false
reader:
sheet_mode: all
strip_blank_rows: false
[validation]
require_numeric_sbd = false
require_nonempty_name = true
require_nonempty_sbd = true
validation:
require_numeric_sbd: false
require_nonempty_name: true
require_nonempty_sbd: true
[header]
# Tokens that identify a header row by first-cell content (uppercased).
# Covers both SOBAODANH-style and SBD-style headers.
tokens = ["SOBAODANH", "SBD", "HO_TEN", "HOTEN", "HỌ TÊN", "STT"]
header:
# Tokens that identify a header row by first-cell content (uppercased).
# Covers both SOBAODANH-style and SBD-style headers.
tokens: ["SOBAODANH", "SBD", "HO_TEN", "HOTEN", "HỌ TÊN", "STT"]
@@ -6,20 +6,20 @@
#
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
[reader]
sheet_mode = "first"
strip_blank_rows = false
reader:
sheet_mode: first
strip_blank_rows: false
[columns]
ho_ten = 0
ngay_sinh = 1
so_bao_danh = 2
diem_thi = 3
columns:
ho_ten: 0
ngay_sinh: 1
so_bao_danh: 2
diem_thi: 3
[validation]
require_numeric_sbd = true
require_nonempty_name = true
require_nonempty_sbd = true
validation:
require_numeric_sbd: true
require_nonempty_name: true
require_nonempty_sbd: true
[header]
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
header:
tokens: ["HO_TEN", "HỌ TÊN", "STT"]
@@ -7,20 +7,20 @@
#
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
[reader]
sheet_mode = "all"
strip_blank_rows = true
reader:
sheet_mode: all
strip_blank_rows: true
[columns]
ho_ten = 0
ngay_sinh = 1
so_bao_danh = 2
diem_thi = 3
columns:
ho_ten: 0
ngay_sinh: 1
so_bao_danh: 2
diem_thi: 3
[validation]
require_numeric_sbd = true
require_nonempty_name = true
require_nonempty_sbd = true
validation:
require_numeric_sbd: true
require_nonempty_name: true
require_nonempty_sbd: true
[header]
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
header:
tokens: ["HO_TEN", "HỌ TÊN", "STT"]
@@ -6,20 +6,20 @@
#
# Table shape, INSERT and subject regexes are canonical — see src/schema.rs.
[reader]
sheet_mode = "all"
strip_blank_rows = false
reader:
sheet_mode: all
strip_blank_rows: false
[columns]
ho_ten = 0
ngay_sinh = 1
so_bao_danh = 2
diem_thi = 3
columns:
ho_ten: 0
ngay_sinh: 1
so_bao_danh: 2
diem_thi: 3
[validation]
require_numeric_sbd = false
require_nonempty_name = true
require_nonempty_sbd = true
validation:
require_numeric_sbd: false
require_nonempty_name: true
require_nonempty_sbd: true
[header]
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
header:
tokens: ["HO_TEN", "HỌ TÊN", "STT"]
+106
View File
@@ -0,0 +1,106 @@
//! Ground-truth cell dumper for the Go reader fidelity gate.
//!
//! Emits a canonical, reader-agnostic text rendering of every sheet and every
//! cell of a spreadsheet exactly as calamine sees it. The Go reader must
//! reproduce this byte-for-byte; the two dumps are compared by hash.
//!
//! Deliberately dumps the RAW used range: every sheet, every row, header rows
//! included, no config applied. This gate is about cell fidelity, not build
//! semantics — sheet selection and header skipping are exercised later.
//!
//! Usage: cargo run --release --example dump_cells -- <spreadsheet> [out-file]
//! With no out-file the dump goes to stdout.
use std::env;
use std::fs::File;
use std::io::{self, BufWriter, Write};
use calamine::{open_workbook_auto, Data, Reader, Sheets};
/// Escapes the field separators so a cell value can never break the line format.
fn escape(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for ch in s.chars() {
match ch {
'\\' => out.push_str("\\\\"),
'\t' => out.push_str("\\t"),
'\n' => out.push_str("\\n"),
'\r' => out.push_str("\\r"),
_ => out.push(ch),
}
}
out
}
/// Discriminates the calamine variant so a Go port can be checked against the
/// actual type, not just the rendered string.
fn kind(d: &Data) -> &'static str {
match d {
Data::Empty => "empty",
Data::String(_) => "str",
Data::Float(_) => "float",
Data::Int(_) => "int",
Data::Bool(_) => "bool",
Data::Error(_) => "err",
Data::DateTime(_) => "datetime",
Data::DateTimeIso(_) => "datetimeiso",
Data::DurationIso(_) => "durationiso",
}
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
let args: Vec<String> = env::args().collect();
if args.len() < 2 {
eprintln!("usage: dump_cells <spreadsheet> [out-file]");
std::process::exit(2);
}
let path = &args[1];
let mut out: Box<dyn Write> = match args.get(2) {
Some(p) => Box::new(BufWriter::new(File::create(p)?)),
None => Box::new(BufWriter::new(io::stdout())),
};
let mut workbook: Sheets<_> = open_workbook_auto(path)?;
let sheet_names: Vec<String> = workbook.sheet_names().to_vec();
writeln!(out, "FILE\t{}", escape(path))?;
writeln!(out, "SHEETCOUNT\t{}", sheet_names.len())?;
for (idx, name) in sheet_names.iter().enumerate() {
let range = workbook.worksheet_range(name)?;
// start() is the used-range origin — the key question for any Go reader,
// which may index absolutely from A1 instead.
let (srow, scol) = range.start().unwrap_or((0, 0));
writeln!(
out,
"SHEET\t{}\t{}\t{}\t{}\t{}\t{}",
idx,
escape(name),
range.height(),
range.width(),
srow,
scol
)?;
for (r, row) in range.rows().enumerate() {
writeln!(out, "ROW\t{}\t{}\t{}", idx, r, row.len())?;
for (c, cell) in row.iter().enumerate() {
let s = cell.to_string();
writeln!(
out,
"CELL\t{}\t{}\t{}\t{}\t{}\t{}",
idx,
r,
c,
kind(cell),
if matches!(cell, Data::Empty) { 1 } else { 0 },
escape(&s)
)?;
}
}
}
out.flush()?;
Ok(())
}
+76
View File
@@ -0,0 +1,76 @@
//! Corpus-wide cell-kind and sheet-geometry scan, for the Go reader fidelity gate.
//!
//! For every file given, prints one TSV line per sheet and one summary line per
//! file. Aggregates only — never materialises the cell text — so the whole
//! 418 MB corpus can be scanned quickly.
//!
//! The point is to find out which calamine `Data` variants actually occur in
//! real inputs. `DateTime` and `Float` are the variants whose rendering differs
//! between readers; if they never appear, the divergence risk is theoretical.
//!
//! Usage: cargo run --release --example scan_kinds -- <file>...
use std::env;
use calamine::{open_workbook_auto, Data, Reader, Sheets};
fn main() {
let args: Vec<String> = env::args().skip(1).collect();
if args.is_empty() {
eprintln!("usage: scan_kinds <file>...");
std::process::exit(2);
}
println!("#TYPE\tpath\tsheet_idx\tname\theight\twidth\tstart_row\tstart_col\tempty\tstr\tfloat\tint\tbool\tdatetime\tdtiso\tduriso\terr");
for path in &args {
let mut wb: Sheets<_> = match open_workbook_auto(path) {
Ok(w) => w,
Err(e) => {
println!("ERR\t{path}\t{e}");
continue;
}
};
let names: Vec<String> = wb.sheet_names().to_vec();
for (idx, name) in names.iter().enumerate() {
let range = match wb.worksheet_range(name) {
Ok(r) => r,
Err(e) => {
println!("SHEETERR\t{path}\t{idx}\t{e}");
continue;
}
};
let (sr, sc) = range.start().unwrap_or((0, 0));
let mut k = [0usize; 9]; // empty,str,float,int,bool,datetime,dtiso,duriso,err
for row in range.rows() {
for cell in row {
let i = match cell {
Data::Empty => 0,
Data::String(_) => 1,
Data::Float(_) => 2,
Data::Int(_) => 3,
Data::Bool(_) => 4,
Data::DateTime(_) => 5,
Data::DateTimeIso(_) => 6,
Data::DurationIso(_) => 7,
Data::Error(_) => 8,
};
k[i] += 1;
}
}
println!(
"SHEET\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
path,
idx,
name.replace('\t', " "),
range.height(),
range.width(),
sr,
sc,
k[0], k[1], k[2], k[3], k[4], k[5], k[6], k[7], k[8]
);
}
println!("FILE\t{}\t{}", path, names.len());
}
}
+1 -1
View File
@@ -50,7 +50,7 @@ for (const id of targets) {
[
"build",
"--schema",
resolve(ROOT, `parser/configs/${id}.toml`),
resolve(ROOT, `parser/configs/${id}.yml`),
"--input",
resolve(ROOT, `data/${id}`),
"--output",
+36 -35
View File
@@ -6,7 +6,7 @@ use serde::Deserialize;
use crate::error::BuildError;
// ---------------------------------------------------------------------------
// Top-level dataset configuration loaded from a .toml file
// Top-level dataset configuration loaded from a .yml file
// ---------------------------------------------------------------------------
/// Per-dataset parse rules.
@@ -79,7 +79,7 @@ pub fn load_config(path: &Path) -> Result<DatasetConfig, BuildError> {
path: path.display().to_string(),
source: e,
})?;
let cfg: DatasetConfig = toml::from_str(&text)?;
let cfg: DatasetConfig = serde_yaml::from_str(&text)?;
Ok(cfg)
}
@@ -91,29 +91,29 @@ pub fn load_config(path: &Path) -> Result<DatasetConfig, BuildError> {
mod tests {
use super::*;
const SAMPLE_TOML: &str = r#"
[reader]
sheet_mode = "all"
strip_blank_rows = false
const SAMPLE_YAML: &str = r#"
reader:
sheet_mode: all
strip_blank_rows: false
[columns]
ho_ten = 0
ngay_sinh = 1
so_bao_danh = 2
diem_thi = 3
columns:
ho_ten: 0
ngay_sinh: 1
so_bao_danh: 2
diem_thi: 3
[validation]
require_numeric_sbd = false
require_nonempty_name = true
require_nonempty_sbd = true
validation:
require_numeric_sbd: false
require_nonempty_name: true
require_nonempty_sbd: true
[header]
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
header:
tokens: ["HO_TEN", "HỌ TÊN", "STT"]
"#;
#[test]
fn config_round_trip() {
let cfg: DatasetConfig = toml::from_str(SAMPLE_TOML).expect("parse failed");
let cfg: DatasetConfig = serde_yaml::from_str(SAMPLE_YAML).expect("parse failed");
assert_eq!(cfg.reader.sheet_mode, SheetMode::All);
assert!(!cfg.reader.strip_blank_rows);
let cols = cfg.columns.as_ref().unwrap();
@@ -131,37 +131,38 @@ tokens = ["HO_TEN", "HỌ TÊN", "STT"]
#[test]
fn config_rejects_leftover_sql_sections() {
let with_ddl = format!(
"{SAMPLE_TOML}\n[schema]\nddl = \"CREATE TABLE student (so_bao_danh TEXT);\"\n"
"{SAMPLE_YAML}\nschema:\n ddl: \"CREATE TABLE student (so_bao_danh TEXT);\"\n"
);
assert!(toml::from_str::<DatasetConfig>(&with_ddl).is_err());
assert!(serde_yaml::from_str::<DatasetConfig>(&with_ddl).is_err());
}
#[test]
fn config_first_sheet_mode() {
let toml_str = SAMPLE_TOML.replace(r#"sheet_mode = "all""#, r#"sheet_mode = "first""#);
let cfg: DatasetConfig = toml::from_str(&toml_str).expect("parse failed");
let yaml_str = SAMPLE_YAML.replace("sheet_mode: all", "sheet_mode: first");
let cfg: DatasetConfig = serde_yaml::from_str(&yaml_str).expect("parse failed");
assert_eq!(cfg.reader.sheet_mode, SheetMode::First);
}
#[test]
fn config_format_detection_field() {
// Configs without [columns] and with format_detection = "thptqg2016" parse correctly
let toml_str = r#"
format_detection = "thptqg2016"
// Configs without a `columns:` mapping and with format_detection:
// thptqg2016 parse correctly
let yaml_str = r#"
format_detection: thptqg2016
[reader]
sheet_mode = "all"
strip_blank_rows = false
reader:
sheet_mode: all
strip_blank_rows: false
[validation]
require_numeric_sbd = false
require_nonempty_name = true
require_nonempty_sbd = true
validation:
require_numeric_sbd: false
require_nonempty_name: true
require_nonempty_sbd: true
[header]
tokens = ["SBD", "SOBAODANH", "STT"]
header:
tokens: ["SBD", "SOBAODANH", "STT"]
"#;
let cfg: DatasetConfig = toml::from_str(toml_str).expect("parse failed");
let cfg: DatasetConfig = serde_yaml::from_str(yaml_str).expect("parse failed");
assert_eq!(cfg.format_detection.as_deref(), Some("thptqg2016"));
assert!(cfg.columns.is_none());
}
+1 -1
View File
@@ -20,7 +20,7 @@ pub enum BuildError {
Sqlite(#[from] rusqlite::Error),
#[error("Config parse error: {0}")]
Config(#[from] toml::de::Error),
Config(#[from] serde_yaml::Error),
#[error("Regex compile error for pattern '{pattern}': {source}")]
Regex {
+9 -9
View File
@@ -267,7 +267,7 @@ fn ensure_fixtures() {
fn make_data_config() -> xlsxread::config::DatasetConfig {
let cfg_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("configs")
.join("2017.toml");
.join("2017.yml");
xlsxread::config::load_config(&cfg_path).expect("load data config")
}
@@ -284,7 +284,7 @@ fn province_100_builds_100_rows() {
std::fs::create_dir_all(&fixture_dir).unwrap();
std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap();
run_build_cmd(&fixture_dir, &db_path, "2017.toml");
run_build_cmd(&fixture_dir, &db_path, "2017.yml");
let count = query_count(&db_path);
assert_eq!(count, 100, "expected 100 rows from province-100 fixture");
@@ -299,7 +299,7 @@ fn hcm_overflow_builds_400_rows() {
std::fs::create_dir_all(&fixture_dir).unwrap();
std::fs::copy(hcm_overflow_fixture_path(), fixture_dir.join("hcm.xlsx")).unwrap();
run_build_cmd(&fixture_dir, &db_path, "2017.toml");
run_build_cmd(&fixture_dir, &db_path, "2017.yml");
let count = query_count(&db_path);
assert_eq!(
@@ -318,7 +318,7 @@ fn data_old_first_sheet_only_100_rows() {
// Use the overflow file but with data-old config (first sheet only → 200 rows)
std::fs::copy(hcm_overflow_fixture_path(), fixture_dir.join("hcm.xlsx")).unwrap();
run_build_cmd(&fixture_dir, &db_path, "2017-old.toml");
run_build_cmd(&fixture_dir, &db_path, "2017-old.yml");
// data-old: sheet_mode=first → only 200 rows from sheet1; but SBDs "1000NNNN" are
// all digits so all pass the numeric guard
@@ -356,7 +356,7 @@ fn numeric_sbd_guard_rejects_non_numeric() {
let mixed_path = fixture_dir.join("mixed.xlsx");
write_xlsx(&mixed_path, &[("Sheet1".to_owned(), rows)]);
run_build_cmd(&fixture_dir, &db_path, "2017-old.toml");
run_build_cmd(&fixture_dir, &db_path, "2017-old.yml");
// Row i=5 has non-numeric SBD → rejected by data-old config
let count = query_count(&db_path);
@@ -390,7 +390,7 @@ fn scores_parsed_correctly_into_db() {
&[("Sheet1".to_owned(), rows)],
);
run_build_cmd(&fixture_dir, &db_path, "2017.toml");
run_build_cmd(&fixture_dir, &db_path, "2017.yml");
let conn = rusqlite::Connection::open(&db_path).unwrap();
let (toan, van, anh): (f64, f64, f64) = conn
@@ -429,7 +429,7 @@ fn to_ascii_stored_correctly() {
&[("Sheet1".to_owned(), rows)],
);
run_build_cmd(&fixture_dir, &db_path, "2017.toml");
run_build_cmd(&fixture_dir, &db_path, "2017.yml");
let conn = rusqlite::Connection::open(&db_path).unwrap();
let ascii: String = conn
@@ -451,7 +451,7 @@ fn audit_subcommand_matches_after_build() {
std::fs::create_dir_all(&fixture_dir).unwrap();
std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap();
run_build_cmd(&fixture_dir, &db_path, "2017.toml");
run_build_cmd(&fixture_dir, &db_path, "2017.yml");
// audit should match (100 distinct SBDs in xlsx == 100 rows in DB)
let cfg = make_data_config();
@@ -491,7 +491,7 @@ fn audit_subcommand_mismatch_detected() {
&[("Sheet1".to_owned(), five_rows)],
);
run_build_cmd(&build_dir, &db_path, "2017.toml");
run_build_cmd(&build_dir, &db_path, "2017.yml");
// audit against fixture_dir (10 xlsx rows) but DB has 5 rows → mismatch
let cfg = make_data_config();