diff --git a/2017/.gitignore b/2017/.gitignore index 8116dfc..6f4752f 100644 --- a/2017/.gitignore +++ b/2017/.gitignore @@ -22,3 +22,6 @@ public-old2/thptqg2017.db.gz ### Build output ### dist/ + +### Rust build artefacts ### +tools/xlsxread/target/ diff --git a/2017/tools/xlsxread/Cargo.toml b/2017/tools/xlsxread/Cargo.toml new file mode 100644 index 0000000..d23dcca --- /dev/null +++ b/2017/tools/xlsxread/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "xlsxread" +version = "0.1.0" +edition = "2021" +description = "Rust CLI replacing SheetJS xlsx build scripts for thptqg2017/thptqg2016" + +[dependencies] +calamine = "0.26" +rusqlite = { version = "0.32", features = ["bundled"] } +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +toml = "0.8" +regex = "1" +unicode-normalization = "0.1" +thiserror = "1" +anyhow = "1" +glob = "0.3" + +# zip is already a transitive dep of calamine; pin explicitly so tests can use it +[dev-dependencies] +zip = "2" +rusqlite = { version = "0.32", features = ["bundled"] } + +[[bin]] +name = "xlsxread" +path = "src/main.rs" + +[[test]] +name = "golden" +path = "tests/golden.rs" diff --git a/2017/tools/xlsxread/src/cli.rs b/2017/tools/xlsxread/src/cli.rs new file mode 100644 index 0000000..bce8285 --- /dev/null +++ b/2017/tools/xlsxread/src/cli.rs @@ -0,0 +1,48 @@ +/// CLI argument structs via clap derive. +use std::path::PathBuf; + +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command( + name = "xlsxread", + version, + about = "Read .xls/.xlsx files and build SQLite databases for thptqg datasets" +)] +pub struct Cli { + #[command(subcommand)] + pub cmd: Cmd, +} + +#[derive(Subcommand)] +pub enum Cmd { + /// Read input spreadsheets and write a SQLite database + Build { + /// Path to the dataset TOML config file + #[arg(long)] + schema: PathBuf, + + /// Directory containing the .xls / .xlsx source files + #[arg(long)] + input: PathBuf, + + /// Output SQLite database path + #[arg(long)] + output: PathBuf, + }, + + /// Audit: compare distinct SBD count from xlsx files vs DB row count + Audit { + /// Path to the dataset TOML config file + #[arg(long)] + schema: PathBuf, + + /// Directory containing the .xlsx source files + #[arg(long)] + input: PathBuf, + + /// SQLite database to compare against + #[arg(long)] + db: PathBuf, + }, +} diff --git a/2017/tools/xlsxread/src/lib.rs b/2017/tools/xlsxread/src/lib.rs new file mode 100644 index 0000000..948fa08 --- /dev/null +++ b/2017/tools/xlsxread/src/lib.rs @@ -0,0 +1,9 @@ +/// Public library interface for integration tests. +/// The binary entry point is src/main.rs; this file re-exports the internal +/// modules so tests/golden.rs can call them without going through the CLI. +pub mod audit; +pub mod config; +pub mod error; +pub mod reader; +pub mod transform; +pub mod writer; diff --git a/2017/tools/xlsxread/src/main.rs b/2017/tools/xlsxread/src/main.rs new file mode 100644 index 0000000..963f8f3 --- /dev/null +++ b/2017/tools/xlsxread/src/main.rs @@ -0,0 +1,193 @@ +/// xlsxread — Rust CLI replacing the SheetJS xlsx build scripts. +/// +/// Subcommands: +/// build — read .xls/.xlsx files → write SQLite DB +/// audit — compare distinct SBD count from xlsx vs DB row count +/// +/// Library modules are declared in lib.rs; main.rs only adds the CLI layer. +mod cli; + +use std::path::Path; + +use anyhow::{Context, Result}; +use clap::Parser; + +use cli::{Cli, Cmd}; +use xlsxread::audit; +use xlsxread::config::load_config; +use xlsxread::reader::{is_all_blank, process_file}; +use xlsxread::transform::{validate_row, CompiledPatterns, SkipReason}; +use xlsxread::writer::{finish_db, insert_row, open_db, SCORE_FIELDS}; + +fn main() -> Result<()> { + let cli = Cli::parse(); + + match cli.cmd { + Cmd::Build { + schema, + input, + output, + } => { + run_build(&schema, &input, &output)?; + } + Cmd::Audit { schema, input, db } => { + let cfg = load_config(&schema) + .with_context(|| format!("Failed to load config: {}", schema.display()))?; + let result = audit::run_audit(&input, &db, &cfg).with_context(|| "Audit failed")?; + audit::print_audit_report(&result); + if !result.matched { + std::process::exit(1); + } + } + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Build subcommand +// --------------------------------------------------------------------------- + +fn run_build(schema_path: &Path, input_dir: &Path, output_path: &Path) -> Result<()> { + let cfg = load_config(schema_path) + .with_context(|| format!("Failed to load config: {}", schema_path.display()))?; + + // Compile score regexes once at startup + let patterns = + CompiledPatterns::new(&cfg.scores).with_context(|| "Failed to compile score regexes")?; + + // Collect input files (.xls and .xlsx), sorted for deterministic order + let mut files: Vec = std::fs::read_dir(input_dir) + .with_context(|| format!("Cannot read input dir: {}", input_dir.display()))? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| { + p.is_file() + && p.extension() + .and_then(|e| e.to_str()) + .map(|e| { + let lower = e.to_lowercase(); + lower == "xls" || lower == "xlsx" + }) + .unwrap_or(false) + }) + .collect(); + files.sort(); + + let dataset_label = input_dir + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("data"); + + println!( + "[build] {dataset_label}/ → {} ({} files)", + output_path.display(), + files.len() + ); + + // Open (or recreate) DB and apply DDL + let conn = open_db(output_path, &cfg) + .with_context(|| format!("Failed to open DB: {}", output_path.display()))?; + + let mut total_source_rows: u64 = 0; + let mut total_skipped: u64 = 0; + let mut total_errors: u64 = 0; + + let is_old2 = dataset_label.contains("old2"); + let strip_blank = cfg.reader.strip_blank_rows; + + // Single transaction over all files — mirrors the Node `db.transaction(() => { ... })()` + conn.execute_batch("BEGIN")?; + + for file in &files { + let base = file + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("?") + .to_owned(); + let mut file_rows: u64 = 0; + let mut file_skipped: u64 = 0; + let mut file_errors: u64 = 0; + + let process_result = process_file(file, &cfg, |_sheet_idx, raw| { + // data-old2: skip fully blank rows BEFORE counting sourceRows + let all_blank = is_all_blank(raw); + if strip_blank && all_blank { + return; + } + + total_source_rows += 1; + + let ho_ten = raw + .get(cfg.columns.ho_ten) + .map(|c| c.to_string().trim().to_owned()) + .unwrap_or_default(); + let so_bao_danh = raw + .get(cfg.columns.so_bao_danh) + .map(|c| c.to_string().trim().to_owned()) + .unwrap_or_default(); + + match validate_row( + &ho_ten, + &so_bao_danh, + &cfg.validation, + strip_blank, + all_blank, + ) { + Err(SkipReason::BlankRow) => { + // Already guarded above; won't reach here + } + Err(_) => { + file_skipped += 1; + return; + } + Ok(()) => {} + } + + let parsed = xlsxread::transform::transform_row(raw, &cfg, &patterns); + + match insert_row(&conn, &cfg.insert.sql, &parsed, SCORE_FIELDS) { + Ok(()) => { + file_rows += 1; + } + Err(e) => { + file_errors += 1; + if total_errors + file_errors <= 5 { + eprintln!(" [warn] {base}: {e}"); + } + } + } + }); + + match process_result { + Ok(_) => {} + Err(e) => { + eprintln!(" [error] {base}: {e}"); + file_errors += 1; + } + } + + total_skipped += file_skipped; + total_errors += file_errors; + + // Per-file row count line — mirrors `console.log(` ${base}: ${fileRows} rows`)` + println!(" {base}: {file_rows} rows"); + } + + conn.execute_batch("COMMIT")?; + + // VACUUM + stats output + finish_db( + &conn, + output_path, + total_source_rows, + total_skipped, + total_errors, + dataset_label, + files.len(), + is_old2, + ) + .with_context(|| "Failed to finalise DB")?; + + Ok(()) +}