feat(xlsxread): stage 0 scaffold with pinned deps and clap CLI skeleton

2017/.gitignore

@@ -22,3 +22,6 @@ public-old2/thptqg2017.db.gz
 
 ### Build output ###
 dist/
+
+### Rust build artefacts ###
+tools/xlsxread/target/

2017/tools/xlsxread/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "xlsxread"
+version = "0.1.0"
+edition = "2021"
+description = "Rust CLI replacing SheetJS xlsx build scripts for thptqg2017/thptqg2016"
+
+[dependencies]
+calamine       = "0.26"
+rusqlite       = { version = "0.32", features = ["bundled"] }
+clap           = { version = "4", features = ["derive"] }
+serde          = { version = "1", features = ["derive"] }
+toml           = "0.8"
+regex          = "1"
+unicode-normalization = "0.1"
+thiserror      = "1"
+anyhow         = "1"
+glob           = "0.3"
+
+# zip is already a transitive dep of calamine; pin explicitly so tests can use it
+[dev-dependencies]
+zip = "2"
+rusqlite = { version = "0.32", features = ["bundled"] }
+
+[[bin]]
+name = "xlsxread"
+path = "src/main.rs"
+
+[[test]]
+name = "golden"
+path = "tests/golden.rs"

2017/tools/xlsxread/src/cli.rs

@@ -0,0 +1,48 @@
+/// CLI argument structs via clap derive.
+use std::path::PathBuf;
+
+use clap::{Parser, Subcommand};
+
+#[derive(Parser)]
+#[command(
+    name = "xlsxread",
+    version,
+    about = "Read .xls/.xlsx files and build SQLite databases for thptqg datasets"
+)]
+pub struct Cli {
+    #[command(subcommand)]
+    pub cmd: Cmd,
+}
+
+#[derive(Subcommand)]
+pub enum Cmd {
+    /// Read input spreadsheets and write a SQLite database
+    Build {
+        /// Path to the dataset TOML config file
+        #[arg(long)]
+        schema: PathBuf,
+
+        /// Directory containing the .xls / .xlsx source files
+        #[arg(long)]
+        input: PathBuf,
+
+        /// Output SQLite database path
+        #[arg(long)]
+        output: PathBuf,
+    },
+
+    /// Audit: compare distinct SBD count from xlsx files vs DB row count
+    Audit {
+        /// Path to the dataset TOML config file
+        #[arg(long)]
+        schema: PathBuf,
+
+        /// Directory containing the .xlsx source files
+        #[arg(long)]
+        input: PathBuf,
+
+        /// SQLite database to compare against
+        #[arg(long)]
+        db: PathBuf,
+    },
+}

2017/tools/xlsxread/src/lib.rs

@@ -0,0 +1,9 @@
+/// Public library interface for integration tests.
+/// The binary entry point is src/main.rs; this file re-exports the internal
+/// modules so tests/golden.rs can call them without going through the CLI.
+pub mod audit;
+pub mod config;
+pub mod error;
+pub mod reader;
+pub mod transform;
+pub mod writer;

2017/tools/xlsxread/src/main.rs

@@ -0,0 +1,193 @@
+/// xlsxread — Rust CLI replacing the SheetJS xlsx build scripts.
+///
+/// Subcommands:
+///   build  — read .xls/.xlsx files → write SQLite DB
+///   audit  — compare distinct SBD count from xlsx vs DB row count
+///
+/// Library modules are declared in lib.rs; main.rs only adds the CLI layer.
+mod cli;
+
+use std::path::Path;
+
+use anyhow::{Context, Result};
+use clap::Parser;
+
+use cli::{Cli, Cmd};
+use xlsxread::audit;
+use xlsxread::config::load_config;
+use xlsxread::reader::{is_all_blank, process_file};
+use xlsxread::transform::{validate_row, CompiledPatterns, SkipReason};
+use xlsxread::writer::{finish_db, insert_row, open_db, SCORE_FIELDS};
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+
+    match cli.cmd {
+        Cmd::Build {
+            schema,
+            input,
+            output,
+        } => {
+            run_build(&schema, &input, &output)?;
+        }
+        Cmd::Audit { schema, input, db } => {
+            let cfg = load_config(&schema)
+                .with_context(|| format!("Failed to load config: {}", schema.display()))?;
+            let result = audit::run_audit(&input, &db, &cfg).with_context(|| "Audit failed")?;
+            audit::print_audit_report(&result);
+            if !result.matched {
+                std::process::exit(1);
+            }
+        }
+    }
+
+    Ok(())
+}
+
+// ---------------------------------------------------------------------------
+// Build subcommand
+// ---------------------------------------------------------------------------
+
+fn run_build(schema_path: &Path, input_dir: &Path, output_path: &Path) -> Result<()> {
+    let cfg = load_config(schema_path)
+        .with_context(|| format!("Failed to load config: {}", schema_path.display()))?;
+
+    // Compile score regexes once at startup
+    let patterns =
+        CompiledPatterns::new(&cfg.scores).with_context(|| "Failed to compile score regexes")?;
+
+    // Collect input files (.xls and .xlsx), sorted for deterministic order
+    let mut files: Vec<std::path::PathBuf> = std::fs::read_dir(input_dir)
+        .with_context(|| format!("Cannot read input dir: {}", input_dir.display()))?
+        .filter_map(|e| e.ok())
+        .map(|e| e.path())
+        .filter(|p| {
+            p.is_file()
+                && p.extension()
+                    .and_then(|e| e.to_str())
+                    .map(|e| {
+                        let lower = e.to_lowercase();
+                        lower == "xls" || lower == "xlsx"
+                    })
+                    .unwrap_or(false)
+        })
+        .collect();
+    files.sort();
+
+    let dataset_label = input_dir
+        .file_name()
+        .and_then(|n| n.to_str())
+        .unwrap_or("data");
+
+    println!(
+        "[build] {dataset_label}/ → {}  ({} files)",
+        output_path.display(),
+        files.len()
+    );
+
+    // Open (or recreate) DB and apply DDL
+    let conn = open_db(output_path, &cfg)
+        .with_context(|| format!("Failed to open DB: {}", output_path.display()))?;
+
+    let mut total_source_rows: u64 = 0;
+    let mut total_skipped: u64 = 0;
+    let mut total_errors: u64 = 0;
+
+    let is_old2 = dataset_label.contains("old2");
+    let strip_blank = cfg.reader.strip_blank_rows;
+
+    // Single transaction over all files — mirrors the Node `db.transaction(() => { ... })()`
+    conn.execute_batch("BEGIN")?;
+
+    for file in &files {
+        let base = file
+            .file_name()
+            .and_then(|n| n.to_str())
+            .unwrap_or("?")
+            .to_owned();
+        let mut file_rows: u64 = 0;
+        let mut file_skipped: u64 = 0;
+        let mut file_errors: u64 = 0;
+
+        let process_result = process_file(file, &cfg, |_sheet_idx, raw| {
+            // data-old2: skip fully blank rows BEFORE counting sourceRows
+            let all_blank = is_all_blank(raw);
+            if strip_blank && all_blank {
+                return;
+            }
+
+            total_source_rows += 1;
+
+            let ho_ten = raw
+                .get(cfg.columns.ho_ten)
+                .map(|c| c.to_string().trim().to_owned())
+                .unwrap_or_default();
+            let so_bao_danh = raw
+                .get(cfg.columns.so_bao_danh)
+                .map(|c| c.to_string().trim().to_owned())
+                .unwrap_or_default();
+
+            match validate_row(
+                &ho_ten,
+                &so_bao_danh,
+                &cfg.validation,
+                strip_blank,
+                all_blank,
+            ) {
+                Err(SkipReason::BlankRow) => {
+                    // Already guarded above; won't reach here
+                }
+                Err(_) => {
+                    file_skipped += 1;
+                    return;
+                }
+                Ok(()) => {}
+            }
+
+            let parsed = xlsxread::transform::transform_row(raw, &cfg, &patterns);
+
+            match insert_row(&conn, &cfg.insert.sql, &parsed, SCORE_FIELDS) {
+                Ok(()) => {
+                    file_rows += 1;
+                }
+                Err(e) => {
+                    file_errors += 1;
+                    if total_errors + file_errors <= 5 {
+                        eprintln!("  [warn] {base}: {e}");
+                    }
+                }
+            }
+        });
+
+        match process_result {
+            Ok(_) => {}
+            Err(e) => {
+                eprintln!("  [error] {base}: {e}");
+                file_errors += 1;
+            }
+        }
+
+        total_skipped += file_skipped;
+        total_errors += file_errors;
+
+        // Per-file row count line — mirrors `console.log(`  ${base}: ${fileRows} rows`)`
+        println!("  {base}: {file_rows} rows");
+    }
+
+    conn.execute_batch("COMMIT")?;
+
+    // VACUUM + stats output
+    finish_db(
+        &conn,
+        output_path,
+        total_source_rows,
+        total_skipped,
+        total_errors,
+        dataset_label,
+        files.len(),
+        is_old2,
+    )
+    .with_context(|| "Failed to finalise DB")?;
+
+    Ok(())
+}