/// xlsxread — Rust CLI replacing the SheetJS xlsx build scripts. /// /// Subcommands: /// build — read .xls/.xlsx files → write SQLite DB /// audit — compare distinct SBD count from xlsx vs DB row count /// /// Library modules are declared in lib.rs; main.rs only adds the CLI layer. /// /// When config contains `format_detection = "thptqg2016"` the build subcommand /// uses per-file header inspection to pick the right column layout, replicating /// the `detectFormat` logic from scripts/build-database.js (lines 63–87). mod cli; use std::path::Path; use anyhow::{Context, Result}; use calamine::Data; use clap::Parser; use cli::{Cli, Cmd}; use xlsxread::audit; use xlsxread::config::load_config; use xlsxread::format_detect_2016::{ detect_format, is_header_row_2016, process_row_2016, DetectedFormat, }; use xlsxread::reader::{is_all_blank, process_file}; use xlsxread::transform::{validate_row, CompiledPatterns, SkipReason}; use xlsxread::writer::{finish_db, insert_row, open_db}; fn main() -> Result<()> { let cli = Cli::parse(); match cli.cmd { Cmd::Build { schema, input, output, } => { run_build(&schema, &input, &output)?; } Cmd::Audit { schema, input, db } => { let cfg = load_config(&schema) .with_context(|| format!("Failed to load config: {}", schema.display()))?; let result = audit::run_audit(&input, &db, &cfg).with_context(|| "Audit failed")?; audit::print_audit_report(&result); if !result.matched { std::process::exit(1); } } } Ok(()) } // --------------------------------------------------------------------------- // Build subcommand — dispatches to thptqg2016 or standard path // --------------------------------------------------------------------------- fn run_build(schema_path: &Path, input_dir: &Path, output_path: &Path) -> Result<()> { let cfg = load_config(schema_path) .with_context(|| format!("Failed to load config: {}", schema_path.display()))?; if cfg.format_detection.as_deref() == Some("thptqg2016") { run_build_2016(&cfg, input_dir, output_path) } else { run_build_standard(&cfg, input_dir, output_path) } } // --------------------------------------------------------------------------- // Standard build path (thptqg2017 and similar fixed-column configs) // --------------------------------------------------------------------------- fn run_build_standard( cfg: &xlsxread::config::DatasetConfig, input_dir: &Path, output_path: &Path, ) -> Result<()> { let patterns = CompiledPatterns::new().with_context(|| "Failed to compile score regexes")?; let mut files: Vec = std::fs::read_dir(input_dir) .with_context(|| format!("Cannot read input dir: {}", input_dir.display()))? .filter_map(|e| e.ok()) .map(|e| e.path()) .filter(|p| { p.is_file() && p.extension() .and_then(|e| e.to_str()) .map(|e| { let lower = e.to_lowercase(); lower == "xls" || lower == "xlsx" }) .unwrap_or(false) }) .collect(); files.sort(); let dataset_label = input_dir .file_name() .and_then(|n| n.to_str()) .unwrap_or("data"); println!( "[build] {dataset_label}/ → {} ({} files)", output_path.display(), files.len() ); let conn = open_db(output_path) .with_context(|| format!("Failed to open DB: {}", output_path.display()))?; let mut total_source_rows: u64 = 0; let mut total_skipped: u64 = 0; let mut total_errors: u64 = 0; let is_old2 = dataset_label.contains("old2"); let strip_blank = cfg.reader.strip_blank_rows; conn.execute_batch("BEGIN")?; for file in &files { let base = file .file_name() .and_then(|n| n.to_str()) .unwrap_or("?") .to_owned(); let mut file_rows: u64 = 0; let mut file_skipped: u64 = 0; let mut file_errors: u64 = 0; let process_result = process_file(file, cfg, |_sheet_idx, raw| { let all_blank = is_all_blank(raw); if strip_blank && all_blank { return; } total_source_rows += 1; let ho_ten = raw .get(cfg.columns.as_ref().unwrap().ho_ten) .map(|c| c.to_string().trim().to_owned()) .unwrap_or_default(); let so_bao_danh = raw .get(cfg.columns.as_ref().unwrap().so_bao_danh) .map(|c| c.to_string().trim().to_owned()) .unwrap_or_default(); match validate_row(&ho_ten, &so_bao_danh, &cfg.validation, strip_blank, all_blank) { Err(SkipReason::BlankRow) => {} Err(_) => { file_skipped += 1; return; } Ok(()) => {} } let parsed = xlsxread::transform::transform_row(raw, cfg, &patterns); match insert_row(&conn, &parsed) { Ok(()) => file_rows += 1, Err(e) => { file_errors += 1; if total_errors + file_errors <= 5 { eprintln!(" [warn] {base}: {e}"); } } } }); match process_result { Ok(_) => {} Err(e) => { eprintln!(" [error] {base}: {e}"); file_errors += 1; } } total_skipped += file_skipped; total_errors += file_errors; println!(" {base}: {file_rows} rows"); } conn.execute_batch("COMMIT")?; finish_db( &conn, output_path, total_source_rows, total_skipped, total_errors, dataset_label, files.len(), is_old2, ) .with_context(|| "Failed to finalise DB")?; Ok(()) } // --------------------------------------------------------------------------- // thptqg2016 build path — per-file format detection // --------------------------------------------------------------------------- /// Build the thptqg2016 database. /// /// Each file is processed independently: the first row is inspected to determine /// which of the three column layouts applies (separate-scores / mapped / default). /// This mirrors `detectFormat` in scripts/build-database.js lines 63–87, called /// once per file inside the file loop at build-database.js:218–219. fn run_build_2016( cfg: &xlsxread::config::DatasetConfig, input_dir: &Path, output_path: &Path, ) -> Result<()> { let patterns = CompiledPatterns::new().with_context(|| "Failed to compile score regexes")?; let mut files: Vec = std::fs::read_dir(input_dir) .with_context(|| format!("Cannot read input dir: {}", input_dir.display()))? .filter_map(|e| e.ok()) .map(|e| e.path()) .filter(|p| { p.is_file() && p.extension() .and_then(|e| e.to_str()) .map(|e| { let lower = e.to_lowercase(); lower == "xls" || lower == "xlsx" }) .unwrap_or(false) }) .collect(); files.sort(); let dataset_label = input_dir .file_name() .and_then(|n| n.to_str()) .unwrap_or("data"); println!( "[build:2016] {dataset_label}/ → {} ({} files)", output_path.display(), files.len() ); let conn = open_db(output_path) .with_context(|| format!("Failed to open DB: {}", output_path.display()))?; let mut total_source_rows: u64 = 0; let total_skipped: u64 = 0; let mut total_errors: u64 = 0; conn.execute_batch("BEGIN")?; for file in &files { let base = file .file_name() .and_then(|n| n.to_str()) .unwrap_or("?") .to_owned(); match process_file_2016( file, cfg, &patterns, &conn, &base, &mut total_source_rows, &mut total_errors, ) { Ok(file_rows) => { println!(" {base}: {file_rows} rows"); } Err(e) => { eprintln!(" [error] {base}: {e}"); total_errors += 1; } } } conn.execute_batch("COMMIT")?; finish_db( &conn, output_path, total_source_rows, total_skipped, total_errors, dataset_label, files.len(), false, ) .with_context(|| "Failed to finalise DB")?; Ok(()) } /// Process one file in the thptqg2016 format-detection path. /// /// Reads the file, uses the first row to detect the column layout, then processes /// all subsequent data rows. Returns the count of successfully inserted rows. fn process_file_2016( file: &Path, cfg: &xlsxread::config::DatasetConfig, patterns: &CompiledPatterns, conn: &rusqlite::Connection, base: &str, total_source_rows: &mut u64, total_errors: &mut u64, ) -> Result { use calamine::{open_workbook_auto, Reader, Sheets}; let path_str = file.display().to_string(); let mut workbook: Sheets<_> = open_workbook_auto(file).with_context(|| format!("Cannot open {path_str}"))?; let sheet_names: Vec = workbook.sheet_names().to_vec(); if sheet_names.is_empty() { return Ok(0); } // Sheet selection: thptqg2016 data/ has all-sheets mode to handle // HCM/HN overflow (same reason as thptqg2017 data/). let sheets_to_read: Vec = match cfg.reader.sheet_mode { xlsxread::config::SheetMode::All => sheet_names.clone(), xlsxread::config::SheetMode::First => vec![sheet_names[0].clone()], }; let mut file_rows: u64 = 0; for sheet_name in &sheets_to_read { let range = workbook .worksheet_range(sheet_name) .with_context(|| format!("Cannot read sheet {sheet_name} in {path_str}"))?; let rows: Vec> = range.rows().map(|r| r.to_vec()).collect(); if rows.is_empty() { continue; } // Detect format from first row, then determine start index. // Mirrors build-database.js:215–220: isHeaderRow check + detectFormat. let (fmt, start_idx) = if is_header_row_2016(&rows[0]) { (detect_format(&rows[0]), 1) } else { (DetectedFormat::Default, 0) }; for row in rows.iter().skip(start_idx) { if row.len() < 2 { continue; } *total_source_rows += 1; match process_row_2016(row, &fmt, patterns) { None => { // Row was empty/invalid — skipped (mirrors JS `if (!record) continue`) } Some(parsed) => { match insert_row(conn, &parsed) { Ok(()) => file_rows += 1, Err(e) => { *total_errors += 1; if *total_errors <= 5 { eprintln!(" [warn] {base}: {e}"); } } } } } } } Ok(file_rows) }