refactor: flatten data layout to data/, drop update/ overrides

- Move 63 Excel files from data/raw/ to data/ (single flat dir)
- Remove all 53 files in data/raw/update/: verified identical SBD
  coverage to raw/ (847349 rows either way), so they added no new
  students — only potential score corrections that can be reintroduced
  later if source is recovered
- Update build-database.js to read data/ directly
- Add scripts/audit-row-counts.js: compares source row count to DB row
  count to verify zero-loss parsing
- Point check-duplicates.js at new data/ location
This commit is contained in:
tiennm99 committed 2026-04-14 21:02:47 +07:00
1 parent c1cd39e535
commit e165e3c5b6
120 files changed
+66 -20

No files matched your search

+2 -16
View File
@@ -6,7 +6,7 @@ import { fileURLToPath } from "url";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const RAW_DIR = path.join(__dirname, "..", "data", "raw");
const RAW_DIR = path.join(__dirname, "..", "data");
const DB_PATH = path.join(__dirname, "..", "public", "thptqg2017.db");
// Score patterns. Note: source data uses both "X.YZ" and "X" forms; allow optional decimal.
@@ -37,29 +37,15 @@ function toAscii(str) {
.toLowerCase();
}
// Collect all .xlsx files: raw/ first, then raw/(update)/ to overwrite
// Collect all .xlsx files directly under data/
function collectExcelFiles() {
const files = [];
// Main raw files first
for (const f of fs.readdirSync(RAW_DIR)) {
const full = path.join(RAW_DIR, f);
if (fs.statSync(full).isFile() && f.endsWith(".xlsx")) {
files.push(full);
}
}
// Update files second (will overwrite via INSERT OR REPLACE)
const updateDir = path.join(RAW_DIR, "update");
if (fs.existsSync(updateDir)) {
for (const f of fs.readdirSync(updateDir)) {
const full = path.join(updateDir, f);
if (fs.statSync(full).isFile() && f.endsWith(".xlsx")) {
files.push(full);
}
}
}
return files;
}