mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-18 00:15:40 +00:00
refactor: flatten data layout to data/, drop update/ overrides
- Move 63 Excel files from data/raw/ to data/ (single flat dir) - Remove all 53 files in data/raw/update/: verified identical SBD coverage to raw/ (847349 rows either way), so they added no new students — only potential score corrections that can be reintroduced later if source is recovered - Update build-database.js to read data/ directly - Add scripts/audit-row-counts.js: compares source row count to DB row count to verify zero-loss parsing - Point check-duplicates.js at new data/ location
This commit is contained in:
1 parent
c1cd39e535
commit
e165e3c5b6
120 files changed
+66
-20
No files matched your search
@@ -6,7 +6,7 @@ import { fileURLToPath } from "url";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const RAW_DIR = path.join(__dirname, "..", "data", "raw");
|
||||
const RAW_DIR = path.join(__dirname, "..", "data");
|
||||
const DB_PATH = path.join(__dirname, "..", "public", "thptqg2017.db");
|
||||
|
||||
// Score patterns. Note: source data uses both "X.YZ" and "X" forms; allow optional decimal.
|
||||
@@ -37,29 +37,15 @@ function toAscii(str) {
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
// Collect all .xlsx files: raw/ first, then raw/(update)/ to overwrite
|
||||
// Collect all .xlsx files directly under data/
|
||||
function collectExcelFiles() {
|
||||
const files = [];
|
||||
|
||||
// Main raw files first
|
||||
for (const f of fs.readdirSync(RAW_DIR)) {
|
||||
const full = path.join(RAW_DIR, f);
|
||||
if (fs.statSync(full).isFile() && f.endsWith(".xlsx")) {
|
||||
files.push(full);
|
||||
}
|
||||
}
|
||||
|
||||
// Update files second (will overwrite via INSERT OR REPLACE)
|
||||
const updateDir = path.join(RAW_DIR, "update");
|
||||
if (fs.existsSync(updateDir)) {
|
||||
for (const f of fs.readdirSync(updateDir)) {
|
||||
const full = path.join(updateDir, f);
|
||||
if (fs.statSync(full).isFile() && f.endsWith(".xlsx")) {
|
||||
files.push(full);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user