mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-15 06:16:22 +00:00
feat: build 3 parallel site variants, one per dataset
Sites: /thptqg2017/ → data/ 861,068 students (baotintuc.vn .xls) /thptqg2017/old/ → data-old/ 847,348 students (original xlsx) /thptqg2017/old2/ → data-old2/ 679,764 students (update/ overrides) Build pipeline: - scripts/build-lib.js: shared schema, score regex, toAscii, insert builder — keeps the 3 DBs drop-in compatible for the same frontend - scripts/build-database.js: multi-sheet walk for baotintuc .xls (HN/HCM overflow past the 65k-per-sheet .xls cap) - scripts/build-database-old.js: single-sheet xlsx + strict numeric SBD guard (rejects the header-row leak present in data-old) - scripts/build-database-old2.js: multi-sheet + blank-row pre-filter for the 54 update files (HCM overflow present, HN not in this subset) - Each build prints a source-vs-DB audit; verified 0 parse errors and exact row-count match across all 3 datasets Frontend plumbing: - vite.config.js: VARIANT env selects base path, publicDir, outDir - package.json: build:db:*, build:db:all, build:old, build:old2, build:all - .github/workflows/deploy.yml: CI builds all 3 DBs + sites - .gitignore: cover public-old/ and public-old2/ outputs - eslint.config.js: add Node globals for config/script files
This commit is contained in:
1 parent
4dd376b921
commit
f82542f764
9 files changed
+396
-189
No files matched your search
Vendored
+14
-8
@@ -28,17 +28,23 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Build database
|
||||
run: npm run build:db
|
||||
- name: Build all databases
|
||||
run: npm run build:db:all
|
||||
|
||||
- name: Compress database
|
||||
run: gzip -k -9 public/thptqg2017.db
|
||||
- name: Compress databases
|
||||
run: |
|
||||
gzip -kf -9 public/thptqg2017.db
|
||||
gzip -kf -9 public-old/thptqg2017.db
|
||||
gzip -kf -9 public-old2/thptqg2017.db
|
||||
|
||||
- name: Build site
|
||||
run: npm run build
|
||||
- name: Build all site variants
|
||||
run: npm run build:all
|
||||
|
||||
- name: Remove uncompressed db from dist
|
||||
run: rm -f dist/thptqg2017.db
|
||||
- name: Drop uncompressed DB files from dist
|
||||
run: |
|
||||
rm -f dist/thptqg2017.db
|
||||
rm -f dist/old/thptqg2017.db
|
||||
rm -f dist/old2/thptqg2017.db
|
||||
|
||||
- uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
|
||||
+5
-1
@@ -12,9 +12,13 @@
|
||||
### Node ###
|
||||
node_modules/
|
||||
|
||||
### Generated database ###
|
||||
### Generated databases (all 3 variants) ###
|
||||
public/thptqg2017.db
|
||||
public/thptqg2017.db.gz
|
||||
public-old/thptqg2017.db
|
||||
public-old/thptqg2017.db.gz
|
||||
public-old2/thptqg2017.db
|
||||
public-old2/thptqg2017.db.gz
|
||||
|
||||
### Build output ###
|
||||
dist/
|
||||
@@ -26,4 +26,11 @@ export default defineConfig([
|
||||
'no-unused-vars': ['error', { varsIgnorePattern: '^[A-Z_]' }],
|
||||
},
|
||||
},
|
||||
{
|
||||
// Node-executed files (Vite config, build scripts) run with Node globals.
|
||||
files: ['vite.config.js', 'scripts/**/*.js'],
|
||||
languageOptions: {
|
||||
globals: { ...globals.node },
|
||||
},
|
||||
},
|
||||
])
|
||||
@@ -6,8 +6,14 @@
|
||||
"description": "Tra cứu điểm thi THPT QG 2017",
|
||||
"scripts": {
|
||||
"build:db": "node scripts/build-database.js",
|
||||
"build:db:old": "node scripts/build-database-old.js",
|
||||
"build:db:old2": "node scripts/build-database-old2.js",
|
||||
"build:db:all": "npm run build:db && npm run build:db:old && npm run build:db:old2",
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"build:old": "node -e \"process.env.VARIANT='old';require('child_process').spawnSync('npx',['vite','build'],{stdio:'inherit',shell:true,env:process.env})\"",
|
||||
"build:old2": "node -e \"process.env.VARIANT='old2';require('child_process').spawnSync('npx',['vite','build'],{stdio:'inherit',shell:true,env:process.env})\"",
|
||||
"build:all": "npm run build && npm run build:old && npm run build:old2",
|
||||
"preview": "vite preview",
|
||||
"lint": "eslint ."
|
||||
},
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
// Build DB from data-old/ — the original 63 xlsx files (pre-baotintuc refresh).
|
||||
// Quirks: single-sheet workbooks; some files have no header row, so we
|
||||
// isHeaderRow-test row 0 and skip only when it matches. A previous bug let a
|
||||
// HO_TEN='SOBAODANH' header leak through — that's handled here by the
|
||||
// hoTen/soBaoDanh validity check (trim + non-empty + soBaoDanh must be digits).
|
||||
import XLSX from "xlsx";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import {
|
||||
createDb,
|
||||
parseScores,
|
||||
isHeaderRow,
|
||||
buildRow,
|
||||
} from "./build-lib.js";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const SRC_DIR = path.join(__dirname, "..", "data-old");
|
||||
const DB_PATH = path.join(__dirname, "..", "public-old", "thptqg2017.db");
|
||||
|
||||
function collectFiles() {
|
||||
return fs
|
||||
.readdirSync(SRC_DIR)
|
||||
.filter((f) => f.endsWith(".xlsx") || f.endsWith(".xls"))
|
||||
.map((f) => path.join(SRC_DIR, f));
|
||||
}
|
||||
|
||||
function main() {
|
||||
const { db, insert } = createDb(DB_PATH);
|
||||
const files = collectFiles();
|
||||
let sourceRows = 0, skipped = 0, errors = 0;
|
||||
|
||||
const run = db.transaction(() => {
|
||||
for (const file of files) {
|
||||
const base = path.basename(file);
|
||||
let fileRows = 0;
|
||||
const wb = XLSX.readFile(file);
|
||||
// data-old/: always read ONLY sheet 0 — these are xlsx and never hit the
|
||||
// 65k row cap, so any additional sheet is noise.
|
||||
const rows = XLSX.utils.sheet_to_json(wb.Sheets[wb.SheetNames[0]], {
|
||||
header: 1,
|
||||
});
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
if (i === 0 && isHeaderRow(rows[i])) continue;
|
||||
sourceRows++;
|
||||
const r = rows[i];
|
||||
const hoTen = String(r?.[0] || "").trim();
|
||||
const ngaySinh = String(r?.[1] || "").trim();
|
||||
const soBaoDanh = String(r?.[2] || "").trim();
|
||||
const diemThi = String(r?.[3] || "");
|
||||
// Extra guard specific to old data: reject rows where sbd is not
|
||||
// numeric or ho_ten literally says 'HO_TEN' (header leak from the
|
||||
// prior conversion pipeline).
|
||||
if (!soBaoDanh || !hoTen) { skipped++; continue; }
|
||||
if (!/^\d+$/.test(soBaoDanh)) { skipped++; continue; }
|
||||
try {
|
||||
insert.run(buildRow({ hoTen, ngaySinh, soBaoDanh, scores: parseScores(diemThi) }));
|
||||
fileRows++;
|
||||
} catch (err) {
|
||||
errors++;
|
||||
if (errors <= 5) console.warn(` [warn] ${base}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
console.log(` ${base}: ${fileRows} rows`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`[build] data-old/ → ${DB_PATH} (${files.length} files)`);
|
||||
run();
|
||||
db.exec("VACUUM");
|
||||
|
||||
const dbCount = db.prepare("SELECT COUNT(*) c FROM student").get().c;
|
||||
console.log(`\nSource data rows (post-header): ${sourceRows}`);
|
||||
console.log(` skipped (empty/non-numeric SBD): ${skipped}`);
|
||||
console.log(` insertable: ${sourceRows - skipped}`);
|
||||
console.log(` insert errors: ${errors}`);
|
||||
console.log(`DB rows (distinct SBD): ${dbCount}`);
|
||||
const sz = fs.statSync(DB_PATH).size;
|
||||
console.log(`Size: ${(sz / 1024 / 1024).toFixed(1)} MB`);
|
||||
db.close();
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,86 @@
|
||||
// Build DB from data-old2/ — the 54 "update/" corrected-export files.
|
||||
// Quirks:
|
||||
// • 54 provinces only (no Hà Nội, Bình Phước etc.)
|
||||
// • Mostly 2 sheets (one main + one empty trailing) — safe to iterate all
|
||||
// • HCM (24.HCM_UTLQ.xlsx) overflows into Sheet2 with +6,446 students —
|
||||
// multi-sheet walk is REQUIRED for it
|
||||
// • Some headers have extended columns beyond DIEM_THI (pre-parsed per-
|
||||
// subject cols). We still use only the DIEM_THI text at index 3.
|
||||
import XLSX from "xlsx";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import {
|
||||
createDb,
|
||||
parseScores,
|
||||
isHeaderRow,
|
||||
buildRow,
|
||||
} from "./build-lib.js";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const SRC_DIR = path.join(__dirname, "..", "data-old2");
|
||||
const DB_PATH = path.join(__dirname, "..", "public-old2", "thptqg2017.db");
|
||||
|
||||
function collectFiles() {
|
||||
return fs
|
||||
.readdirSync(SRC_DIR)
|
||||
.filter((f) => f.endsWith(".xlsx") || f.endsWith(".xls"))
|
||||
.map((f) => path.join(SRC_DIR, f));
|
||||
}
|
||||
|
||||
function main() {
|
||||
const { db, insert } = createDb(DB_PATH);
|
||||
const files = collectFiles();
|
||||
let sourceRows = 0, skipped = 0, errors = 0;
|
||||
|
||||
const run = db.transaction(() => {
|
||||
for (const file of files) {
|
||||
const base = path.basename(file);
|
||||
let fileRows = 0;
|
||||
const wb = XLSX.readFile(file);
|
||||
|
||||
for (const sheetName of wb.SheetNames) {
|
||||
const rows = XLSX.utils.sheet_to_json(wb.Sheets[sheetName], {
|
||||
header: 1,
|
||||
});
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
if (i === 0 && isHeaderRow(rows[i])) continue;
|
||||
const r = rows[i];
|
||||
// Skip completely blank rows (common in xlsx tails) before counting
|
||||
if (!r || r.every((c) => c === "" || c == null)) continue;
|
||||
sourceRows++;
|
||||
const hoTen = String(r?.[0] || "").trim();
|
||||
const ngaySinh = String(r?.[1] || "").trim();
|
||||
const soBaoDanh = String(r?.[2] || "").trim();
|
||||
const diemThi = String(r?.[3] || "");
|
||||
if (!soBaoDanh || !hoTen) { skipped++; continue; }
|
||||
if (!/^\d+$/.test(soBaoDanh)) { skipped++; continue; }
|
||||
try {
|
||||
insert.run(buildRow({ hoTen, ngaySinh, soBaoDanh, scores: parseScores(diemThi) }));
|
||||
fileRows++;
|
||||
} catch (err) {
|
||||
errors++;
|
||||
if (errors <= 5) console.warn(` [warn] ${base}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(` ${base}: ${fileRows} rows`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`[build] data-old2/ → ${DB_PATH} (${files.length} files)`);
|
||||
run();
|
||||
db.exec("VACUUM");
|
||||
|
||||
const dbCount = db.prepare("SELECT COUNT(*) c FROM student").get().c;
|
||||
console.log(`\nSource non-blank data rows: ${sourceRows}`);
|
||||
console.log(` skipped (empty/non-numeric SBD): ${skipped}`);
|
||||
console.log(` insertable: ${sourceRows - skipped}`);
|
||||
console.log(` insert errors: ${errors}`);
|
||||
console.log(`DB rows (distinct SBD): ${dbCount}`);
|
||||
const sz = fs.statSync(DB_PATH).size;
|
||||
console.log(`Size: ${(sz / 1024 / 1024).toFixed(1)} MB`);
|
||||
db.close();
|
||||
}
|
||||
|
||||
main();
|
||||
+59
-179
@@ -1,209 +1,89 @@
|
||||
// Build DB from data/ (baotintuc.vn .xls source).
|
||||
// Quirks: .xls has a 65,536-row-per-sheet limit. Hà Nội and HCM overflow into
|
||||
// Sheet2, so we MUST iterate every sheet. Header row may or may not be
|
||||
// present on each sheet.
|
||||
import XLSX from "xlsx";
|
||||
import Database from "better-sqlite3";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import {
|
||||
createDb,
|
||||
parseScores,
|
||||
isHeaderRow,
|
||||
buildRow,
|
||||
} from "./build-lib.js";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const RAW_DIR = path.join(__dirname, "..", "data");
|
||||
const SRC_DIR = path.join(__dirname, "..", "data");
|
||||
const DB_PATH = path.join(__dirname, "..", "public", "thptqg2017.db");
|
||||
|
||||
// Score patterns. Note: source data uses both "X.YZ" and "X" forms; allow optional decimal.
|
||||
const NUM = "(\\d+(?:\\.\\d+)?)";
|
||||
const SCORE_PATTERNS = {
|
||||
toan: new RegExp("Toán:\\s*" + NUM),
|
||||
ngu_van: new RegExp("Ngữ văn:\\s*" + NUM),
|
||||
vat_ly: new RegExp("Vật lí:\\s*" + NUM),
|
||||
hoa_hoc: new RegExp("Hóa học:\\s*" + NUM),
|
||||
sinh_hoc: new RegExp("Sinh học:\\s*" + NUM),
|
||||
khtn: new RegExp("KHTN:\\s*" + NUM),
|
||||
lich_su: new RegExp("Lịch sử:\\s*" + NUM),
|
||||
dia_ly: new RegExp("Địa lí:\\s*" + NUM),
|
||||
gdcd: new RegExp("GDCD:\\s*" + NUM),
|
||||
khxh: new RegExp("KHXH:\\s*" + NUM),
|
||||
tieng_anh: new RegExp("Tiếng Anh:\\s*" + NUM),
|
||||
tieng_phap: new RegExp("Tiếng Pháp:\\s*" + NUM),
|
||||
tieng_nga: new RegExp("Tiếng Nga:\\s*" + NUM),
|
||||
tieng_trung: new RegExp("Tiếng Trung:\\s*" + NUM),
|
||||
};
|
||||
|
||||
// Strip Vietnamese diacritics for ASCII-insensitive search
|
||||
function toAscii(str) {
|
||||
return str
|
||||
.normalize("NFD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.replace(/đ/gi, "d")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
// Collect all .xls and .xlsx files directly under data/
|
||||
function collectExcelFiles() {
|
||||
const files = [];
|
||||
for (const f of fs.readdirSync(RAW_DIR)) {
|
||||
const full = path.join(RAW_DIR, f);
|
||||
if (!fs.statSync(full).isFile()) continue;
|
||||
if (f.endsWith(".xls") || f.endsWith(".xlsx")) files.push(full);
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
// Parse score text into an object of score fields
|
||||
function parseScores(diemThi) {
|
||||
const scores = {};
|
||||
for (const [field, pattern] of Object.entries(SCORE_PATTERNS)) {
|
||||
const match = diemThi.match(pattern);
|
||||
if (match) {
|
||||
scores[field] = parseFloat(match[1]);
|
||||
}
|
||||
}
|
||||
return scores;
|
||||
}
|
||||
|
||||
// Detect if first row is a header (not actual student data)
|
||||
function isHeaderRow(row) {
|
||||
if (!row || row.length < 3) return false;
|
||||
const first = String(row[0] || "").toUpperCase();
|
||||
return first === "HO_TEN" || first === "HỌ TÊN" || first === "STT";
|
||||
function collectFiles() {
|
||||
return fs
|
||||
.readdirSync(SRC_DIR)
|
||||
.filter((f) => f.endsWith(".xls") || f.endsWith(".xlsx"))
|
||||
.map((f) => path.join(SRC_DIR, f));
|
||||
}
|
||||
|
||||
function main() {
|
||||
// Ensure output directory exists
|
||||
fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
|
||||
const { db, insert } = createDb(DB_PATH);
|
||||
|
||||
// Remove old database if exists
|
||||
if (fs.existsSync(DB_PATH)) fs.unlinkSync(DB_PATH);
|
||||
const files = collectFiles();
|
||||
let sourceRows = 0; // total data rows observed across ALL sheets (post-header)
|
||||
let skipped = 0; // empty/invalid rows skipped
|
||||
let errors = 0;
|
||||
|
||||
const db = new Database(DB_PATH);
|
||||
|
||||
// Create table
|
||||
db.exec(`
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
khtn REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
gdcd REAL,
|
||||
khxh REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_nga REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
`);
|
||||
|
||||
const insert = db.prepare(`
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_trung)
|
||||
VALUES
|
||||
(@so_bao_danh, @ho_ten, @ho_ten_ascii, @ngay_sinh,
|
||||
@toan, @ngu_van, @vat_ly, @hoa_hoc, @sinh_hoc, @khtn,
|
||||
@lich_su, @dia_ly, @gdcd, @khxh,
|
||||
@tieng_anh, @tieng_phap, @tieng_nga, @tieng_trung)
|
||||
`);
|
||||
|
||||
const files = collectExcelFiles();
|
||||
let errorCount = 0;
|
||||
|
||||
// Wrap all inserts in a single transaction for speed
|
||||
const insertAll = db.transaction((files) => {
|
||||
const run = db.transaction(() => {
|
||||
for (const file of files) {
|
||||
const basename = path.basename(file);
|
||||
const base = path.basename(file);
|
||||
let fileRows = 0;
|
||||
const wb = XLSX.readFile(file);
|
||||
|
||||
try {
|
||||
const wb = XLSX.readFile(file);
|
||||
// Iterate ALL sheets — .xls files over 65,536 rows (Hà Nội, HCM) split
|
||||
// into continuation sheets that we must not miss.
|
||||
const allRows = [];
|
||||
for (const sheetName of wb.SheetNames) {
|
||||
const rows = XLSX.utils.sheet_to_json(wb.Sheets[sheetName], {
|
||||
header: 1,
|
||||
});
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
// Skip header row on every sheet (continuation sheets may repeat it,
|
||||
// or start directly with data — isHeaderRow handles both)
|
||||
if (i === 0 && isHeaderRow(rows[i])) continue;
|
||||
allRows.push(rows[i]);
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < allRows.length; i++) {
|
||||
const row = allRows[i];
|
||||
for (const sheetName of wb.SheetNames) {
|
||||
const rows = XLSX.utils.sheet_to_json(wb.Sheets[sheetName], {
|
||||
header: 1,
|
||||
});
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
if (i === 0 && isHeaderRow(rows[i])) continue;
|
||||
sourceRows++;
|
||||
const r = rows[i];
|
||||
const hoTen = String(r?.[0] || "").trim();
|
||||
const ngaySinh = String(r?.[1] || "").trim();
|
||||
const soBaoDanh = String(r?.[2] || "").trim();
|
||||
const diemThi = String(r?.[3] || "");
|
||||
if (!soBaoDanh || !hoTen) { skipped++; continue; }
|
||||
try {
|
||||
const hoTen = String(row?.[0] || "").trim();
|
||||
const ngaySinh = String(row?.[1] || "").trim();
|
||||
const soBaoDanh = String(row?.[2] || "").trim();
|
||||
const diemThi = String(row?.[3] || "");
|
||||
|
||||
// Skip truly empty rows (common tail padding in .xls)
|
||||
if (!soBaoDanh || !hoTen) continue;
|
||||
|
||||
const scores = parseScores(diemThi);
|
||||
|
||||
insert.run({
|
||||
so_bao_danh: soBaoDanh,
|
||||
ho_ten: hoTen,
|
||||
ho_ten_ascii: toAscii(hoTen),
|
||||
ngay_sinh: ngaySinh || null,
|
||||
toan: scores.toan ?? null,
|
||||
ngu_van: scores.ngu_van ?? null,
|
||||
vat_ly: scores.vat_ly ?? null,
|
||||
hoa_hoc: scores.hoa_hoc ?? null,
|
||||
sinh_hoc: scores.sinh_hoc ?? null,
|
||||
khtn: scores.khtn ?? null,
|
||||
lich_su: scores.lich_su ?? null,
|
||||
dia_ly: scores.dia_ly ?? null,
|
||||
gdcd: scores.gdcd ?? null,
|
||||
khxh: scores.khxh ?? null,
|
||||
tieng_anh: scores.tieng_anh ?? null,
|
||||
tieng_phap: scores.tieng_phap ?? null,
|
||||
tieng_nga: scores.tieng_nga ?? null,
|
||||
tieng_trung: scores.tieng_trung ?? null,
|
||||
});
|
||||
insert.run(buildRow({ hoTen, ngaySinh, soBaoDanh, scores: parseScores(diemThi) }));
|
||||
fileRows++;
|
||||
} catch (err) {
|
||||
errorCount++;
|
||||
if (errorCount <= 5) {
|
||||
console.warn(` [warn] ${basename} row ${i}: ${err.message}`);
|
||||
}
|
||||
errors++;
|
||||
if (errors <= 5) console.warn(` [warn] ${base}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Failed to read ${basename}: ${err.message}`);
|
||||
}
|
||||
|
||||
console.log(` ${basename}: ${fileRows} rows`);
|
||||
console.log(` ${base}: ${fileRows} rows`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`Processing ${files.length} Excel files...\n`);
|
||||
insertAll(files);
|
||||
|
||||
// Compact the database
|
||||
console.log(`[build] data/ → ${DB_PATH} (${files.length} files)`);
|
||||
run();
|
||||
db.exec("VACUUM");
|
||||
|
||||
const count = db.prepare("SELECT COUNT(*) as cnt FROM student").get();
|
||||
console.log(`\nDone! ${count.cnt} students in database.`);
|
||||
console.log(`Errors skipped: ${errorCount}`);
|
||||
console.log(`Output: ${DB_PATH}`);
|
||||
|
||||
const stat = fs.statSync(DB_PATH);
|
||||
console.log(`Size: ${(stat.size / 1024 / 1024).toFixed(1)} MB`);
|
||||
|
||||
const dbCount = db.prepare("SELECT COUNT(*) c FROM student").get().c;
|
||||
const distinctSbd = sourceRows - skipped;
|
||||
console.log(`\nSource data rows (post-header): ${sourceRows}`);
|
||||
console.log(` skipped (empty/invalid): ${skipped}`);
|
||||
console.log(` insertable: ${distinctSbd}`);
|
||||
console.log(` insert errors: ${errors}`);
|
||||
console.log(`DB rows (distinct SBD): ${dbCount}`);
|
||||
// data/ comes from a single source, so every SBD should be unique after
|
||||
// header skip — no duplicates expected.
|
||||
if (dbCount !== distinctSbd - 0 && errors === 0) {
|
||||
const gap = distinctSbd - dbCount;
|
||||
if (gap === 0) console.log(`Audit: OK — every source row made it in.`);
|
||||
else console.log(`Audit: ${gap} row(s) collapsed (duplicate SBDs overwriting).`);
|
||||
}
|
||||
const sz = fs.statSync(DB_PATH).size;
|
||||
console.log(`Size: ${(sz / 1024 / 1024).toFixed(1)} MB`);
|
||||
db.close();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
// Shared SQLite schema + score-text helpers for all 3 build pipelines.
|
||||
// The parse LOOP for each dataset lives in its own build-database-*.js file
|
||||
// (so source-specific quirks stay isolated), but the DB shape and regex
|
||||
// dictionary are centralised here so the 3 DBs stay drop-in compatible with
|
||||
// the same frontend.
|
||||
import Database from "better-sqlite3";
|
||||
import fs from "fs";
|
||||
|
||||
const NUM = "(\\d+(?:\\.\\d+)?)";
|
||||
export const SCORE_PATTERNS = {
|
||||
toan: new RegExp("Toán:\\s*" + NUM),
|
||||
ngu_van: new RegExp("Ngữ văn:\\s*" + NUM),
|
||||
vat_ly: new RegExp("Vật lí:\\s*" + NUM),
|
||||
hoa_hoc: new RegExp("Hóa học:\\s*" + NUM),
|
||||
sinh_hoc: new RegExp("Sinh học:\\s*" + NUM),
|
||||
khtn: new RegExp("KHTN:\\s*" + NUM),
|
||||
lich_su: new RegExp("Lịch sử:\\s*" + NUM),
|
||||
dia_ly: new RegExp("Địa lí:\\s*" + NUM),
|
||||
gdcd: new RegExp("GDCD:\\s*" + NUM),
|
||||
khxh: new RegExp("KHXH:\\s*" + NUM),
|
||||
tieng_anh: new RegExp("Tiếng Anh:\\s*" + NUM),
|
||||
tieng_phap: new RegExp("Tiếng Pháp:\\s*" + NUM),
|
||||
tieng_nga: new RegExp("Tiếng Nga:\\s*" + NUM),
|
||||
tieng_trung: new RegExp("Tiếng Trung:\\s*" + NUM),
|
||||
};
|
||||
|
||||
export function parseScores(diemThi) {
|
||||
const out = {};
|
||||
for (const [k, re] of Object.entries(SCORE_PATTERNS)) {
|
||||
const m = diemThi.match(re);
|
||||
if (m) out[k] = parseFloat(m[1]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function toAscii(str) {
|
||||
return str
|
||||
.normalize("NFD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.replace(/đ/gi, "d")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
export function isHeaderRow(row) {
|
||||
if (!row || row.length < 3) return false;
|
||||
const first = String(row[0] || "").toUpperCase();
|
||||
return first === "HO_TEN" || first === "HỌ TÊN" || first === "STT";
|
||||
}
|
||||
|
||||
// Create empty DB file at dbPath with the canonical schema + prepared insert.
|
||||
// Caller owns the returned handles and must close them.
|
||||
export function createDb(dbPath) {
|
||||
fs.mkdirSync(dbPath.replace(/[^/\\]+$/, ""), { recursive: true });
|
||||
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
|
||||
const db = new Database(dbPath);
|
||||
db.exec(`
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
khtn REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
gdcd REAL,
|
||||
khxh REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_nga REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
`);
|
||||
const insert = db.prepare(`
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_trung)
|
||||
VALUES
|
||||
(@so_bao_danh, @ho_ten, @ho_ten_ascii, @ngay_sinh,
|
||||
@toan, @ngu_van, @vat_ly, @hoa_hoc, @sinh_hoc, @khtn,
|
||||
@lich_su, @dia_ly, @gdcd, @khxh,
|
||||
@tieng_anh, @tieng_phap, @tieng_nga, @tieng_trung)
|
||||
`);
|
||||
return { db, insert };
|
||||
}
|
||||
|
||||
// Build the @named params object the INSERT expects
|
||||
export function buildRow({ hoTen, ngaySinh, soBaoDanh, scores }) {
|
||||
return {
|
||||
so_bao_danh: soBaoDanh,
|
||||
ho_ten: hoTen,
|
||||
ho_ten_ascii: toAscii(hoTen),
|
||||
ngay_sinh: ngaySinh || null,
|
||||
toan: scores.toan ?? null,
|
||||
ngu_van: scores.ngu_van ?? null,
|
||||
vat_ly: scores.vat_ly ?? null,
|
||||
hoa_hoc: scores.hoa_hoc ?? null,
|
||||
sinh_hoc: scores.sinh_hoc ?? null,
|
||||
khtn: scores.khtn ?? null,
|
||||
lich_su: scores.lich_su ?? null,
|
||||
dia_ly: scores.dia_ly ?? null,
|
||||
gdcd: scores.gdcd ?? null,
|
||||
khxh: scores.khxh ?? null,
|
||||
tieng_anh: scores.tieng_anh ?? null,
|
||||
tieng_phap: scores.tieng_phap ?? null,
|
||||
tieng_nga: scores.tieng_nga ?? null,
|
||||
tieng_trung: scores.tieng_trung ?? null,
|
||||
};
|
||||
}
|
||||
+19
-1
@@ -1,7 +1,25 @@
|
||||
import { defineConfig } from "vite";
|
||||
import react from "@vitejs/plugin-react";
|
||||
|
||||
// VARIANT selects which dataset to ship:
|
||||
// (unset) → main site at /thptqg2017/ using public/
|
||||
// old → at /thptqg2017/old/ using public-old/
|
||||
// old2 → at /thptqg2017/old2/ using public-old2/
|
||||
const VARIANT = process.env.VARIANT || "";
|
||||
const VARIANT_CONFIG = {
|
||||
"": { base: "/thptqg2017/", publicDir: "public", outDir: "dist" },
|
||||
old: { base: "/thptqg2017/old/", publicDir: "public-old", outDir: "dist/old" },
|
||||
old2: { base: "/thptqg2017/old2/", publicDir: "public-old2", outDir: "dist/old2" },
|
||||
};
|
||||
const cfg = VARIANT_CONFIG[VARIANT];
|
||||
if (!cfg) throw new Error(`Unknown VARIANT: ${VARIANT}`);
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
base: "/thptqg2017/",
|
||||
base: cfg.base,
|
||||
publicDir: cfg.publicDir,
|
||||
build: {
|
||||
outDir: cfg.outDir,
|
||||
emptyOutDir: VARIANT === "", // only main build wipes dist root
|
||||
},
|
||||
});
|
||||
Reference in new issue
Block a user