mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-11 03:13:48 +00:00
Merge pull request #5 from tiennm99/refactor/unify-frontend-and-schema
refactor: one frontend, one parser, one schema
This commit is contained in:
421 files changed
+14450
-9833
No files matched your search
@@ -24,57 +24,29 @@ jobs:
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: |
|
||||
2016/tools/xlsxread
|
||||
2017/tools/xlsxread
|
||||
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
package_json_file: 2017/package.json
|
||||
workspaces: parser
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '24'
|
||||
cache: 'pnpm'
|
||||
cache-dependency-path: |
|
||||
2016/pnpm-lock.yaml
|
||||
2017/pnpm-lock.yaml
|
||||
cache: 'npm'
|
||||
cache-dependency-path: package-lock.json
|
||||
|
||||
- name: Build 2016 site
|
||||
working-directory: '2016'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo build --release --manifest-path tools/xlsxread/Cargo.toml
|
||||
./tools/xlsxread/target/release/xlsxread build \
|
||||
--schema tools/xlsxread/configs/thptqg2016-data.toml \
|
||||
--input data --output public/thptqg2016.db
|
||||
pnpm install --frozen-lockfile
|
||||
gzip -k -9 public/thptqg2016.db
|
||||
pnpm build
|
||||
rm -f dist/thptqg2016.db
|
||||
- run: npm ci
|
||||
|
||||
- name: Build 2017 site variants
|
||||
working-directory: '2017'
|
||||
# One parser binary builds every dataset; build-db.js reads the dataset
|
||||
# list from src/datasets.js and gzips each database in place, leaving no
|
||||
# uncompressed file behind.
|
||||
- name: Build databases
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo build --release --manifest-path tools/xlsxread/Cargo.toml
|
||||
./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data.toml --input data --output public/thptqg2017.db
|
||||
./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data-old.toml --input data-old --output public-old/thptqg2017.db
|
||||
./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data-old2.toml --input data-old2 --output public-old2/thptqg2017.db
|
||||
pnpm install --frozen-lockfile
|
||||
gzip -kf -9 public/thptqg2017.db
|
||||
gzip -kf -9 public-old/thptqg2017.db
|
||||
gzip -kf -9 public-old2/thptqg2017.db
|
||||
pnpm build:all
|
||||
rm -f dist/thptqg2017.db dist/old/thptqg2017.db dist/old2/thptqg2017.db
|
||||
npm run build:rust
|
||||
npm run build:db
|
||||
|
||||
- name: Assemble site
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p _site
|
||||
cp index.html _site/
|
||||
cp -r 2016/dist _site/2016
|
||||
cp -r 2017/dist _site/2017
|
||||
# One Vite build produces every page. scripts/assemble-site.js copies the
|
||||
# emitted index.html to each dataset path (and the legacy nested URLs),
|
||||
# then fails the job if any uncompressed database reached the artifact.
|
||||
- name: Build and assemble site
|
||||
run: npm run build:site
|
||||
|
||||
- uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
### Editors ###
|
||||
.idea
|
||||
*.iws
|
||||
*.iml
|
||||
*.ipr
|
||||
.vscode/
|
||||
|
||||
### Mac OS ###
|
||||
.DS_Store
|
||||
|
||||
### Node ###
|
||||
node_modules/
|
||||
|
||||
### Build output ###
|
||||
dist/
|
||||
|
||||
### Generated databases and Vite publicDir staging ###
|
||||
.build/
|
||||
|
||||
### Rust build artefacts ###
|
||||
parser/target/
|
||||
|
||||
### Assembled Pages artifact ###
|
||||
_site/
|
||||
-146
@@ -1,146 +0,0 @@
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||
|
||||
# Runtime data
|
||||
pids
|
||||
*.pid
|
||||
*.seed
|
||||
*.pid.lock
|
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||
lib-cov
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# nyc test coverage
|
||||
.nyc_output
|
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||
.grunt
|
||||
|
||||
# Bower dependency directory (https://bower.io/)
|
||||
bower_components
|
||||
|
||||
# node-waf configuration
|
||||
.lock-wscript
|
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||
build/Release
|
||||
|
||||
# Dependency directories
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
|
||||
# Snowpack dependency directory (https://snowpack.dev/)
|
||||
web_modules/
|
||||
|
||||
# TypeScript cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# Optional npm cache directory
|
||||
.npm
|
||||
|
||||
# Optional eslint cache
|
||||
.eslintcache
|
||||
|
||||
# Optional stylelint cache
|
||||
.stylelintcache
|
||||
|
||||
# Optional REPL history
|
||||
.node_repl_history
|
||||
|
||||
# Output of 'npm pack'
|
||||
*.tgz
|
||||
|
||||
# Yarn Integrity file
|
||||
.yarn-integrity
|
||||
|
||||
# dotenv environment variable files
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
.cache
|
||||
.parcel-cache
|
||||
|
||||
# Next.js build output
|
||||
.next
|
||||
out
|
||||
|
||||
# Nuxt.js build / generate output
|
||||
.nuxt
|
||||
dist
|
||||
|
||||
# Gatsby files
|
||||
.cache/
|
||||
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||
# public
|
||||
|
||||
# vuepress build output
|
||||
.vuepress/dist
|
||||
|
||||
# vuepress v2.x temp and cache directory
|
||||
.temp
|
||||
.cache
|
||||
|
||||
# Sveltekit cache directory
|
||||
.svelte-kit/
|
||||
|
||||
# vitepress build output
|
||||
**/.vitepress/dist
|
||||
|
||||
# vitepress cache directory
|
||||
**/.vitepress/cache
|
||||
|
||||
# Docusaurus cache and generated files
|
||||
.docusaurus
|
||||
|
||||
# Serverless directories
|
||||
.serverless/
|
||||
|
||||
# FuseBox cache
|
||||
.fusebox/
|
||||
|
||||
# DynamoDB Local files
|
||||
.dynamodb/
|
||||
|
||||
# Firebase cache directory
|
||||
.firebase/
|
||||
|
||||
# TernJS port file
|
||||
.tern-port
|
||||
|
||||
# Stores VSCode versions used for testing VSCode extensions
|
||||
.vscode-test
|
||||
|
||||
# yarn v3
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/sdks
|
||||
!.yarn/versions
|
||||
|
||||
# Vite logs files
|
||||
vite.config.js.timestamp-*
|
||||
vite.config.ts.timestamp-*
|
||||
|
||||
# Generated database files
|
||||
public/*.db
|
||||
public/*.db.gz
|
||||
|
||||
# Rust build artifacts
|
||||
tools/xlsxread/target/
|
||||
@@ -1,57 +0,0 @@
|
||||
# thptqg2016
|
||||
|
||||
Lookup tool for Vietnam's 2016 National High School Graduation Exam (THPT Quốc gia) scores — 877,461 candidates nationwide.
|
||||
|
||||
Fully static app running entirely in the browser (SQLite via `sql.js`). No backend, no query logging.
|
||||
|
||||
## Features
|
||||
|
||||
- **Quick lookup** by exam ID (`số báo danh`) or full name (diacritics-insensitive)
|
||||
- **Custom read-only SQL** (SELECT / PRAGMA / EXPLAIN / WITH) with 7 built-in preset queries
|
||||
- **Complete data**: 12 subjects (Math, Literature, Physics, Chemistry, Biology, History, Geography, English, French, German, Japanese, Chinese), exam cluster, date of birth, gender
|
||||
- Safety caps: 100 rows for lookup, 1000 rows for custom SQL
|
||||
- Dark mode, `Ctrl+Enter` shortcut to run queries
|
||||
|
||||
## Demo
|
||||
|
||||
<https://tiennm99.github.io/thptqg/2016/>
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
# Build the SQLite database from source Excel files (requires Rust stable)
|
||||
pnpm run build:db
|
||||
|
||||
# Or build in two steps:
|
||||
pnpm run build:rust # compile the xlsxread binary
|
||||
./tools/xlsxread/target/release/xlsxread build \
|
||||
--schema tools/xlsxread/configs/thptqg2016-data.toml \
|
||||
--input data \
|
||||
--output public/thptqg2016.db
|
||||
|
||||
pnpm run dev # Vite dev server
|
||||
pnpm run build # Production bundle → dist/
|
||||
pnpm run lint # ESLint
|
||||
```
|
||||
|
||||
The database is built by the `xlsxread` Rust binary (`tools/xlsxread/`), which
|
||||
reads the 119 mixed `.xls`/`.xlsx` files and auto-detects the column layout per
|
||||
file (`separate-scores`, `mapped`, or positional default). No Node.js Excel
|
||||
library is required at build time.
|
||||
|
||||
The GitHub Actions workflow (`.github/workflows/deploy.yml`) compiles
|
||||
`xlsxread`, builds the DB, gzips it, and deploys to GitHub Pages on every push
|
||||
to `main`.
|
||||
|
||||
## Tech stack
|
||||
|
||||
React 19 · Vite · sql.js (WASM) · xlsxread (Rust, build-time) · GitHub Pages
|
||||
|
||||
## Documentation
|
||||
|
||||
- [Project overview (PDR)](./docs/project-overview-pdr.md)
|
||||
- [Codebase summary](./docs/codebase-summary.md)
|
||||
- [System architecture](./docs/system-architecture.md)
|
||||
- [Deployment guide](./docs/deployment-guide.md)
|
||||
|
||||
**Source**: Originally published at <https://dtntbacgiang.edu.vn/tin-tuc/tin-tuc-su-kien/cong-bo-diem-thi-thptqg-2016-toan-bo-120-cum-thi-da-co-diem.html> (currently inaccessible). Data is for reference only.
|
||||
@@ -1,63 +0,0 @@
|
||||
# Codebase Summary
|
||||
|
||||
## Directory layout
|
||||
|
||||
```
|
||||
thptqg2016/
|
||||
├── data/ # Source Excel files (~100, mixed formats)
|
||||
├── scripts/
|
||||
│ └── build-database.js # Parse Excel → SQLite (build-time, Node + better-sqlite3)
|
||||
├── public/
|
||||
│ └── thptqg2016.db # Generated DB, gzipped during CI
|
||||
├── src/
|
||||
│ ├── main.jsx # React entry
|
||||
│ ├── App.jsx # Root: tabs, lookup logic, useSqlite wiring
|
||||
│ ├── App.css / index.css # Design tokens, dark mode, a11y styles
|
||||
│ ├── hooks/
|
||||
│ │ └── use-sqlite.js # Fetch .db.gz + decompress + init sql.js
|
||||
│ └── components/
|
||||
│ ├── search-form.jsx # Input for exam ID / full name
|
||||
│ ├── score-table.jsx # Result table for lookups
|
||||
│ └── custom-query.jsx # SQL editor + presets + result grid
|
||||
├── .github/workflows/deploy.yml # CI: build db → gzip → vite build → Pages
|
||||
├── vite.config.js # base: "/thptqg2016/"
|
||||
└── eslint.config.js
|
||||
```
|
||||
|
||||
## Key modules
|
||||
|
||||
### `scripts/build-database.js`
|
||||
Build-time only. Reads every `.xlsx/.xls` in `data/`, detects the header format (three variants), parses the `DIEM_THI` string via regex or separate score columns, normalizes gender, derives a diacritics-stripped `ho_ten_ascii` column for accent-insensitive search, and inserts into SQLite with three indexes (`ho_ten`, `ho_ten_ascii`, `ten_cum_thi`).
|
||||
|
||||
### `src/hooks/use-sqlite.js`
|
||||
Streams `.db.gz` with download progress, decompresses via `DecompressionStream("gzip")`, loads `sql.js` (WASM served from the `sql.js.org` CDN), and returns `{ db, loading, error, progress }`.
|
||||
|
||||
### `src/App.jsx`
|
||||
Two tabs: **Lookup** and **Custom SQL**. Lookup auto-detects exam IDs (regex `^[A-Z]{2,4}\d+$`) vs names and picks one of three query paths: exact exam ID / ASCII LIKE / original + ASCII LIKE. Capped at 100 rows.
|
||||
|
||||
### `src/components/custom-query.jsx`
|
||||
Whitelists leading keywords (`SELECT`, `PRAGMA`, `EXPLAIN`, `WITH`), auto-appends `LIMIT 1000` when missing, measures `performance.now()` execution time, and ships 7 preset analytics queries.
|
||||
|
||||
## `student` table schema
|
||||
|
||||
```sql
|
||||
so_bao_danh TEXT PRIMARY KEY -- exam ID
|
||||
ho_ten TEXT NOT NULL -- full name
|
||||
ho_ten_ascii TEXT NOT NULL -- diacritics stripped, lowercased
|
||||
ngay_sinh TEXT -- date of birth
|
||||
ten_cum_thi TEXT -- exam cluster name
|
||||
gioi_tinh TEXT -- "Nam" | "Nữ" | NULL
|
||||
toan, ngu_van, vat_ly, hoa_hoc, -- REAL (nullable) subject scores
|
||||
sinh_hoc, lich_su, dia_ly,
|
||||
tieng_anh, tieng_phap, tieng_duc,
|
||||
tieng_nhat, tieng_trung
|
||||
```
|
||||
|
||||
Indexes: `idx_ho_ten`, `idx_ho_ten_ascii`, `idx_ten_cum_thi`.
|
||||
|
||||
## Conventions
|
||||
|
||||
- JS/JSX filenames: **kebab-case** (e.g., `search-form.jsx`, `use-sqlite.js`)
|
||||
- React components: `PascalCase` named exports
|
||||
- UI strings: Vietnamese (target audience)
|
||||
- Code comments: English; explain *why*, not *what*
|
||||
@@ -1,50 +0,0 @@
|
||||
# Deployment Guide
|
||||
|
||||
## Automatic (recommended)
|
||||
|
||||
Push to `main` → GitHub Actions builds and deploys to GitHub Pages automatically.
|
||||
|
||||
Workflow: `.github/workflows/deploy.yml`
|
||||
|
||||
CI steps:
|
||||
1. `npm ci`
|
||||
2. `npm run build:db` — generate `public/thptqg2016.db` from `data/*.xlsx`
|
||||
3. `gzip -k -9 public/thptqg2016.db` — max compression, keep original
|
||||
4. `npm run build` — Vite bundles `dist/`
|
||||
5. `rm -f dist/thptqg2016.db` — ship only the gzipped copy
|
||||
6. `actions/upload-pages-artifact@v3` + `actions/deploy-pages@v4`
|
||||
|
||||
One-time setup: **Settings → Pages → Source: GitHub Actions**.
|
||||
|
||||
## Manual (local verification)
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run build:db
|
||||
gzip -k -9 public/thptqg2016.db # Linux/macOS; on Windows use 7zip or WSL
|
||||
npm run build
|
||||
rm dist/thptqg2016.db # optional, shrinks artifact
|
||||
npm run preview # serve dist/ locally
|
||||
```
|
||||
|
||||
Open <http://localhost:4173/thptqg2016/>.
|
||||
|
||||
## Base path
|
||||
|
||||
`vite.config.js` sets `base: "/thptqg2016/"`. If you fork under a different repo name, update this to match `<repo-name>` so assets resolve correctly on GitHub Pages.
|
||||
|
||||
## Updating data
|
||||
|
||||
1. Add the new Excel file to `data/`.
|
||||
2. If its header is unfamiliar, open `scripts/build-database.js` and extend `detectFormat()` or the `processSeparateScoresRow` / `processMappedRow` helpers.
|
||||
3. Run `npm run build:db` locally to check row counts and error skips.
|
||||
4. Commit + push → CI redeploys.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Typical cause |
|
||||
|---------|---------------|
|
||||
| Blank page, 404 on assets | `base` in `vite.config.js` doesn't match the repo name |
|
||||
| `Failed to fetch database: 404` | gzip step skipped, or `.db.gz` removed from `dist/` |
|
||||
| WASM fails to load | `sql.js.org` blocked / offline — self-host `sql-wasm.wasm` in `public/` and update `SQL_WASM_URL` in `use-sqlite.js` |
|
||||
| Missing rows after build | Excel file has an unknown header — check console for `Failed to read` or `errorCount` |
|
||||
@@ -1,34 +0,0 @@
|
||||
# Project Overview — thptqg2016
|
||||
|
||||
## Goal
|
||||
|
||||
Provide a public lookup tool for Vietnam's 2016 National High School Graduation Exam scores (877,461 candidates), running entirely on the client, hosted for free on GitHub Pages.
|
||||
|
||||
## Scope
|
||||
|
||||
- Lookup by exam ID or full name (with Vietnamese diacritics handling)
|
||||
- Read-only SQL queries against a single `student` table
|
||||
- Static dataset — no updates (the 2016 exam is long over)
|
||||
|
||||
## Target users
|
||||
|
||||
- Former 2016 candidates checking their scores
|
||||
- Education researchers / data journalists running aggregate stats
|
||||
- Developers exploring SQL on a real-world dataset
|
||||
|
||||
## Constraints
|
||||
|
||||
- **Zero backend**: the full DB (tens of MB gzipped) is downloaded to the browser
|
||||
- **Read-only**: INSERT/UPDATE/DELETE rejected to avoid the illusion that user edits persist
|
||||
- **Row caps**: 100 rows (lookup), 1000 rows (SQL) to prevent browser hangs
|
||||
- **Vietnamese-first UI**: app labels and data are Vietnamese; documentation is English
|
||||
|
||||
## Data sources
|
||||
|
||||
Excel files (`.xlsx`/`.xls`) collected from newspapers and exam clusters in 2016, stored in `data/`. One file per cluster, with several different column layouts (see `scripts/build-database.js`).
|
||||
|
||||
Original aggregator link: <https://dtntbacgiang.edu.vn/tin-tuc/tin-tuc-su-kien/cong-bo-diem-thi-thptqg-2016-toan-bo-120-cum-thi-da-co-diem.html> — no longer accessible, so the raw Excel files are mirrored in this repo under `data/`.
|
||||
|
||||
## Status
|
||||
|
||||
Stable. Data is frozen. Recent work focuses on UX polish (dark mode, accessibility, diacritics-insensitive search).
|
||||
@@ -1,83 +0,0 @@
|
||||
# System Architecture
|
||||
|
||||
## Overview
|
||||
|
||||
A **static serverless** design: the entire dataset is packaged into a single SQLite file, gzip-compressed, and served as a static asset via GitHub Pages. The browser downloads it, decompresses it, and queries it in-process using `sql.js` (SQLite compiled to WebAssembly).
|
||||
|
||||
```
|
||||
┌─────────────────┐ build ┌──────────────────────┐
|
||||
│ data/*.xlsx │ ───────────▶ │ scripts/ │
|
||||
│ (mixed formats)│ │ build-database.js │
|
||||
└─────────────────┘ │ (Node + xlsx + │
|
||||
│ better-sqlite3) │
|
||||
└──────────┬───────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────────────┐
|
||||
│ public/thptqg2016.db │
|
||||
└──────────┬───────────┘
|
||||
│ gzip -9 (CI)
|
||||
▼
|
||||
┌──────────────────────┐
|
||||
│ dist/thptqg2016.db.gz│
|
||||
│ dist/assets/* │ ◀── Vite build
|
||||
└──────────┬───────────┘
|
||||
│ upload-pages-artifact
|
||||
▼
|
||||
GitHub Pages CDN
|
||||
│
|
||||
▼
|
||||
┌──────────────────────────────────┐
|
||||
│ Browser │
|
||||
│ ┌────────────────────────────┐ │
|
||||
│ │ useSqlite hook │ │
|
||||
│ │ fetch(.db.gz) + stream │ │
|
||||
│ │ DecompressionStream gzip │ │
|
||||
│ │ sql.js WASM (from CDN) │ │
|
||||
│ └─────────────┬──────────────┘ │
|
||||
│ ▼ │
|
||||
│ ┌────────────────────────────┐ │
|
||||
│ │ React UI │ │
|
||||
│ │ - SearchForm / ScoreTable │ │
|
||||
│ │ - CustomQuery (SQL editor)│ │
|
||||
│ └────────────────────────────┘ │
|
||||
└──────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Build-time data flow
|
||||
|
||||
1. Developer drops Excel files into `data/`.
|
||||
2. `npm run build:db` reads every file; for each one:
|
||||
- Detects the header row against a `KNOWN_HEADERS` set.
|
||||
- Picks a format: `separate-scores` (one column per subject) vs. `mapped` (single `DIEM_THI` string).
|
||||
- Parses each row into a canonical 18-column record.
|
||||
- `INSERT OR REPLACE` into SQLite (primary key = `so_bao_danh` handles duplicates).
|
||||
3. `VACUUM` shrinks the file.
|
||||
4. CI runs `gzip -k -9` → `.db.gz`.
|
||||
|
||||
## Runtime flow
|
||||
|
||||
1. Page loads → React mounts → `useSqlite("thptqg2016.db.gz")`.
|
||||
2. Streaming fetch with a progress bar (driven by `Content-Length`).
|
||||
3. `DecompressionStream("gzip")` decompresses on the fly.
|
||||
4. `sql.js` loads its WASM from `https://sql.js.org/dist/sql-wasm.wasm`.
|
||||
5. `new SQL.Database(Uint8Array)` — DB is now in RAM.
|
||||
6. Each search / query → `db.prepare()` + `stmt.step()` loop → render.
|
||||
|
||||
## Design decisions
|
||||
|
||||
| Concern | Choice | Rationale |
|
||||
|---------|--------|-----------|
|
||||
| Storage | Static SQLite file | No backend needed; dataset is frozen |
|
||||
| Compression | gzip in CI, `DecompressionStream` in browser | Native browser API; no extra library |
|
||||
| WASM hosting | `sql.js.org` CDN | Smaller self-hosted artifact |
|
||||
| Diacritics search | Pre-computed `ho_ten_ascii` column | `LOWER(REPLACE(...))` at query time defeats the index |
|
||||
| SQL safety | Leading-keyword allowlist | `sql.js` is in-memory so writes don't persist, but the allowlist prevents user confusion |
|
||||
| Row caps | 100 (lookup), 1000 (SQL) | Keep DOM render sizes reasonable |
|
||||
|
||||
## Risks and limitations
|
||||
|
||||
- **DB size**: tens of MB gzipped — slow links have a visible wait; mitigated by the progress bar.
|
||||
- **Browser memory**: the full DB lives in RAM; older mobile devices may OOM.
|
||||
- **Dependency on `sql.js.org`**: if that CDN is unreachable, WASM fails to load.
|
||||
- **Excel format drift**: a new source file with an unseen header layout needs a new branch in `detectFormat()`.
|
||||
@@ -1,31 +0,0 @@
|
||||
import js from "@eslint/js";
|
||||
import globals from "globals";
|
||||
import reactHooks from "eslint-plugin-react-hooks";
|
||||
import reactRefresh from "eslint-plugin-react-refresh";
|
||||
|
||||
export default [
|
||||
{ ignores: ["dist"] },
|
||||
{
|
||||
files: ["**/*.{js,jsx}"],
|
||||
languageOptions: {
|
||||
ecmaVersion: 2024,
|
||||
globals: globals.browser,
|
||||
parserOptions: {
|
||||
ecmaVersion: "latest",
|
||||
ecmaFeatures: { jsx: true },
|
||||
sourceType: "module",
|
||||
},
|
||||
},
|
||||
plugins: {
|
||||
"react-hooks": reactHooks,
|
||||
"react-refresh": reactRefresh,
|
||||
},
|
||||
rules: {
|
||||
...reactHooks.configs.recommended.rules,
|
||||
"react-refresh/only-export-components": [
|
||||
"warn",
|
||||
{ allowConstantExport: true },
|
||||
],
|
||||
},
|
||||
},
|
||||
];
|
||||
@@ -1,13 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="vi">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Tra cứu điểm thi THPT QG 2016</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.jsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
Generated
-1624
File diff suppressed because it is too large.
Load diff
@@ -1,2 +0,0 @@
|
||||
allowBuilds:
|
||||
better-sqlite3: true
|
||||
@@ -1,474 +0,0 @@
|
||||
.app {
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
padding: 2rem 1rem 3rem;
|
||||
}
|
||||
|
||||
/* Header */
|
||||
header {
|
||||
text-align: center;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
header h1 {
|
||||
font-size: clamp(1.4rem, 2.4vw, 2rem);
|
||||
color: var(--text);
|
||||
margin: 0 0 0.4rem;
|
||||
letter-spacing: -0.01em;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.subtitle {
|
||||
color: var(--text-muted);
|
||||
font-size: 0.95rem;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.subtitle .pill {
|
||||
display: inline-block;
|
||||
padding: 0.1rem 0.55rem;
|
||||
background: var(--accent-soft);
|
||||
color: var(--accent);
|
||||
border-radius: 999px;
|
||||
font-weight: 600;
|
||||
font-size: 0.8rem;
|
||||
margin-right: 0.35rem;
|
||||
}
|
||||
|
||||
/* Tabs */
|
||||
.tabs {
|
||||
display: flex;
|
||||
gap: 0;
|
||||
max-width: 600px;
|
||||
margin: 0 auto 1.5rem;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.tab {
|
||||
flex: 1;
|
||||
padding: 0.75rem 1rem;
|
||||
font-size: 0.95rem;
|
||||
background: none;
|
||||
border: none;
|
||||
border-bottom: 2px solid transparent;
|
||||
margin-bottom: -1px;
|
||||
cursor: pointer;
|
||||
color: var(--text-muted);
|
||||
font-weight: 500;
|
||||
transition: color 0.15s, border-color 0.15s;
|
||||
}
|
||||
|
||||
.tab:hover {
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.tab[aria-selected="true"] {
|
||||
color: var(--accent);
|
||||
border-bottom-color: var(--accent);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* Search form */
|
||||
.search-form {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
max-width: 600px;
|
||||
margin: 0 auto 1.5rem;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.search-field {
|
||||
flex: 1;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.search-form input {
|
||||
width: 100%;
|
||||
padding: 0.75rem 2.25rem 0.75rem 1rem;
|
||||
font-size: 1rem;
|
||||
border: 1px solid var(--border-strong);
|
||||
border-radius: var(--radius);
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
outline: none;
|
||||
transition: border-color 0.15s, box-shadow 0.15s;
|
||||
}
|
||||
|
||||
.search-form input:focus {
|
||||
border-color: var(--accent);
|
||||
box-shadow: 0 0 0 3px var(--accent-soft);
|
||||
}
|
||||
|
||||
.search-clear {
|
||||
position: absolute;
|
||||
right: 0.4rem;
|
||||
top: 50%;
|
||||
transform: translateY(-50%);
|
||||
width: 1.75rem;
|
||||
height: 1.75rem;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: var(--text-faint);
|
||||
font-size: 1.1rem;
|
||||
border-radius: 50%;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.search-clear:hover {
|
||||
background: var(--surface-alt);
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.search-form button[type="submit"] {
|
||||
padding: 0.75rem 1.5rem;
|
||||
font-size: 0.95rem;
|
||||
font-weight: 600;
|
||||
background: var(--accent);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: var(--radius);
|
||||
cursor: pointer;
|
||||
transition: background 0.15s, transform 0.05s;
|
||||
}
|
||||
|
||||
.search-form button[type="submit"]:hover:not(:disabled) {
|
||||
background: var(--accent-hover);
|
||||
}
|
||||
|
||||
.search-form button[type="submit"]:active:not(:disabled) {
|
||||
transform: translateY(1px);
|
||||
}
|
||||
|
||||
.search-form button:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
/* Custom query */
|
||||
.custom-query {
|
||||
max-width: 960px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.preset-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.4rem;
|
||||
margin-bottom: 0.85rem;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.preset-label {
|
||||
font-size: 0.8rem;
|
||||
color: var(--text-muted);
|
||||
white-space: nowrap;
|
||||
margin-right: 0.25rem;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.preset-btn {
|
||||
padding: 0.35rem 0.75rem;
|
||||
font-size: 0.8rem;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--text);
|
||||
border-radius: 999px;
|
||||
cursor: pointer;
|
||||
transition: background 0.15s, border-color 0.15s, color 0.15s;
|
||||
}
|
||||
|
||||
.preset-btn:hover:not(:disabled) {
|
||||
background: var(--accent-soft);
|
||||
border-color: var(--accent);
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.preset-btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.query-form {
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.query-form textarea {
|
||||
width: 100%;
|
||||
padding: 0.85rem 1rem;
|
||||
font-size: 0.9rem;
|
||||
font-family:
|
||||
"JetBrains Mono", "Cascadia Code", "Fira Code", ui-monospace, monospace;
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
border: 1px solid var(--border-strong);
|
||||
border-radius: var(--radius);
|
||||
outline: none;
|
||||
resize: vertical;
|
||||
min-height: 130px;
|
||||
transition: border-color 0.15s, box-shadow 0.15s;
|
||||
tab-size: 2;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.query-form textarea:focus {
|
||||
border-color: var(--accent);
|
||||
box-shadow: 0 0 0 3px var(--accent-soft);
|
||||
}
|
||||
|
||||
.query-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
margin-top: 0.6rem;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.query-actions button {
|
||||
padding: 0.6rem 1.4rem;
|
||||
font-size: 0.9rem;
|
||||
font-weight: 600;
|
||||
background: var(--accent);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: var(--radius);
|
||||
cursor: pointer;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
|
||||
.query-actions button:hover:not(:disabled) {
|
||||
background: var(--accent-hover);
|
||||
}
|
||||
|
||||
.query-actions button:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.exec-time {
|
||||
font-size: 0.85rem;
|
||||
color: var(--text-muted);
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.exec-time.running {
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.kbd {
|
||||
display: inline-block;
|
||||
padding: 0.05rem 0.4rem;
|
||||
font-family: ui-monospace, monospace;
|
||||
font-size: 0.75rem;
|
||||
background: var(--surface-alt);
|
||||
border: 1px solid var(--border);
|
||||
border-bottom-width: 2px;
|
||||
border-radius: 4px;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
/* Loading */
|
||||
.loading {
|
||||
max-width: 600px;
|
||||
margin: 0.5rem auto 1.5rem;
|
||||
padding: 0.85rem 1rem;
|
||||
text-align: center;
|
||||
color: var(--text-muted);
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.loading p {
|
||||
margin: 0 0 0.5rem;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.progress-bar {
|
||||
width: 100%;
|
||||
max-width: 300px;
|
||||
height: 6px;
|
||||
background: var(--surface-alt);
|
||||
border-radius: 4px;
|
||||
margin: 0 auto;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.progress-fill {
|
||||
height: 100%;
|
||||
background: linear-gradient(90deg, var(--accent), var(--accent-hover));
|
||||
border-radius: 4px;
|
||||
transition: width 0.3s;
|
||||
}
|
||||
|
||||
/* Results table */
|
||||
.table-wrapper {
|
||||
overflow-x: auto;
|
||||
margin-top: 1rem;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
box-shadow: var(--shadow-sm);
|
||||
}
|
||||
|
||||
.result-count {
|
||||
color: var(--text-muted);
|
||||
font-size: 0.9rem;
|
||||
margin: 0.5rem 0;
|
||||
padding: 0 0.25rem;
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
thead th {
|
||||
background: var(--surface-alt);
|
||||
padding: 0.65rem 0.6rem;
|
||||
text-align: left;
|
||||
white-space: nowrap;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
border-bottom: 1px solid var(--border-strong);
|
||||
color: var(--text-muted);
|
||||
font-weight: 600;
|
||||
font-size: 0.8rem;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.03em;
|
||||
}
|
||||
|
||||
tbody td {
|
||||
padding: 0.55rem 0.6rem;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
tbody tr:last-child td {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
tbody tr:nth-child(even) td {
|
||||
background: color-mix(in srgb, var(--surface-alt) 45%, transparent);
|
||||
}
|
||||
|
||||
tbody tr:hover td {
|
||||
background: var(--accent-soft);
|
||||
}
|
||||
|
||||
.sbd-cell {
|
||||
font-family: ui-monospace, monospace;
|
||||
font-size: 0.85rem;
|
||||
color: var(--text-muted);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.name-cell {
|
||||
white-space: nowrap;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.cumthi-cell {
|
||||
font-size: 0.82rem;
|
||||
color: var(--text-muted);
|
||||
max-width: 220px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.score-cell {
|
||||
text-align: center;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.score-cell.score-high {
|
||||
color: var(--ok);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.score-cell.score-low {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
.score-cell.score-empty {
|
||||
color: var(--text-faint);
|
||||
}
|
||||
|
||||
/* Messages */
|
||||
.no-results,
|
||||
.error,
|
||||
.warning {
|
||||
text-align: center;
|
||||
padding: 0.9rem 1rem;
|
||||
border-radius: var(--radius);
|
||||
margin: 1rem auto;
|
||||
max-width: 600px;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.no-results {
|
||||
color: var(--text-muted);
|
||||
background: var(--surface);
|
||||
border: 1px dashed var(--border-strong);
|
||||
}
|
||||
|
||||
.error {
|
||||
color: var(--danger);
|
||||
background: var(--danger-bg);
|
||||
border: 1px solid color-mix(in srgb, var(--danger) 25%, transparent);
|
||||
}
|
||||
|
||||
.warning {
|
||||
color: var(--warn);
|
||||
background: var(--warn-bg);
|
||||
}
|
||||
|
||||
/* Footer */
|
||||
footer {
|
||||
text-align: center;
|
||||
margin-top: 3rem;
|
||||
padding-top: 1rem;
|
||||
border-top: 1px solid var(--border);
|
||||
color: var(--text-faint);
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
footer a {
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
/* Responsive */
|
||||
@media (max-width: 640px) {
|
||||
.app {
|
||||
padding: 1.25rem 0.75rem 2rem;
|
||||
}
|
||||
|
||||
.search-form {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.search-form button[type="submit"] {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
table {
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
thead th,
|
||||
tbody td {
|
||||
padding: 0.45rem 0.4rem;
|
||||
}
|
||||
|
||||
.preset-list {
|
||||
flex-direction: column;
|
||||
align-items: stretch;
|
||||
}
|
||||
|
||||
.preset-btn {
|
||||
text-align: left;
|
||||
}
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
import { useState, useCallback, useEffect } from "react";
|
||||
import { useSqlite } from "./hooks/use-sqlite";
|
||||
import { SearchForm } from "./components/search-form";
|
||||
import { ScoreTable } from "./components/score-table";
|
||||
import { CustomQuery } from "./components/custom-query";
|
||||
import "./App.css";
|
||||
|
||||
const DB_URL = import.meta.env.BASE_URL + "thptqg2016.db.gz";
|
||||
const MAX_RESULTS = 100;
|
||||
|
||||
// Strip Vietnamese diacritics for search: "nguyễn bữu lộc" → "nguyen buu loc"
|
||||
function toAscii(str) {
|
||||
return str
|
||||
.normalize("NFD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.replace(/đ/gi, "d")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
// Check if string contains only ASCII (no Vietnamese diacritics)
|
||||
function isAsciiOnly(str) {
|
||||
return /^[\x00-\x7F]*$/.test(str);
|
||||
}
|
||||
|
||||
function App() {
|
||||
const { db, loading, error, progress } = useSqlite(DB_URL);
|
||||
const [results, setResults] = useState(null);
|
||||
const [searchError, setSearchError] = useState(null);
|
||||
const [activeTab, setActiveTab] = useState("search");
|
||||
|
||||
const handleSearch = useCallback(
|
||||
(query) => {
|
||||
if (!db) return;
|
||||
setSearchError(null);
|
||||
|
||||
try {
|
||||
const isExamId = /^[A-Z]{2,4}\d+$/i.test(query) || /^\d+$/.test(query);
|
||||
let stmt;
|
||||
|
||||
if (isExamId) {
|
||||
stmt = db.prepare(
|
||||
"SELECT * FROM student WHERE so_bao_danh = $q LIMIT $limit",
|
||||
);
|
||||
stmt.bind({ $q: query.toUpperCase(), $limit: MAX_RESULTS });
|
||||
} else if (isAsciiOnly(query)) {
|
||||
// ASCII input: search against normalized column (diacritics-insensitive)
|
||||
stmt = db.prepare(
|
||||
"SELECT * FROM student WHERE ho_ten_ascii LIKE $q LIMIT $limit",
|
||||
);
|
||||
stmt.bind({ $q: `%${toAscii(query)}%`, $limit: MAX_RESULTS });
|
||||
} else {
|
||||
// Vietnamese input: search both original and normalized
|
||||
const normalized = toAscii(query);
|
||||
stmt = db.prepare(
|
||||
`SELECT * FROM student
|
||||
WHERE ho_ten LIKE $q OR ho_ten_ascii LIKE $qn
|
||||
LIMIT $limit`,
|
||||
);
|
||||
stmt.bind({
|
||||
$q: `%${query}%`,
|
||||
$qn: `%${normalized}%`,
|
||||
$limit: MAX_RESULTS,
|
||||
});
|
||||
}
|
||||
|
||||
const rows = [];
|
||||
while (stmt.step()) {
|
||||
rows.push(stmt.getAsObject());
|
||||
}
|
||||
stmt.free();
|
||||
setResults(rows);
|
||||
} catch (err) {
|
||||
setSearchError(err.message);
|
||||
}
|
||||
},
|
||||
[db],
|
||||
);
|
||||
|
||||
// Ctrl+Enter shortcut to execute query in SQL tab
|
||||
useEffect(() => {
|
||||
function handleKeyDown(e) {
|
||||
if (e.ctrlKey && e.key === "Enter" && activeTab === "sql") {
|
||||
const form = document.querySelector(".query-form");
|
||||
if (form) form.requestSubmit();
|
||||
}
|
||||
}
|
||||
window.addEventListener("keydown", handleKeyDown);
|
||||
return () => window.removeEventListener("keydown", handleKeyDown);
|
||||
}, [activeTab]);
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<header>
|
||||
<h1>Tra cứu điểm thi THPT Quốc gia 2016</h1>
|
||||
<p className="subtitle">
|
||||
<span className="pill">877.461 thí sinh</span>
|
||||
Dữ liệu toàn quốc · Hỗ trợ truy vấn SQL tùy chỉnh
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
{loading && (
|
||||
<div className="loading" role="status" aria-live="polite">
|
||||
<p>
|
||||
Đang tải cơ sở dữ liệu{progress > 0 ? ` · ${progress}%` : "..."}
|
||||
</p>
|
||||
<div className="progress-bar">
|
||||
<div
|
||||
className="progress-fill"
|
||||
style={{ width: `${progress}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{error && <p className="error">Lỗi: {error}</p>}
|
||||
|
||||
<div className="tabs" role="tablist">
|
||||
<button
|
||||
role="tab"
|
||||
aria-selected={activeTab === "search"}
|
||||
className="tab"
|
||||
onClick={() => setActiveTab("search")}
|
||||
>
|
||||
Tra cứu
|
||||
</button>
|
||||
<button
|
||||
role="tab"
|
||||
aria-selected={activeTab === "sql"}
|
||||
className="tab"
|
||||
onClick={() => setActiveTab("sql")}
|
||||
>
|
||||
Truy vấn SQL
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{activeTab === "search" && (
|
||||
<>
|
||||
<SearchForm
|
||||
onSearch={handleSearch}
|
||||
disabled={loading || !!error}
|
||||
/>
|
||||
|
||||
{searchError && (
|
||||
<p className="error">Lỗi truy vấn: {searchError}</p>
|
||||
)}
|
||||
|
||||
<ScoreTable results={results} />
|
||||
|
||||
{results && results.length >= MAX_RESULTS && (
|
||||
<p className="warning">
|
||||
Hiển thị tối đa {MAX_RESULTS} kết quả. Vui lòng tìm kiếm cụ
|
||||
thể hơn.
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{activeTab === "sql" && (
|
||||
<CustomQuery db={db} disabled={loading || !!error} />
|
||||
)}
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>
|
||||
Nguồn: Sưu tầm từ trang báo thời đó · Dữ liệu chỉ mang tính tham
|
||||
khảo
|
||||
</p>
|
||||
</footer>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default App;
|
||||
@@ -1,74 +0,0 @@
|
||||
const SUBJECT_COLUMNS = [
|
||||
{ key: "toan", label: "Toán" },
|
||||
{ key: "ngu_van", label: "Ngữ văn" },
|
||||
{ key: "vat_ly", label: "Vật lí" },
|
||||
{ key: "hoa_hoc", label: "Hóa học" },
|
||||
{ key: "sinh_hoc", label: "Sinh học" },
|
||||
{ key: "lich_su", label: "Lịch sử" },
|
||||
{ key: "dia_ly", label: "Địa lí" },
|
||||
{ key: "tieng_anh", label: "T.Anh" },
|
||||
{ key: "tieng_phap", label: "T.Pháp" },
|
||||
{ key: "tieng_duc", label: "T.Đức" },
|
||||
{ key: "tieng_nhat", label: "T.Nhật" },
|
||||
{ key: "tieng_trung", label: "T.Trung" },
|
||||
];
|
||||
|
||||
function formatScore(val) {
|
||||
if (val === null || val === undefined) return "—";
|
||||
return Number(val).toFixed(2);
|
||||
}
|
||||
|
||||
function scoreClass(val) {
|
||||
if (val === null || val === undefined) return "score-cell score-empty";
|
||||
const num = Number(val);
|
||||
if (num >= 8) return "score-cell score-high";
|
||||
if (num < 4) return "score-cell score-low";
|
||||
return "score-cell";
|
||||
}
|
||||
|
||||
export function ScoreTable({ results }) {
|
||||
if (!results) return null;
|
||||
if (results.length === 0) {
|
||||
return <p className="no-results">Không tìm thấy kết quả.</p>;
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<p className="result-count">Tìm thấy {results.length} kết quả</p>
|
||||
<div className="table-wrapper">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>SBD</th>
|
||||
<th>Họ tên</th>
|
||||
<th>Ngày sinh</th>
|
||||
<th>Cụm thi</th>
|
||||
<th>GT</th>
|
||||
{SUBJECT_COLUMNS.map((col) => (
|
||||
<th key={col.key}>{col.label}</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{results.map((row) => (
|
||||
<tr key={row.so_bao_danh}>
|
||||
<td className="sbd-cell">{row.so_bao_danh}</td>
|
||||
<td className="name-cell">{row.ho_ten}</td>
|
||||
<td>{row.ngay_sinh || "—"}</td>
|
||||
<td className="cumthi-cell" title={row.ten_cum_thi || ""}>
|
||||
{row.ten_cum_thi || "—"}
|
||||
</td>
|
||||
<td>{row.gioi_tinh || "—"}</td>
|
||||
{SUBJECT_COLUMNS.map((col) => (
|
||||
<td key={col.key} className={scoreClass(row[col.key])}>
|
||||
{formatScore(row[col.key])}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
import { useState } from "react";
|
||||
|
||||
export function SearchForm({ onSearch, disabled }) {
|
||||
const [query, setQuery] = useState("");
|
||||
|
||||
function handleSubmit(e) {
|
||||
e.preventDefault();
|
||||
const trimmed = query.trim();
|
||||
if (trimmed) onSearch(trimmed);
|
||||
}
|
||||
|
||||
return (
|
||||
<form onSubmit={handleSubmit} className="search-form" role="search">
|
||||
<div className="search-field">
|
||||
<label htmlFor="search-input" className="visually-hidden" hidden>
|
||||
Tìm theo số báo danh hoặc họ tên
|
||||
</label>
|
||||
<input
|
||||
id="search-input"
|
||||
type="text"
|
||||
value={query}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
placeholder="Nhập SBD hoặc họ tên (VD: nguyen van a)..."
|
||||
disabled={disabled}
|
||||
autoFocus
|
||||
autoComplete="off"
|
||||
spellCheck={false}
|
||||
/>
|
||||
{query && (
|
||||
<button
|
||||
type="button"
|
||||
className="search-clear"
|
||||
onClick={() => setQuery("")}
|
||||
aria-label="Xóa từ khóa"
|
||||
tabIndex={-1}
|
||||
>
|
||||
×
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<button type="submit" disabled={disabled || !query.trim()}>
|
||||
Tra cứu
|
||||
</button>
|
||||
</form>
|
||||
);
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
import { useState, useEffect, useRef } from "react";
|
||||
import initSqlJs from "sql.js";
|
||||
|
||||
const SQL_WASM_URL = "https://sql.js.org/dist/sql-wasm.wasm";
|
||||
|
||||
/**
|
||||
* Hook to load a SQLite database from a gzipped URL into sql.js.
|
||||
* Returns { db, loading, error, progress }.
|
||||
*/
|
||||
export function useSqlite(dbUrl) {
|
||||
const [db, setDb] = useState(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState(null);
|
||||
const [progress, setProgress] = useState(0);
|
||||
const dbRef = useRef(null);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
async function load() {
|
||||
try {
|
||||
const SQL = await initSqlJs({ locateFile: () => SQL_WASM_URL });
|
||||
|
||||
const response = await fetch(dbUrl);
|
||||
if (!response.ok)
|
||||
throw new Error(`Failed to fetch database: ${response.status}`);
|
||||
|
||||
const contentLength = +response.headers.get("Content-Length") || 0;
|
||||
const reader = response.body.getReader();
|
||||
const chunks = [];
|
||||
let received = 0;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
received += value.length;
|
||||
if (contentLength > 0) {
|
||||
setProgress(Math.round((received / contentLength) * 100));
|
||||
}
|
||||
}
|
||||
|
||||
if (cancelled) return;
|
||||
|
||||
const blob = new Blob(chunks);
|
||||
let arrayBuffer;
|
||||
|
||||
if (dbUrl.endsWith(".gz")) {
|
||||
const ds = new DecompressionStream("gzip");
|
||||
const decompressed = blob.stream().pipeThrough(ds);
|
||||
const decompressedBlob = await new Response(decompressed).blob();
|
||||
arrayBuffer = await decompressedBlob.arrayBuffer();
|
||||
} else {
|
||||
arrayBuffer = await blob.arrayBuffer();
|
||||
}
|
||||
|
||||
if (cancelled) return;
|
||||
|
||||
const database = new SQL.Database(new Uint8Array(arrayBuffer));
|
||||
dbRef.current = database;
|
||||
setDb(database);
|
||||
setLoading(false);
|
||||
} catch (err) {
|
||||
if (!cancelled) {
|
||||
setError(err.message);
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
load();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (dbRef.current) {
|
||||
dbRef.current.close();
|
||||
dbRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [dbUrl]);
|
||||
|
||||
return { db, loading, error, progress };
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
:root {
|
||||
/* Light theme tokens */
|
||||
--bg: #f7f8fb;
|
||||
--surface: #ffffff;
|
||||
--surface-alt: #f2f4f9;
|
||||
--border: #e3e6ee;
|
||||
--border-strong: #cfd4e1;
|
||||
--text: #1a1a2e;
|
||||
--text-muted: #5c6275;
|
||||
--text-faint: #8a8fa3;
|
||||
--accent: #4361ee;
|
||||
--accent-hover: #3a56d4;
|
||||
--accent-soft: #e9edff;
|
||||
--danger: #d32f2f;
|
||||
--danger-bg: #fdecea;
|
||||
--warn: #b45309;
|
||||
--warn-bg: #fff4d6;
|
||||
--ok: #0f8a4f;
|
||||
--ok-bg: #e4f7ec;
|
||||
--shadow-sm: 0 1px 2px rgba(20, 22, 38, 0.06);
|
||||
--shadow-md: 0 4px 14px rgba(20, 22, 38, 0.08);
|
||||
--radius: 10px;
|
||||
--radius-sm: 6px;
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root {
|
||||
--bg: #0f1117;
|
||||
--surface: #181b24;
|
||||
--surface-alt: #21252f;
|
||||
--border: #2a2f3d;
|
||||
--border-strong: #3a4052;
|
||||
--text: #eef0f6;
|
||||
--text-muted: #a7adbe;
|
||||
--text-faint: #6e7486;
|
||||
--accent: #8aa1ff;
|
||||
--accent-hover: #a6b8ff;
|
||||
--accent-soft: #1e2647;
|
||||
--danger: #ff7a7a;
|
||||
--danger-bg: #3a1d1d;
|
||||
--warn: #ffc875;
|
||||
--warn-bg: #3a2a12;
|
||||
--ok: #6ee2a4;
|
||||
--ok-bg: #133524;
|
||||
--shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.5);
|
||||
--shadow-md: 0 4px 14px rgba(0, 0, 0, 0.4);
|
||||
}
|
||||
}
|
||||
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
html {
|
||||
color-scheme: light dark;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
font-family:
|
||||
"Inter", system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
:focus-visible {
|
||||
outline: 2px solid var(--accent);
|
||||
outline-offset: 2px;
|
||||
border-radius: 4px;
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
import { StrictMode } from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import "./index.css";
|
||||
import App from "./App.jsx";
|
||||
|
||||
createRoot(document.getElementById("root")).render(
|
||||
<StrictMode>
|
||||
<App />
|
||||
</StrictMode>,
|
||||
);
|
||||
@@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Re-sync xlsxread Rust source from thptqg2017.
|
||||
#
|
||||
# Source SHA: 8b4a755c115595bf1b937d749eb1133efb3a6e22 (chore/xlsxread-rust)
|
||||
#
|
||||
# Re-run this when xlsxread is updated upstream. The configs/ directory is
|
||||
# NOT synced — it contains thptqg2016-specific configs and test stubs that
|
||||
# must be maintained here independently.
|
||||
#
|
||||
# Usage:
|
||||
# ./tools/sync-from-thptqg2017.sh /path/to/thptqg2017
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
SRC="${1:?Usage: $0 /path/to/thptqg2017}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEST="$SCRIPT_DIR/xlsxread"
|
||||
|
||||
if [ ! -d "$SRC/tools/xlsxread" ]; then
|
||||
echo "Error: $SRC/tools/xlsxread not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Syncing from: $SRC/tools/xlsxread"
|
||||
echo "Syncing to: $DEST"
|
||||
echo ""
|
||||
|
||||
# Sync source, tests, and Cargo manifests — exclude build artifacts and dataset configs
|
||||
for item in src tests Cargo.toml Cargo.lock; do
|
||||
if [ -e "$SRC/tools/xlsxread/$item" ]; then
|
||||
cp -r "$SRC/tools/xlsxread/$item" "$DEST/"
|
||||
echo " synced: $item"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Sync complete."
|
||||
echo "Next steps:"
|
||||
echo " 1. Review src/ for breaking changes to config.rs / writer.rs that"
|
||||
echo " may affect format_detect_2016.rs or the thptqg2016-data.toml config."
|
||||
echo " 2. Rebuild: cargo build --release --manifest-path $DEST/Cargo.toml"
|
||||
echo " 3. Test: cargo test --manifest-path $DEST/Cargo.toml"
|
||||
echo " 4. Commit on a chore/xlsxread-sync-... branch."
|
||||
@@ -1,86 +0,0 @@
|
||||
# Config for thptqg2016 data/ — 4 .xls + 115 .xlsx mixed files.
|
||||
#
|
||||
# Three column layouts exist across the 119 files; the binary selects the
|
||||
# right one per-file at runtime via format_detection = "thptqg2016":
|
||||
#
|
||||
# separate-scores header SBD(0)/HOTEN(1)/TOAN(2)... — dhhanghai files
|
||||
# mapped header SOBAODANH|SBD + DIEM_THI — most provinces
|
||||
# default no header; positional 6-col layout — remaining files
|
||||
#
|
||||
# Schema differences from thptqg2017:
|
||||
# + ten_cum_thi TEXT (exam-cluster name, TEN_CUMTHI column)
|
||||
# + gioi_tinh TEXT (gender: "Nam"/"Nữ", GIOI_TINH column)
|
||||
# + tieng_duc REAL (German)
|
||||
# + tieng_nhat REAL (Japanese)
|
||||
# - khtn, khxh, tieng_nga (not in 2016 dataset)
|
||||
#
|
||||
# sheet_mode = "all": several provinces overflow into Sheet2 (65k Excel row cap).
|
||||
# strip_blank_rows = false: no blank-row anomaly observed in this dataset.
|
||||
|
||||
format_detection = "thptqg2016"
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
# Tokens that identify a header row by first-cell content (uppercased).
|
||||
# Covers both SOBAODANH-style and SBD-style headers.
|
||||
tokens = ["SOBAODANH", "SBD", "HO_TEN", "HOTEN", "HỌ TÊN", "STT"]
|
||||
|
||||
[schema]
|
||||
ddl = """
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
ten_cum_thi TEXT,
|
||||
gioi_tinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_duc REAL,
|
||||
tieng_nhat REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi);
|
||||
"""
|
||||
|
||||
[scores]
|
||||
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
||||
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
||||
vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)'
|
||||
hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)'
|
||||
sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)'
|
||||
lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)'
|
||||
dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_duc = 'Tiếng Đức:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_nhat = 'Tiếng Nhật:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)'
|
||||
|
||||
[insert]
|
||||
sql = """
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, ten_cum_thi, gioi_tinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc,
|
||||
lich_su, dia_ly,
|
||||
tieng_anh, tieng_phap, tieng_duc, tieng_nhat, tieng_trung)
|
||||
VALUES
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
@@ -1,76 +0,0 @@
|
||||
# Test-only config used by golden integration tests (data-old variant).
|
||||
# Same column layout as thptqg2017-data.toml but with:
|
||||
# - sheet_mode = "first" (reads only first sheet)
|
||||
# - require_numeric_sbd = true (rejects non-digit SBDs)
|
||||
# Schema and INSERT match SCORE_FIELDS_2017 (14 score cols).
|
||||
|
||||
[reader]
|
||||
sheet_mode = "first"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = true
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
|
||||
[schema]
|
||||
ddl = """
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
khtn REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
gdcd REAL,
|
||||
khxh REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_nga REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
"""
|
||||
|
||||
[scores]
|
||||
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
||||
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
||||
vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)'
|
||||
hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)'
|
||||
sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)'
|
||||
khtn = 'KHTN:\s*(\d+(?:\.\d+)?)'
|
||||
lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)'
|
||||
dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)'
|
||||
gdcd = 'GDCD:\s*(\d+(?:\.\d+)?)'
|
||||
khxh = 'KHXH:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_nga = 'Tiếng Nga:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)'
|
||||
|
||||
[insert]
|
||||
sql = """
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_trung)
|
||||
VALUES
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
@@ -1,76 +0,0 @@
|
||||
# Test-only config used by golden integration tests.
|
||||
# Matches the fixture row layout produced by tests/golden.rs: write_xlsx()
|
||||
# header: HO_TEN(0) NGAY_SINH(1) SO_BAO_DANH(2) DIEM_THI(3)
|
||||
# Schema and INSERT match SCORE_FIELDS_2017 (14 score cols) so run_build_cmd
|
||||
# in golden.rs can call insert_row(..., SCORE_FIELDS) without modification.
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
|
||||
[schema]
|
||||
ddl = """
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
khtn REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
gdcd REAL,
|
||||
khxh REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_nga REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
"""
|
||||
|
||||
[scores]
|
||||
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
||||
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
||||
vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)'
|
||||
hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)'
|
||||
sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)'
|
||||
khtn = 'KHTN:\s*(\d+(?:\.\d+)?)'
|
||||
lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)'
|
||||
dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)'
|
||||
gdcd = 'GDCD:\s*(\d+(?:\.\d+)?)'
|
||||
khxh = 'KHXH:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_nga = 'Tiếng Nga:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)'
|
||||
|
||||
[insert]
|
||||
sql = """
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_trung)
|
||||
VALUES
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
@@ -1,201 +0,0 @@
|
||||
/// SQLite writer: DDL setup, batched INSERT OR REPLACE, VACUUM, stats output.
|
||||
///
|
||||
/// Mirrors build-lib.js createDb + the transaction loop in each build-database*.js.
|
||||
/// Stats output lines match the JS stdout exactly so existing CI log-greps still work.
|
||||
///
|
||||
/// thptqg2016 differences vs thptqg2017:
|
||||
/// - INSERT includes ten_cum_thi and gioi_tinh after ngay_sinh
|
||||
/// - Score columns are toan/ngu_van/vat_ly/hoa_hoc/sinh_hoc/lich_su/dia_ly/
|
||||
/// tieng_anh/tieng_phap/tieng_duc/tieng_nhat/tieng_trung
|
||||
/// (no khtn, khxh, tieng_nga)
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use rusqlite::{params_from_iter, Connection, ToSql};
|
||||
|
||||
use crate::config::DatasetConfig;
|
||||
use crate::error::BuildError;
|
||||
use crate::transform::ParsedRow;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DB initialisation — mirrors build-lib.js createDb (delete + recreate)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Open (or recreate) the output SQLite database, execute the DDL from config,
|
||||
/// and return the open connection ready for inserts.
|
||||
pub fn open_db(db_path: &Path, cfg: &DatasetConfig) -> Result<Connection, BuildError> {
|
||||
// Mirror Node behaviour: delete existing file before creating (build-lib.js:54)
|
||||
if db_path.exists() {
|
||||
fs::remove_file(db_path).map_err(|e| BuildError::Io {
|
||||
path: db_path.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
if let Some(parent) = db_path.parent() {
|
||||
if !parent.as_os_str().is_empty() {
|
||||
fs::create_dir_all(parent).map_err(|e| BuildError::Io {
|
||||
path: parent.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
}
|
||||
}
|
||||
|
||||
let conn = Connection::open(db_path)?;
|
||||
conn.execute_batch(&cfg.schema.ddl)?;
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Score field lists — one per dataset schema
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// thptqg2017 score columns (14 fields including khtn/khxh/tieng_nga).
|
||||
pub const SCORE_FIELDS_2017: &[&str] = &[
|
||||
"toan",
|
||||
"ngu_van",
|
||||
"vat_ly",
|
||||
"hoa_hoc",
|
||||
"sinh_hoc",
|
||||
"khtn",
|
||||
"lich_su",
|
||||
"dia_ly",
|
||||
"gdcd",
|
||||
"khxh",
|
||||
"tieng_anh",
|
||||
"tieng_phap",
|
||||
"tieng_nga",
|
||||
"tieng_trung",
|
||||
];
|
||||
|
||||
/// thptqg2016 score columns (12 fields: tieng_duc/tieng_nhat present; no khtn/khxh/tieng_nga).
|
||||
pub const SCORE_FIELDS_2016: &[&str] = &[
|
||||
"toan",
|
||||
"ngu_van",
|
||||
"vat_ly",
|
||||
"hoa_hoc",
|
||||
"sinh_hoc",
|
||||
"lich_su",
|
||||
"dia_ly",
|
||||
"tieng_anh",
|
||||
"tieng_phap",
|
||||
"tieng_duc",
|
||||
"tieng_nhat",
|
||||
"tieng_trung",
|
||||
];
|
||||
|
||||
/// Alias kept so thptqg2017 callers that import SCORE_FIELDS continue to compile.
|
||||
pub const SCORE_FIELDS: &[&str] = SCORE_FIELDS_2017;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Insert a single parsed row (thptqg2017 schema — no ten_cum_thi / gioi_tinh)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Bind all fields from `row` into the prepared statement and execute it.
|
||||
/// Positional params: so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, <scores...>
|
||||
pub fn insert_row(
|
||||
conn: &Connection,
|
||||
sql: &str,
|
||||
row: &ParsedRow,
|
||||
score_fields: &[&str],
|
||||
) -> Result<(), BuildError> {
|
||||
let mut params: Vec<Box<dyn ToSql>> = Vec::with_capacity(4 + score_fields.len());
|
||||
params.push(Box::new(row.so_bao_danh.clone()));
|
||||
params.push(Box::new(row.ho_ten.clone()));
|
||||
params.push(Box::new(row.ho_ten_ascii.clone()));
|
||||
params.push(Box::new(row.ngay_sinh.clone()));
|
||||
|
||||
for field in score_fields {
|
||||
let val: Option<f64> = row.scores.get(*field).copied();
|
||||
params.push(Box::new(val));
|
||||
}
|
||||
|
||||
conn.execute(sql, params_from_iter(params.iter().map(|p| p.as_ref())))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Insert a single parsed row (thptqg2016 schema — includes ten_cum_thi + gioi_tinh)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Bind all fields from `row` into the prepared statement for the thptqg2016 schema.
|
||||
/// Positional params: so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh, ten_cum_thi,
|
||||
/// gioi_tinh, <scores...>
|
||||
pub fn insert_row_2016(
|
||||
conn: &Connection,
|
||||
sql: &str,
|
||||
row: &ParsedRow,
|
||||
score_fields: &[&str],
|
||||
) -> Result<(), BuildError> {
|
||||
let mut params: Vec<Box<dyn ToSql>> = Vec::with_capacity(6 + score_fields.len());
|
||||
params.push(Box::new(row.so_bao_danh.clone()));
|
||||
params.push(Box::new(row.ho_ten.clone()));
|
||||
params.push(Box::new(row.ho_ten_ascii.clone()));
|
||||
params.push(Box::new(row.ngay_sinh.clone()));
|
||||
params.push(Box::new(row.ten_cum_thi.clone()));
|
||||
params.push(Box::new(row.gioi_tinh.clone()));
|
||||
|
||||
for field in score_fields {
|
||||
let val: Option<f64> = row.scores.get(*field).copied();
|
||||
params.push(Box::new(val));
|
||||
}
|
||||
|
||||
conn.execute(sql, params_from_iter(params.iter().map(|p| p.as_ref())))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Post-build: VACUUM + stats output
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Run VACUUM and print statistics lines that mirror the Node scripts' stdout.
|
||||
/// The exact prefix tokens ("Source data rows", "DB rows", "Size:") are preserved
|
||||
/// so any log-grep in the deploy pipeline keeps working.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn finish_db(
|
||||
conn: &Connection,
|
||||
db_path: &Path,
|
||||
source_rows: u64,
|
||||
skipped: u64,
|
||||
errors: u64,
|
||||
dataset_label: &str,
|
||||
_file_count: usize,
|
||||
is_old2: bool,
|
||||
) -> Result<(), BuildError> {
|
||||
conn.execute_batch("VACUUM")?;
|
||||
|
||||
let db_count: i64 = conn.query_row("SELECT COUNT(*) FROM student", [], |row| row.get(0))?;
|
||||
|
||||
let insertable = source_rows - skipped;
|
||||
|
||||
println!();
|
||||
if is_old2 {
|
||||
println!("Source non-blank data rows: {source_rows}");
|
||||
println!(" skipped (empty/non-numeric SBD): {skipped}");
|
||||
} else {
|
||||
println!("Source data rows (post-header): {source_rows}");
|
||||
if dataset_label.contains("old") {
|
||||
println!(" skipped (empty/non-numeric SBD): {skipped}");
|
||||
} else {
|
||||
println!(" skipped (empty/invalid): {skipped}");
|
||||
}
|
||||
}
|
||||
println!(" insertable: {insertable}");
|
||||
println!(" insert errors: {errors}");
|
||||
println!("DB rows (distinct SBD): {db_count}");
|
||||
|
||||
if !dataset_label.contains("old") && errors == 0 {
|
||||
let gap = insertable as i64 - db_count;
|
||||
if gap == 0 {
|
||||
println!("Audit: OK — every source row made it in.");
|
||||
} else {
|
||||
println!("Audit: {gap} row(s) collapsed (duplicate SBDs overwriting).");
|
||||
}
|
||||
}
|
||||
|
||||
let sz = fs::metadata(db_path).map(|m| m.len()).unwrap_or(0);
|
||||
println!("Size: {:.1} MB", sz as f64 / 1024.0 / 1024.0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
import { defineConfig } from "vite";
|
||||
import react from "@vitejs/plugin-react";
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
base: "/thptqg/2016/",
|
||||
});
|
||||
@@ -1,27 +0,0 @@
|
||||
.idea
|
||||
*.iws
|
||||
*.iml
|
||||
*.ipr
|
||||
|
||||
### VS Code ###
|
||||
.vscode/
|
||||
|
||||
### Mac OS ###
|
||||
.DS_Store
|
||||
|
||||
### Node ###
|
||||
node_modules/
|
||||
|
||||
### Generated databases (all 3 variants) ###
|
||||
public/thptqg2017.db
|
||||
public/thptqg2017.db.gz
|
||||
public-old/thptqg2017.db
|
||||
public-old/thptqg2017.db.gz
|
||||
public-old2/thptqg2017.db
|
||||
public-old2/thptqg2017.db.gz
|
||||
|
||||
### Build output ###
|
||||
dist/
|
||||
|
||||
### Rust build artefacts ###
|
||||
tools/xlsxread/target/
|
||||
-201
@@ -1,201 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-113
@@ -1,113 +0,0 @@
|
||||
# Vietnam THPT 2017 Score Lookup
|
||||
|
||||
Static React + SQLite site for looking up Vietnamese high school graduation exam scores (2017). The full database (~861k students, 63 provinces) ships to the browser as a compressed SQLite file and queries run client-side via `sql.js` — no backend.
|
||||
|
||||
## Live sites
|
||||
|
||||
Three deployments, one per dataset:
|
||||
|
||||
| URL | Data source | Rows |
|
||||
|---|---|---|
|
||||
| `/thptqg2017/` | `data/` — [baotintuc.vn](https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm) CDN, `.xls` | **861,068** |
|
||||
| `/thptqg2017/old/` | `data-old/` — earlier `.xlsx` collection from Vietnamese news sites at the time (exact origin no longer known) | 847,348 |
|
||||
| `/thptqg2017/old2/` | `data-old2/` — partial re-export (54 provinces) also from contemporary news sites, exact origin unrecorded | 679,764 |
|
||||
|
||||
## Features
|
||||
|
||||
- Diacritics-insensitive name search (`"nguyen"` matches `"Nguyễn"`)
|
||||
- Live debounced search with URL deep-link (`?q=49008235`)
|
||||
- Single-result detail card with per-subject TFT rarity-tiered scores (6 tiers, ≤1 → 9-10) and all 49 admission blocks (A00 – D15)
|
||||
- Share button: copies a formatted summary plus deep-link URL
|
||||
- SQL query tab with grouped presets (rankings, Long An filters, statistics, schema)
|
||||
- Light + dark mode (follows OS preference)
|
||||
- Keyboard shortcut `/` to focus search
|
||||
|
||||
## Requirements
|
||||
|
||||
- Node.js 24+
|
||||
- pnpm
|
||||
- Rust (stable) — installed via [rustup](https://rustup.rs/); required for `build:db*` scripts
|
||||
|
||||
## Quickstart
|
||||
|
||||
```bash
|
||||
pnpm install
|
||||
pnpm build:db # compile xlsxread + parse data/ → public/thptqg2017.db (~2 min, 159 MB)
|
||||
gzip -kf -9 public/thptqg2017.db
|
||||
pnpm dev # http://localhost:5173
|
||||
```
|
||||
|
||||
The database build pipeline uses the `xlsxread` Rust CLI located at
|
||||
`tools/xlsxread/`. It reads `.xls`/`.xlsx` source files, strips
|
||||
diacritics, parses score text, and writes a SQLite database — replacing
|
||||
the former Node.js + `xlsx` (SheetJS) pipeline. See
|
||||
`tools/xlsxread/README.md` for CLI invocation details and config schema.
|
||||
|
||||
## Scripts
|
||||
|
||||
| Command | Action |
|
||||
|---|---|
|
||||
| `pnpm dev` | Vite dev server |
|
||||
| `pnpm build` | Production build (main variant → `dist/`) |
|
||||
| `pnpm build:old` / `build:old2` | Build variant sites to `dist/old/`, `dist/old2/` |
|
||||
| `pnpm build:all` | All 3 web variants |
|
||||
| `pnpm build:rust` | Compile `tools/xlsxread` release binary (run once; auto-called by `build:db*`) |
|
||||
| `pnpm build:db` | Build main DB from `data/` via xlsxread |
|
||||
| `pnpm build:db:old` / `build:db:old2` | Build old / old2 variant DBs via xlsxread |
|
||||
| `pnpm build:db:all` | All 3 DBs via xlsxread |
|
||||
| `pnpm lint` | ESLint |
|
||||
| `node scripts/crawl-baotintuc.js` | Re-download all 63 province files from baotintuc.vn |
|
||||
| `node scripts/check-duplicates.js` | MD5 + row-content duplicate audit |
|
||||
| `node scripts/diff-datasets.js` | Compare `public/` vs `backup/` DB (when backup present) |
|
||||
|
||||
## Project layout
|
||||
|
||||
```
|
||||
.
|
||||
├── data/ # 63 .xls files (source)
|
||||
├── data-old/ # 63 .xlsx (previous export)
|
||||
├── data-old2/ # 54 .xlsx (update/ overrides)
|
||||
├── public/ # main variant assets + thptqg2017.db.gz
|
||||
├── public-old/ # old variant assets
|
||||
├── public-old2/ # old2 variant assets
|
||||
├── scripts/
|
||||
│ ├── crawl-baotintuc.js # downloader
|
||||
│ ├── check-duplicates.js # md5 dup detector
|
||||
│ └── diff-datasets.js # DB-to-DB comparator
|
||||
├── src/
|
||||
│ ├── App.jsx
|
||||
│ ├── App.css
|
||||
│ ├── components/{search-form, score-table, student-detail, custom-query}.jsx
|
||||
│ ├── hooks/use-sqlite.js
|
||||
│ └── lib/admission-blocks.js # 49 admission-block definitions + score-tier helper
|
||||
├── tools/
|
||||
│ └── xlsxread/ # Rust CLI — reads .xls/.xlsx, writes SQLite
|
||||
│ ├── configs/
|
||||
│ │ ├── thptqg2017-data.toml # config for data/ (63 .xls, all sheets)
|
||||
│ │ ├── thptqg2017-data-old.toml # config for data-old/ (63 .xlsx, sheet 0)
|
||||
│ │ └── thptqg2017-data-old2.toml # config for data-old2/ (54 .xlsx, all sheets)
|
||||
│ ├── src/ # Rust source
|
||||
│ └── README.md # CLI reference + config schema
|
||||
├── docs/ # see docs/README.md
|
||||
├── index.html
|
||||
├── vite.config.js
|
||||
└── package.json
|
||||
```
|
||||
|
||||
See `docs/` for architecture + deployment details.
|
||||
|
||||
## License
|
||||
|
||||
See `LICENSE`.
|
||||
|
||||
## Source
|
||||
|
||||
- **`data/`** — scraped in full from the baotintuc.vn 2017 announcement article:
|
||||
`https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm`
|
||||
Reproducible via `node scripts/crawl-baotintuc.js`.
|
||||
- **`data-old/`** and **`data-old2/`** — collected from Vietnamese news sites at the
|
||||
time of the 2017 exam. The specific publisher URLs were not recorded and cannot
|
||||
be recovered now. These datasets are preserved for comparison and historical
|
||||
reference only; `data/` is the canonical source for the main deployment.
|
||||
|
||||
Intended for reference only.
|
||||
@@ -1,5 +0,0 @@
|
||||
# Docs
|
||||
|
||||
- [`system-architecture.md`](./system-architecture.md) — data flow, schema, 3-variant deploy, score-tier model, admission-block catalog
|
||||
- [`data-pipeline.md`](./data-pipeline.md) — Excel parse quirks, overflow-sheet gotcha, audit scripts, refresh flow
|
||||
- [`deployment-guide.md`](./deployment-guide.md) — GitHub Pages workflow, adding a new variant, rollback
|
||||
@@ -1,110 +0,0 @@
|
||||
# Data Pipeline
|
||||
|
||||
From raw Excel files to a compressed SQLite file the browser can load.
|
||||
|
||||
## Canonical source (data/)
|
||||
|
||||
63 Excel files from baotintuc.vn CDN. Source article:
|
||||
`https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm`
|
||||
|
||||
Re-download anytime:
|
||||
|
||||
```bash
|
||||
node scripts/crawl-baotintuc.js
|
||||
```
|
||||
|
||||
Idempotent — skips files already present. Saves to `data/<ascii-kebab-province>.xls`.
|
||||
|
||||
## Reference sources (data-old/, data-old2/)
|
||||
|
||||
Both were collected from Vietnamese news sites around the time of the 2017 exam.
|
||||
Specific publisher URLs were not recorded and cannot be recovered, so these
|
||||
datasets are **not reproducible** — the files committed in git are the only
|
||||
copy. They are kept for historical comparison and to expose the deploy matrix;
|
||||
the main deployment always uses the reproducible `data/` source.
|
||||
|
||||
## Source Excel shape
|
||||
|
||||
Every file is a single sheet (for `data-old/`) or up to 3 sheets (for `data/` .xls) with columns:
|
||||
|
||||
| Col | Name | Content |
|
||||
|---|---|---|
|
||||
| 0 | HO_TEN | full name in Vietnamese |
|
||||
| 1 | NGAY_SINH | `dd/mm/yyyy` |
|
||||
| 2 | SOBAODANH | 8-digit string, first 2 digits = province |
|
||||
| 3 | DIEM_THI | concatenated per-subject score string in the source language, e.g. `"Toan: 6.80 Van: 5.25 ... English: 5.80"` |
|
||||
|
||||
Some files lack a header row. `isHeaderRow()` in `build-lib.js` detects and skips.
|
||||
|
||||
## Score text parsing
|
||||
|
||||
`SCORE_PATTERNS` in `build-lib.js` defines one regex per subject. The patterns must match the exact subject labels the source files use (Vietnamese).
|
||||
|
||||
Subjects supported (DB column / label in source text):
|
||||
|
||||
| Column | Source label | English |
|
||||
|---|---|---|
|
||||
| toan | Toán | Math |
|
||||
| ngu_van | Ngữ văn | Literature |
|
||||
| vat_ly | Vật lí | Physics |
|
||||
| hoa_hoc | Hóa học | Chemistry |
|
||||
| sinh_hoc | Sinh học | Biology |
|
||||
| khtn | KHTN | Natural Sciences combined |
|
||||
| lich_su | Lịch sử | History |
|
||||
| dia_ly | Địa lí | Geography |
|
||||
| gdcd | GDCD | Civic Education |
|
||||
| khxh | KHXH | Social Sciences combined |
|
||||
| tieng_anh | Tiếng Anh | English |
|
||||
| tieng_phap | Tiếng Pháp | French |
|
||||
| tieng_nga | Tiếng Nga | Russian |
|
||||
| tieng_trung | Tiếng Trung | Chinese |
|
||||
|
||||
Missing-in-string → `NULL` in DB (student didn't take that subject).
|
||||
|
||||
## Why three parsers
|
||||
|
||||
One shared lib + three thin parsers, not one parser with flags. Each dataset's quirks stay visible in its own file:
|
||||
|
||||
| File | Source dir | Sheet strategy | Extra guard |
|
||||
|---|---|---|---|
|
||||
| `build-database.js` | `data/` | iterate ALL sheets (.xls row overflow) | — |
|
||||
| `build-database-old.js` | `data-old/` | sheet 0 only | reject rows where `so_bao_danh` isn't digits-only (rejects an `HO_TEN` → `SOBAODANH` header leak present in this export) |
|
||||
| `build-database-old2.js` | `data-old2/` | iterate ALL sheets (HCM overflow) | skip fully blank rows before counting |
|
||||
|
||||
## Overflow-sheet gotcha
|
||||
|
||||
The old `.xls` binary format caps at 65,536 rows per sheet. Hanoi and Ho Chi Minh City exceed that in the baotintuc export, so Excel auto-splits the overflow into `Sheet2`. **Only reading `Sheet1` silently drops 13,720 students** (Hanoi +7,275, HCM +6,445).
|
||||
|
||||
The audit `node scripts/audit-row-counts.js` catches this by comparing source-row-count (across all sheets) to DB-row-count.
|
||||
|
||||
## Audits
|
||||
|
||||
Run any time after a rebuild:
|
||||
|
||||
```bash
|
||||
node scripts/audit-row-counts.js # source rows vs DB rows (zero-loss check)
|
||||
node scripts/check-duplicates.js # md5 + row-content duplicates in data/
|
||||
node scripts/diff-datasets.js # compare two DBs (needs backup/ dir)
|
||||
```
|
||||
|
||||
Expected baseline:
|
||||
|
||||
| DB | Source rows | Skipped | DB rows |
|
||||
|---|---|---|---|
|
||||
| main | 861,131 | 63 empty rows | **861,068** |
|
||||
| old | 847,349 | 1 header-leak | **847,348** |
|
||||
| old2 | 679,764 | 0 | **679,764** |
|
||||
|
||||
## Refreshing the data
|
||||
|
||||
```bash
|
||||
rm data/*.xls # clear current snapshot
|
||||
node scripts/crawl-baotintuc.js # re-download from CDN
|
||||
npm run build:db # rebuild
|
||||
node scripts/audit-row-counts.js # verify
|
||||
gzip -kf -9 public/thptqg2017.db # compress for shipping
|
||||
```
|
||||
|
||||
## Adding a new source dataset
|
||||
|
||||
See `docs/deployment-guide.md` → "Adding a new variant".
|
||||
@@ -1,52 +0,0 @@
|
||||
# Deployment Guide
|
||||
|
||||
Deploys to GitHub Pages via `.github/workflows/deploy.yml`. Every push to `main` rebuilds and redeploys all 3 variants.
|
||||
|
||||
## What the workflow does
|
||||
|
||||
1. Checkout, setup Node 20, `npm ci`
|
||||
2. `npm run build:db:all` — builds 3 SQLite DBs (main, old, old2)
|
||||
3. Gzip each DB at level 9
|
||||
4. `npm run build:all` — builds 3 Vite bundles into `dist/`, `dist/old/`, `dist/old2/`
|
||||
5. Remove uncompressed `.db` files from `dist/` (only `.gz` ships)
|
||||
6. `actions/upload-pages-artifact` + `actions/deploy-pages`
|
||||
|
||||
Total CI time ≈ 4–6 min (DB build dominates).
|
||||
|
||||
## Resulting URLs
|
||||
|
||||
- `https://<user>.github.io/thptqg2017/`
|
||||
- `https://<user>.github.io/thptqg2017/old/`
|
||||
- `https://<user>.github.io/thptqg2017/old2/`
|
||||
|
||||
## Local reproduction
|
||||
|
||||
```bash
|
||||
npm ci
|
||||
npm run build:db:all
|
||||
gzip -kf -9 public/thptqg2017.db public-old/thptqg2017.db public-old2/thptqg2017.db
|
||||
npm run build:all
|
||||
npx serve dist # or any static server
|
||||
```
|
||||
|
||||
## Adding a new variant
|
||||
|
||||
1. Drop source Excel files into a new `data-vN/` folder
|
||||
2. Add `public-vN/` to `.gitignore` patterns for the `.db` + `.db.gz`
|
||||
3. Copy `scripts/build-database-old.js` → `scripts/build-database-vN.js`; update `SRC_DIR` / `DB_PATH`; adjust parse logic for the new quirks (multi-sheet? blank rows? numeric guard?)
|
||||
4. Add variant to `VARIANT_CONFIG` in `vite.config.js`:
|
||||
```js
|
||||
vN: { base: "/thptqg2017/vN/", publicDir: "public-vN", outDir: "dist/vN" }
|
||||
```
|
||||
5. Add `build:db:vN` and `build:vN` npm scripts; wire them into `build:db:all` and `build:all`
|
||||
6. Update `.github/workflows/deploy.yml` — add gzip step + rm step for the new DB
|
||||
|
||||
## Notes
|
||||
|
||||
- **DB gzip is non-cacheable across deploys** — every rebuild produces a new `.db.gz` (not byte-identical due to SQLite page shuffling). First-visit users pay the 47 MB download; subsequent visits hit browser cache until the next deploy.
|
||||
- **GitHub Pages file size limit**: individual files ≤ 100 MB. Main gzipped DB is ~47 MB — safe. Uncompressed 159 MB DB would not fit; that's why we ship `.gz` and the frontend decompresses via `DecompressionStream`.
|
||||
- **No server-side compression assumption.** The app reads the `.gz` file directly (not `Content-Encoding: gzip`). GH Pages does not reliably gzip on-the-fly for arbitrary paths; shipping pre-gzipped bytes is deterministic.
|
||||
|
||||
## Rollback
|
||||
|
||||
Deploys are stateless snapshots. To rollback, revert the commit on `main` and push — the next workflow rebuilds the older state. There's no state to migrate.
|
||||
@@ -1,97 +0,0 @@
|
||||
# System Architecture
|
||||
|
||||
Static site. No backend. Browser downloads a compressed SQLite file at boot, then every query runs locally via `sql.js` (WASM).
|
||||
|
||||
## Data flow
|
||||
|
||||
```
|
||||
Excel files (.xls / .xlsx)
|
||||
│
|
||||
▼ build-database*.js (Node)
|
||||
SQLite DB (public*/thptqg2017.db)
|
||||
│
|
||||
▼ gzip -9
|
||||
thptqg2017.db.gz (~47 MB for main variant)
|
||||
│
|
||||
▼ Vite build copies public*/ into dist/
|
||||
Static site on GitHub Pages
|
||||
│
|
||||
▼ browser loads
|
||||
sql.js (WASM) opens the .db → queries run client-side
|
||||
```
|
||||
|
||||
## Three deployment variants
|
||||
|
||||
One repo → three independent sites, same frontend, different dataset.
|
||||
|
||||
| Variant | Route | Source dir | Public dir | Build cmd |
|
||||
|---|---|---|---|---|
|
||||
| main | `/thptqg2017/` | `data/` | `public/` | `npm run build` |
|
||||
| old | `/thptqg2017/old/` | `data-old/` | `public-old/` | `npm run build:old` |
|
||||
| old2 | `/thptqg2017/old2/` | `data-old2/` | `public-old2/` | `npm run build:old2` |
|
||||
|
||||
`vite.config.js` reads `process.env.VARIANT` and switches `base`, `publicDir`, `outDir` accordingly. `emptyOutDir` is on only for the main build so the variant builds merge into `dist/old/`, `dist/old2/` cleanly.
|
||||
|
||||
## Schema (all 3 DBs share this)
|
||||
|
||||
```sql
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY, -- exam ID (8 digits, first 2 = province)
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL, -- NFD-stripped lowercase for accent-insensitive search
|
||||
ngay_sinh TEXT, -- dd/mm/yyyy
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
```
|
||||
|
||||
No German / Japanese language columns — neither appears in any source file.
|
||||
|
||||
Each score column is `NULL` when the student didn't take that subject. A student's "admission block" total is only computed when all 3 required subjects are non-null.
|
||||
|
||||
## Parse quirks (why 3 builder scripts)
|
||||
|
||||
Each dataset had a different quirk. Rather than one mega-parser with mode flags, each builder script is ~70 lines and isolates its own workarounds.
|
||||
|
||||
| Dataset | Quirk | Mitigation |
|
||||
|---|---|---|
|
||||
| data/ (baotintuc .xls) | Hanoi + HCM overflow past the 65,536 row-per-sheet .xls limit | Iterate all `wb.SheetNames` |
|
||||
| data-old/ (xlsx) | Single-sheet always; one bogus header row (`SOBAODANH`/`HO_TEN`) leaked into an earlier DB | Strict numeric-SBD guard rejects the leak |
|
||||
| data-old2/ (xlsx) | HCM overflow + many blank trailing rows | Multi-sheet walk + blank-row pre-filter |
|
||||
|
||||
Shared concerns (regex, schema, `toAscii`, header detection) live in `scripts/build-lib.js`.
|
||||
|
||||
## Admission blocks
|
||||
|
||||
Vietnamese universities admit students based on 3-subject combinations called "admission blocks" (khối thi). `src/lib/admission-blocks.js` lists all 49 official 2017 blocks computable from our schema (A00–A11, B00–B08, C00–C20, D01–D15). Each entry is `{ code, subjects: [3 keys], label }`.
|
||||
|
||||
`computeBlocks(student)` returns only blocks where the student has all 3 subject scores, sorted by total desc. Used by the student detail card.
|
||||
|
||||
The SQL preset "Top 10 best-block in Long An" materialises all 49 blocks as a `UNION ALL` CTE, picks each student's best block via `ROW_NUMBER() OVER (PARTITION BY so_bao_danh ORDER BY s DESC, k)`, then ranks across students.
|
||||
|
||||
## Score tiers (UI coding)
|
||||
|
||||
6-level TFT rarity ladder (white → green → blue → purple → gold → prismatic). Defined in `scoreTier()` in `src/lib/admission-blocks.js`.
|
||||
|
||||
| Tier | Range | UI label (Vietnamese) | English |
|
||||
|---|---|---|---|
|
||||
| common | ≤ 1 | Điểm liệt | Disqualifying |
|
||||
| uncommon | < 5 | Chưa đạt | Below passing |
|
||||
| rare | 5–6.5 | Trung bình | Average |
|
||||
| epic | 6.5–8 | Khá | Good |
|
||||
| legendary | 8–9 | Giỏi | Very good |
|
||||
| prismatic | 9–10 | Xuất sắc | Excellent |
|
||||
|
||||
Color + icon + text label — never color-only.
|
||||
|
||||
## Frontend key behaviors
|
||||
|
||||
- Query state lives in `App.jsx` and is synced to URL `?q=...` via `history.replaceState` — bookmarkable, shareable.
|
||||
- `SearchForm` is a controlled component that 300ms-debounces input changes and auto-triggers search once the query is long enough (3+ digits for SBD, 2+ chars for name).
|
||||
- Detection: all-digits → exact `so_bao_danh` match; ASCII-only → folded `ho_ten_ascii LIKE`; has Vietnamese diacritics → search both `ho_ten` and folded column.
|
||||
- Exactly 1 result → `StudentDetail` card; 0 or >1 → `ScoreTable`.
|
||||
- Global `/` key focuses the search input when not already typing.
|
||||
- Share button on the detail card uses Web Share API when present, else clipboard with a pre-formatted summary + deep-link URL.
|
||||
@@ -1,19 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="vi">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
||||
<link
|
||||
rel="stylesheet"
|
||||
href="https://fonts.googleapis.com/css2?family=Be+Vietnam+Pro:wght@400;500;600;700&display=swap"
|
||||
/>
|
||||
<title>Tra cứu điểm thi THPT QG 2017</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.jsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,43 +0,0 @@
|
||||
{
|
||||
"name": "thptqg2017",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@11.1.1",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"description": "Tra cứu điểm thi THPT QG 2017",
|
||||
"scripts": {
|
||||
"build:rust": "cargo build --release --manifest-path tools/xlsxread/Cargo.toml",
|
||||
"build:db": "pnpm build:rust && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data.toml --input data --output public/thptqg2017.db",
|
||||
"build:db:old": "pnpm build:rust && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data-old.toml --input data-old --output public-old/thptqg2017.db",
|
||||
"build:db:old2": "pnpm build:rust && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data-old2.toml --input data-old2 --output public-old2/thptqg2017.db",
|
||||
"build:db:all": "pnpm build:rust && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data.toml --input data --output public/thptqg2017.db && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data-old.toml --input data-old --output public-old/thptqg2017.db && ./tools/xlsxread/target/release/xlsxread build --schema tools/xlsxread/configs/thptqg2017-data-old2.toml --input data-old2 --output public-old2/thptqg2017.db",
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"build:old": "node -e \"process.env.VARIANT='old';require('child_process').spawnSync('npx',['vite','build'],{stdio:'inherit',shell:true,env:process.env})\"",
|
||||
"build:old2": "node -e \"process.env.VARIANT='old2';require('child_process').spawnSync('npx',['vite','build'],{stdio:'inherit',shell:true,env:process.env})\"",
|
||||
"build:all": "pnpm build && pnpm build:old && pnpm build:old2",
|
||||
"preview": "vite preview",
|
||||
"lint": "eslint ."
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/tiennm99/thptqg2017.git"
|
||||
},
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"react": "^19.2.4",
|
||||
"react-dom": "^19.2.4",
|
||||
"sql.js": "^1.14.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.39.4",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^6.0.1",
|
||||
"eslint": "^9.39.4",
|
||||
"eslint-plugin-react-hooks": "^7.0.1",
|
||||
"eslint-plugin-react-refresh": "^0.5.2",
|
||||
"globals": "^17.4.0",
|
||||
"vite": "^8.0.16"
|
||||
}
|
||||
}
|
||||
Generated
-1624
File diff suppressed because it is too large.
Load diff
@@ -1,2 +0,0 @@
|
||||
allowBuilds:
|
||||
better-sqlite3: true
|
||||
Generated
-1159
File diff suppressed because it is too large.
Load diff
@@ -1,30 +0,0 @@
|
||||
[package]
|
||||
name = "xlsxread"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Rust CLI replacing SheetJS xlsx build scripts for thptqg2017/thptqg2016"
|
||||
|
||||
[dependencies]
|
||||
calamine = "0.26"
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
toml = "0.8"
|
||||
regex = "1"
|
||||
unicode-normalization = "0.1"
|
||||
thiserror = "1"
|
||||
anyhow = "1"
|
||||
glob = "0.3"
|
||||
|
||||
# zip is already a transitive dep of calamine; pin explicitly so tests can use it
|
||||
[dev-dependencies]
|
||||
zip = "2"
|
||||
rusqlite = { version = "0.32", features = ["bundled"] }
|
||||
|
||||
[[bin]]
|
||||
name = "xlsxread"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[test]]
|
||||
name = "golden"
|
||||
path = "tests/golden.rs"
|
||||
@@ -1,75 +0,0 @@
|
||||
# Config for data-old/ — 63 .xlsx files (pre-baotintuc refresh)
|
||||
# Sheet mode: "first" — single-sheet workbooks, never hit 65k row cap
|
||||
# SBD validation: require ^\d+$ (build-database-old.js:55 guard)
|
||||
# Blank row strip: off (no explicit blank-skip in build-database-old.js)
|
||||
|
||||
[reader]
|
||||
sheet_mode = "first"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = true
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
|
||||
[schema]
|
||||
ddl = """
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
khtn REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
gdcd REAL,
|
||||
khxh REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_nga REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
"""
|
||||
|
||||
[scores]
|
||||
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
||||
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
||||
vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)'
|
||||
hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)'
|
||||
sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)'
|
||||
khtn = 'KHTN:\s*(\d+(?:\.\d+)?)'
|
||||
lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)'
|
||||
dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)'
|
||||
gdcd = 'GDCD:\s*(\d+(?:\.\d+)?)'
|
||||
khxh = 'KHXH:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_nga = 'Tiếng Nga:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)'
|
||||
|
||||
[insert]
|
||||
sql = """
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_trung)
|
||||
VALUES
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
@@ -1,76 +0,0 @@
|
||||
# Config for data-old2/ — 54 .xlsx files (corrected-export set)
|
||||
# Sheet mode: "all" — HCM (24.HCM_UTLQ.xlsx) overflows into Sheet2 (+6,446 rows)
|
||||
# SBD validation: require ^\d+$ (build-database-old2.js:57 guard)
|
||||
# Blank row strip: true — skip fully blank rows BEFORE counting sourceRows
|
||||
# (build-database-old2.js:50-51: blank row check before sourceRows++)
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = true
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = true
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
|
||||
[schema]
|
||||
ddl = """
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
khtn REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
gdcd REAL,
|
||||
khxh REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_nga REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
"""
|
||||
|
||||
[scores]
|
||||
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
||||
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
||||
vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)'
|
||||
hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)'
|
||||
sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)'
|
||||
khtn = 'KHTN:\s*(\d+(?:\.\d+)?)'
|
||||
lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)'
|
||||
dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)'
|
||||
gdcd = 'GDCD:\s*(\d+(?:\.\d+)?)'
|
||||
khxh = 'KHXH:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_nga = 'Tiếng Nga:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)'
|
||||
|
||||
[insert]
|
||||
sql = """
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_trung)
|
||||
VALUES
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
@@ -1,75 +0,0 @@
|
||||
# Config for data/ — 63 .xls files from baotintuc.vn
|
||||
# Sheet mode: "all" because Hà Nội and HCM overflow into Sheet2 (65k row cap)
|
||||
# SBD validation: no numeric guard (build-database.js does not apply ^\d+$)
|
||||
# Blank row strip: off
|
||||
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
|
||||
[schema]
|
||||
ddl = """
|
||||
CREATE TABLE student (
|
||||
so_bao_danh TEXT PRIMARY KEY,
|
||||
ho_ten TEXT NOT NULL,
|
||||
ho_ten_ascii TEXT NOT NULL,
|
||||
ngay_sinh TEXT,
|
||||
toan REAL,
|
||||
ngu_van REAL,
|
||||
vat_ly REAL,
|
||||
hoa_hoc REAL,
|
||||
sinh_hoc REAL,
|
||||
khtn REAL,
|
||||
lich_su REAL,
|
||||
dia_ly REAL,
|
||||
gdcd REAL,
|
||||
khxh REAL,
|
||||
tieng_anh REAL,
|
||||
tieng_phap REAL,
|
||||
tieng_nga REAL,
|
||||
tieng_trung REAL
|
||||
);
|
||||
CREATE INDEX idx_ho_ten ON student(ho_ten);
|
||||
CREATE INDEX idx_ho_ten_ascii ON student(ho_ten_ascii);
|
||||
"""
|
||||
|
||||
[scores]
|
||||
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
||||
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
||||
vat_ly = 'Vật lí:\s*(\d+(?:\.\d+)?)'
|
||||
hoa_hoc = 'Hóa học:\s*(\d+(?:\.\d+)?)'
|
||||
sinh_hoc = 'Sinh học:\s*(\d+(?:\.\d+)?)'
|
||||
khtn = 'KHTN:\s*(\d+(?:\.\d+)?)'
|
||||
lich_su = 'Lịch sử:\s*(\d+(?:\.\d+)?)'
|
||||
dia_ly = 'Địa lí:\s*(\d+(?:\.\d+)?)'
|
||||
gdcd = 'GDCD:\s*(\d+(?:\.\d+)?)'
|
||||
khxh = 'KHXH:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_anh = 'Tiếng Anh:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_phap = 'Tiếng Pháp:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_nga = 'Tiếng Nga:\s*(\d+(?:\.\d+)?)'
|
||||
tieng_trung = 'Tiếng Trung:\s*(\d+(?:\.\d+)?)'
|
||||
|
||||
[insert]
|
||||
sql = """
|
||||
INSERT OR REPLACE INTO student
|
||||
(so_bao_danh, ho_ten, ho_ten_ascii, ngay_sinh,
|
||||
toan, ngu_van, vat_ly, hoa_hoc, sinh_hoc, khtn,
|
||||
lich_su, dia_ly, gdcd, khxh,
|
||||
tieng_anh, tieng_phap, tieng_nga, tieng_trung)
|
||||
VALUES
|
||||
(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
@@ -1,174 +0,0 @@
|
||||
/// Audit subcommand: replicates audit-row-counts.js exactly.
|
||||
///
|
||||
/// Reads all .xlsx files from the input directory (sheet 0 only, matching the
|
||||
/// JS script's behaviour at audit-row-counts.js:33), collects distinct SBDs
|
||||
/// into a HashSet, then queries `SELECT COUNT(*) FROM student` from the DB.
|
||||
/// Prints the same lines as audit-row-counts.js:54-62 and exits 0 on match,
|
||||
/// 1 on mismatch.
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
|
||||
use calamine::{open_workbook_auto, Data, Reader};
|
||||
|
||||
use crate::config::DatasetConfig;
|
||||
use crate::error::BuildError;
|
||||
use crate::reader::is_header_row;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Audit result
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct AuditResult {
|
||||
pub total_data_rows: u64,
|
||||
pub both_empty: u64,
|
||||
pub empty_name: u64,
|
||||
pub empty_sbd: u64,
|
||||
pub distinct_sbds: usize,
|
||||
pub db_count: i64,
|
||||
pub matched: bool,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main audit logic
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Collect distinct SBDs from all xlsx files in `input_dir`, query `db_path`,
|
||||
/// print the audit report and return the result.
|
||||
///
|
||||
/// The JS script reads only sheet 0 for every file (audit-row-counts.js:33).
|
||||
/// Unlike build-database.js, the audit script does NOT iterate all sheets.
|
||||
pub fn run_audit(
|
||||
input_dir: &Path,
|
||||
db_path: &Path,
|
||||
cfg: &DatasetConfig,
|
||||
) -> Result<AuditResult, BuildError> {
|
||||
// Collect .xlsx files (audit-row-counts.js only checks .xlsx — line 15)
|
||||
let mut files: Vec<std::path::PathBuf> = std::fs::read_dir(input_dir)
|
||||
.map_err(|e| BuildError::Io {
|
||||
path: input_dir.display().to_string(),
|
||||
source: e,
|
||||
})?
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.is_file()
|
||||
&& p.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| e.eq_ignore_ascii_case("xlsx"))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
|
||||
let mut all_sbd: HashSet<String> = HashSet::new();
|
||||
let mut total_data_rows: u64 = 0;
|
||||
let mut empty_name: u64 = 0;
|
||||
let mut empty_sbd: u64 = 0;
|
||||
let mut both_empty: u64 = 0;
|
||||
|
||||
for file in &files {
|
||||
let path_str = file.display().to_string();
|
||||
let mut workbook = open_workbook_auto(file).map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let sheet_names = workbook.sheet_names().to_vec();
|
||||
if sheet_names.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// audit-row-counts.js reads only sheet 0 (line 33: wb.SheetNames[0])
|
||||
let range =
|
||||
workbook
|
||||
.worksheet_range(&sheet_names[0])
|
||||
.map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let mut first_row = true;
|
||||
for raw in range.rows() {
|
||||
let row: Vec<Data> = raw.to_vec();
|
||||
|
||||
// Skip header row on first row only
|
||||
if first_row {
|
||||
first_row = false;
|
||||
if is_header_row(&row, &cfg.header) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
total_data_rows += 1;
|
||||
|
||||
let ho_ten = row
|
||||
.get(cfg.columns.ho_ten)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
let sbd = row
|
||||
.get(cfg.columns.so_bao_danh)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
|
||||
if ho_ten.is_empty() && sbd.is_empty() {
|
||||
both_empty += 1;
|
||||
continue;
|
||||
}
|
||||
if ho_ten.is_empty() {
|
||||
empty_name += 1;
|
||||
}
|
||||
if sbd.is_empty() {
|
||||
empty_sbd += 1;
|
||||
}
|
||||
if !sbd.is_empty() {
|
||||
all_sbd.insert(sbd);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Query DB count
|
||||
let conn =
|
||||
rusqlite::Connection::open_with_flags(db_path, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY)?;
|
||||
let db_count: i64 = conn.query_row("SELECT COUNT(*) FROM student", [], |row| row.get(0))?;
|
||||
|
||||
let distinct_sbds = all_sbd.len();
|
||||
let matched = distinct_sbds as i64 == db_count;
|
||||
|
||||
Ok(AuditResult {
|
||||
total_data_rows,
|
||||
both_empty,
|
||||
empty_name,
|
||||
empty_sbd,
|
||||
distinct_sbds,
|
||||
db_count,
|
||||
matched,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Print audit report — mirrors audit-row-counts.js:54-62 exactly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn print_audit_report(r: &AuditResult) {
|
||||
println!("=== Source vs DB ===");
|
||||
println!(
|
||||
"Source: total data rows across all files: {}",
|
||||
r.total_data_rows
|
||||
);
|
||||
println!(
|
||||
"Source: rows with empty name AND sbd (skipped): {}",
|
||||
r.both_empty
|
||||
);
|
||||
println!("Source: rows with missing name only: {}", r.empty_name);
|
||||
println!("Source: rows with missing sbd only: {}", r.empty_sbd);
|
||||
println!("Source: distinct SBDs: {}", r.distinct_sbds);
|
||||
println!("DB: row count: {}", r.db_count);
|
||||
println!(
|
||||
"Match: {}",
|
||||
if r.matched {
|
||||
"YES — all unique SBDs accounted for".to_string()
|
||||
} else {
|
||||
format!("NO — gap of {}", r.distinct_sbds as i64 - r.db_count)
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
/// CLI argument structs via clap derive.
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "xlsxread",
|
||||
version,
|
||||
about = "Read .xls/.xlsx files and build SQLite databases for thptqg datasets"
|
||||
)]
|
||||
pub struct Cli {
|
||||
#[command(subcommand)]
|
||||
pub cmd: Cmd,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
pub enum Cmd {
|
||||
/// Read input spreadsheets and write a SQLite database
|
||||
Build {
|
||||
/// Path to the dataset TOML config file
|
||||
#[arg(long)]
|
||||
schema: PathBuf,
|
||||
|
||||
/// Directory containing the .xls / .xlsx source files
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
|
||||
/// Output SQLite database path
|
||||
#[arg(long)]
|
||||
output: PathBuf,
|
||||
},
|
||||
|
||||
/// Audit: compare distinct SBD count from xlsx files vs DB row count
|
||||
Audit {
|
||||
/// Path to the dataset TOML config file
|
||||
#[arg(long)]
|
||||
schema: PathBuf,
|
||||
|
||||
/// Directory containing the .xlsx source files
|
||||
#[arg(long)]
|
||||
input: PathBuf,
|
||||
|
||||
/// SQLite database to compare against
|
||||
#[arg(long)]
|
||||
db: PathBuf,
|
||||
},
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BuildError;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Top-level dataset configuration loaded from a .toml file
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct DatasetConfig {
|
||||
pub reader: ReaderCfg,
|
||||
pub columns: ColumnMap,
|
||||
pub validation: ValidationCfg,
|
||||
pub header: HeaderCfg,
|
||||
pub schema: SchemaCfg,
|
||||
/// field name → regex source string (one entry per scoreable subject)
|
||||
pub scores: HashMap<String, String>,
|
||||
pub insert: InsertCfg,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct ReaderCfg {
|
||||
/// "all" → iterate every sheet (handles HCM/HN overflow); "first" → sheet 0 only
|
||||
pub sheet_mode: SheetMode,
|
||||
/// If true, skip rows where every cell is empty/null before counting (data-old2 quirk)
|
||||
pub strip_blank_rows: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone, PartialEq, Eq)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum SheetMode {
|
||||
All,
|
||||
First,
|
||||
}
|
||||
|
||||
/// Zero-indexed column positions in the source spreadsheet row.
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct ColumnMap {
|
||||
pub ho_ten: usize,
|
||||
pub ngay_sinh: usize,
|
||||
pub so_bao_danh: usize,
|
||||
pub diem_thi: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct ValidationCfg {
|
||||
/// build-database-old.js / -old2.js require soBaoDanh to match ^\d+$
|
||||
pub require_numeric_sbd: bool,
|
||||
pub require_nonempty_name: bool,
|
||||
pub require_nonempty_sbd: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct HeaderCfg {
|
||||
/// Tokens to match against row[0].to_uppercase() to detect a header row
|
||||
pub tokens: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct SchemaCfg {
|
||||
/// DDL executed verbatim before inserts (CREATE TABLE + CREATE INDEX)
|
||||
pub ddl: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct InsertCfg {
|
||||
/// Parameterised INSERT OR REPLACE SQL using :named_param style
|
||||
pub sql: String,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Loader
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn load_config(path: &Path) -> Result<DatasetConfig, BuildError> {
|
||||
let text = fs::read_to_string(path).map_err(|e| BuildError::Io {
|
||||
path: path.display().to_string(),
|
||||
source: e,
|
||||
})?;
|
||||
let cfg: DatasetConfig = toml::from_str(&text)?;
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SAMPLE_TOML: &str = r#"
|
||||
[reader]
|
||||
sheet_mode = "all"
|
||||
strip_blank_rows = false
|
||||
|
||||
[columns]
|
||||
ho_ten = 0
|
||||
ngay_sinh = 1
|
||||
so_bao_danh = 2
|
||||
diem_thi = 3
|
||||
|
||||
[validation]
|
||||
require_numeric_sbd = false
|
||||
require_nonempty_name = true
|
||||
require_nonempty_sbd = true
|
||||
|
||||
[header]
|
||||
tokens = ["HO_TEN", "HỌ TÊN", "STT"]
|
||||
|
||||
[schema]
|
||||
ddl = "CREATE TABLE student (so_bao_danh TEXT PRIMARY KEY);"
|
||||
|
||||
[scores]
|
||||
toan = 'Toán:\s*(\d+(?:\.\d+)?)'
|
||||
ngu_van = 'Ngữ văn:\s*(\d+(?:\.\d+)?)'
|
||||
|
||||
[insert]
|
||||
sql = "INSERT OR REPLACE INTO student (so_bao_danh) VALUES (:so_bao_danh)"
|
||||
"#;
|
||||
|
||||
#[test]
|
||||
fn config_round_trip() {
|
||||
let cfg: DatasetConfig = toml::from_str(SAMPLE_TOML).expect("parse failed");
|
||||
assert_eq!(cfg.reader.sheet_mode, SheetMode::All);
|
||||
assert!(!cfg.reader.strip_blank_rows);
|
||||
assert_eq!(cfg.columns.ho_ten, 0);
|
||||
assert_eq!(cfg.columns.diem_thi, 3);
|
||||
assert!(!cfg.validation.require_numeric_sbd);
|
||||
assert!(cfg.validation.require_nonempty_name);
|
||||
assert_eq!(cfg.header.tokens.len(), 3);
|
||||
assert!(cfg.scores.contains_key("toan"));
|
||||
assert!(cfg.scores.contains_key("ngu_van"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_first_sheet_mode() {
|
||||
let toml_str = SAMPLE_TOML.replace(r#"sheet_mode = "all""#, r#"sheet_mode = "first""#);
|
||||
let cfg: DatasetConfig = toml::from_str(&toml_str).expect("parse failed");
|
||||
assert_eq!(cfg.reader.sheet_mode, SheetMode::First);
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum BuildError {
|
||||
#[error("I/O error for {path}: {source}")]
|
||||
Io {
|
||||
path: String,
|
||||
#[source]
|
||||
source: std::io::Error,
|
||||
},
|
||||
|
||||
#[error("Calamine error for {path}: {source}")]
|
||||
Calamine {
|
||||
path: String,
|
||||
#[source]
|
||||
source: calamine::Error,
|
||||
},
|
||||
|
||||
#[error("SQLite error: {0}")]
|
||||
Sqlite(#[from] rusqlite::Error),
|
||||
|
||||
#[error("Config parse error: {0}")]
|
||||
Config(#[from] toml::de::Error),
|
||||
|
||||
#[error("Regex compile error for pattern '{pattern}': {source}")]
|
||||
Regex {
|
||||
pattern: String,
|
||||
#[source]
|
||||
source: regex::Error,
|
||||
},
|
||||
|
||||
#[error("Schema has no sheets in file: {0}")]
|
||||
NoSheets(String),
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
/// Public library interface for integration tests.
|
||||
/// The binary entry point is src/main.rs; this file re-exports the internal
|
||||
/// modules so tests/golden.rs can call them without going through the CLI.
|
||||
pub mod audit;
|
||||
pub mod config;
|
||||
pub mod error;
|
||||
pub mod reader;
|
||||
pub mod transform;
|
||||
pub mod writer;
|
||||
@@ -1,193 +0,0 @@
|
||||
/// xlsxread — Rust CLI replacing the SheetJS xlsx build scripts.
|
||||
///
|
||||
/// Subcommands:
|
||||
/// build — read .xls/.xlsx files → write SQLite DB
|
||||
/// audit — compare distinct SBD count from xlsx vs DB row count
|
||||
///
|
||||
/// Library modules are declared in lib.rs; main.rs only adds the CLI layer.
|
||||
mod cli;
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser;
|
||||
|
||||
use cli::{Cli, Cmd};
|
||||
use xlsxread::audit;
|
||||
use xlsxread::config::load_config;
|
||||
use xlsxread::reader::{is_all_blank, process_file};
|
||||
use xlsxread::transform::{validate_row, CompiledPatterns, SkipReason};
|
||||
use xlsxread::writer::{finish_db, insert_row, open_db, SCORE_FIELDS};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
match cli.cmd {
|
||||
Cmd::Build {
|
||||
schema,
|
||||
input,
|
||||
output,
|
||||
} => {
|
||||
run_build(&schema, &input, &output)?;
|
||||
}
|
||||
Cmd::Audit { schema, input, db } => {
|
||||
let cfg = load_config(&schema)
|
||||
.with_context(|| format!("Failed to load config: {}", schema.display()))?;
|
||||
let result = audit::run_audit(&input, &db, &cfg).with_context(|| "Audit failed")?;
|
||||
audit::print_audit_report(&result);
|
||||
if !result.matched {
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Build subcommand
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn run_build(schema_path: &Path, input_dir: &Path, output_path: &Path) -> Result<()> {
|
||||
let cfg = load_config(schema_path)
|
||||
.with_context(|| format!("Failed to load config: {}", schema_path.display()))?;
|
||||
|
||||
// Compile score regexes once at startup
|
||||
let patterns =
|
||||
CompiledPatterns::new(&cfg.scores).with_context(|| "Failed to compile score regexes")?;
|
||||
|
||||
// Collect input files (.xls and .xlsx), sorted for deterministic order
|
||||
let mut files: Vec<std::path::PathBuf> = std::fs::read_dir(input_dir)
|
||||
.with_context(|| format!("Cannot read input dir: {}", input_dir.display()))?
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.is_file()
|
||||
&& p.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| {
|
||||
let lower = e.to_lowercase();
|
||||
lower == "xls" || lower == "xlsx"
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
|
||||
let dataset_label = input_dir
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("data");
|
||||
|
||||
println!(
|
||||
"[build] {dataset_label}/ → {} ({} files)",
|
||||
output_path.display(),
|
||||
files.len()
|
||||
);
|
||||
|
||||
// Open (or recreate) DB and apply DDL
|
||||
let conn = open_db(output_path, &cfg)
|
||||
.with_context(|| format!("Failed to open DB: {}", output_path.display()))?;
|
||||
|
||||
let mut total_source_rows: u64 = 0;
|
||||
let mut total_skipped: u64 = 0;
|
||||
let mut total_errors: u64 = 0;
|
||||
|
||||
let is_old2 = dataset_label.contains("old2");
|
||||
let strip_blank = cfg.reader.strip_blank_rows;
|
||||
|
||||
// Single transaction over all files — mirrors the Node `db.transaction(() => { ... })()`
|
||||
conn.execute_batch("BEGIN")?;
|
||||
|
||||
for file in &files {
|
||||
let base = file
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("?")
|
||||
.to_owned();
|
||||
let mut file_rows: u64 = 0;
|
||||
let mut file_skipped: u64 = 0;
|
||||
let mut file_errors: u64 = 0;
|
||||
|
||||
let process_result = process_file(file, &cfg, |_sheet_idx, raw| {
|
||||
// data-old2: skip fully blank rows BEFORE counting sourceRows
|
||||
let all_blank = is_all_blank(raw);
|
||||
if strip_blank && all_blank {
|
||||
return;
|
||||
}
|
||||
|
||||
total_source_rows += 1;
|
||||
|
||||
let ho_ten = raw
|
||||
.get(cfg.columns.ho_ten)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
let so_bao_danh = raw
|
||||
.get(cfg.columns.so_bao_danh)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
|
||||
match validate_row(
|
||||
&ho_ten,
|
||||
&so_bao_danh,
|
||||
&cfg.validation,
|
||||
strip_blank,
|
||||
all_blank,
|
||||
) {
|
||||
Err(SkipReason::BlankRow) => {
|
||||
// Already guarded above; won't reach here
|
||||
}
|
||||
Err(_) => {
|
||||
file_skipped += 1;
|
||||
return;
|
||||
}
|
||||
Ok(()) => {}
|
||||
}
|
||||
|
||||
let parsed = xlsxread::transform::transform_row(raw, &cfg, &patterns);
|
||||
|
||||
match insert_row(&conn, &cfg.insert.sql, &parsed, SCORE_FIELDS) {
|
||||
Ok(()) => {
|
||||
file_rows += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
file_errors += 1;
|
||||
if total_errors + file_errors <= 5 {
|
||||
eprintln!(" [warn] {base}: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
match process_result {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
eprintln!(" [error] {base}: {e}");
|
||||
file_errors += 1;
|
||||
}
|
||||
}
|
||||
|
||||
total_skipped += file_skipped;
|
||||
total_errors += file_errors;
|
||||
|
||||
// Per-file row count line — mirrors `console.log(` ${base}: ${fileRows} rows`)`
|
||||
println!(" {base}: {file_rows} rows");
|
||||
}
|
||||
|
||||
conn.execute_batch("COMMIT")?;
|
||||
|
||||
// VACUUM + stats output
|
||||
finish_db(
|
||||
&conn,
|
||||
output_path,
|
||||
total_source_rows,
|
||||
total_skipped,
|
||||
total_errors,
|
||||
dataset_label,
|
||||
files.len(),
|
||||
is_old2,
|
||||
)
|
||||
.with_context(|| "Failed to finalise DB")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
/// Spreadsheet reader: wraps calamine to iterate rows across sheets.
|
||||
///
|
||||
/// Sheet selection mirrors the JS scripts:
|
||||
/// - sheet_mode = "all" → iterate every sheet (handles HCM/HN 65k overflow in data/)
|
||||
/// - sheet_mode = "first" → sheet 0 only (data-old/)
|
||||
///
|
||||
/// Header detection mirrors build-lib.js isHeaderRow:
|
||||
/// row[0].toUpperCase() in {"HO_TEN", "HỌ TÊN", "STT"}
|
||||
use std::path::Path;
|
||||
|
||||
use calamine::{open_workbook_auto, Data, Reader, Sheets};
|
||||
|
||||
use crate::config::{DatasetConfig, HeaderCfg, SheetMode};
|
||||
use crate::error::BuildError;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public row representation from calamine
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub type RawRow = Vec<Data>;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Header detection — mirrors build-lib.js isHeaderRow
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Returns true when the first cell (uppercased) matches one of the configured
|
||||
/// header tokens. Used to skip the header row on the first row of each sheet.
|
||||
pub fn is_header_row(row: &[Data], header_cfg: &HeaderCfg) -> bool {
|
||||
if row.len() < 3 {
|
||||
return false;
|
||||
}
|
||||
let first = row[0].to_string().trim().to_uppercase();
|
||||
header_cfg.tokens.iter().any(|t| t.to_uppercase() == first)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// All-blank row check (data-old2: strip_blank_rows)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn is_all_blank(row: &[Data]) -> bool {
|
||||
row.iter()
|
||||
.all(|c| matches!(c, Data::Empty) || c.to_string().trim().is_empty())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// File processor — yields all data rows from the file
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Process one spreadsheet file, calling `on_row` for each data row.
|
||||
///
|
||||
/// `on_row` receives `(sheet_index, row_index_in_sheet, raw_row)` where
|
||||
/// `row_index_in_sheet` is 0-based AFTER the header has been consumed.
|
||||
/// Returns `(sheets_seen, total_rows_yielded)`.
|
||||
pub fn process_file<F>(
|
||||
path: &Path,
|
||||
cfg: &DatasetConfig,
|
||||
mut on_row: F,
|
||||
) -> Result<(usize, usize), BuildError>
|
||||
where
|
||||
F: FnMut(usize, &RawRow),
|
||||
{
|
||||
let path_str = path.display().to_string();
|
||||
|
||||
// calamine::open_workbook_auto dispatches on file extension
|
||||
let mut workbook: Sheets<_> = open_workbook_auto(path).map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let sheet_names: Vec<String> = workbook.sheet_names().to_vec();
|
||||
if sheet_names.is_empty() {
|
||||
return Err(BuildError::NoSheets(path_str.clone()));
|
||||
}
|
||||
|
||||
// Sheet selection per config
|
||||
let sheets_to_read: Vec<String> = match cfg.reader.sheet_mode {
|
||||
SheetMode::All => sheet_names.clone(),
|
||||
SheetMode::First => vec![sheet_names[0].clone()],
|
||||
};
|
||||
|
||||
let mut total_rows = 0usize;
|
||||
|
||||
for (sheet_idx, sheet_name) in sheets_to_read.iter().enumerate() {
|
||||
let range = workbook
|
||||
.worksheet_range(sheet_name)
|
||||
.map_err(|e| BuildError::Calamine {
|
||||
path: path_str.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
|
||||
let mut first_row = true;
|
||||
|
||||
for raw in range.rows() {
|
||||
let row: RawRow = raw.to_vec();
|
||||
|
||||
// Skip header row on first row of each sheet (matches JS: `if (i === 0 && isHeaderRow(...))`)
|
||||
if first_row {
|
||||
first_row = false;
|
||||
if is_header_row(&row, &cfg.header) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
on_row(sheet_idx, &row);
|
||||
total_rows += 1;
|
||||
}
|
||||
}
|
||||
|
||||
Ok((sheets_to_read.len(), total_rows))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::HeaderCfg;
|
||||
|
||||
fn hdr(tokens: &[&str]) -> HeaderCfg {
|
||||
HeaderCfg {
|
||||
tokens: tokens.iter().map(|s| s.to_string()).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_detects_ho_ten() {
|
||||
let row = vec![
|
||||
Data::String("HO_TEN".into()),
|
||||
Data::String("NGAY_SINH".into()),
|
||||
Data::String("SBD".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_detects_stt() {
|
||||
let row = vec![
|
||||
Data::String("STT".into()),
|
||||
Data::String("B".into()),
|
||||
Data::String("C".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_detects_ho_ten_unicode() {
|
||||
let row = vec![
|
||||
Data::String("HỌ TÊN".into()),
|
||||
Data::String("B".into()),
|
||||
Data::String("C".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_rejects_data_row() {
|
||||
let row = vec![
|
||||
Data::String("Nguyen Van A".into()),
|
||||
Data::String("01/01/2000".into()),
|
||||
Data::String("12345678".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(!is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_rejects_short_row() {
|
||||
let row = vec![Data::String("HO_TEN".into()), Data::Empty];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(!is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_case_insensitive() {
|
||||
let row = vec![
|
||||
Data::String("ho_ten".into()),
|
||||
Data::String("B".into()),
|
||||
Data::String("C".into()),
|
||||
];
|
||||
let cfg = hdr(&["HO_TEN", "HỌ TÊN", "STT"]);
|
||||
assert!(is_header_row(&row, &cfg));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blank_row_detection() {
|
||||
let row = vec![Data::Empty, Data::Empty, Data::String("".into())];
|
||||
assert!(is_all_blank(&row));
|
||||
|
||||
let row2 = vec![Data::String("Nguyen".into()), Data::Empty, Data::Empty];
|
||||
assert!(!is_all_blank(&row2));
|
||||
}
|
||||
}
|
||||
@@ -1,398 +0,0 @@
|
||||
/// Row transformation: ascii normalisation, score regex parsing, validation.
|
||||
///
|
||||
/// `to_ascii` replicates build-lib.js `toAscii` exactly:
|
||||
/// str.normalize("NFD").replace(/[̀-ͯ]/g,"").replace(/đ/gi,"d").toLowerCase()
|
||||
use std::collections::HashMap;
|
||||
|
||||
use regex::Regex;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
use crate::config::{DatasetConfig, ValidationCfg};
|
||||
use crate::error::BuildError;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Compiled score patterns (built once at startup from config)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct CompiledPatterns {
|
||||
/// Ordered list so INSERT column order is deterministic
|
||||
pub patterns: Vec<(String, Regex)>,
|
||||
}
|
||||
|
||||
impl CompiledPatterns {
|
||||
pub fn new(scores: &HashMap<String, String>) -> Result<Self, BuildError> {
|
||||
let mut patterns = Vec::with_capacity(scores.len());
|
||||
for (field, src) in scores {
|
||||
let re = Regex::new(src).map_err(|e| BuildError::Regex {
|
||||
pattern: src.clone(),
|
||||
source: e,
|
||||
})?;
|
||||
patterns.push((field.clone(), re));
|
||||
}
|
||||
// Sort for deterministic order across HashMap iteration
|
||||
patterns.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
Ok(Self { patterns })
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// to_ascii — must be byte-for-byte equivalent to build-lib.js toAscii
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Normalise a Vietnamese name to an ASCII slug.
|
||||
///
|
||||
/// Algorithm mirrors the JavaScript `toAscii` in build-lib.js:
|
||||
/// 1. NFD decompose (splits base + combining diacritics)
|
||||
/// 2. Drop all Unicode combining marks (U+0300–U+036F)
|
||||
/// 3. Replace đ/Đ with d (NFD does not decompose đ)
|
||||
/// 4. Lowercase
|
||||
pub fn to_ascii(s: &str) -> String {
|
||||
// Step 1 + 2: NFD then filter out combining marks (Unicode category M)
|
||||
let decomposed: String = s
|
||||
.nfd()
|
||||
.filter(|c| !('\u{0300}'..='\u{036f}').contains(c))
|
||||
.collect();
|
||||
|
||||
// Step 3: đ/Đ are not decomposed by NFD — replace explicitly
|
||||
let replaced = decomposed.replace(['đ', 'Đ'], "d");
|
||||
|
||||
// Step 4: lowercase
|
||||
replaced.to_lowercase()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parsed row ready for DB insert
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct ParsedRow {
|
||||
pub so_bao_danh: String,
|
||||
pub ho_ten: String,
|
||||
pub ho_ten_ascii: String,
|
||||
pub ngay_sinh: Option<String>,
|
||||
/// Subject field → float value; absent subjects not in map → NULL
|
||||
pub scores: HashMap<String, f64>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Row validation — mirrors the per-script skip logic
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Returns `None` when the row should be skipped entirely (before sourceRows counter).
|
||||
/// Returns `Some(reason)` when the row should be counted as sourceRows but skipped.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum SkipReason {
|
||||
/// Row is fully blank (data-old2 only, before sourceRows counter)
|
||||
BlankRow,
|
||||
/// soBaoDanh or hoTen empty/missing
|
||||
EmptyField,
|
||||
/// soBaoDanh contains non-digit characters (data-old / data-old2 guard)
|
||||
NonNumericSbd,
|
||||
}
|
||||
|
||||
/// Validates a raw cell slice against the dataset's `ValidationCfg`.
|
||||
/// Returns `Ok(())` on pass, `Err(SkipReason)` on fail.
|
||||
pub fn validate_row(
|
||||
ho_ten: &str,
|
||||
so_bao_danh: &str,
|
||||
cfg: &ValidationCfg,
|
||||
strip_blank_rows: bool,
|
||||
all_blank: bool,
|
||||
) -> Result<(), SkipReason> {
|
||||
// data-old2: skip fully blank rows BEFORE counting sourceRows
|
||||
if strip_blank_rows && all_blank {
|
||||
return Err(SkipReason::BlankRow);
|
||||
}
|
||||
|
||||
if cfg.require_nonempty_sbd && so_bao_danh.is_empty() {
|
||||
return Err(SkipReason::EmptyField);
|
||||
}
|
||||
if cfg.require_nonempty_name && ho_ten.is_empty() {
|
||||
return Err(SkipReason::EmptyField);
|
||||
}
|
||||
if cfg.require_numeric_sbd && !so_bao_danh.chars().all(|c| c.is_ascii_digit()) {
|
||||
return Err(SkipReason::NonNumericSbd);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Score parsing — mirrors build-lib.js parseScores
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Parse a DIEM_THI cell string and extract matching subject scores.
|
||||
pub fn parse_scores(diem_thi: &str, patterns: &CompiledPatterns) -> HashMap<String, f64> {
|
||||
let mut out = HashMap::new();
|
||||
for (field, re) in &patterns.patterns {
|
||||
if let Some(caps) = re.captures(diem_thi) {
|
||||
if let Some(m) = caps.get(1) {
|
||||
if let Ok(v) = m.as_str().parse::<f64>() {
|
||||
if v.is_finite() {
|
||||
out.insert(field.clone(), v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Full row transform
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Extract and transform one spreadsheet row into a `ParsedRow`.
|
||||
/// `raw` is the full cell slice; column indices come from `cfg.columns`.
|
||||
pub fn transform_row(
|
||||
raw: &[calamine::Data],
|
||||
cfg: &DatasetConfig,
|
||||
patterns: &CompiledPatterns,
|
||||
) -> ParsedRow {
|
||||
let get = |idx: usize| -> String {
|
||||
raw.get(idx)
|
||||
.map(|cell| cell.to_string().trim().to_owned())
|
||||
.unwrap_or_default()
|
||||
};
|
||||
|
||||
let ho_ten = get(cfg.columns.ho_ten);
|
||||
let ngay_sinh = get(cfg.columns.ngay_sinh);
|
||||
let so_bao_danh = get(cfg.columns.so_bao_danh);
|
||||
let diem_thi = raw
|
||||
.get(cfg.columns.diem_thi)
|
||||
.map(|c| c.to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let ho_ten_ascii = to_ascii(&ho_ten);
|
||||
let scores = parse_scores(&diem_thi, patterns);
|
||||
let ngay_sinh_opt = if ngay_sinh.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(ngay_sinh)
|
||||
};
|
||||
|
||||
ParsedRow {
|
||||
so_bao_danh,
|
||||
ho_ten,
|
||||
ho_ten_ascii,
|
||||
ngay_sinh: ngay_sinh_opt,
|
||||
scores,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unit tests — 20 cases for to_ascii (real Vietnamese names)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// Helper: assert to_ascii(input) == expected
|
||||
fn check(input: &str, expected: &str) {
|
||||
assert_eq!(
|
||||
to_ascii(input),
|
||||
expected,
|
||||
"to_ascii({input:?}) expected {expected:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_plain_latin() {
|
||||
check("Nguyen Van A", "nguyen van a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_thi_hoa() {
|
||||
check("Nguyễn Thị Hoa", "nguyen thi hoa");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_tran_van_duc() {
|
||||
// đ/Đ replacement
|
||||
check("Trần Văn Đức", "tran van duc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_le_thi_my_duyen() {
|
||||
check("Lê Thị Mỹ Duyên", "le thi my duyen");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_pham_thi_lan() {
|
||||
check("Phạm Thị Lan", "pham thi lan");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_bui_thi_thu() {
|
||||
check("Bùi Thị Thu", "bui thi thu");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_hoang_van_truong() {
|
||||
check("Hoàng Văn Trường", "hoang van truong");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_do_thi_ngan() {
|
||||
// Đ uppercase at start
|
||||
check("Đỗ Thị Ngân", "do thi ngan");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_van_khanh() {
|
||||
check("Nguyễn Văn Khánh", "nguyen van khanh");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_trinh_thi_bich_ngoc() {
|
||||
check("Trịnh Thị Bích Ngọc", "trinh thi bich ngoc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_vu_thi_dieu() {
|
||||
// ề = e + combining grave + combining circumflex (after NFD)
|
||||
check("Vũ Thị Diệu", "vu thi dieu");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_thi_tuong_vi() {
|
||||
check("Nguyễn Thị Tường Vi", "nguyen thi tuong vi");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_lowercase_d_stroke() {
|
||||
// Lowercase đ → d
|
||||
check("đặng thị hằng", "dang thi hang");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_uppercase_d_stroke() {
|
||||
check("ĐẶNG THỊ HẰNG", "dang thi hang");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_mixed_case() {
|
||||
check("NGUYỄN VĂN AN", "nguyen van an");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_tran_thi_kim_anh() {
|
||||
check("Trần Thị Kim Anh", "tran thi kim anh");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_nguyen_thi_phuong_thao() {
|
||||
check("Nguyễn Thị Phương Thảo", "nguyen thi phuong thao");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_le_van_long() {
|
||||
check("Lê Văn Long", "le van long");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_vo_thi_xuan_mai() {
|
||||
check("Võ Thị Xuân Mai", "vo thi xuan mai");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ascii_empty_string() {
|
||||
check("", "");
|
||||
}
|
||||
|
||||
// --- Score parsing tests ---
|
||||
|
||||
fn make_patterns() -> CompiledPatterns {
|
||||
let mut map = HashMap::new();
|
||||
map.insert("toan".into(), r"Toán:\s*(\d+(?:\.\d+)?)".into());
|
||||
map.insert("ngu_van".into(), r"Ngữ văn:\s*(\d+(?:\.\d+)?)".into());
|
||||
map.insert("vat_ly".into(), r"Vật lí:\s*(\d+(?:\.\d+)?)".into());
|
||||
CompiledPatterns::new(&map).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_scores_single() {
|
||||
let p = make_patterns();
|
||||
let s = "Toán: 8.5";
|
||||
let scores = parse_scores(s, &p);
|
||||
assert_eq!(scores.get("toan"), Some(&8.5));
|
||||
assert!(scores.get("ngu_van").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_scores_multiple() {
|
||||
let p = make_patterns();
|
||||
let s = "Toán: 7.25 Ngữ văn: 6.0 Vật lí: 9";
|
||||
let scores = parse_scores(s, &p);
|
||||
assert_eq!(scores.get("toan"), Some(&7.25));
|
||||
assert_eq!(scores.get("ngu_van"), Some(&6.0));
|
||||
assert_eq!(scores.get("vat_ly"), Some(&9.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_scores_empty_cell() {
|
||||
let p = make_patterns();
|
||||
let scores = parse_scores("", &p);
|
||||
assert!(scores.is_empty());
|
||||
}
|
||||
|
||||
// --- Validation tests ---
|
||||
|
||||
fn default_validation() -> ValidationCfg {
|
||||
ValidationCfg {
|
||||
require_numeric_sbd: false,
|
||||
require_nonempty_name: true,
|
||||
require_nonempty_sbd: true,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_ok() {
|
||||
let v = default_validation();
|
||||
assert!(validate_row("Nguyen Van A", "12345678", &v, false, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_empty_sbd() {
|
||||
let v = default_validation();
|
||||
assert_eq!(
|
||||
validate_row("Nguyen Van A", "", &v, false, false),
|
||||
Err(SkipReason::EmptyField)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_empty_name() {
|
||||
let v = default_validation();
|
||||
assert_eq!(
|
||||
validate_row("", "12345678", &v, false, false),
|
||||
Err(SkipReason::EmptyField)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_non_numeric_sbd_rejected() {
|
||||
let mut v = default_validation();
|
||||
v.require_numeric_sbd = true;
|
||||
assert_eq!(
|
||||
validate_row("Nguyen Van A", "12AB5678", &v, false, false),
|
||||
Err(SkipReason::NonNumericSbd)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_numeric_sbd_accepted() {
|
||||
let mut v = default_validation();
|
||||
v.require_numeric_sbd = true;
|
||||
assert!(validate_row("Nguyen Van A", "12345678", &v, false, false).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_blank_row_skipped() {
|
||||
let v = default_validation();
|
||||
// strip_blank_rows=true AND all_blank=true → BlankRow
|
||||
assert_eq!(
|
||||
validate_row("", "", &v, true, true),
|
||||
Err(SkipReason::BlankRow)
|
||||
);
|
||||
}
|
||||
}
|
||||
-18
@@ -1,18 +0,0 @@
|
||||
# Test Fixtures
|
||||
|
||||
Anonymised `.xlsx` files for integration testing. All student PII has been replaced:
|
||||
|
||||
- `ho_ten` replaced with `Nguyen Van Test NNN` / `Tran Thi Test NNN` patterns
|
||||
- `so_bao_danh` replaced with sequential synthetic numbers (e.g. `10000001`)
|
||||
- `ngay_sinh` replaced with fixed synthetic dates
|
||||
- Scores are realistic random values in the 0–10 range
|
||||
|
||||
Files:
|
||||
|
||||
- `province-100.xlsx` — 100-row single-sheet file (simulates a normal province)
|
||||
- `hcm-overflow.xlsx` — 2-sheet file (200 rows Sheet1 + 200 rows Sheet2, simulating HCM overflow)
|
||||
- `province-numeric-sbd.xlsx` — 100 rows with strictly numeric SBDs (for data-old variant)
|
||||
|
||||
These files are generated by `tests/golden.rs` `generate_fixtures()` if they do not already exist
|
||||
on disk. The generator is pure Rust (uses the `zip` crate already pulled in via calamine).
|
||||
No external Python or Node tooling required for unit/integration tests.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,690 +0,0 @@
|
||||
/// Stage 5 golden tests — integration tests using anonymised fixture files.
|
||||
///
|
||||
/// Fixture files are generated in-process via raw OOXML + zip if they do not
|
||||
/// already exist on disk. No external Python or Node tooling required for the
|
||||
/// Rust-side tests. The Node golden comparison is marked #[ignore] when pnpm
|
||||
/// is not in PATH.
|
||||
use std::io::Write as IoWrite;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Minimal OOXML xlsx generator
|
||||
//
|
||||
// Produces a valid .xlsx that calamine can read. Only uses the `zip` crate
|
||||
// which is already pulled in as a transitive dependency of calamine.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One row of cell data for a fixture sheet.
|
||||
struct XlsxRow {
|
||||
values: Vec<String>,
|
||||
}
|
||||
|
||||
/// Write a minimal .xlsx to `path` with the given sheets.
|
||||
/// `sheets`: Vec<(sheet_name, rows)> where rows[0] is the header.
|
||||
fn write_xlsx(path: &Path, sheets: &[(String, Vec<XlsxRow>)]) {
|
||||
use zip::{write::SimpleFileOptions, ZipWriter};
|
||||
|
||||
let file = std::fs::File::create(path).expect("create fixture xlsx");
|
||||
let mut zip = ZipWriter::new(file);
|
||||
let opts = SimpleFileOptions::default();
|
||||
|
||||
// [Content_Types].xml
|
||||
let mut content_types = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
|
||||
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
|
||||
<Default Extension="xml" ContentType="application/xml"/>
|
||||
<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>
|
||||
"#,
|
||||
);
|
||||
for (i, _) in sheets.iter().enumerate() {
|
||||
content_types.push_str(&format!(
|
||||
r#" <Override PartName="/xl/worksheets/sheet{}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>
|
||||
"#,
|
||||
i + 1
|
||||
));
|
||||
}
|
||||
content_types.push_str("</Types>");
|
||||
zip.start_file("[Content_Types].xml", opts).unwrap();
|
||||
zip.write_all(content_types.as_bytes()).unwrap();
|
||||
|
||||
// _rels/.rels
|
||||
zip.start_file("_rels/.rels", opts).unwrap();
|
||||
zip.write_all(
|
||||
br#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>
|
||||
</Relationships>"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// xl/_rels/workbook.xml.rels
|
||||
let mut wb_rels = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
"#,
|
||||
);
|
||||
for (i, _) in sheets.iter().enumerate() {
|
||||
wb_rels.push_str(&format!(
|
||||
r#" <Relationship Id="rId{}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet{}.xml"/>
|
||||
"#,
|
||||
i + 1,
|
||||
i + 1
|
||||
));
|
||||
}
|
||||
wb_rels.push_str("</Relationships>");
|
||||
zip.start_file("xl/_rels/workbook.xml.rels", opts).unwrap();
|
||||
zip.write_all(wb_rels.as_bytes()).unwrap();
|
||||
|
||||
// xl/workbook.xml
|
||||
let mut wb = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
|
||||
xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
|
||||
<sheets>
|
||||
"#,
|
||||
);
|
||||
for (i, (name, _)) in sheets.iter().enumerate() {
|
||||
let escaped = xml_escape(name);
|
||||
wb.push_str(&format!(
|
||||
r#" <sheet name="{}" sheetId="{}" r:id="rId{}"/>
|
||||
"#,
|
||||
escaped,
|
||||
i + 1,
|
||||
i + 1
|
||||
));
|
||||
}
|
||||
wb.push_str(" </sheets>\n</workbook>");
|
||||
zip.start_file("xl/workbook.xml", opts).unwrap();
|
||||
zip.write_all(wb.as_bytes()).unwrap();
|
||||
|
||||
// xl/worksheets/sheetN.xml
|
||||
for (i, (_, rows)) in sheets.iter().enumerate() {
|
||||
let mut ws = String::from(
|
||||
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<sheetData>
|
||||
"#,
|
||||
);
|
||||
for (row_idx, row) in rows.iter().enumerate() {
|
||||
ws.push_str(&format!(
|
||||
r#" <row r="{}">
|
||||
"#,
|
||||
row_idx + 1
|
||||
));
|
||||
for (col_idx, val) in row.values.iter().enumerate() {
|
||||
let col_letter = col_letter(col_idx);
|
||||
let cell_ref = format!("{}{}", col_letter, row_idx + 1);
|
||||
let escaped = xml_escape(val);
|
||||
ws.push_str(&format!(
|
||||
r#" <c r="{}" t="inlineStr"><is><t>{}</t></is></c>
|
||||
"#,
|
||||
cell_ref, escaped
|
||||
));
|
||||
}
|
||||
ws.push_str(" </row>\n");
|
||||
}
|
||||
ws.push_str(" </sheetData>\n</worksheet>");
|
||||
zip.start_file(&format!("xl/worksheets/sheet{}.xml", i + 1), opts)
|
||||
.unwrap();
|
||||
zip.write_all(ws.as_bytes()).unwrap();
|
||||
}
|
||||
|
||||
zip.finish().unwrap();
|
||||
}
|
||||
|
||||
fn col_letter(idx: usize) -> &'static str {
|
||||
const LETTERS: &[&str] = &[
|
||||
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R",
|
||||
"S", "T", "U", "V", "W", "X", "Y", "Z",
|
||||
];
|
||||
LETTERS[idx % 26]
|
||||
}
|
||||
|
||||
fn xml_escape(s: &str) -> String {
|
||||
s.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('>', ">")
|
||||
.replace('"', """)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture data builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn header_row() -> XlsxRow {
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
"HO_TEN".into(),
|
||||
"NGAY_SINH".into(),
|
||||
"SO_BAO_DANH".into(),
|
||||
"DIEM_THI".into(),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn data_row(idx: usize, scores: &str) -> XlsxRow {
|
||||
// Anonymised: name uses sequential pattern, SBD is purely synthetic
|
||||
let name = if idx % 2 == 0 {
|
||||
format!("Nguyen Van Test {:03}", idx)
|
||||
} else {
|
||||
format!("Tran Thi Test {:03}", idx)
|
||||
};
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
name,
|
||||
format!("15/0{}/{}", (idx % 9) + 1, 1999 + (idx % 5)),
|
||||
format!("1000{:04}", idx),
|
||||
scores.to_owned(),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_scores(idx: usize) -> String {
|
||||
// Realistic scores in 0–10 range, varies by idx
|
||||
let toan = 4.0 + (idx % 60) as f64 / 10.0;
|
||||
let van = 3.5 + (idx % 65) as f64 / 10.0;
|
||||
format!("Toán: {toan:.1} Ngữ văn: {van:.1} Tiếng Anh: 7.5")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture file paths
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
// tests/fixtures/ relative to the crate root
|
||||
let mut p = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
p.push("tests");
|
||||
p.push("fixtures");
|
||||
p
|
||||
}
|
||||
|
||||
fn province_fixture_path() -> PathBuf {
|
||||
fixtures_dir().join("province-100.xlsx")
|
||||
}
|
||||
|
||||
fn hcm_overflow_fixture_path() -> PathBuf {
|
||||
fixtures_dir().join("hcm-overflow.xlsx")
|
||||
}
|
||||
|
||||
fn numeric_sbd_fixture_path() -> PathBuf {
|
||||
fixtures_dir().join("province-numeric-sbd.xlsx")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixture generation — called once per test run if files missing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn ensure_fixtures() {
|
||||
let dir = fixtures_dir();
|
||||
std::fs::create_dir_all(&dir).expect("create fixtures dir");
|
||||
|
||||
// province-100.xlsx — 100 data rows, single sheet, with header
|
||||
if !province_fixture_path().exists() {
|
||||
let mut rows = vec![header_row()];
|
||||
for i in 0..100 {
|
||||
rows.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(&province_fixture_path(), &[("Sheet1".to_owned(), rows)]);
|
||||
}
|
||||
|
||||
// hcm-overflow.xlsx — 2 sheets × 200 rows each (no header on sheet 2)
|
||||
if !hcm_overflow_fixture_path().exists() {
|
||||
let mut sheet1 = vec![header_row()];
|
||||
for i in 0..200 {
|
||||
sheet1.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
// Sheet2: continuation rows, no header row (as in real HCM overflow)
|
||||
let mut sheet2 = Vec::new();
|
||||
for i in 200..400 {
|
||||
sheet2.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(
|
||||
&hcm_overflow_fixture_path(),
|
||||
&[("Sheet1".to_owned(), sheet1), ("Sheet2".to_owned(), sheet2)],
|
||||
);
|
||||
}
|
||||
|
||||
// province-numeric-sbd.xlsx — strictly numeric SBDs for data-old config
|
||||
if !numeric_sbd_fixture_path().exists() {
|
||||
let mut rows = vec![header_row()];
|
||||
for i in 0..100 {
|
||||
rows.push(XlsxRow {
|
||||
values: vec![
|
||||
format!("Nguyen Van Test {:03}", i),
|
||||
"01/01/2000".to_owned(),
|
||||
format!("{:08}", 20000000 + i), // pure digits
|
||||
sample_scores(i),
|
||||
],
|
||||
});
|
||||
}
|
||||
write_xlsx(&numeric_sbd_fixture_path(), &[("Sheet1".to_owned(), rows)]);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn make_data_config() -> xlsxread::config::DatasetConfig {
|
||||
let cfg_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("configs")
|
||||
.join("thptqg2017-data.toml");
|
||||
xlsxread::config::load_config(&cfg_path).expect("load data config")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Integration tests — pure Rust, no Node dependency
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn province_100_builds_100_rows() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(count, 100, "expected 100 rows from province-100 fixture");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hcm_overflow_builds_400_rows() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
std::fs::copy(hcm_overflow_fixture_path(), fixture_dir.join("hcm.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(
|
||||
count, 400,
|
||||
"expected 400 rows (200 × 2 sheets) from hcm-overflow fixture"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_old_first_sheet_only_100_rows() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
// Use the overflow file but with data-old config (first sheet only → 200 rows)
|
||||
std::fs::copy(hcm_overflow_fixture_path(), fixture_dir.join("hcm.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data-old.toml");
|
||||
|
||||
// data-old: sheet_mode=first → only 200 rows from sheet1; but SBDs "1000NNNN" are
|
||||
// all digits so all pass the numeric guard
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(
|
||||
count, 200,
|
||||
"data-old config should read only first sheet (200 rows)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn numeric_sbd_guard_rejects_non_numeric() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
// Write a fixture with one non-numeric SBD mixed in
|
||||
let mut rows = vec![header_row()];
|
||||
for i in 0..10 {
|
||||
rows.push(XlsxRow {
|
||||
values: vec![
|
||||
format!("Test {:03}", i),
|
||||
"01/01/2000".to_owned(),
|
||||
if i == 5 {
|
||||
"ABC123".to_owned()
|
||||
} else {
|
||||
format!("{:08}", 20000000 + i)
|
||||
},
|
||||
sample_scores(i),
|
||||
],
|
||||
});
|
||||
}
|
||||
let mixed_path = fixture_dir.join("mixed.xlsx");
|
||||
write_xlsx(&mixed_path, &[("Sheet1".to_owned(), rows)]);
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data-old.toml");
|
||||
|
||||
// Row i=5 has non-numeric SBD → rejected by data-old config
|
||||
let count = query_count(&db_path);
|
||||
assert_eq!(
|
||||
count, 9,
|
||||
"non-numeric SBD row should be skipped by data-old config"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scores_parsed_correctly_into_db() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
let rows = vec![
|
||||
header_row(),
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
"Nguyen Van Test 001".to_owned(),
|
||||
"01/01/2000".to_owned(),
|
||||
"10000001".to_owned(),
|
||||
"Toán: 8.5 Ngữ văn: 7.0 Tiếng Anh: 9.25".to_owned(),
|
||||
],
|
||||
},
|
||||
];
|
||||
write_xlsx(
|
||||
&fixture_dir.join("one.xlsx"),
|
||||
&[("Sheet1".to_owned(), rows)],
|
||||
);
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
let (toan, van, anh): (f64, f64, f64) = conn
|
||||
.query_row(
|
||||
"SELECT toan, ngu_van, tieng_anh FROM student WHERE so_bao_danh = '10000001'",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
)
|
||||
.expect("row not found");
|
||||
assert!((toan - 8.5).abs() < 1e-9);
|
||||
assert!((van - 7.0).abs() < 1e-9);
|
||||
assert!((anh - 9.25).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_ascii_stored_correctly() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
let rows = vec![
|
||||
header_row(),
|
||||
XlsxRow {
|
||||
values: vec![
|
||||
"Nguyễn Văn Đức".to_owned(),
|
||||
"".to_owned(),
|
||||
"20000001".to_owned(),
|
||||
"".to_owned(),
|
||||
],
|
||||
},
|
||||
];
|
||||
write_xlsx(
|
||||
&fixture_dir.join("one.xlsx"),
|
||||
&[("Sheet1".to_owned(), rows)],
|
||||
);
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
let conn = rusqlite::Connection::open(&db_path).unwrap();
|
||||
let ascii: String = conn
|
||||
.query_row(
|
||||
"SELECT ho_ten_ascii FROM student WHERE so_bao_danh = '20000001'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.expect("row not found");
|
||||
assert_eq!(ascii, "nguyen van duc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audit_subcommand_matches_after_build() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap();
|
||||
|
||||
run_build_cmd(&fixture_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
// audit should match (100 distinct SBDs in xlsx == 100 rows in DB)
|
||||
let cfg = make_data_config();
|
||||
let result = xlsxread::audit::run_audit(&fixture_dir, &db_path, &cfg).expect("audit failed");
|
||||
assert!(result.matched, "audit should match after build");
|
||||
assert_eq!(result.distinct_sbds, 100);
|
||||
assert_eq!(result.db_count, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audit_subcommand_mismatch_detected() {
|
||||
ensure_fixtures();
|
||||
let dir = tempdir();
|
||||
let db_path = dir.join("test.db");
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
|
||||
// Write 10 rows to xlsx but build DB from only 5 rows
|
||||
let mut all_rows = vec![header_row()];
|
||||
for i in 0..10 {
|
||||
all_rows.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(
|
||||
&fixture_dir.join("all.xlsx"),
|
||||
&[("Sheet1".to_owned(), all_rows)],
|
||||
);
|
||||
|
||||
// Build DB with only first 5 rows in a different file
|
||||
let build_dir = dir.join("build_input");
|
||||
std::fs::create_dir_all(&build_dir).unwrap();
|
||||
let mut five_rows = vec![header_row()];
|
||||
for i in 0..5 {
|
||||
five_rows.push(data_row(i, &sample_scores(i)));
|
||||
}
|
||||
write_xlsx(
|
||||
&build_dir.join("five.xlsx"),
|
||||
&[("Sheet1".to_owned(), five_rows)],
|
||||
);
|
||||
|
||||
run_build_cmd(&build_dir, &db_path, "thptqg2017-data.toml");
|
||||
|
||||
// audit against fixture_dir (10 xlsx rows) but DB has 5 rows → mismatch
|
||||
let cfg = make_data_config();
|
||||
let result = xlsxread::audit::run_audit(&fixture_dir, &db_path, &cfg).expect("audit failed");
|
||||
assert!(!result.matched, "audit should not match (10 xlsx vs 5 db)");
|
||||
assert_eq!(result.distinct_sbds, 10);
|
||||
assert_eq!(result.db_count, 5);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Golden test: compare Rust DB vs Node DB on identical fixture
|
||||
// Marked #[ignore] when pnpm / node is not in PATH — CI installs them first.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn golden_rust_matches_node_db() {
|
||||
// This test requires: pnpm, node, and the thptqg2017 package to be installed
|
||||
// Run with: cargo test -- --ignored golden_rust_matches_node_db
|
||||
let which_pnpm = std::process::Command::new("which")
|
||||
.arg("pnpm")
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false);
|
||||
if !which_pnpm {
|
||||
eprintln!("pnpm not in PATH — skipping golden test");
|
||||
return;
|
||||
}
|
||||
|
||||
let dir = tempdir();
|
||||
let fixture_dir = dir.join("input");
|
||||
std::fs::create_dir_all(&fixture_dir).unwrap();
|
||||
std::fs::copy(province_fixture_path(), fixture_dir.join("province.xlsx")).unwrap();
|
||||
|
||||
// Build with Rust
|
||||
let rust_db = dir.join("rust.db");
|
||||
run_build_cmd(&fixture_dir, &rust_db, "thptqg2017-data.toml");
|
||||
|
||||
// Build with Node (run build-database.js with DATA_DIR / DB_PATH overrides)
|
||||
// Node script reads env-vars via a thin wrapper — see scripts/build-database.js
|
||||
// For now: diff via SELECT * ORDER BY so_bao_danh
|
||||
let node_db = dir.join("node.db");
|
||||
let status = std::process::Command::new("pnpm")
|
||||
.args(["exec", "node", "scripts/build-database.js"])
|
||||
.env("OVERRIDE_SRC_DIR", fixture_dir.to_str().unwrap())
|
||||
.env("OVERRIDE_DB_PATH", node_db.to_str().unwrap())
|
||||
.current_dir(
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.parent()
|
||||
.unwrap()
|
||||
.parent()
|
||||
.unwrap(),
|
||||
)
|
||||
.status()
|
||||
.expect("failed to run node build script");
|
||||
|
||||
if !status.success() {
|
||||
panic!("Node build script failed with: {status}");
|
||||
}
|
||||
|
||||
// Row-by-row comparison
|
||||
diff_dbs(&rust_db, &node_db);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn tempdir() -> PathBuf {
|
||||
let base = std::env::temp_dir().join(format!(
|
||||
"xlsxread-test-{}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.subsec_nanos()
|
||||
));
|
||||
std::fs::create_dir_all(&base).unwrap();
|
||||
base
|
||||
}
|
||||
|
||||
fn run_build_cmd(input_dir: &Path, db_path: &Path, config_name: &str) {
|
||||
let cfg_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("configs")
|
||||
.join(config_name);
|
||||
|
||||
let cfg = xlsxread::config::load_config(&cfg_path)
|
||||
.unwrap_or_else(|e| panic!("load config {config_name}: {e}"));
|
||||
let patterns =
|
||||
xlsxread::transform::CompiledPatterns::new(&cfg.scores).expect("compile patterns");
|
||||
|
||||
// Collect files
|
||||
let mut files: Vec<PathBuf> = std::fs::read_dir(input_dir)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| {
|
||||
p.is_file()
|
||||
&& p.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.map(|e| {
|
||||
let l = e.to_lowercase();
|
||||
l == "xls" || l == "xlsx"
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.collect();
|
||||
files.sort();
|
||||
|
||||
let conn = xlsxread::writer::open_db(db_path, &cfg).expect("open db");
|
||||
conn.execute_batch("BEGIN").unwrap();
|
||||
|
||||
for file in &files {
|
||||
xlsxread::reader::process_file(file, &cfg, |_, raw| {
|
||||
let all_blank = xlsxread::reader::is_all_blank(raw);
|
||||
if cfg.reader.strip_blank_rows && all_blank {
|
||||
return;
|
||||
}
|
||||
let ho_ten = raw
|
||||
.get(cfg.columns.ho_ten)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
let so_bao_danh = raw
|
||||
.get(cfg.columns.so_bao_danh)
|
||||
.map(|c| c.to_string().trim().to_owned())
|
||||
.unwrap_or_default();
|
||||
if xlsxread::transform::validate_row(
|
||||
&ho_ten,
|
||||
&so_bao_danh,
|
||||
&cfg.validation,
|
||||
cfg.reader.strip_blank_rows,
|
||||
all_blank,
|
||||
)
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
let row = xlsxread::transform::transform_row(raw, &cfg, &patterns);
|
||||
let _ = xlsxread::writer::insert_row(
|
||||
&conn,
|
||||
&cfg.insert.sql,
|
||||
&row,
|
||||
xlsxread::writer::SCORE_FIELDS,
|
||||
);
|
||||
})
|
||||
.expect("process file");
|
||||
}
|
||||
|
||||
conn.execute_batch("COMMIT").unwrap();
|
||||
conn.execute_batch("VACUUM").unwrap();
|
||||
}
|
||||
|
||||
fn query_count(db_path: &Path) -> i64 {
|
||||
let conn = rusqlite::Connection::open(db_path).expect("open db for count");
|
||||
conn.query_row("SELECT COUNT(*) FROM student", [], |r| r.get(0))
|
||||
.expect("count query")
|
||||
}
|
||||
|
||||
fn diff_dbs(a: &Path, b: &Path) {
|
||||
let conn_a = rusqlite::Connection::open(a).unwrap();
|
||||
|
||||
// Attach b as "other"
|
||||
conn_a
|
||||
.execute_batch(&format!("ATTACH DATABASE '{}' AS other", b.display()))
|
||||
.unwrap();
|
||||
|
||||
// Rows in a not in b
|
||||
let missing_in_b: i64 = conn_a
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM main.student s
|
||||
WHERE NOT EXISTS (SELECT 1 FROM other.student o WHERE o.so_bao_danh = s.so_bao_danh)",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Rows in b not in a
|
||||
let missing_in_a: i64 = conn_a
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM other.student o
|
||||
WHERE NOT EXISTS (SELECT 1 FROM main.student s WHERE s.so_bao_danh = o.so_bao_danh)",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
missing_in_b, 0,
|
||||
"{missing_in_b} rows in Rust DB missing from Node DB"
|
||||
);
|
||||
assert_eq!(
|
||||
missing_in_a, 0,
|
||||
"{missing_in_a} rows in Node DB missing from Rust DB"
|
||||
);
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
import { defineConfig } from "vite";
|
||||
import react from "@vitejs/plugin-react";
|
||||
|
||||
// VARIANT selects which dataset to ship:
|
||||
// (unset) → main site at /thptqg2017/ using public/
|
||||
// old → at /thptqg2017/old/ using public-old/
|
||||
// old2 → at /thptqg2017/old2/ using public-old2/
|
||||
const VARIANT = process.env.VARIANT || "";
|
||||
const VARIANT_CONFIG = {
|
||||
"": { base: "/thptqg/2017/", publicDir: "public", outDir: "dist" },
|
||||
old: { base: "/thptqg/2017/old/", publicDir: "public-old", outDir: "dist/old" },
|
||||
old2: { base: "/thptqg/2017/old2/", publicDir: "public-old2", outDir: "dist/old2" },
|
||||
};
|
||||
const cfg = VARIANT_CONFIG[VARIANT];
|
||||
if (!cfg) throw new Error(`Unknown VARIANT: ${VARIANT}`);
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
base: cfg.base,
|
||||
publicDir: cfg.publicDir,
|
||||
build: {
|
||||
outDir: cfg.outDir,
|
||||
emptyOutDir: VARIANT === "", // only main build wipes dist root
|
||||
},
|
||||
});
|
||||
File renamed without changes.
@@ -2,18 +2,67 @@
|
||||
|
||||
Tra cứu điểm thi THPT Quốc gia — exam-score lookup for Vietnam's national high
|
||||
school graduation exam. Client-side SQL (sql.js) over a SQLite database built
|
||||
from the ministry's raw `.xls` score files by the Rust `xlsxread` CLI.
|
||||
from the ministry's raw `.xls` score files by the Rust `xlsxread` parser.
|
||||
|
||||
Served as one site at **[tiennm99.github.io/thptqg](https://tiennm99.github.io/thptqg/)**.
|
||||
Live at **[tiennm99.github.io/thptqg](https://tiennm99.github.io/thptqg/)**.
|
||||
|
||||
| Directory | Year | Size | Site |
|
||||
| Dataset | Exam | Candidates | Site |
|
||||
| --- | --- | --- | --- |
|
||||
| [`2016/`](./2016/) | THPT QG 2016 — 877,461 candidates | ~50 MB raw data | [/2016/](https://tiennm99.github.io/thptqg/2016/) |
|
||||
| [`2017/`](./2017/) | THPT QG 2017 — ~861,000 candidates | ~117 MB raw data | [/2017/](https://tiennm99.github.io/thptqg/2017/) (+ [old](https://tiennm99.github.io/thptqg/2017/old/), [old2](https://tiennm99.github.io/thptqg/2017/old2/) generations) |
|
||||
| `2016` | 2016 | 877,461 | [/2016/](https://tiennm99.github.io/thptqg/2016/) |
|
||||
| `2017` | 2017 | 861,068 | [/2017/](https://tiennm99.github.io/thptqg/2017/) |
|
||||
| `2017-old` | 2017 | 847,348 | [/2017-old/](https://tiennm99.github.io/thptqg/2017-old/) |
|
||||
| `2017-old2` | 2017 | 679,764 | [/2017-old2/](https://tiennm99.github.io/thptqg/2017-old2/) |
|
||||
|
||||
Each year was a standalone repository (`thptqg2016`, `thptqg2017`), merged here
|
||||
with full history. Each keeps its own `tools/xlsxread` copy because the raw
|
||||
data formats differ per year.
|
||||
The three 2017 datasets are successive publications of the same exam and they
|
||||
disagree; all three are kept so the differences stay inspectable.
|
||||
|
||||
The `Deploy to GitHub Pages` workflow builds both years' databases and site
|
||||
bundles and publishes them behind a root index.
|
||||
## Layout
|
||||
|
||||
```
|
||||
index.html + src/ the frontend — one app serving all four datasets and the hub
|
||||
datasets.js the four dataset ids and their per-dataset content
|
||||
router.js pathname → dataset
|
||||
data/<id>/ raw Excel files, one directory per dataset
|
||||
parser/ the Rust parser
|
||||
src/schema.rs canonical 22-column table: DDL, INSERT, subject regexes
|
||||
configs/<id>.toml per-dataset parse rules only, no SQL
|
||||
scripts/ database build, crawler, parity verification
|
||||
scripts/ site assembly
|
||||
docs/ architecture, data pipeline, deployment
|
||||
```
|
||||
|
||||
The dataset id is one identifier end to end:
|
||||
|
||||
```
|
||||
data/2017-old/ → parser/configs/2017-old.toml → db/2017-old.db.gz → /thptqg/2017-old/
|
||||
```
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
npm ci
|
||||
npm run build:rust # compile the parser
|
||||
npm run build:db # build + gzip all four databases (add an id for just one)
|
||||
npm run build:site # one Vite build, then assemble into _site/
|
||||
npx serve _site
|
||||
```
|
||||
|
||||
Pushing to `main` runs the same steps in
|
||||
`.github/workflows/deploy-pages.yml` and publishes to GitHub Pages.
|
||||
|
||||
## Adding a dataset
|
||||
|
||||
1. Put the Excel files in `data/<id>/`
|
||||
2. Add `parser/configs/<id>.toml` — sheet mode, column indices, validation
|
||||
guards. No SQL; the schema is canonical.
|
||||
3. Add an entry to `DATASETS` in `src/datasets.js`
|
||||
|
||||
Everything else follows: the build script, the site assembly and the router all
|
||||
read that one list, and the UI adapts to whichever columns the dataset fills.
|
||||
|
||||
## Docs
|
||||
|
||||
See [`docs/`](./docs/) — [overview](./docs/project-overview.md),
|
||||
[architecture](./docs/system-architecture.md),
|
||||
[data pipeline](./docs/data-pipeline.md),
|
||||
[deployment](./docs/deployment-guide.md).
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Loaded 100 of 421 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user