diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index 02abd54..80bb772 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -49,25 +49,33 @@ jobs: - name: Test parser run: npm run test:go + # The crawler is not part of the build — it only refreshes data/ by hand. + # It is still compiled and tested here so it cannot rot unnoticed, and + # because its filename test guards the parser: input filenames decide + # which row survives a duplicate exam number. + - name: Test crawler + run: npm run test:crawler + # excelize carries an open advisory, and the 2017 refresh runbook feeds # network-downloaded spreadsheets straight into the parser. - name: Vulnerability scan - working-directory: go-parser run: | go install golang.org/x/vuln/cmd/govulncheck@latest - "$(go env GOPATH)/bin/govulncheck" ./... + GOVULNCHECK="$(go env GOPATH)/bin/govulncheck" + (cd go-parser && "$GOVULNCHECK" ./...) + (cd crawler && "$GOVULNCHECK" ./...) # One parser binary builds every dataset; build-db.js reads the dataset - # list from src/datasets.js, verifies each database against its known row + # list from web/src/datasets.js, verifies each database against its row # count, then gzips it in place leaving no uncompressed file behind. - name: Build databases run: | npm run build:go npm run build:db - # One Vite build produces every page. scripts/assemble-site.js copies the - # emitted index.html to each dataset path (and the legacy nested URLs), - # then fails the job if any uncompressed database reached the artifact. + # One Vite build produces every page. web/scripts/assemble-site.js copies + # the emitted index.html to each dataset path (and the legacy nested + # URLs), then fails the job if an uncompressed database reached _site. - name: Build and assemble site run: npm run build:site diff --git a/README.md b/README.md index 02c569d..192fdf8 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Tra cứu điểm thi THPT Quốc gia — exam-score lookup for Vietnam's national high school graduation exam. Client-side SQL (sql.js) over a SQLite database built -from the ministry's raw `.xls` score files by the Rust `xlsxread` parser. +from the ministry's raw `.xls` score files by the Go `xlsxread` parser. Live at **[tiennm99.github.io/thptqg](https://tiennm99.github.io/thptqg/)**. @@ -10,43 +10,60 @@ Live at **[tiennm99.github.io/thptqg](https://tiennm99.github.io/thptqg/)**. | --- | --- | --- | --- | | `2016` | 2016 | 877,461 | [/2016/](https://tiennm99.github.io/thptqg/2016/) | | `2017` | 2017 | 861,068 | [/2017/](https://tiennm99.github.io/thptqg/2017/) | -| `2017-old` | 2017 | 847,348 | [/2017-old/](https://tiennm99.github.io/thptqg/2017-old/) | -| `2017-old2` | 2017 | 679,764 | [/2017-old2/](https://tiennm99.github.io/thptqg/2017-old2/) | -The three 2017 datasets are successive publications of the same exam and they -disagree; all three are kept so the differences stay inspectable. +Two earlier 2017 publications (`2017-old`, `2017-old2`) were kept for a while +because they disagreed with the current one. They have been removed; they remain +in git history. ## Layout ``` -index.html + src/ the frontend — one app serving all four datasets and the hub - datasets.js the four dataset ids and their per-dataset content - router.js pathname → dataset +web/ the frontend — one Vite app serving both datasets and the hub + src/datasets.js the dataset ids and their per-dataset content + src/router.js pathname → dataset + scripts/ site assembly +crawler/ Go — re-fetches the source spreadsheets + internal/sources/ one file per data source: its links and local filenames + internal/fetch/ concurrent, resumable downloading +go-parser/ Go — Excel to SQLite + internal/schema/ canonical 22-column table: DDL, INSERT, subject regexes + configs/.yml per-dataset parse rules only, no SQL + scripts/ database build, parity verification data// raw Excel files, one directory per dataset -parser/ the Rust parser - src/schema.rs canonical 22-column table: DDL, INSERT, subject regexes - configs/.yml per-dataset parse rules only, no SQL - scripts/ database build, crawler, parity verification -scripts/ site assembly docs/ architecture, data pipeline, deployment ``` +`web/` is the only npm workspace; `crawler/` and `go-parser/` are independent Go +modules. The one cross-boundary import is `web/src/datasets.js`, which +`go-parser/scripts/build-db.js` reads for the dataset list and expected sizes. + The dataset id is one identifier end to end: ``` -data/2017-old/ → go-parser/configs/2017-old.yml → db/2017-old.db.gz → /thptqg/2017-old/ +data/2017/ → go-parser/configs/2017.yml → db/2017.db.gz → /thptqg/2017/ ``` ## Build ```bash npm ci -npm run build:go # compile the parser -npm run build:db # build + gzip all four databases (add an id for just one) +npm run build:go # compile the parser +npm run build:db # build + gzip both databases (add an id for just one) npm run build:site # one Vite build, then assemble into _site/ npx serve _site ``` +The source spreadsheets are committed, so a crawl is only needed to refresh +them: + +```bash +npm run crawl:2016 # re-fetch data/2016/ +npm run crawl:2017 # re-fetch data/2017/ from the baotintuc.vn CDN +``` + +Crawling is idempotent — files already present are skipped — and is never part +of the build. + Pushing to `main` runs the same steps in `.github/workflows/deploy-pages.yml` and publishes to GitHub Pages. @@ -55,7 +72,7 @@ Pushing to `main` runs the same steps in 1. Put the Excel files in `data//` 2. Add `go-parser/configs/.yml` — sheet mode, column indices, validation guards. No SQL; the schema is canonical. -3. Add an entry to `DATASETS` in `src/datasets.js` +3. Add an entry to `DATASETS` in `web/src/datasets.js` Everything else follows: the build script, the site assembly and the router all read that one list, and the UI adapts to whichever columns the dataset fills. diff --git a/crawler/cmd/crawl/main.go b/crawler/cmd/crawl/main.go new file mode 100644 index 0000000..ac95d6d --- /dev/null +++ b/crawler/cmd/crawl/main.go @@ -0,0 +1,145 @@ +// Command crawl downloads a dataset's source spreadsheets into data//. +// +// The argument is the dataset id, the same one used by go-parser's configs and +// the published site paths. +// +// crawl 2017 # from the baotintuc.vn CDN +// crawl 2016 # source not yet configured +// crawl 2017 --list # show what would be downloaded +// +// Runs are idempotent: a file already present and non-empty is skipped, so an +// interrupted crawl can simply be re-run. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/tiennm99/thptqg/crawler/internal/fetch" + "github.com/tiennm99/thptqg/crawler/internal/sources" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintf(os.Stderr, "crawl: %v\n", err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprint(os.Stderr, "usage: crawl [flags]\n\nDatasets:\n") + for _, s := range sources.All() { + fmt.Fprintf(os.Stderr, " %-6s %s\n", s.ID, s.Summary) + } + fmt.Fprint(os.Stderr, "\nFlags:\n") + fmt.Fprint(os.Stderr, " --out string output directory (default ../data/)\n") + fmt.Fprint(os.Stderr, " --concurrency int parallel downloads (default 6)\n") + fmt.Fprint(os.Stderr, " --timeout duration per-file timeout (default 2m)\n") + fmt.Fprint(os.Stderr, " --list print the file list and exit\n") +} + +func run(args []string) error { + if len(args) == 0 || args[0] == "-h" || args[0] == "--help" { + usage() + if len(args) == 0 { + return fmt.Errorf("no dataset given") + } + return nil + } + + src, err := sources.Lookup(args[0]) + if err != nil { + usage() + return err + } + + fs := flag.NewFlagSet(src.ID, flag.ContinueOnError) + // The default is relative to the crawler module directory, which is where + // both `go -C crawler run ./cmd/crawl` and a manual `cd crawler` land. + out := fs.String("out", filepath.Join("..", "data", src.ID), "output directory") + concurrency := fs.Int("concurrency", 6, "parallel downloads") + timeout := fs.Duration("timeout", 2*time.Minute, "per-file timeout") + list := fs.Bool("list", false, "print the file list and exit") + if err := fs.Parse(args[1:]); err != nil { + return err + } + + files, err := src.Files() + if err != nil { + return err + } + if len(files) == 0 { + return fmt.Errorf("source %q produced no files", src.ID) + } + + outDir, err := filepath.Abs(*out) + if err != nil { + return err + } + + items := make([]fetch.Item, 0, len(files)) + for _, f := range files { + items = append(items, fetch.Item{ + Name: f.Name, + URL: f.URL, + Path: filepath.Join(outDir, f.Dest), + }) + } + + if *list { + for _, it := range items { + fmt.Printf("%s\t%s\n", filepath.Base(it.Path), it.URL) + } + return nil + } + + // Ctrl-C cancels in-flight requests; each worker deletes its partial file + // on the way out, so an interrupted run leaves nothing half-written. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + fmt.Printf("Downloading %d files to %s...\n", len(items), outDir) + + results, runErr := fetch.Run(ctx, items, fetch.Options{ + Concurrency: *concurrency, + Timeout: *timeout, + Headers: src.Headers, + OnResult: printResult, + }) + if runErr != nil { + return runErr + } + + ok, skip, failed := fetch.Tally(results) + fmt.Printf("\nDone. ok=%d skip=%d fail=%d\n", ok, skip, len(failed)) + if len(failed) > 0 { + for _, r := range failed { + fmt.Fprintf(os.Stderr, " %s: %v\n", r.Item.Name, r.Err) + } + return fmt.Errorf("%d file(s) failed", len(failed)) + } + return nil +} + +func printResult(done, total int, r fetch.Result) { + tag := map[fetch.Status]string{ + fetch.StatusOK: "✓", + fetch.StatusSkip: "·", + fetch.StatusFail: "✗", + }[r.Status] + + detail := "" + switch r.Status { + case fetch.StatusFail: + detail = r.Err.Error() + default: + detail = fmt.Sprintf("%.0f KB", float64(r.Size)/1024) + } + fmt.Printf(" %s [%d/%d] %-20s %s\n", tag, done, total, r.Item.Name, detail) +} diff --git a/crawler/go.mod b/crawler/go.mod new file mode 100644 index 0000000..dd80379 --- /dev/null +++ b/crawler/go.mod @@ -0,0 +1,3 @@ +module github.com/tiennm99/thptqg/crawler + +go 1.26.5 diff --git a/crawler/internal/fetch/fetch.go b/crawler/internal/fetch/fetch.go new file mode 100644 index 0000000..5afff19 --- /dev/null +++ b/crawler/internal/fetch/fetch.go @@ -0,0 +1,205 @@ +// Package fetch downloads a list of files concurrently, skipping any that are +// already on disk. +// +// It is deliberately source-agnostic: it knows nothing about provinces, exam +// years or spreadsheet formats. Each source in internal/sources produces a +// []Item and this package moves the bytes. +package fetch + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sync" + "time" +) + +// Item is one file to download. +type Item struct { + // Name is the human label used in progress output, e.g. "An Giang". + Name string + URL string + // Path is the absolute destination, including filename. + Path string +} + +// Status is the outcome of one item. +type Status int + +const ( + // StatusOK means the file was downloaded during this run. + StatusOK Status = iota + // StatusSkip means a non-empty file was already present. + StatusSkip + // StatusFail means the download did not complete; Result.Err says why. + StatusFail +) + +// Result records what happened to one item. +type Result struct { + Item Item + Status Status + Size int64 + Err error +} + +// Options configures a run. The zero value is usable: Concurrency and Timeout +// fall back to defaults. +type Options struct { + // Concurrency is the number of files downloaded at once. + Concurrency int + // Timeout bounds each individual request, headers and body together. + // The original JS crawler had no timeout, so one stalled connection could + // hang the whole run indefinitely. + Timeout time.Duration + // Headers are sent with every request. The CDN this was written against + // rejects requests without a browser User-Agent and a matching Referer. + Headers map[string]string + // OnResult, if set, is called once per completed item. Calls are + // serialised, so it does not need its own locking, but they arrive in + // completion order rather than list order. + OnResult func(done, total int, r Result) +} + +const ( + defaultConcurrency = 6 + defaultTimeout = 2 * time.Minute +) + +// Run downloads every item, returning one Result each. It does not return an +// error for a failed download — that is reported per item — only for a problem +// that stops the run as a whole, such as an unusable output directory. +// +// Results come back in completion order, not input order. +func Run(ctx context.Context, items []Item, opts Options) ([]Result, error) { + if opts.Concurrency <= 0 { + opts.Concurrency = defaultConcurrency + } + if opts.Timeout <= 0 { + opts.Timeout = defaultTimeout + } + + // Every item's parent directory must exist before any worker starts, so a + // missing output directory fails once here rather than N times in parallel. + for _, it := range items { + if err := os.MkdirAll(filepath.Dir(it.Path), 0o755); err != nil { + return nil, fmt.Errorf("cannot create output directory: %w", err) + } + } + + client := &http.Client{Timeout: opts.Timeout} + + var ( + mu sync.Mutex + results = make([]Result, 0, len(items)) + next int + ) + + work := make(chan Item) + var wg sync.WaitGroup + for range opts.Concurrency { + wg.Add(1) + go func() { + defer wg.Done() + for it := range work { + r := download(ctx, client, it, opts.Headers) + mu.Lock() + results = append(results, r) + next++ + if opts.OnResult != nil { + opts.OnResult(next, len(items), r) + } + mu.Unlock() + } + }() + } + + for _, it := range items { + select { + case work <- it: + case <-ctx.Done(): + close(work) + wg.Wait() + return results, ctx.Err() + } + } + close(work) + wg.Wait() + + return results, nil +} + +// download fetches one item, or reports that it was already present. +// +// The body is streamed to a .part file and renamed into place only once it is +// complete. Without that, an interrupted run leaves a truncated file at the +// final path — and because the skip check only tests for a non-empty file, +// every later run would skip it and the corruption would persist silently. +func download(ctx context.Context, client *http.Client, it Item, headers map[string]string) Result { + if st, err := os.Stat(it.Path); err == nil && st.Size() > 0 { + return Result{Item: it, Status: StatusSkip, Size: st.Size()} + } + + fail := func(err error) Result { + return Result{Item: it, Status: StatusFail, Err: err} + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, it.URL, nil) + if err != nil { + return fail(err) + } + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := client.Do(req) + if err != nil { + return fail(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fail(fmt.Errorf("HTTP %d", resp.StatusCode)) + } + + part := it.Path + ".part" + f, err := os.Create(part) + if err != nil { + return fail(err) + } + n, err := io.Copy(f, resp.Body) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + os.Remove(part) + return fail(err) + } + if n == 0 { + os.Remove(part) + return fail(fmt.Errorf("empty response body")) + } + if err := os.Rename(part, it.Path); err != nil { + os.Remove(part) + return fail(err) + } + + return Result{Item: it, Status: StatusOK, Size: n} +} + +// Tally counts results by status. +func Tally(results []Result) (ok, skip int, failed []Result) { + for _, r := range results { + switch r.Status { + case StatusOK: + ok++ + case StatusSkip: + skip++ + case StatusFail: + failed = append(failed, r) + } + } + return ok, skip, failed +} diff --git a/crawler/internal/fetch/fetch_test.go b/crawler/internal/fetch/fetch_test.go new file mode 100644 index 0000000..f4214a1 --- /dev/null +++ b/crawler/internal/fetch/fetch_test.go @@ -0,0 +1,184 @@ +package fetch + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func serve(t *testing.T, h http.HandlerFunc) string { + t.Helper() + s := httptest.NewServer(h) + t.Cleanup(s.Close) + return s.URL +} + +func TestDownloadsAndSkips(t *testing.T) { + var hits int + url := serve(t, func(w http.ResponseWriter, _ *http.Request) { + hits++ + w.Write([]byte("payload")) + }) + + dir := t.TempDir() + items := []Item{{Name: "one", URL: url, Path: filepath.Join(dir, "one.xls")}} + + results, err := Run(context.Background(), items, Options{}) + if err != nil { + t.Fatal(err) + } + if results[0].Status != StatusOK { + t.Fatalf("status = %v, err = %v", results[0].Status, results[0].Err) + } + if got, _ := os.ReadFile(items[0].Path); string(got) != "payload" { + t.Errorf("content = %q", got) + } + + // Second run must not re-fetch. + results, err = Run(context.Background(), items, Options{}) + if err != nil { + t.Fatal(err) + } + if results[0].Status != StatusSkip { + t.Errorf("status = %v, want StatusSkip", results[0].Status) + } + if hits != 1 { + t.Errorf("server hit %d times, want 1", hits) + } +} + +func TestNon200Fails(t *testing.T) { + url := serve(t, func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "gone", http.StatusNotFound) + }) + + dir := t.TempDir() + path := filepath.Join(dir, "missing.xls") + results, err := Run(context.Background(), []Item{{Name: "x", URL: url, Path: path}}, Options{}) + if err != nil { + t.Fatal(err) + } + if results[0].Status != StatusFail { + t.Fatalf("status = %v, want StatusFail", results[0].Status) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Error("a failed download must leave no file at the destination") + } + assertNoPartFiles(t, dir) +} + +// TestFailureLeavesNoPartial is the reason downloads land on a .part file +// first. Writing straight to the destination would leave a truncated file +// there, and the skip check — which only tests for a non-empty file — would +// skip it on every later run, so the corruption would never be re-fetched. +func TestFailureLeavesNoPartial(t *testing.T) { + url := serve(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "1000") + w.Write([]byte("short")) + // Closing early makes the body read fail mid-copy. + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + panic(http.ErrAbortHandler) + }) + + dir := t.TempDir() + path := filepath.Join(dir, "truncated.xls") + results, err := Run(context.Background(), []Item{{Name: "x", URL: url, Path: path}}, Options{}) + if err != nil { + t.Fatal(err) + } + if results[0].Status != StatusFail { + t.Fatalf("status = %v, want StatusFail", results[0].Status) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Error("a truncated download must not be left at the destination") + } + assertNoPartFiles(t, dir) +} + +func TestEmptyBodyFails(t *testing.T) { + url := serve(t, func(w http.ResponseWriter, _ *http.Request) {}) + + dir := t.TempDir() + path := filepath.Join(dir, "empty.xls") + results, err := Run(context.Background(), []Item{{Name: "x", URL: url, Path: path}}, Options{}) + if err != nil { + t.Fatal(err) + } + if results[0].Status != StatusFail { + t.Errorf("an empty body must fail, got %v", results[0].Status) + } + assertNoPartFiles(t, dir) +} + +func TestHeadersAreSent(t *testing.T) { + var gotUA, gotRef string + url := serve(t, func(w http.ResponseWriter, r *http.Request) { + gotUA, gotRef = r.Header.Get("User-Agent"), r.Header.Get("Referer") + w.Write([]byte("ok")) + }) + + _, err := Run(context.Background(), + []Item{{Name: "x", URL: url, Path: filepath.Join(t.TempDir(), "x.xls")}}, + Options{Headers: map[string]string{"User-Agent": "test-agent", "Referer": "https://example.test/"}}) + if err != nil { + t.Fatal(err) + } + if gotUA != "test-agent" || gotRef != "https://example.test/" { + t.Errorf("headers not sent: ua=%q referer=%q", gotUA, gotRef) + } +} + +func TestAllItemsRun(t *testing.T) { + url := serve(t, func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte("x")) + }) + + dir := t.TempDir() + var items []Item + for i := range 20 { + items = append(items, Item{ + Name: "f", + URL: url, + Path: filepath.Join(dir, "sub", string(rune('a'+i))+".xls"), + }) + } + + results, err := Run(context.Background(), items, Options{Concurrency: 4}) + if err != nil { + t.Fatal(err) + } + if len(results) != len(items) { + t.Fatalf("got %d results, want %d", len(results), len(items)) + } + ok, _, failed := Tally(results) + if ok != len(items) { + t.Errorf("ok = %d, want %d (failures: %v)", ok, len(items), failed) + } +} + +func TestTally(t *testing.T) { + ok, skip, failed := Tally([]Result{ + {Status: StatusOK}, {Status: StatusOK}, {Status: StatusSkip}, {Status: StatusFail}, + }) + if ok != 2 || skip != 1 || len(failed) != 1 { + t.Errorf("ok=%d skip=%d fail=%d", ok, skip, len(failed)) + } +} + +func assertNoPartFiles(t *testing.T, dir string) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if filepath.Ext(e.Name()) == ".part" { + t.Errorf("leftover partial file: %s", e.Name()) + } + } +} diff --git a/crawler/internal/sources/source_2016.go b/crawler/internal/sources/source_2016.go new file mode 100644 index 0000000..58f1764 --- /dev/null +++ b/crawler/internal/sources/source_2016.go @@ -0,0 +1,182 @@ +package sources + +// source2016 fetches the 2016 dataset: one spreadsheet per exam cluster +// (cụm thi), 4 .xls and 115 .xlsx. +// +// # Provenance +// +// The list below was recovered from the aggregator article that originally +// published these files: +// +// cong-bo-diem-thi-thptqg-2016-toan-bo-120-cum-thi-da-co-diem.html +// +// The site that first carried it (dtntbacgiang.edu.vn) no longer resolves, so +// the list was read from the Internet Archive's copy. It is verified, not +// guessed: all 119 filenames match data/2016/ exactly in both directions, which +// TestSource2016MatchesFilesOnDisk asserts. +// +// # Filenames are load-bearing — do not "tidy" them +// +// Dest is the server-assigned name, a 32-hex content hash followed by the +// cluster slug and a millisecond timestamp. It is kept verbatim because +// go-parser sorts input files bytewise and inserts last-wins, so filenames +// decide which row survives a duplicate exam number. That is live for 2016, not +// hypothetical: 877,464 source rows collapse to 877,461, so three rows' contents +// depend on this sort order. The hashes in +// go-parser/testdata/reader-fidelity-hashes.tsv are keyed by full path as well, +// and they are frozen — they were produced by the Rust reader, which no longer +// exists. +// +// # The host is not verified +// +// Only the paths are known to be right; the host that still serves them is not. +// The original is gone and the Internet Archive captured the article but none of +// the spreadsheets, so this could not be confirmed by fetching. baseURL points +// at a mirror of the same article that is still online. If it serves these files +// from a different directory, uploadPath is the single constant to change. +const ( + baseURL = "https://dtnt.bacninh.edu.vn" + uploadPath = "/upload/s/20180102/" +) + +var source2016 = Source{ + ID: "2016", + Summary: "119 exam-cluster files (4 .xls + 115 .xlsx)", + Files: func() ([]File, error) { + out := make([]File, 0, len(clusters2016)) + for _, c := range clusters2016 { + out = append(out, File{ + Name: c.name, + URL: baseURL + uploadPath + c.file, + Dest: c.file, + }) + } + return out, nil + }, +} + +type cluster struct{ name, file string } + +// clusters2016 is in the article's publication order. That order is incidental — +// go-parser re-sorts by filename — but it is kept as published so the list can +// be diffed against the source article. +var clusters2016 = []cluster{ + {"Trường Đại học Bách khoa Hà Nội", "b34c777942ca8a4de91f23a35cce1c6bdhbachkhoahn-1468901491486.xlsx"}, + {"Trường Đại học Sư phạm Hà Nội", "76f19098881e9c79f66ac4f1b42a8a55dhsuphamhanoi-1468901144203.xlsx"}, + {"Trường Đại học Thuỷ lợi * Cơ sở 1 ở phía Bắc", "4a3fa95a83fe9b3b2fafab309323fd22dhthuyloi-1468901840919.xlsx"}, + {"Học viện Kỹ thuật Quân sự * Cơ sở 1 ở phía Bắc (Quân đội)", "4334ea476a2b8196d54ba40a341831echvkythuatquansu-1468920342994.xlsx"}, + {"Trường Đại học Lâm nghiệp", "c5ecc58a731fd1a54b7e6cc3bc99449ddhlamnghiep-1468939300790.xlsx"}, + {"Trường Đại học Bách khoa - Đại học Quốc gia Thành phố Hồ Chí Minh", "357c6aaf59aeaa6c11c3e83c595d38cfdhbachkhoa-tphcm-1468939678829.xlsx"}, + {"Trường Đại học Khoa học tự nhiên - Đại học Quốc gia Thành phố Hồ Chí Minh", "64e44466ffeff9a587260795644805d0dhkhtunhien-dhqgtphcm-1468921015146.xlsx"}, + {"Trường Đại học Khoa học xã hội và Nhân văn - Đại học Quốc gia Thành phố Hồ Chí Minh", "5fd13ae5287cbf0149bd733b507e0608dhkhxhnv-dhqgtphcm-1468920630193.xlsx"}, + {"Trường Đại học Sư phạm Tp.HCM", "a995de3f7a7055cf10b2f002f0e194f7dhsptphcm-1468920792291.xlsx"}, + {"Trường Đại học Hàng Hải", "c6ef16bc9f75ce0fd16c1229afad4e71dhhanghai-1468906457479.xlsx"}, + {"Trường Đại học Sư phạm - Đại học Thái Nguyên", "ee09c723da4e86cdbe26213413450af2dhsupham-dhthainguyen-1468939061872.xlsx"}, + {"Trường Đại học Kỹ thuật công nghiệp - Đại học Thái Nguyên", "8af8f1b29b6a40e26ccb4a2e19e8eededhkythuatcongnghiep-dhthainguyen-1468901919568.xlsx"}, + {"Trường Đại học Nông lâm - Đại học Thái Nguyên", "53e5e20ebd74d84741513b703fbd13bfdhnonglam-dhthainguyen-1468939023637.xlsx"}, + {"Học viện Ngân hàng", "f2846894bd5533e02b122a1e3b4333f6hvnganhang-1468939472850.xlsx"}, + {"Trường Đại học Luật Hà Nội", "9e0e564ae801e347bd7649da8b89ad90dhluathn-1468920377978.xlsx"}, + {"Trường Đại học Tân Trào", "20db5eaace927596f1806f151649ee6edhtantrao-1468901296040.xlsx"}, + {"Trường Đại học Xây dựng Hà Nội", "7b5d74564ab9f6e0e137f6d161fab6f0dhxaydung-1468940561682.xlsx"}, + {"Trường Đại học Khoa học - Đại học Thái Nguyên", "c2cfaaf287fb1e7120894e545771b027dhkhoahoc-dhthainguyen-1468900365545.xlsx"}, + {"Đại học Thái Nguyên", "3daa6ee33fdafadc3c112c18bff18c2ddhthainguyen-1468940268065.xlsx"}, + {"Học viện Tài chính", "96c95c85c5549a06f2e92ffcacf8ff21hvtaichinh-1468934470924.xlsx"}, + {"Trường Đại học Tây Bắc", "74e6cd0dc78a3fb7dcf56ae57283b9b9dhtaybac-1468940302052.xlsx"}, + {"Trường Đại học Hùng Vương", "778ab074ed6a95d209fa901fd5d68843dhhungvuong-1468901964127.xlsx"}, + {"Trường Đại học Sư phạm Hà Nội 2", "a11e707e6ed5fe4d29a8e32380d7558cdhsphanoi-2-1468902006284.xlsx"}, + {"Trường Đại học Ngoại thương * Cơ sở 1 ở phía Bắc", "ec84067268f47d523b1e80b493248377dhngoaithuong-1468920480753.xlsx"}, + {"Trường Đại học Kinh tế Quốc dân", "8f948c2a5ab49a3c5fd03b6ea04887a7dhkinhtequocdanfix-1468902083782.xlsx"}, + {"Trường Đại học Giao thông Vận tải", "51bb2f9b6e9f565c2d705606fd56958bdhgiaothongvantai-1468939107463.xlsx"}, + {"Học viện Nông Nghiệp Việt Nam", "5aede4951e7af8c1e1056c747a553386hvnongnghiepvn-1468900610577.xlsx"}, + {"Trường Đại học Sư phạm Kỹ thuật Hưng Yên", "f85f04bb2bbd84457b83ac846728a2a6dhspkythuathungyen-1468920667091.xlsx"}, + {"Trường Đại học Hải Phòng", "deca197a916a8f633f6894b4e13c3ca4dhhaiphong-1468901178368.xlsx"}, + {"Trường Đại học Thương mại", "b2552e20c9ff4b237244dfbf2e278a47dhthuongmai-1468902876264.xlsx"}, + {"Trường Đại học Công nghiệp Hà Nội", "82338a4a93ea1e0cde26e0061066ca8adhcongnghiephanoi-1468920154757.xlsx"}, + {"Y Dược Thái Bình", "22e8617886412abd90ae3d33cc6d7db4dhyduocthaibinh-1468941020145.xlsx"}, + {"Trường Đại học Mỏ Địa chất", "c174631d90303276ac63ff38b31157bbdhmodiachat-1468900697349.xlsx"}, + {"Trường Đại học Hồng Đức", "bd857add0cfe92c1c247469ff2104527dh-hong-duc-1468900427119.xlsx"}, + {"Trường Đại học Vinh", "64d2635efd30a0d0db4f7c30b9735261dhvinh-1468940230027.xlsx"}, + {"Trường ĐH Sư phạm - ĐH Huế", "ab2f259ef55cb08b3a436d3f15d42cc7dhsupham-dhhue-1468934431828.xlsx"}, + {"Trường ĐH Khoa học - ĐH Huế", "87ffaf3197ba20f0919f98a0b3f33914dhkhoahoc-dhhue-1468938729472.xlsx"}, + {"Trường ĐH Kinh tế - ĐH Huế", "8916da7b3c4d58cffc1dd29dc7a8aabfdhkinhte-dhhue-1468938560205.xlsx"}, + {"Đại học Huế", "919276a0348634459f8937dfcd6c7129dhhue-1468938792115.xlsx"}, + {"ĐH Đà Nẵng", "b42fd58ead4c54279d7df5f53160a851dhdanang-1468920187203.xlsx"}, + {"Trường ĐH Bách khoa - ĐH Đà Nẵng", "43213f61074068d9177685eb6c3f2d3bdhbachkhoa-dhdanang-1468938494437.xlsx"}, + {"Trường ĐH Sư phạm - ĐH Đà Nẵng", "54bbcf865c137d58cf62af15733bd376dhsupham-dhdanang-1468938526558.xlsx"}, + {"Trường Đại học Quy Nhơn", "70bf8cd2b6e87523219e55149a8524c5dhquynhon-1468938878328.xlsx"}, + {"Trường ĐH Xây dựng miền Trung", "4e9bb192d56c9fa92b3e0ec13d8236c8dhxaydungmientrung-1468940978346.xlsx"}, + {"Trường ĐH Nông Lâm TPHCM", "b2a739584a8861502573d1c6372fd6e9dhnonglamtphcm-1468939555326.xlsx"}, + {"Trường ĐH Ngoại ngữ - ĐH Đà Nẵng", "9ba098a7913ad837a66e173b3fc41fd7dhngoaingu-dhdanang-1468902750424.xlsx"}, + {"Trường ĐH Tây Nguyên", "216ee4e385dbc236767be899b068bf98dhtaynguyen-1468940490240.xlsx"}, + {"Trường ĐH Tài chính Marketing", "af6828f837a80e03d77a65f3d310ec60dh-tai-chinh-marketing-1468924711596.xlsx"}, + {"Trường ĐH Nha Trang", "6375fd13557bee08d20f1f3e46dac130dhnhatrang-coso1-1468901350425.xlsx"}, + {"Trường ĐH Giao thông vận tải TP.HCM", "c5c28959fcd0b4e39dd9fd538db5448adhgtvttphcm-1468939150821.xlsx"}, + {"Trường ĐH Sư phạm Kĩ thuật TP.HCM", "514877c1cfebdc9c2251f651b5f02c7edhspkythuattphcm-1468906568655.xlsx"}, + {"Trường ĐH Đà Lạt", "2b33640b0dd154490f2809d96be81797dhdalat-1468902130616.xlsx"}, + {"Trường ĐH Kinh tế TP.HCM", "f0aa5a19c2c8c1fde8211341f0e98e26dhkinhtetphcm-1468939229043.xlsx"}, + {"Trường ĐH Kinh tế - Luật - ĐHQG TP.HCM", "ff877788e43bd84f0119dae026996892dhkinhteluat-dhqgtphcm-1468939860894.xlsx"}, + {"Trường ĐH Công nghiệp Thực phẩm TP.HCM", "167116af6c8a096c2a9307edfa115f77dhcnthucpham-tphcm-1468910578187.xls"}, + {"Trường ĐH Công nghiệp TP.HCM", "487c7287c1c9735722b31d877942becddhcongnghieptphcm-1468983373303.xlsx"}, + {"Trường ĐH Sài Gòn", "d75282bff0a4d8cd1421c3fb74f789c3dhsaigon-1468900867561.xlsx"}, + {"Trường ĐH Đồng Tháp", "388fcdfdac18ef6eee6125a2a015714ddhdongthap-1468934545359.xlsx"}, + {"Trường Đại học An Giang", "fc511852659150f6305f6f624445300ddhangiang-1468940189321.xlsx"}, + {"Trường ĐH Tôn Đức Thắng", "9d78f6dce7965fb4b7a337279920d4b1dhtonducthang-1468902168623.xlsx"}, + {"Trường ĐH Tiền Giang", "bacd1b2133b37d0ea9fbf46445265036dhtiengiang-1468940391597.xlsx"}, + {"Trường ĐH Cần Thơ", "b35a3f26a162bec3740e2da5639327afdhcantho-1468907131146.xls"}, + {"Trường ĐH Cần Thơ - Hậu Giang", "7f1cbe7f35d1f539ca817a0ab8bd00c2dhcantho-haugiang-1468907167921.xls"}, + {"Trường ĐH Luật TP.HCM", "d652d24e8bd4fe6a48d634b116f37c27dhluattphcm-1468939398928.xlsx"}, + {"Trường ĐH Sư phạm Kỹ thuật Vĩnh Long", "4c2fd846ed2a0670ba6d1200d1d68e1dspkythuatvinhlong-1468902640438.xlsx"}, + {"Trường ĐH Trà Vinh", "7418e83ff9d07b146216a3beafa927bbdhtravinh-1468902716174.xlsx"}, + {"Trường ĐH Ngân hàng TP.HCM", "86a692bebf9b0b7c58adb9876b9befd5dhnganhangtphcm-1468902244566.xlsx"}, + {"Trường ĐH Cần Thơ - Bạc Liêu", "5ca241aa1de8d25a3e6ff6db75234025dhcantho-baclieu-1468907071460.xls"}, + {"Trường ĐH Kiên Giang", "cdc912664d7217c75eb9b67c6ef3797adhkiengiang-1468901222446.xlsx"}, + {"Trường ĐH Y Dược Cần Thơ", "023718c7d3cf7ace3a7116fabb12bd9cdhyduoccantho-1468920829104.xlsx"}, + {"Sở GDĐT Hà Nội", "1c66db0265bf45df237b37a7bcaeaef3hanoi-1468932790746.xlsx"}, + {"Sở GDĐT Hà Giang", "178814c33c8b85a77d0fc295fa6d2be1hagiang-1468932847230.xlsx"}, + {"Sở GDĐT Cao Bằng", "ddb631f833af894200ef10cd408d2ef0caobang-1468932877769.xlsx"}, + {"Sở GDĐT Lai Châu", "a3aa106104f114c6426deb2d5e5da5fdlaichau-1468932954079.xlsx"}, + {"Sở GDĐT Lào Cai", "d035f470c13adbf5785797c6ccdcb204laocai-1468919805337.xlsx"}, + {"xem TẠI ĐÂY ) 07 Sở GDĐT Tuyên Quang", "d5ecd2574b5ab36c62dffad435262cb8tuyenquang-1468902296807.xlsx"}, + {"Sở GDĐT Lạng Sơn", "7afd5d7fdff7c170a6342cd8f9474b22langson-1468932984587.xlsx"}, + {"Sở GDĐT Bắc Kạn", "17843bccde10f362c2b9600bc49cd3f8backan-1468933016076.xlsx"}, + {"Sở GDĐT Thái Nguyên", "5fb3a1b66aff077ee3da482be08cd391thainguyen-1468902797020.xlsx"}, + {"Sở GDĐT Yên Bái", "97aa741a4c2b1276559eb021779daa32yenbai-1468933116055.xlsx"}, + {"Sở GDĐT Sơn La", "4b98fdd7300b86a94636e4d15394fc19sonla-1468899319124.xlsx"}, + {"Sở GDĐT Phú Thọ", "5465cca59495dcd9a48f689e2d38b85ephutho-1468902409682.xlsx"}, + {"xem TẠI ĐÂY ) 14 Sở GDĐT Vĩnh Phúc", "4e0ccee19eb64513d23a6dff96ff7f9cvinhphuc-1468919874814.xlsx"}, + {"Sở GDĐT Quảng Ninh", "a31b3446bae5a17c517348d8c38c5b7equangninh-1468919920197.xlsx"}, + {"Sở GDĐT Bắc Giang", "b7a0c1ecce7c8445b3a1021091a5e654bacgiang-1468983153932.xlsx"}, + {"Sở GDĐT Bắc Ninh", "7e0d0b6fb981d407cffdb79d724e8aecbacninh-1468919984998.xlsx"}, + {"Sở GDĐT Hải Dương", "7cfc91bd6a1f5ae9d7917e920f469b90haiduong-1468899640007.xlsx"}, + {"Sở GDĐT Hưng Yên", "92045214159cb0335b2836aa1c4bb25ahungyen-1468899732002.xlsx"}, + {"Sở GDĐT Hoà Bình", "505a876dfd1ea321291e29192aebe95ehoabinh-1468983220125.xlsx"}, + {"Sở GDĐT Hà Nam", "e693bc9de67acc916032c6a3b2fc9eedhanam-1468933146526.xlsx"}, + {"Sở GDĐT Nam Định", "dfa63c5b68a8f638379ca0e9c5a1c556namdinh-1468933234061.xlsx"}, + {"Sở GDĐT Thái Bình", "6960b6fa493e1365a183af461e945389thaibinh-1468920019273.xlsx"}, + {"xem TẠI ĐÂY ) 24 Sở GDĐT Ninh Bình", "0b583c9875443e65ffa449d5ca76fae8ninhbinh-1468899778557.xlsx"}, + {"xem TẠI ĐÂY ) 25 Sở GDĐT Thanh Hoá", "238adc19e80c0daef089a307af23edcethanhhoa-1468933304999.xlsx"}, + {"Sở GDĐT Nghệ An", "7f12b90225e91f9d00aac3bcdf9974e1nghean-1468920043121.xlsx"}, + {"Sở GDĐT Quảng Bình", "9c730c43bf4309171528c3853ed53b19quangbinh-1468933376032.xlsx"}, + {"Sở GDĐT Quảng Trị", "2208903e1fa12220d100f93e40d5ff3bquangtri-1468933440097.xlsx"}, + {"Sở GDĐT Thừa Thiên -Huế", "6badc7cfa258b59ad67e2b64f713b95fthuathienhue-1468983258438.xlsx"}, + {"Sở GDĐT Quảng Nam", "ec765c32190773f5a18cd8202a5eaf0dquangnam-1468933501467.xlsx"}, + {"Sở GDĐT Quảng Ngãi", "e80dc028d53fa49ee247391a0653dee5quangngai-1468933592400.xlsx"}, + {"Sở GDĐT Kon Tum", "cc57dc986acb525086b86e85df54949dkontum-1468933632113.xlsx"}, + {"Sở GDĐT Bình Định", "412e62265f35ca0078a0d94af5d1ed8abinhdinh-1468899884192.xlsx"}, + {"Sở GDĐT Gia Lai", "698b930ab7697a0672bbc39168024c9cgialai-1468933753098.xlsx"}, + {"Sở GDĐT Đắk Lắk", "2004a8d225fd87524e324d2d519e87a4daclac-1468899960505.xlsx"}, + {"Sở GDĐT Khánh Hoà", "4ad416ccf014bd5d01f415238566b210khanhhoa-1468933789968.xlsx"}, + {"Sở GDĐT Lâm Đồng", "2fa6d9a308bd11bc09f02d787eef2c15lamdong-1468933821082.xlsx"}, + {"Sở GDĐT Ninh Thuận", "15c85d901a73dd49f2bae71aadcdbb5cninhthuan-1468920071763.xlsx"}, + {"Sở GDĐT Đồng Nai", "538c9676b70f2509573245d58fba93bcdongnai-1468938241935.xlsx"}, + {"Sở GDĐT Đồng Tháp", "575df8a4df74e87abb8f55d807324374dongthap-1468933902699.xlsx"}, + {"Sở GDĐT Kiên Giang", "7435bb054e8e4ec34c173af1b6ad8402kiengiang-1468900147395.xlsx"}, + {"Sở GDĐT Cần Thơ", "5b16d7156a03bb7ed0618796bae87bc3cantho-1468933954714.xlsx"}, + {"Sở GDĐT Bến Tre", "74145979ee30186bf4e40ee3e6a3ac74bentre-1468934010040.xlsx"}, + {"Sở GDĐT Vĩnh Long", "7f4362ef567cdf74495368e0fc11072avinhlong-1468934067981.xlsx"}, + {"Sở GDĐT Trà Vinh", "08cdeb3636dcb2adaef829d62968274atravinh-1468902443859.xlsx"}, + {"Sở GDĐT Sóc Trăng", "a8ff80d9478747e36d21ec95cc9ecb8fsoctrang-1468902686409.xlsx"}, + {"Sở GDĐT Bạc Liêu", "18055d17e46854ecc6c8be585bcb6e5cbaclieu-1468934144150.xlsx"}, + {"Sở GDĐT Điện Biên", "3c0e56abd8c5334f4e1ff7df3ac23a15dienbien-1468920113820.xlsx"}, + {"Sở GDĐT Đăk Nông", "bf259a908ff8a8dcc067a761204459f4daknong-1468934173803.xlsx"}, + {"Sở GDĐT Hậu Giang", "964c4368131f6be6058fabb1caefba1chaugiang-1468934247324.xlsx"}} diff --git a/crawler/internal/sources/source_2017.go b/crawler/internal/sources/source_2017.go new file mode 100644 index 0000000..c3a8738 --- /dev/null +++ b/crawler/internal/sources/source_2017.go @@ -0,0 +1,121 @@ +package sources + +import ( + "regexp" + "strings" +) + +// article is the page the links were taken from. The CDN rejects requests that +// do not carry it as a Referer. +const article = "https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm" + +// browserUA is required too: the CDN 403s an unrecognised User-Agent. +const browserUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" + +// source2017 fetches the 2017 dataset from the baotintuc.vn CDN: one .xls per +// province. +// +// This is the only dataset that can be re-fetched from its original source. +// The CDN filenames are inconsistent (Angiang.xls, 1BaRiaVungTau.xls, +// 23HaiPhong.xls, gia-lai.xls), so local names are derived from the province +// name instead — which is what produced the data/2017/ files currently on disk, +// and what any re-crawl must keep producing. +var source2017 = Source{ + ID: "2017", + Summary: "63 province .xls files from the baotintuc.vn CDN", + Headers: map[string]string{ + "User-Agent": browserUA, + "Referer": article, + }, + Files: func() ([]File, error) { + out := make([]File, 0, len(provinces2017)) + for _, p := range provinces2017 { + out = append(out, File{ + Name: p.name, + URL: p.url, + Dest: slug(p.name) + ".xls", + }) + } + return out, nil + }, +} + +var nonAlphanumeric = regexp.MustCompile(`[^a-z0-9]+`) + +// slug converts a province name to its local filename stem: lowercase, with +// every run of non-alphanumeric characters collapsed to a single hyphen. +// +// "Ba Ria - Vung Tau" -> "ba-ria-vung-tau" +// +// The names are already unaccented ASCII, so no transliteration is involved. +func slug(name string) string { + return strings.Trim(nonAlphanumeric.ReplaceAllString(strings.ToLower(name), "-"), "-") +} + +type province struct{ name, url string } + +var provinces2017 = []province{ + {"An Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/17/57/Angiang.xls"}, + {"Bac Lieu", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Baclieu.xls"}, + {"Ba Ria - Vung Tau", "https://cdnmedia.baotintuc.vn/2017/07/06/08/17/1BaRiaVungTau.xls"}, + {"Bac Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacGiang.xls"}, + {"Bac Kan", "https://cdnmedia.baotintuc.vn/2017/07/06/08/27/BacKan.xls"}, + {"Bac Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacNinh.xls"}, + {"Ben Tre", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BenTre.xls"}, + {"Binh Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BinhDuong.xls"}, + {"Binh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/BinhThuan.xls"}, + {"Binh Phuoc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/BinhPhuoc.xls"}, + {"Binh Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Binhdinh.xls"}, + {"Ca Mau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Camau.xls"}, + {"Cao Bang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Caobang.xls"}, + {"Can Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/13/07/Cantho.xls"}, + {"Da Nang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/03/Danang.xls"}, + {"Dak Nong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/DakNong.xls"}, + {"Dak Lak", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Daklak.xls"}, + {"Dong Nai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/08/dongnai.xls"}, + {"Dong Thap", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Dongthap.xls"}, + {"Dien Bien", "https://cdnmedia.baotintuc.vn/2017/07/06/09/08/DienBien.xls"}, + {"Gia Lai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/09/Gia-Lai.xls"}, + {"Ha Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/18/Hagiang.xls"}, + {"Ha Noi", "https://cdnmedia.baotintuc.vn/2017/07/07/08/16/HaNoi.xls"}, + {"Ha Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/07/Hanam.xls"}, + {"Ha Tinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaTinh.xls"}, + {"Hai Phong", "https://cdnmedia.baotintuc.vn/2017/07/06/08/18/23HaiPhong.xls"}, + {"Hai Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaiDuong.xls"}, + {"Hau Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Haugiang.xls"}, + {"Ho Chi Minh", "https://cdnmedia.baotintuc.vn/2017/07/06/08/26/HCM.xls"}, + {"Hoa Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/06/HoaBinh.xls"}, + {"Hung Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/08/25/HungYen.xls"}, + {"Khanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/18/04/Khanhhoa.xls"}, + {"Kien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/08/28/KienGiang.xls"}, + {"Kon Tum", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/KonTum.xls"}, + {"Nam Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/13NamDinh.xls"}, + {"Nghe An", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/14NgheAn.xls"}, + {"Ninh Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/37/NinhBinh.xls"}, + {"Ninh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Ninhthuan.xls"}, + {"Lao Cai", "https://cdnmedia.baotintuc.vn/2017/07/06/08/52/11LaoCai.xls"}, + {"Lai Chau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/33/LaiChau.xls"}, + {"Lang Son", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/Langson.xls"}, + {"Lam Dong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/00/LamDong.xls"}, + {"Long An", "https://cdnmedia.baotintuc.vn/2017/07/06/08/53/12LongAn.xls"}, + {"Quang Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/QuangBinh.xls"}, + {"Quang Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNam.xls"}, + {"Quang Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Quangninh.xls"}, + {"Quang Ngai", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNgai.xls"}, + {"Quang Tri", "https://cdnmedia.baotintuc.vn/2017/07/06/09/25/QuangTri.xls"}, + {"Phu Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/PhuTho.xls"}, + {"Phu Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/PhuYen.xls"}, + {"Son La", "https://cdnmedia.baotintuc.vn/2017/07/06/18/01/Sonla.xls"}, + {"Soc Trang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Soctrang.xls"}, + {"Tay Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/26/TayNinh.xls"}, + {"Thai Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/ThaiBinh.xls"}, + {"Thai Nguyen", "https://cdnmedia.baotintuc.vn/2017/07/06/13/35/ThaiNguyen.xls"}, + {"Thanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thanhoa.xls"}, + {"Tra Vinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/TraVinh.xls"}, + {"Thua Thien Hue", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thuathienhue.xls"}, + {"Tien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/13/11/tiengiang.xls"}, + {"Tuyen Quang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/39/TuyenQuang.xls"}, + {"Vinh Phuc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/40/VinhPhuc.xls"}, + {"Vinh Long", "https://cdnmedia.baotintuc.vn/2017/07/06/13/13/vinhlong.xls"}, + {"Yen Bai", "https://cdnmedia.baotintuc.vn/2017/07/06/18/07/Yenbai.xls"}, +} diff --git a/crawler/internal/sources/sources.go b/crawler/internal/sources/sources.go new file mode 100644 index 0000000..0efafc7 --- /dev/null +++ b/crawler/internal/sources/sources.go @@ -0,0 +1,53 @@ +// Package sources describes where each dataset's spreadsheets come from. +// +// A source is a dataset id plus the list of remote files that fill it. +// Downloading is internal/fetch's job; deciding what to download and what to +// call it locally is this package's. +package sources + +import "fmt" + +// File is one remote spreadsheet. +type File struct { + // Name is the human label used in progress output, e.g. "An Giang". + Name string + URL string + // Dest is the filename within the dataset directory. + // + // This is not cosmetic. go-parser sorts its input files and inserts with + // INSERT OR REPLACE, which is last-wins, so filenames decide which row + // survives a duplicate exam number. A re-crawl that names files differently + // can produce a database with the same row count and different content. + Dest string +} + +// Source is one crawlable dataset. +type Source struct { + // ID is the dataset id: the subcommand, the directory under data/, and the + // go-parser config name, all at once. Keeping it single means a source + // cannot be pointed at the wrong dataset's directory. + ID string + Summary string + // Headers are sent with every request for this source. + Headers map[string]string + // Files lists what to download. It returns an error rather than an empty + // list when a source is known but not yet configured, so "nothing to do" + // can never be mistaken for success. + Files func() ([]File, error) +} + +// registry is ordered: it drives the help output. +var registry = []Source{source2016, source2017} + +// All returns every known source, in help order. +func All() []Source { return registry } + +// Lookup finds a source by ID. +func Lookup(id string) (Source, error) { + for _, s := range registry { + if s.ID == id { + return s, nil + } + } + return Source{}, fmt.Errorf("unknown dataset %q", id) +} diff --git a/crawler/internal/sources/sources_test.go b/crawler/internal/sources/sources_test.go new file mode 100644 index 0000000..dca1379 --- /dev/null +++ b/crawler/internal/sources/sources_test.go @@ -0,0 +1,156 @@ +package sources + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestSlug(t *testing.T) { + for in, want := range map[string]string{ + "An Giang": "an-giang", + "Ba Ria - Vung Tau": "ba-ria-vung-tau", + "Ho Chi Minh": "ho-chi-minh", + "Thua Thien Hue": "thua-thien-hue", + } { + if got := slug(in); got != want { + t.Errorf("slug(%q) = %q, want %q", in, got, want) + } + } +} + +func TestSource2017IsComplete(t *testing.T) { + files, err := source2017.Files() + if err != nil { + t.Fatal(err) + } + if len(files) != 63 { + t.Errorf("got %d provinces, want 63", len(files)) + } + + // A duplicate Dest would silently overwrite one province with another and + // leave the dataset one file short, with no error anywhere. + seenDest := map[string]string{} + seenURL := map[string]string{} + for _, f := range files { + if prev, dup := seenDest[f.Dest]; dup { + t.Errorf("%q and %q both write to %s", prev, f.Name, f.Dest) + } + seenDest[f.Dest] = f.Name + if prev, dup := seenURL[f.URL]; dup { + t.Errorf("%q and %q share a URL: %s", prev, f.Name, f.URL) + } + seenURL[f.URL] = f.Name + + if !strings.HasPrefix(f.URL, "https://") { + t.Errorf("%s: URL is not https: %s", f.Name, f.URL) + } + if !strings.HasSuffix(f.Dest, ".xls") { + t.Errorf("%s: Dest %q does not end in .xls", f.Name, f.Dest) + } + } +} + +// TestMatchesFilesOnDisk is the guard that matters. +// +// go-parser sorts its inputs and inserts last-wins, so filenames decide which +// row survives a duplicate exam number. If a re-crawl started naming files +// differently, the rebuilt database could differ in content while still passing +// the row-count guard in build-db.js. The committed data// is the oracle: +// whatever a source would write must match it exactly, in both directions. +// +// For 2016 this is also the only evidence that the recovered link list is the +// right one — the host that served those files is gone, so the list cannot be +// checked by fetching it. +func TestMatchesFilesOnDisk(t *testing.T) { + for _, src := range All() { + t.Run(src.ID, func(t *testing.T) { + dir := filepath.Join("..", "..", "..", "data", src.ID) + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + t.Skipf("%s not present", dir) + } + if err != nil { + t.Fatal(err) + } + + onDisk := map[string]bool{} + for _, e := range entries { + if !e.IsDir() { + onDisk[e.Name()] = true + } + } + + files, err := src.Files() + if err != nil { + t.Fatal(err) + } + for _, f := range files { + if !onDisk[f.Dest] { + t.Errorf("crawler would write %q, which is not in %s — "+ + "a renamed input can change which duplicate row survives", f.Dest, dir) + } + delete(onDisk, f.Dest) + } + for name := range onDisk { + t.Errorf("%s/%s exists but no source produces it", dir, name) + } + }) + } +} + +func TestLookup(t *testing.T) { + for _, s := range All() { + got, err := Lookup(s.ID) + if err != nil { + t.Errorf("Lookup(%q): %v", s.ID, err) + } + if got.ID != s.ID || got.Summary == "" { + t.Errorf("Lookup(%q) returned %+v", s.ID, got) + } + } + if _, err := Lookup("nope"); err == nil { + t.Error("Lookup of an unknown source must fail") + } +} + +// TestIDsAreDatasetIDs: the ID doubles as the directory under data/, so a typo +// would send a crawl into a new directory the parser never reads. +func TestIDsAreDatasetIDs(t *testing.T) { + for _, s := range All() { + dir := filepath.Join("..", "..", "..", "data", s.ID) + if _, err := os.Stat(dir); os.IsNotExist(err) { + t.Errorf("source %q has no dataset directory at %s", s.ID, dir) + } + } +} + +// TestSource2016IsComplete: 4 .xls + 115 .xlsx, matching what the source +// article published and what data/2016/ holds. +func TestSource2016IsComplete(t *testing.T) { + files, err := source2016.Files() + if err != nil { + t.Fatal(err) + } + if len(files) != 119 { + t.Errorf("got %d clusters, want 119", len(files)) + } + var xls, xlsx int + for _, f := range files { + switch filepath.Ext(f.Dest) { + case ".xls": + xls++ + case ".xlsx": + xlsx++ + default: + t.Errorf("%s: unexpected extension in %q", f.Name, f.Dest) + } + if !strings.HasPrefix(f.URL, baseURL+uploadPath) { + t.Errorf("%s: URL does not use the configured base: %s", f.Name, f.URL) + } + } + if xls != 4 || xlsx != 115 { + t.Errorf("got %d .xls and %d .xlsx, want 4 and 115", xls, xlsx) + } +} diff --git a/data/2017-old/10_BinhThuan_RIIW.xls.xlsx b/data/2017-old/10_BinhThuan_RIIW.xls.xlsx deleted file mode 100644 index 8ecaccd..0000000 Binary files a/data/2017-old/10_BinhThuan_RIIW.xls.xlsx and /dev/null differ diff --git a/data/2017-old/10_Ca_Mau_BKXT.xls.xlsx b/data/2017-old/10_Ca_Mau_BKXT.xls.xlsx deleted file mode 100644 index 8fe2b4a..0000000 Binary files a/data/2017-old/10_Ca_Mau_BKXT.xls.xlsx and /dev/null differ diff --git a/data/2017-old/10_LamDong_GNFT.xls.xlsx b/data/2017-old/10_LamDong_GNFT.xls.xlsx deleted file mode 100644 index 54b89a4..0000000 Binary files a/data/2017-old/10_LamDong_GNFT.xls.xlsx and /dev/null differ diff --git a/data/2017-old/10_Soc_Trang_XCGJ.xls.xlsx b/data/2017-old/10_Soc_Trang_XCGJ.xls.xlsx deleted file mode 100644 index 70a0ff8..0000000 Binary files a/data/2017-old/10_Soc_Trang_XCGJ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/11_BinhDuong_RYQL.xls.xlsx b/data/2017-old/11_BinhDuong_RYQL.xls.xlsx deleted file mode 100644 index ccb2951..0000000 Binary files a/data/2017-old/11_BinhDuong_RYQL.xls.xlsx and /dev/null differ diff --git a/data/2017-old/11_LaoCai_GMSU.xls.xlsx b/data/2017-old/11_LaoCai_GMSU.xls.xlsx deleted file mode 100644 index 33ed6f0..0000000 Binary files a/data/2017-old/11_LaoCai_GMSU.xls.xlsx and /dev/null differ diff --git a/data/2017-old/12_BenTre_DKWF.xls.xlsx b/data/2017-old/12_BenTre_DKWF.xls.xlsx deleted file mode 100644 index 8a7cddb..0000000 Binary files a/data/2017-old/12_BenTre_DKWF.xls.xlsx and /dev/null differ diff --git a/data/2017-old/12_LongAn_ZZUK.xls.xlsx b/data/2017-old/12_LongAn_ZZUK.xls.xlsx deleted file mode 100644 index 8f11d78..0000000 Binary files a/data/2017-old/12_LongAn_ZZUK.xls.xlsx and /dev/null differ diff --git a/data/2017-old/13_NamDinh_ESEL.xls.xlsx b/data/2017-old/13_NamDinh_ESEL.xls.xlsx deleted file mode 100644 index b296577..0000000 Binary files a/data/2017-old/13_NamDinh_ESEL.xls.xlsx and /dev/null differ diff --git a/data/2017-old/13_TraVinh_LKUJ.xls.xlsx b/data/2017-old/13_TraVinh_LKUJ.xls.xlsx deleted file mode 100644 index 340c20e..0000000 Binary files a/data/2017-old/13_TraVinh_LKUJ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/14_NgheAn_BSLY.xls.xlsx b/data/2017-old/14_NgheAn_BSLY.xls.xlsx deleted file mode 100644 index b1f2891..0000000 Binary files a/data/2017-old/14_NgheAn_BSLY.xls.xlsx and /dev/null differ diff --git a/data/2017-old/15_PhuTho_ABWQ.xls.xlsx b/data/2017-old/15_PhuTho_ABWQ.xls.xlsx deleted file mode 100644 index 5135dac..0000000 Binary files a/data/2017-old/15_PhuTho_ABWQ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/16_QuangBinh_KGEU.xls.xlsx b/data/2017-old/16_QuangBinh_KGEU.xls.xlsx deleted file mode 100644 index c12f598..0000000 Binary files a/data/2017-old/16_QuangBinh_KGEU.xls.xlsx and /dev/null differ diff --git a/data/2017-old/17_QuangNam_AMTK.xls.xlsx b/data/2017-old/17_QuangNam_AMTK.xls.xlsx deleted file mode 100644 index 761a8df..0000000 Binary files a/data/2017-old/17_QuangNam_AMTK.xls.xlsx and /dev/null differ diff --git a/data/2017-old/18_QuangNgai_KOFP.xls.xlsx b/data/2017-old/18_QuangNgai_KOFP.xls.xlsx deleted file mode 100644 index 349bede..0000000 Binary files a/data/2017-old/18_QuangNgai_KOFP.xls.xlsx and /dev/null differ diff --git a/data/2017-old/19_QuangTri_OMZF.xls.xlsx b/data/2017-old/19_QuangTri_OMZF.xls.xlsx deleted file mode 100644 index f66a8bb..0000000 Binary files a/data/2017-old/19_QuangTri_OMZF.xls.xlsx and /dev/null differ diff --git a/data/2017-old/1_BaRia_VungTau_HJKG.xls.xlsx b/data/2017-old/1_BaRia_VungTau_HJKG.xls.xlsx deleted file mode 100644 index 35be1a9..0000000 Binary files a/data/2017-old/1_BaRia_VungTau_HJKG.xls.xlsx and /dev/null differ diff --git a/data/2017-old/1_Da_Nang_AHWJ.xls.xlsx b/data/2017-old/1_Da_Nang_AHWJ.xls.xlsx deleted file mode 100644 index 4a63447..0000000 Binary files a/data/2017-old/1_Da_Nang_AHWJ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/1_Ha_Noi_CVXG.xls.xlsx b/data/2017-old/1_Ha_Noi_CVXG.xls.xlsx deleted file mode 100644 index e55102c..0000000 Binary files a/data/2017-old/1_Ha_Noi_CVXG.xls.xlsx and /dev/null differ diff --git a/data/2017-old/1_Son_La_JIDP.xls.xlsx b/data/2017-old/1_Son_La_JIDP.xls.xlsx deleted file mode 100644 index 62d7b18..0000000 Binary files a/data/2017-old/1_Son_La_JIDP.xls.xlsx and /dev/null differ diff --git a/data/2017-old/1_TuyenQuang_JBYF.xls.xlsx b/data/2017-old/1_TuyenQuang_JBYF.xls.xlsx deleted file mode 100644 index 601bb9e..0000000 Binary files a/data/2017-old/1_TuyenQuang_JBYF.xls.xlsx and /dev/null differ diff --git a/data/2017-old/20_TayNinh_ILFA.xls.xlsx b/data/2017-old/20_TayNinh_ILFA.xls.xlsx deleted file mode 100644 index 2b636fc..0000000 Binary files a/data/2017-old/20_TayNinh_ILFA.xls.xlsx and /dev/null differ diff --git a/data/2017-old/21_ThaiBinh_FTVG.xls.xlsx b/data/2017-old/21_ThaiBinh_FTVG.xls.xlsx deleted file mode 100644 index a9f5bae..0000000 Binary files a/data/2017-old/21_ThaiBinh_FTVG.xls.xlsx and /dev/null differ diff --git a/data/2017-old/22_ThaiNguyen_TLTW.xls.xlsx b/data/2017-old/22_ThaiNguyen_TLTW.xls.xlsx deleted file mode 100644 index 8f9c2bb..0000000 Binary files a/data/2017-old/22_ThaiNguyen_TLTW.xls.xlsx and /dev/null differ diff --git a/data/2017-old/23_HaiPhong_HXBV.xls.xlsx b/data/2017-old/23_HaiPhong_HXBV.xls.xlsx deleted file mode 100644 index c219f1f..0000000 Binary files a/data/2017-old/23_HaiPhong_HXBV.xls.xlsx and /dev/null differ diff --git a/data/2017-old/24_HCM_XULN.xls.xlsx b/data/2017-old/24_HCM_XULN.xls.xlsx deleted file mode 100644 index a526aac..0000000 Binary files a/data/2017-old/24_HCM_XULN.xls.xlsx and /dev/null differ diff --git a/data/2017-old/2_BacKan_GFVQ.xls.xlsx b/data/2017-old/2_BacKan_GFVQ.xls.xlsx deleted file mode 100644 index 115d551..0000000 Binary files a/data/2017-old/2_BacKan_GFVQ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/2_Ha_Giang_QNCM.xls.xlsx b/data/2017-old/2_Ha_Giang_QNCM.xls.xlsx deleted file mode 100644 index 2750bed..0000000 Binary files a/data/2017-old/2_Ha_Giang_QNCM.xls.xlsx and /dev/null differ diff --git a/data/2017-old/2_Ninh_Thuan_VHLY.xls.xlsx b/data/2017-old/2_Ninh_Thuan_VHLY.xls.xlsx deleted file mode 100644 index 042f58d..0000000 Binary files a/data/2017-old/2_Ninh_Thuan_VHLY.xls.xlsx and /dev/null differ diff --git a/data/2017-old/2_Thanh_Hoa_AUFV.xls.xlsx b/data/2017-old/2_Thanh_Hoa_AUFV.xls.xlsx deleted file mode 100644 index 351d53e..0000000 Binary files a/data/2017-old/2_Thanh_Hoa_AUFV.xls.xlsx and /dev/null differ diff --git a/data/2017-old/2_VinhPhuc_GUDK.xls.xlsx b/data/2017-old/2_VinhPhuc_GUDK.xls.xlsx deleted file mode 100644 index cfb98f9..0000000 Binary files a/data/2017-old/2_VinhPhuc_GUDK.xls.xlsx and /dev/null differ diff --git a/data/2017-old/3_BacGiang_TOIF.xls.xlsx b/data/2017-old/3_BacGiang_TOIF.xls.xlsx deleted file mode 100644 index 1e9661b..0000000 Binary files a/data/2017-old/3_BacGiang_TOIF.xls.xlsx and /dev/null differ diff --git a/data/2017-old/3_BinhPhuoc_YFMU.xls.xlsx b/data/2017-old/3_BinhPhuoc_YFMU.xls.xlsx deleted file mode 100644 index 592d676..0000000 Binary files a/data/2017-old/3_BinhPhuoc_YFMU.xls.xlsx and /dev/null differ diff --git a/data/2017-old/3_Cao_Bang_CIEY.xls.xlsx b/data/2017-old/3_Cao_Bang_CIEY.xls.xlsx deleted file mode 100644 index f68c4e7..0000000 Binary files a/data/2017-old/3_Cao_Bang_CIEY.xls.xlsx and /dev/null differ diff --git a/data/2017-old/3_Dong_Thap_GSXQ.xls.xlsx b/data/2017-old/3_Dong_Thap_GSXQ.xls.xlsx deleted file mode 100644 index 5afdc2d..0000000 Binary files a/data/2017-old/3_Dong_Thap_GSXQ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/3_Thua_Thien_Hue_HMDB.xls.xlsx b/data/2017-old/3_Thua_Thien_Hue_HMDB.xls.xlsx deleted file mode 100644 index 34d591f..0000000 Binary files a/data/2017-old/3_Thua_Thien_Hue_HMDB.xls.xlsx and /dev/null differ diff --git a/data/2017-old/4_An_Giang_JNOS.xls.xlsx b/data/2017-old/4_An_Giang_JNOS.xls.xlsx deleted file mode 100644 index b4032d3..0000000 Binary files a/data/2017-old/4_An_Giang_JNOS.xls.xlsx and /dev/null differ diff --git a/data/2017-old/4_BacNinh_STLR.xls.xlsx b/data/2017-old/4_BacNinh_STLR.xls.xlsx deleted file mode 100644 index cc08868..0000000 Binary files a/data/2017-old/4_BacNinh_STLR.xls.xlsx and /dev/null differ diff --git a/data/2017-old/4_Binh_Dinh_WWZW.xls.xlsx b/data/2017-old/4_Binh_Dinh_WWZW.xls.xlsx deleted file mode 100644 index ddb0fc5..0000000 Binary files a/data/2017-old/4_Binh_Dinh_WWZW.xls.xlsx and /dev/null differ diff --git a/data/2017-old/4_DienBien_SOJG.xls.xlsx b/data/2017-old/4_DienBien_SOJG.xls.xlsx deleted file mode 100644 index 0dddaac..0000000 Binary files a/data/2017-old/4_DienBien_SOJG.xls.xlsx and /dev/null differ diff --git a/data/2017-old/4_Lang_Son_NYQL.xls.xlsx b/data/2017-old/4_Lang_Son_NYQL.xls.xlsx deleted file mode 100644 index a845623..0000000 Binary files a/data/2017-old/4_Lang_Son_NYQL.xls.xlsx and /dev/null differ diff --git a/data/2017-old/5_Bac_Lieu_XEKH.xls.xlsx b/data/2017-old/5_Bac_Lieu_XEKH.xls.xlsx deleted file mode 100644 index 73c1bef..0000000 Binary files a/data/2017-old/5_Bac_Lieu_XEKH.xls.xlsx and /dev/null differ diff --git a/data/2017-old/5_Gia_Lai_ABZI.xls.xlsx b/data/2017-old/5_Gia_Lai_ABZI.xls.xlsx deleted file mode 100644 index c0f9294..0000000 Binary files a/data/2017-old/5_Gia_Lai_ABZI.xls.xlsx and /dev/null differ diff --git a/data/2017-old/5_HaiDuong_WWWG.xls.xlsx b/data/2017-old/5_HaiDuong_WWWG.xls.xlsx deleted file mode 100644 index 190b19a..0000000 Binary files a/data/2017-old/5_HaiDuong_WWWG.xls.xlsx and /dev/null differ diff --git a/data/2017-old/5_Hanam_QGJS.xls.xlsx b/data/2017-old/5_Hanam_QGJS.xls.xlsx deleted file mode 100644 index 1740b8b..0000000 Binary files a/data/2017-old/5_Hanam_QGJS.xls.xlsx and /dev/null differ diff --git a/data/2017-old/5_Yen_Bai_FAQR.xls.xlsx b/data/2017-old/5_Yen_Bai_FAQR.xls.xlsx deleted file mode 100644 index 09a6e5f..0000000 Binary files a/data/2017-old/5_Yen_Bai_FAQR.xls.xlsx and /dev/null differ diff --git a/data/2017-old/6_Dong_Nai_WOTM.xls.xlsx b/data/2017-old/6_Dong_Nai_WOTM.xls.xlsx deleted file mode 100644 index 35fbd77..0000000 Binary files a/data/2017-old/6_Dong_Nai_WOTM.xls.xlsx and /dev/null differ diff --git a/data/2017-old/6_Hau_Giang_KWDM.xls.xlsx b/data/2017-old/6_Hau_Giang_KWDM.xls.xlsx deleted file mode 100644 index 7543315..0000000 Binary files a/data/2017-old/6_Hau_Giang_KWDM.xls.xlsx and /dev/null differ diff --git a/data/2017-old/6_HoaBinh_TPZY.xls.xlsx b/data/2017-old/6_HoaBinh_TPZY.xls.xlsx deleted file mode 100644 index 932db3e..0000000 Binary files a/data/2017-old/6_HoaBinh_TPZY.xls.xlsx and /dev/null differ diff --git a/data/2017-old/6_NinhBinh_IGFT.xls.xlsx b/data/2017-old/6_NinhBinh_IGFT.xls.xlsx deleted file mode 100644 index 2553671..0000000 Binary files a/data/2017-old/6_NinhBinh_IGFT.xls.xlsx and /dev/null differ diff --git a/data/2017-old/6_Quang_Ninh_DQCJ.xls.xlsx b/data/2017-old/6_Quang_Ninh_DQCJ.xls.xlsx deleted file mode 100644 index c04950f..0000000 Binary files a/data/2017-old/6_Quang_Ninh_DQCJ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/7_HaTinh_DDHD.xls.xlsx b/data/2017-old/7_HaTinh_DDHD.xls.xlsx deleted file mode 100644 index 9327104..0000000 Binary files a/data/2017-old/7_HaTinh_DDHD.xls.xlsx and /dev/null differ diff --git a/data/2017-old/7_HungYen_LTIK.xls.xlsx b/data/2017-old/7_HungYen_LTIK.xls.xlsx deleted file mode 100644 index bfff635..0000000 Binary files a/data/2017-old/7_HungYen_LTIK.xls.xlsx and /dev/null differ diff --git a/data/2017-old/7_Kon_Tum_RSLR.xls.xlsx b/data/2017-old/7_Kon_Tum_RSLR.xls.xlsx deleted file mode 100644 index 3563aab..0000000 Binary files a/data/2017-old/7_Kon_Tum_RSLR.xls.xlsx and /dev/null differ diff --git a/data/2017-old/7_Tien_Giang_EFHX.xls.xlsx b/data/2017-old/7_Tien_Giang_EFHX.xls.xlsx deleted file mode 100644 index 8e51dda..0000000 Binary files a/data/2017-old/7_Tien_Giang_EFHX.xls.xlsx and /dev/null differ diff --git a/data/2017-old/8_Can_Tho_RQZM.xls.xlsx b/data/2017-old/8_Can_Tho_RQZM.xls.xlsx deleted file mode 100644 index 172a18d..0000000 Binary files a/data/2017-old/8_Can_Tho_RQZM.xls.xlsx and /dev/null differ diff --git a/data/2017-old/8_Dak_Lak_YKPR.xls.xlsx b/data/2017-old/8_Dak_Lak_YKPR.xls.xlsx deleted file mode 100644 index 98c80a6..0000000 Binary files a/data/2017-old/8_Dak_Lak_YKPR.xls.xlsx and /dev/null differ diff --git a/data/2017-old/8_KienGiang_OTOB.xls.xlsx b/data/2017-old/8_KienGiang_OTOB.xls.xlsx deleted file mode 100644 index fa07efa..0000000 Binary files a/data/2017-old/8_KienGiang_OTOB.xls.xlsx and /dev/null differ diff --git a/data/2017-old/8_PhuYen_OGIM.xls.xlsx b/data/2017-old/8_PhuYen_OGIM.xls.xlsx deleted file mode 100644 index 02e391c..0000000 Binary files a/data/2017-old/8_PhuYen_OGIM.xls.xlsx and /dev/null differ diff --git a/data/2017-old/9_DakNong_FBOP.xls.xlsx b/data/2017-old/9_DakNong_FBOP.xls.xlsx deleted file mode 100644 index 6ae062c..0000000 Binary files a/data/2017-old/9_DakNong_FBOP.xls.xlsx and /dev/null differ diff --git a/data/2017-old/9_Khanh_Hoa_KPKQ.xls.xlsx b/data/2017-old/9_Khanh_Hoa_KPKQ.xls.xlsx deleted file mode 100644 index d30f3f7..0000000 Binary files a/data/2017-old/9_Khanh_Hoa_KPKQ.xls.xlsx and /dev/null differ diff --git a/data/2017-old/9_LaiChau_ALKN.xls.xlsx b/data/2017-old/9_LaiChau_ALKN.xls.xlsx deleted file mode 100644 index 9806b8e..0000000 Binary files a/data/2017-old/9_LaiChau_ALKN.xls.xlsx and /dev/null differ diff --git a/data/2017-old/9_Vinh_Long_OLWA.xls.xlsx b/data/2017-old/9_Vinh_Long_OLWA.xls.xlsx deleted file mode 100644 index 7ba2885..0000000 Binary files a/data/2017-old/9_Vinh_Long_OLWA.xls.xlsx and /dev/null differ diff --git a/data/2017-old2/1.BaRia-VungTau_PGZT.xlsx b/data/2017-old2/1.BaRia-VungTau_PGZT.xlsx deleted file mode 100644 index 3a26735..0000000 Binary files a/data/2017-old2/1.BaRia-VungTau_PGZT.xlsx and /dev/null differ diff --git a/data/2017-old2/1.Da Nang_ABWU.xlsx b/data/2017-old2/1.Da Nang_ABWU.xlsx deleted file mode 100644 index 5396aef..0000000 Binary files a/data/2017-old2/1.Da Nang_ABWU.xlsx and /dev/null differ diff --git a/data/2017-old2/1.Son La_XLFN.xlsx b/data/2017-old2/1.Son La_XLFN.xlsx deleted file mode 100644 index 7eafde3..0000000 Binary files a/data/2017-old2/1.Son La_XLFN.xlsx and /dev/null differ diff --git a/data/2017-old2/1.TuyenQuang_PTMR.xlsx b/data/2017-old2/1.TuyenQuang_PTMR.xlsx deleted file mode 100644 index 06f7c27..0000000 Binary files a/data/2017-old2/1.TuyenQuang_PTMR.xlsx and /dev/null differ diff --git a/data/2017-old2/10.BinhThuan_MVVG.xlsx b/data/2017-old2/10.BinhThuan_MVVG.xlsx deleted file mode 100644 index 8cabd6d..0000000 Binary files a/data/2017-old2/10.BinhThuan_MVVG.xlsx and /dev/null differ diff --git a/data/2017-old2/10.LamDong_YUQA.xlsx b/data/2017-old2/10.LamDong_YUQA.xlsx deleted file mode 100644 index e4debd8..0000000 Binary files a/data/2017-old2/10.LamDong_YUQA.xlsx and /dev/null differ diff --git a/data/2017-old2/10.Soc Trang_LQWU.xlsx b/data/2017-old2/10.Soc Trang_LQWU.xlsx deleted file mode 100644 index c8cd707..0000000 Binary files a/data/2017-old2/10.Soc Trang_LQWU.xlsx and /dev/null differ diff --git a/data/2017-old2/11.BinhDuong_HVAH.xlsx b/data/2017-old2/11.BinhDuong_HVAH.xlsx deleted file mode 100644 index 8b3f7ad..0000000 Binary files a/data/2017-old2/11.BinhDuong_HVAH.xlsx and /dev/null differ diff --git a/data/2017-old2/11.LaoCai_ZTBP.xlsx b/data/2017-old2/11.LaoCai_ZTBP.xlsx deleted file mode 100644 index 63870d1..0000000 Binary files a/data/2017-old2/11.LaoCai_ZTBP.xlsx and /dev/null differ diff --git a/data/2017-old2/12.BenTre_NQTU.xlsx b/data/2017-old2/12.BenTre_NQTU.xlsx deleted file mode 100644 index ae9edb4..0000000 Binary files a/data/2017-old2/12.BenTre_NQTU.xlsx and /dev/null differ diff --git a/data/2017-old2/12.LongAn_PDRH.xlsx b/data/2017-old2/12.LongAn_PDRH.xlsx deleted file mode 100644 index e0b66c8..0000000 Binary files a/data/2017-old2/12.LongAn_PDRH.xlsx and /dev/null differ diff --git a/data/2017-old2/13.NamDinh_NAYR.xlsx b/data/2017-old2/13.NamDinh_NAYR.xlsx deleted file mode 100644 index b16ead2..0000000 Binary files a/data/2017-old2/13.NamDinh_NAYR.xlsx and /dev/null differ diff --git a/data/2017-old2/13.TraVinh_FODZ.xlsx b/data/2017-old2/13.TraVinh_FODZ.xlsx deleted file mode 100644 index eb2c6ce..0000000 Binary files a/data/2017-old2/13.TraVinh_FODZ.xlsx and /dev/null differ diff --git a/data/2017-old2/14.NgheAn_HTKD.xlsx b/data/2017-old2/14.NgheAn_HTKD.xlsx deleted file mode 100644 index f872da1..0000000 Binary files a/data/2017-old2/14.NgheAn_HTKD.xlsx and /dev/null differ diff --git a/data/2017-old2/15.PhuTho_IJZW.xlsx b/data/2017-old2/15.PhuTho_IJZW.xlsx deleted file mode 100644 index e075d3a..0000000 Binary files a/data/2017-old2/15.PhuTho_IJZW.xlsx and /dev/null differ diff --git a/data/2017-old2/17.QuangNam_NQMG.xlsx b/data/2017-old2/17.QuangNam_NQMG.xlsx deleted file mode 100644 index 3e75262..0000000 Binary files a/data/2017-old2/17.QuangNam_NQMG.xlsx and /dev/null differ diff --git a/data/2017-old2/18.QuangNgai_IUPY.xlsx b/data/2017-old2/18.QuangNgai_IUPY.xlsx deleted file mode 100644 index 5e37ea2..0000000 Binary files a/data/2017-old2/18.QuangNgai_IUPY.xlsx and /dev/null differ diff --git a/data/2017-old2/19.QuangTri_MKNN.xlsx b/data/2017-old2/19.QuangTri_MKNN.xlsx deleted file mode 100644 index 454256c..0000000 Binary files a/data/2017-old2/19.QuangTri_MKNN.xlsx and /dev/null differ diff --git a/data/2017-old2/2.BacKan_YQNX.xlsx b/data/2017-old2/2.BacKan_YQNX.xlsx deleted file mode 100644 index 6fa41c9..0000000 Binary files a/data/2017-old2/2.BacKan_YQNX.xlsx and /dev/null differ diff --git a/data/2017-old2/2.Ha Giang_PIYK.xlsx b/data/2017-old2/2.Ha Giang_PIYK.xlsx deleted file mode 100644 index d65b471..0000000 Binary files a/data/2017-old2/2.Ha Giang_PIYK.xlsx and /dev/null differ diff --git a/data/2017-old2/2.Ninh Thuan_BAGG.xlsx b/data/2017-old2/2.Ninh Thuan_BAGG.xlsx deleted file mode 100644 index a77706b..0000000 Binary files a/data/2017-old2/2.Ninh Thuan_BAGG.xlsx and /dev/null differ diff --git a/data/2017-old2/2.Thanh Hoa_UOPE.xlsx b/data/2017-old2/2.Thanh Hoa_UOPE.xlsx deleted file mode 100644 index d40be28..0000000 Binary files a/data/2017-old2/2.Thanh Hoa_UOPE.xlsx and /dev/null differ diff --git a/data/2017-old2/2.VinhPhuc_QZJK.xlsx b/data/2017-old2/2.VinhPhuc_QZJK.xlsx deleted file mode 100644 index be86a48..0000000 Binary files a/data/2017-old2/2.VinhPhuc_QZJK.xlsx and /dev/null differ diff --git a/data/2017-old2/20.TayNinh_KJAQ.xlsx b/data/2017-old2/20.TayNinh_KJAQ.xlsx deleted file mode 100644 index 65eedff..0000000 Binary files a/data/2017-old2/20.TayNinh_KJAQ.xlsx and /dev/null differ diff --git a/data/2017-old2/21.ThaiBinh_KTQN.xlsx b/data/2017-old2/21.ThaiBinh_KTQN.xlsx deleted file mode 100644 index 02a3895..0000000 Binary files a/data/2017-old2/21.ThaiBinh_KTQN.xlsx and /dev/null differ diff --git a/data/2017-old2/22.ThaiNguyen_BKIF.xlsx b/data/2017-old2/22.ThaiNguyen_BKIF.xlsx deleted file mode 100644 index a144e9c..0000000 Binary files a/data/2017-old2/22.ThaiNguyen_BKIF.xlsx and /dev/null differ diff --git a/data/2017-old2/24.HCM_UTLQ.xlsx b/data/2017-old2/24.HCM_UTLQ.xlsx deleted file mode 100644 index cc42e51..0000000 Binary files a/data/2017-old2/24.HCM_UTLQ.xlsx and /dev/null differ diff --git a/data/2017-old2/3.BacGiang_SAVS.xlsx b/data/2017-old2/3.BacGiang_SAVS.xlsx deleted file mode 100644 index bc9b5f6..0000000 Binary files a/data/2017-old2/3.BacGiang_SAVS.xlsx and /dev/null differ diff --git a/data/2017-old2/3.BinhPhuoc_IPHL.xlsx b/data/2017-old2/3.BinhPhuoc_IPHL.xlsx deleted file mode 100644 index 8c5871d..0000000 Binary files a/data/2017-old2/3.BinhPhuoc_IPHL.xlsx and /dev/null differ diff --git a/data/2017-old2/3.Cao Bang_WMUU.xlsx b/data/2017-old2/3.Cao Bang_WMUU.xlsx deleted file mode 100644 index 36200fb..0000000 Binary files a/data/2017-old2/3.Cao Bang_WMUU.xlsx and /dev/null differ diff --git a/data/2017-old2/3.Dong Thap_HKJX.xlsx b/data/2017-old2/3.Dong Thap_HKJX.xlsx deleted file mode 100644 index ae37166..0000000 Binary files a/data/2017-old2/3.Dong Thap_HKJX.xlsx and /dev/null differ diff --git a/data/2017-old2/3.Thua Thien -Hue_MAET.xlsx b/data/2017-old2/3.Thua Thien -Hue_MAET.xlsx deleted file mode 100644 index 587af78..0000000 Binary files a/data/2017-old2/3.Thua Thien -Hue_MAET.xlsx and /dev/null differ diff --git a/data/2017-old2/4.An Giang_PMJD.xlsx b/data/2017-old2/4.An Giang_PMJD.xlsx deleted file mode 100644 index b651ffb..0000000 Binary files a/data/2017-old2/4.An Giang_PMJD.xlsx and /dev/null differ diff --git a/data/2017-old2/4.BacNinh_NNIS.xlsx b/data/2017-old2/4.BacNinh_NNIS.xlsx deleted file mode 100644 index 04de119..0000000 Binary files a/data/2017-old2/4.BacNinh_NNIS.xlsx and /dev/null differ diff --git a/data/2017-old2/4.Binh Dinh_VOMJ.xlsx b/data/2017-old2/4.Binh Dinh_VOMJ.xlsx deleted file mode 100644 index 704c4ae..0000000 Binary files a/data/2017-old2/4.Binh Dinh_VOMJ.xlsx and /dev/null differ diff --git a/data/2017-old2/4.DienBien_FYGN.xlsx b/data/2017-old2/4.DienBien_FYGN.xlsx deleted file mode 100644 index aae73eb..0000000 Binary files a/data/2017-old2/4.DienBien_FYGN.xlsx and /dev/null differ diff --git a/data/2017-old2/4.Lang Son_QWOG.xlsx b/data/2017-old2/4.Lang Son_QWOG.xlsx deleted file mode 100644 index e7e9534..0000000 Binary files a/data/2017-old2/4.Lang Son_QWOG.xlsx and /dev/null differ diff --git a/data/2017-old2/5.Bac Lieu_VIVY.xlsx b/data/2017-old2/5.Bac Lieu_VIVY.xlsx deleted file mode 100644 index 575cac0..0000000 Binary files a/data/2017-old2/5.Bac Lieu_VIVY.xlsx and /dev/null differ diff --git a/data/2017-old2/5.Gia Lai_TAAS.xlsx b/data/2017-old2/5.Gia Lai_TAAS.xlsx deleted file mode 100644 index f358c30..0000000 Binary files a/data/2017-old2/5.Gia Lai_TAAS.xlsx and /dev/null differ diff --git a/data/2017-old2/5.HaiDuong_WNHD.xlsx b/data/2017-old2/5.HaiDuong_WNHD.xlsx deleted file mode 100644 index 7b1169f..0000000 Binary files a/data/2017-old2/5.HaiDuong_WNHD.xlsx and /dev/null differ diff --git a/data/2017-old2/5.Hanam_SDKN.xlsx b/data/2017-old2/5.Hanam_SDKN.xlsx deleted file mode 100644 index 5b9d113..0000000 Binary files a/data/2017-old2/5.Hanam_SDKN.xlsx and /dev/null differ diff --git a/data/2017-old2/5.Yen Bai_BSLV.xlsx b/data/2017-old2/5.Yen Bai_BSLV.xlsx deleted file mode 100644 index 915dc38..0000000 Binary files a/data/2017-old2/5.Yen Bai_BSLV.xlsx and /dev/null differ diff --git a/data/2017-old2/6.Hau Giang_SIAJ.xlsx b/data/2017-old2/6.Hau Giang_SIAJ.xlsx deleted file mode 100644 index f4a67e4..0000000 Binary files a/data/2017-old2/6.Hau Giang_SIAJ.xlsx and /dev/null differ diff --git a/data/2017-old2/6.HoaBinh_HLYQ.xlsx b/data/2017-old2/6.HoaBinh_HLYQ.xlsx deleted file mode 100644 index 768a885..0000000 Binary files a/data/2017-old2/6.HoaBinh_HLYQ.xlsx and /dev/null differ diff --git a/data/2017-old2/6.NinhBinh_PKMQ.xlsx b/data/2017-old2/6.NinhBinh_PKMQ.xlsx deleted file mode 100644 index 7fab324..0000000 Binary files a/data/2017-old2/6.NinhBinh_PKMQ.xlsx and /dev/null differ diff --git a/data/2017-old2/6.Quang Ninh_YAKJ.xlsx b/data/2017-old2/6.Quang Ninh_YAKJ.xlsx deleted file mode 100644 index 51ddb21..0000000 Binary files a/data/2017-old2/6.Quang Ninh_YAKJ.xlsx and /dev/null differ diff --git a/data/2017-old2/7.HaTinh_XMFQ.xlsx b/data/2017-old2/7.HaTinh_XMFQ.xlsx deleted file mode 100644 index 0aa1629..0000000 Binary files a/data/2017-old2/7.HaTinh_XMFQ.xlsx and /dev/null differ diff --git a/data/2017-old2/7.HungYen_TCBE.xlsx b/data/2017-old2/7.HungYen_TCBE.xlsx deleted file mode 100644 index be95a39..0000000 Binary files a/data/2017-old2/7.HungYen_TCBE.xlsx and /dev/null differ diff --git a/data/2017-old2/7.Kon Tum_CAQU.xlsx b/data/2017-old2/7.Kon Tum_CAQU.xlsx deleted file mode 100644 index aa2d88a..0000000 Binary files a/data/2017-old2/7.Kon Tum_CAQU.xlsx and /dev/null differ diff --git a/data/2017-old2/7.Tien Giang_AOWZ.xlsx b/data/2017-old2/7.Tien Giang_AOWZ.xlsx deleted file mode 100644 index 43206a0..0000000 Binary files a/data/2017-old2/7.Tien Giang_AOWZ.xlsx and /dev/null differ diff --git a/data/2017-old2/8.PhuYen_MLTQ.xlsx b/data/2017-old2/8.PhuYen_MLTQ.xlsx deleted file mode 100644 index 7af0186..0000000 Binary files a/data/2017-old2/8.PhuYen_MLTQ.xlsx and /dev/null differ diff --git a/data/2017-old2/9.DakNong_ZWJT.xlsx b/data/2017-old2/9.DakNong_ZWJT.xlsx deleted file mode 100644 index cffd8a6..0000000 Binary files a/data/2017-old2/9.DakNong_ZWJT.xlsx and /dev/null differ diff --git a/data/2017-old2/9.Khanh Hoa_IBTX.xlsx b/data/2017-old2/9.Khanh Hoa_IBTX.xlsx deleted file mode 100644 index 058419d..0000000 Binary files a/data/2017-old2/9.Khanh Hoa_IBTX.xlsx and /dev/null differ diff --git a/data/2017-old2/9.Vinh Long_WVKI.xlsx b/data/2017-old2/9.Vinh Long_WVKI.xlsx deleted file mode 100644 index febfad4..0000000 Binary files a/data/2017-old2/9.Vinh Long_WVKI.xlsx and /dev/null differ diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index bbfe48d..b88c711 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -2,32 +2,63 @@ From raw Excel files to a compressed SQLite file the browser can load. -One Rust binary (`go-parser/`) builds every dataset. What differs per dataset is +One Go binary (`go-parser/`) builds every dataset. What differs per dataset is parse rules only — sheet strategy, column layout, validation guards — declared in `go-parser/configs/.yml`. The table shape, the INSERT and the subject regexes are canonical and live in `go-parser/internal/schema/schema.go`. ## Sources -| id | Files | Reproducible | Origin | +| id | Files | Origin | Host live? | | --- | --- | --- | --- | -| `2016` | 4 `.xls` + 115 `.xlsx` | no | Bộ GD&ĐT, collected 2016 | -| `2017` | 63 `.xls` | **yes** | baotintuc.vn CDN | -| `2017-old` | 63 `.xlsx` | no | pre-refresh archive | -| `2017-old2` | 54 `.xlsx` | no | corrected re-export | +| `2016` | 4 `.xls` + 115 `.xlsx` | aggregator article, 119 exam clusters | unconfirmed | +| `2017` | 63 `.xls` | baotintuc.vn CDN | **yes** | -Only `2017` can be re-fetched: +Crawling lives in `crawler/`, a separate Go module. It is never part of the +build — the source files are committed, so a crawl only refreshes them. Both +runs are idempotent: files already present are skipped. ```bash -node go-parser/scripts/crawl-baotintuc.js +npm run crawl:2016 # crawler/internal/sources/source_2016.go +npm run crawl:2017 # crawler/internal/sources/source_2017.go +npm run crawl:2017 -- --list # show what would be downloaded, fetch nothing ``` -Idempotent — skips files already present, saves to `data/2017/`. Source article: +**2017** comes from the article `https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm` +and its CDN is still serving the files. -The other three were collected from Vietnamese news sites at the time and the -publisher URLs were not recorded. **The files committed in git are the only -copy.** +**2016** comes from the aggregator article +`cong-bo-diem-thi-thptqg-2016-toan-bo-120-cum-thi-da-co-diem.html`. The site +that originally published it (`dtntbacgiang.edu.vn`) no longer resolves, so the +link list was read from the Internet Archive's copy of the page. Two caveats +that matter: + +- **The list is verified; the host is not.** All 119 filenames match `data/2016/` + exactly in both directions, which is asserted by a test. But the Archive + captured only the article, not the spreadsheets, so the URLs themselves could + not be confirmed by fetching. `baseURL` in `source_2016.go` points at a mirror + of the same article that is still online; if it serves the files from a + different directory, `uploadPath` is the one constant to change. +- **`data/2016/` is still the only confirmed copy.** Do not delete it on the + assumption that a crawl can restore it. + +### Local filenames are load-bearing + +`go-parser` sorts its input files and inserts with `INSERT OR REPLACE`, which is +last-wins, so **filenames decide which row survives a duplicate exam number**. A +re-crawl that names files differently can produce a database with the same row +count and different content, which the row-count guard in `build-db.js` would +not catch. + +Each source therefore pins its local names, and +`crawler/internal/sources/sources_test.go` checks every source against its +committed `data//` in both directions — every name it would write exists, +and every file present is accounted for. + +2017 derives its names from the province (the CDN's own names are inconsistent: +`Angiang.xls`, `1BaRiaVungTau.xls`, `23HaiPhong.xls`). 2016 keeps the +server-assigned names verbatim, since it did not choose them. ## Source Excel shapes @@ -41,7 +72,7 @@ The 2017 datasets share one layout: | 3 | DIEM_THI | concatenated per-subject scores, e.g. `"Toán: 6.80 Ngữ văn: 5.25 …"` | 2016 has **three** layouts across its 119 files, chosen per file at runtime via -`format_detection = "thptqg2016"` in its config: +`format_detection: thptqg2016` in its config: | Format | Detected by | Notes | | --- | --- | --- | @@ -95,8 +126,6 @@ scores rather than false matches. | --- | --- | --- | | `2016` | all sheets | per-file format detection; header-token rows rejected | | `2017` | all sheets — Hà Nội and HCM overflow | none | -| `2017-old` | first sheet only | reject non-numeric SBD (rejects a header leak in this export) | -| `2017-old2` | all sheets — HCM overflows | reject non-numeric SBD; skip blank rows before counting | ### Overflow-sheet gotcha @@ -111,8 +140,6 @@ silently drops 13,720 students** (Hanoi +7,275, HCM +6,445). That is what | --- | --- | --- | --- | | `2016` | 877,464 | 3 duplicate SBDs collapsed | **877,461** | | `2017` | 861,068 | 0 | **861,068** | -| `2017-old` | 847,349 | 1 header leak | **847,348** | -| `2017-old2` | 679,764 | 0 | **679,764** | ## Verifying a rebuild @@ -143,11 +170,14 @@ cell of any input file reads differently. ```bash rm data/2017/*.xls -node go-parser/scripts/crawl-baotintuc.js -node go-parser/scripts/build-db.js 2017 +npm run crawl:2017 +npm run build:db 2017 ``` -The row-count guard in `build:db` confirms the rebuild matches the expected total. +The row-count guard in `build:db` confirms the rebuild matches the expected +total. That guard checks the count only, so if the crawl was expected to change +the data, compare content with `differential-parity.mjs` against a copy of the +previous database rather than trusting the count. ## Removed scripts @@ -157,3 +187,9 @@ repo was unified (a hardcoded Windows path in one, an undeclared `better-sqlite3` dependency in the other) and neither had any automated caller. The latter two are superseded by `differential-parity.mjs`, which compares more and cannot silently skip a dataset. + +`crawl-baotintuc.js` was not dropped but rewritten in Go as +`crawler/internal/sources/source_2017.go`, keeping the same link list and the same +local filenames. The rewrite added a per-file timeout and made downloads land on +a `.part` file first — writing straight to the destination meant an interrupted +run left a truncated file that the skip check would then skip forever. diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 80710bc..88b8342 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -1,7 +1,7 @@ # Deployment Guide Deploys to GitHub Pages via `.github/workflows/deploy-pages.yml`. Every push to -`main` rebuilds all four datasets and redeploys the whole site. +`main` rebuilds both datasets and redeploys the whole site. One-time setup: **Settings → Pages → Source: GitHub Actions**. @@ -9,9 +9,9 @@ One-time setup: **Settings → Pages → Source: GitHub Actions**. 1. Checkout, Go toolchain, Node 24, `npm ci` 2. `npm run build:go` — one parser binary -3. `npm run build:db` — builds and gzips all four databases into +3. `npm run build:db` — builds and gzips both databases into `.build/public/db/` -4. `npm run build:site` — one Vite build, then `scripts/assemble-site.js` +4. `npm run build:site` — one Vite build, then `web/scripts/assemble-site.js` 5. `actions/upload-pages-artifact` + `actions/deploy-pages` The database build dominates the runtime: roughly 419 MB of Excel is parsed on @@ -23,20 +23,18 @@ every deploy. https://.github.io/thptqg/ https://.github.io/thptqg/2016/ https://.github.io/thptqg/2017/ -https://.github.io/thptqg/2017-old/ -https://.github.io/thptqg/2017-old2/ ``` -`/thptqg/2017/old/` and `/thptqg/2017/old2/` were the pre-flattening URLs. They -are still served, and the router rewrites them to the flat form with the query -string intact. +`/thptqg/2017/old/` and `/thptqg/2017/old2/` were the pre-flattening URLs for +the two removed 2017 archives. They are no longer served; like any unknown path +they now render the hub via `404.html`. ## Local reproduction ```bash npm ci npm run build:go -npm run build:db # all four; pass an id to build just one +npm run build:db # both; pass an id to build just one npm run build:site # vite build + assemble into _site/ npx serve _site ``` @@ -44,7 +42,7 @@ npx serve _site To rebuild a single dataset: ```bash -node go-parser/scripts/build-db.js 2017-old +npm run build:db 2017 ``` ## Base path @@ -59,7 +57,7 @@ up as a blank page with 404s on `/assets/...`. 2. Add `go-parser/configs/.yml` with the parse rules — sheet mode, column indices, SBD validation, header tokens, blank-row stripping. No SQL: the schema is canonical and lives in `go-parser/internal/schema/schema.go` -3. Add an entry to `DATASETS` in `src/datasets.js` +3. Add an entry to `DATASETS` in `web/src/datasets.js` Nothing else. The build script, the site assembly and the router all read that one list, and the frontend adapts to whichever columns the dataset populates. @@ -98,7 +96,7 @@ run rebuilds the older state. There is no data to migrate. | Symptom | Typical cause | | --- | --- | | Blank page, 404 on assets | `base` in `vite.config.js` does not match the repo name | -| `Failed to fetch database: 404` | Dataset id in `src/datasets.js` does not match the file in `db/` | +| `Failed to fetch database: 404` | Dataset id in `web/src/datasets.js` does not match the file in `db/` | | A route 404s | `assemble-site.js` did not run, or the id is missing from `DATASETS` | | WASM fails to load | `sql.js.org` unreachable — self-host `sql-wasm.wasm` and update `SQL_WASM_URL` in `use-sqlite.js` | | Deploy fails on assembly | An uncompressed database artefact reached the output; the error names the files | diff --git a/docs/project-overview.md b/docs/project-overview.md index af8dfdd..7b1cba5 100644 --- a/docs/project-overview.md +++ b/docs/project-overview.md @@ -4,7 +4,7 @@ A public lookup tool for Vietnam's National High School Graduation Exam scores, running entirely in the browser and hosted for free on GitHub Pages. Covers the -2016 and 2017 exams — 1.7 million candidates across four datasets. +2016 and 2017 exams — 1.7 million candidates across two datasets. ## Scope @@ -21,7 +21,7 @@ running entirely in the browser and hosted for free on GitHub Pages. Covers the ## Constraints -- **Zero backend.** The full database (38–48 MB gzipped per dataset) is +- **Zero backend.** The full database (44–48 MB gzipped per dataset) is downloaded to the browser and queried in-process. - **Read-only.** `INSERT`/`UPDATE`/`DELETE` are rejected, so nobody is misled into thinking edits persist. `sql.js` is in-memory anyway. @@ -36,16 +36,17 @@ running entirely in the browser and hosted for free on GitHub Pages. Covers the | --- | --- | --- | --- | | `2016` | 2016 | 877,461 | 119 files, three column layouts | | `2017` | 2017 | 861,068 | current generation, reproducible from source | -| `2017-old` | 2017 | 847,348 | pre-refresh archive | -| `2017-old2` | 2017 | 679,764 | corrected re-export | -The three 2017 datasets are kept side by side because they disagree, and the -disagreement is itself informative. Only `2017` is re-fetchable; the rest exist -solely as the copies committed here. +Two further 2017 datasets (`2017-old`, `2017-old2`) were kept alongside these +because the three publications disagreed. They have been removed; git history +still has them. -Original 2016 aggregator link -() -is no longer accessible, which is why the raw files are mirrored in `data/2016/`. +Both datasets now have a crawler source. 2017 comes from the baotintuc.vn CDN, +which is still live. 2016 comes from an aggregator article whose original host +(`dtntbacgiang.edu.vn`) no longer resolves — its link list was recovered from +the Internet Archive and is pointed at a mirror that is still online. See +[data-pipeline](./data-pipeline.md#sources) for what that does and does not +guarantee. ## History @@ -53,7 +54,7 @@ Each year began as a standalone repository (`thptqg2016`, `thptqg2017`), merged here with full history. They initially kept separate frontends and separate copies of the same Rust parser, synchronised by hand. That duplication was removed: there is now one frontend, one parser, and one canonical schema, with -per-dataset differences confined to four small config files and one registry +per-dataset differences confined to one small config file and one registry entry each. The unification also fixed a latent data-loss bug — neither year's parser diff --git a/docs/system-architecture.md b/docs/system-architecture.md index ab5040d..803b756 100644 --- a/docs/system-architecture.md +++ b/docs/system-architecture.md @@ -3,23 +3,24 @@ Static site, no backend. The browser downloads a compressed SQLite file at boot and every query runs locally via `sql.js` (SQLite compiled to WebAssembly). -One frontend, one parser, one schema, four datasets. +One frontend, one parser, one schema, two datasets. ## Data flow ``` + ▲ crawler/ (Go — manual refresh only, never part of the build) data//*.xls(x) │ - ▼ parser/ (Rust, one binary, one config per dataset) + ▼ go-parser/ (Go, one binary, one config per dataset) .build/public/db/.db │ ▼ gzip -9 (no -k: the raw file does not survive) .build/public/db/.db.gz │ - ▼ vite build (publicDir = .build/public) - dist/ + ▼ vite build (root = web/, publicDir = .build/public) + web/dist/ │ - ▼ scripts/assemble-site.js + ▼ web/scripts/assemble-site.js _site/ → GitHub Pages │ ▼ browser @@ -31,26 +32,28 @@ data//*.xls(x) One identifier ties the whole pipeline together: ``` -data/2017-old/ → go-parser/configs/2017-old.yml → db/2017-old.db.gz → /thptqg/2017-old/ +data/2017/ → go-parser/configs/2017.yml → db/2017.db.gz → /thptqg/2017/ ``` -`src/datasets.js` declares the four ids once. The frontend, the database build -(`go-parser/scripts/build-db.js`) and the site assembly all import that list, so -adding a dataset means adding one entry and one config file. +`web/src/datasets.js` declares the two ids once. The frontend, the database +build (`go-parser/scripts/build-db.js`) and the site assembly all import that +list, so adding a dataset means adding one entry and one config file. + +That import crosses a package boundary — go-parser reaches into the web +workspace for it. It stays there because the same entries also carry the UI's +labels and SQL presets, and splitting them would mean two lists to keep in step. | id | Exam | Rows | Source | | --- | --- | --- | --- | | `2016` | 2016 | 877,461 | Bộ GD&ĐT | | `2017` | 2017 | 861,068 | baotintuc.vn | -| `2017-old` | 2017 | 847,348 | pre-refresh archive | -| `2017-old2` | 2017 | 679,764 | corrected re-export | ## Canonical schema Defined once in `go-parser/internal/schema/schema.go` — DDL, INSERT, column order and the 16 -subject regexes. The four TOML configs carry no SQL at all, only per-dataset -parse rules. Config parsing uses `deny_unknown_fields`, so a leftover `[schema]` -block fails loudly instead of looking effective while `schema.rs` drives the +subject regexes. The two YAML configs carry no SQL at all, only per-dataset +parse rules. Config parsing sets `KnownFields(true)`, so a leftover `schema:` +block fails loudly instead of looking effective while `schema.go` drives the build. ```sql @@ -73,7 +76,7 @@ CREATE INDEX idx_ten_cum_thi ON student(ten_cum_thi) WHERE ten_cum_thi IS NOT N Every dataset gets all 22 columns; ones it has no data for are NULL, costing about a byte per row. `khtn`, `khxh` and `gdcd` are empty on 2016; -`ten_cum_thi` and `gioi_tinh` are empty on the 2017 datasets. +`ten_cum_thi` and `gioi_tinh` are empty on 2017. `idx_ten_cum_thi` is partial, so it holds zero entries where the column is always NULL. @@ -86,15 +89,13 @@ URLs are flat, one segment per dataset, and the segment is the id: /thptqg/ hub /thptqg/2016/ /thptqg/2017/ -/thptqg/2017-old/ -/thptqg/2017-old2/ ``` -`src/router.js` is an exact match on that segment. The nested form used before -(`/thptqg/2017/old/`) would have needed longest-prefix matching, since it also -starts with `/thptqg/2017/`. Those two legacy URLs still resolve: the router -rewrites them to the flat equivalent with `history.replaceState`, preserving the -query string so `?q=` deep links survive. +`web/src/router.js` is an exact match on that segment. The nested form used +before (`/thptqg/2017/old/`) would have needed longest-prefix matching, since it +also starts with `/thptqg/2017/`. Both of those URLs addressed the two removed +2017 archives, so the rewrite that kept them working has gone with them; they +fall through to the hub like any other unknown path. A single Vite build emits one `index.html`. Because `base` is absolute (`/thptqg/`), that file references `/thptqg/assets/...` regardless of the @@ -115,11 +116,11 @@ No component contains a per-dataset conditional. Two mechanisms do the work: self-exclude wherever those languages were not sat. Anything genuinely per-dataset — title, source, database size, search examples, -SQL presets — lives in `src/datasets.js`. +SQL presets — lives in `web/src/datasets.js`. ## Exam ID formats -`src/lib/query-mode.js` decides whether a query is an exam ID or a name, and is +`web/src/lib/query-mode.js` decides whether a query is an exam ID or a name, and is shared by `App.jsx` and `search-form.jsx` (they previously held separate copies and had drifted apart on exactly this rule). @@ -135,7 +136,7 @@ candidates (70.3%). Letter prefixes are upper-cased before lookup, so ## Score tiers -Six-level ladder in `scoreTier()` (`src/lib/admission-blocks.js`), paired with a +Six-level ladder in `scoreTier()` (`web/src/lib/admission-blocks.js`), paired with a symbol so meaning is never colour-only. | Tier | Range | Vietnamese | @@ -150,7 +151,7 @@ symbol so meaning is never colour-only. ## Admission blocks Vietnamese universities admit on three-subject combinations (khối thi). -`src/lib/admission-blocks.js` lists the blocks computable from this schema +`web/src/lib/admission-blocks.js` lists the blocks computable from this schema (A00–A11, B00–B08, C00–C20, D01–D15, plus D05/D06 for German and Japanese). `computeBlocks(student)` returns those where all three scores exist, sorted by total descending. @@ -165,11 +166,11 @@ total descending. | Diacritics search | Pre-computed `ho_ten_ascii` | `LOWER(REPLACE(...))` at query time defeats the index | | SQL safety | Leading-keyword allowlist | `sql.js` is in-memory so writes cannot persist; the allowlist prevents confusion | | Row caps | 100 (lookup), 1000 (SQL) | Keeps DOM render sizes reasonable | -| Routing | Hand-rolled, ~20 lines | Five static routes do not justify a router dependency | +| Routing | Hand-rolled, ~15 lines | Three static routes do not justify a router dependency | ## Risks and limitations -- **Database size.** 38–48 MB gzipped per dataset; slow links wait, mitigated by +- **Database size.** 44–48 MB gzipped per dataset; slow links wait, mitigated by a progress bar. - **Browser memory.** The full database lives in RAM; older mobile devices may run out. @@ -177,4 +178,4 @@ total descending. load. Self-hosting `sql-wasm.wasm` and updating `SQL_WASM_URL` in `use-sqlite.js` is the fix. - **Excel format drift.** A new source file with an unseen header layout needs a - new branch in `format_detect_2016.rs` or a new config. + new branch in `go-parser/internal/ingest/detect2016.go` or a new config. diff --git a/eslint.config.js b/eslint.config.js index 7ce9419..bbaa88e 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -5,7 +5,9 @@ import reactRefresh from 'eslint-plugin-react-refresh' import { defineConfig, globalIgnores } from 'eslint/config' export default defineConfig([ - globalIgnores(['dist', '.build', '_site']), + // '**/dist', not 'dist': the build output moved to web/dist when the app + // became a workspace, and a root-anchored pattern would stop matching it. + globalIgnores(['**/dist', '.build', '_site']), { files: ['**/*.{js,jsx}'], extends: [ @@ -27,8 +29,9 @@ export default defineConfig([ }, }, { - // Node-executed files (Vite config, parser tooling) run with Node globals. - files: ['vite.config.js', 'scripts/**/*.js', 'go-parser/scripts/**/*.js'], + // Node-executed files (Vite config, site assembly, parser tooling) run with + // Node globals. The crawler is Go, so it has nothing here. + files: ['web/vite.config.js', 'web/scripts/**/*.js', 'go-parser/scripts/**/*.js'], languageOptions: { globals: { ...globals.node }, }, diff --git a/go-parser/configs/2016.yml b/go-parser/configs/2016.yml index 412adcf..f823d44 100644 --- a/go-parser/configs/2016.yml +++ b/go-parser/configs/2016.yml @@ -1,7 +1,7 @@ # thptqg2016 data/ — 4 .xls + 115 .xlsx mixed files. # # Three column layouts exist across the 119 files; the binary selects the -# right one per-file at runtime via format_detection = "thptqg2016": +# right one per-file at runtime via format_detection: thptqg2016: # # separate-scores header SBD(0)/HOTEN(1)/TOAN(2)... — dhhanghai files # mapped header SOBAODANH|SBD + DIEM_THI — most provinces @@ -13,7 +13,7 @@ # sheet_mode = "all": several provinces overflow into Sheet2 (65k Excel row cap). # strip_blank_rows = false: no blank-row anomaly observed in this dataset. # -# Table shape, INSERT and subject regexes are canonical — see src/schema.rs. +# Table shape, INSERT and subject regexes are canonical — see go-parser/internal/schema/schema.go. format_detection: thptqg2016 diff --git a/go-parser/configs/2017-old.yml b/go-parser/configs/2017-old.yml deleted file mode 100644 index 4d10c04..0000000 --- a/go-parser/configs/2017-old.yml +++ /dev/null @@ -1,25 +0,0 @@ -# thptqg2017 data-old/ — 63 .xlsx files (pre-baotintuc refresh). -# -# sheet_mode = "first": single-sheet workbooks, never hit the 65k row cap. -# SBD validation: require ^\d+$ (build-database-old.js:55 guard). -# strip_blank_rows = false: no explicit blank-skip in build-database-old.js. -# -# Table shape, INSERT and subject regexes are canonical — see src/schema.rs. - -reader: - sheet_mode: first - strip_blank_rows: false - -columns: - ho_ten: 0 - ngay_sinh: 1 - so_bao_danh: 2 - diem_thi: 3 - -validation: - require_numeric_sbd: true - require_nonempty_name: true - require_nonempty_sbd: true - -header: - tokens: ["HO_TEN", "HỌ TÊN", "STT"] diff --git a/go-parser/configs/2017-old2.yml b/go-parser/configs/2017-old2.yml deleted file mode 100644 index dc9de9a..0000000 --- a/go-parser/configs/2017-old2.yml +++ /dev/null @@ -1,26 +0,0 @@ -# thptqg2017 data-old2/ — 54 .xlsx files (corrected-export set). -# -# sheet_mode = "all": 24.HCM_UTLQ.xlsx overflows into Sheet2 (+6,446 rows). -# SBD validation: require ^\d+$ (build-database-old2.js:57 guard). -# strip_blank_rows = true: skip fully blank rows BEFORE counting sourceRows -# (build-database-old2.js:50-51 checks blank before sourceRows++). -# -# Table shape, INSERT and subject regexes are canonical — see src/schema.rs. - -reader: - sheet_mode: all - strip_blank_rows: true - -columns: - ho_ten: 0 - ngay_sinh: 1 - so_bao_danh: 2 - diem_thi: 3 - -validation: - require_numeric_sbd: true - require_nonempty_name: true - require_nonempty_sbd: true - -header: - tokens: ["HO_TEN", "HỌ TÊN", "STT"] diff --git a/go-parser/configs/2017.yml b/go-parser/configs/2017.yml index 6f28b7b..49dcff6 100644 --- a/go-parser/configs/2017.yml +++ b/go-parser/configs/2017.yml @@ -4,7 +4,7 @@ # SBD validation: no numeric guard — build-database.js did not apply ^\d+$. # strip_blank_rows = false: no blank-row anomaly in this dataset. # -# Table shape, INSERT and subject regexes are canonical — see src/schema.rs. +# Table shape, INSERT and subject regexes are canonical — see go-parser/internal/schema/schema.go. reader: sheet_mode: all diff --git a/go-parser/internal/config/config_test.go b/go-parser/internal/config/config_test.go index c3a1a1c..28f91fd 100644 --- a/go-parser/internal/config/config_test.go +++ b/go-parser/internal/config/config_test.go @@ -135,9 +135,8 @@ header: } } -// TestLoadRealConfigs loads the four shipped configs — the Go binary reads the -// same files as Rust, never a fork — and asserts the per-dataset differences -// recorded during scouting. +// TestLoadRealConfigs loads the shipped configs and asserts the per-dataset +// differences recorded during scouting. func TestLoadRealConfigs(t *testing.T) { root := repoRoot(t) want := map[string]struct { @@ -148,10 +147,8 @@ func TestLoadRealConfigs(t *testing.T) { formatDet string tokenCount int }{ - "2016": {SheetModeAll, false, false, false, "thptqg2016", 6}, - "2017": {SheetModeAll, false, false, true, "", 3}, - "2017-old": {SheetModeFirst, false, true, true, "", 3}, - "2017-old2": {SheetModeAll, true, true, true, "", 3}, + "2016": {SheetModeAll, false, false, false, "thptqg2016", 6}, + "2017": {SheetModeAll, false, false, true, "", 3}, } for id, w := range want { t.Run(id, func(t *testing.T) { diff --git a/go-parser/scripts/build-db.js b/go-parser/scripts/build-db.js index f3ac9da..51bca9d 100644 --- a/go-parser/scripts/build-db.js +++ b/go-parser/scripts/build-db.js @@ -2,8 +2,8 @@ /** * Build the SQLite database for one or all datasets, verify it, then gzip it. * - * The dataset list comes from src/datasets.js so it is written in exactly one - * place. + * The dataset list comes from web/src/datasets.js so it is written in exactly + * one place. * * Output goes to .build/public/db/ — the directory Vite copies as its publicDir. * Only the .gz survives: shipping a 100+ MB uncompressed database is made @@ -14,7 +14,7 @@ * Until now nothing between the parser and the public site asserted that a * database actually had data in it. The parser logs a file-level failure and * continues, returns success regardless, and finishes cleanly even at zero rows; - * this script gzipped whatever it got; and scripts/assemble-site.js only greps + * this script gzipped whatever it got; and web/scripts/assemble-site.js greps * *filenames* for stray .db files. So a reader that silently under-produced * would publish a truncated dataset with green CI and no red signal anywhere. * @@ -32,7 +32,7 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { DatabaseSync } from "node:sqlite"; -import { DATASET_IDS, DATASETS } from "../../src/datasets.js"; +import { DATASET_IDS, DATASETS } from "../../web/src/datasets.js"; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const BIN = resolve(ROOT, "go-parser/bin/xlsxread"); @@ -48,8 +48,6 @@ const OUT_DIR = resolve(ROOT, ".build/public/db"); const EXPECTED_ROWS = { "2016": 877461, "2017": 861068, - "2017-old": 847348, - "2017-old2": 679764, }; /** A gzipped database far below its usual size means a truncated build. */ diff --git a/go-parser/scripts/crawl-baotintuc.js b/go-parser/scripts/crawl-baotintuc.js deleted file mode 100644 index 8f4c20c..0000000 --- a/go-parser/scripts/crawl-baotintuc.js +++ /dev/null @@ -1,137 +0,0 @@ -// Crawl Excel score files for all 63 provinces from baotintuc.vn article: -// https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm -// Saves to data/2017/.xls (63 files). Idempotent: skips already-downloaded files. -import fs from "fs"; -import path from "path"; -import { fileURLToPath } from "url"; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const OUT_DIR = path.join(__dirname, "..", "..", "data", "2017"); - -const LINKS = [ - ["An Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/17/57/Angiang.xls"], - ["Bac Lieu", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Baclieu.xls"], - ["Ba Ria - Vung Tau", "https://cdnmedia.baotintuc.vn/2017/07/06/08/17/1BaRiaVungTau.xls"], - ["Bac Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacGiang.xls"], - ["Bac Kan", "https://cdnmedia.baotintuc.vn/2017/07/06/08/27/BacKan.xls"], - ["Bac Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacNinh.xls"], - ["Ben Tre", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BenTre.xls"], - ["Binh Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BinhDuong.xls"], - ["Binh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/BinhThuan.xls"], - ["Binh Phuoc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/BinhPhuoc.xls"], - ["Binh Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Binhdinh.xls"], - ["Ca Mau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Camau.xls"], - ["Cao Bang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Caobang.xls"], - ["Can Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/13/07/Cantho.xls"], - ["Da Nang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/03/Danang.xls"], - ["Dak Nong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/DakNong.xls"], - ["Dak Lak", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Daklak.xls"], - ["Dong Nai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/08/dongnai.xls"], - ["Dong Thap", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Dongthap.xls"], - ["Dien Bien", "https://cdnmedia.baotintuc.vn/2017/07/06/09/08/DienBien.xls"], - ["Gia Lai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/09/Gia-Lai.xls"], - ["Ha Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/18/Hagiang.xls"], - ["Ha Noi", "https://cdnmedia.baotintuc.vn/2017/07/07/08/16/HaNoi.xls"], - ["Ha Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/07/Hanam.xls"], - ["Ha Tinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaTinh.xls"], - ["Hai Phong", "https://cdnmedia.baotintuc.vn/2017/07/06/08/18/23HaiPhong.xls"], - ["Hai Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaiDuong.xls"], - ["Hau Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Haugiang.xls"], - ["Ho Chi Minh", "https://cdnmedia.baotintuc.vn/2017/07/06/08/26/HCM.xls"], - ["Hoa Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/06/HoaBinh.xls"], - ["Hung Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/08/25/HungYen.xls"], - ["Khanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/18/04/Khanhhoa.xls"], - ["Kien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/08/28/KienGiang.xls"], - ["Kon Tum", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/KonTum.xls"], - ["Nam Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/13NamDinh.xls"], - ["Nghe An", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/14NgheAn.xls"], - ["Ninh Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/37/NinhBinh.xls"], - ["Ninh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Ninhthuan.xls"], - ["Lao Cai", "https://cdnmedia.baotintuc.vn/2017/07/06/08/52/11LaoCai.xls"], - ["Lai Chau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/33/LaiChau.xls"], - ["Lang Son", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/Langson.xls"], - ["Lam Dong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/00/LamDong.xls"], - ["Long An", "https://cdnmedia.baotintuc.vn/2017/07/06/08/53/12LongAn.xls"], - ["Quang Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/QuangBinh.xls"], - ["Quang Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNam.xls"], - ["Quang Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Quangninh.xls"], - ["Quang Ngai", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNgai.xls"], - ["Quang Tri", "https://cdnmedia.baotintuc.vn/2017/07/06/09/25/QuangTri.xls"], - ["Phu Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/PhuTho.xls"], - ["Phu Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/PhuYen.xls"], - ["Son La", "https://cdnmedia.baotintuc.vn/2017/07/06/18/01/Sonla.xls"], - ["Soc Trang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Soctrang.xls"], - ["Tay Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/26/TayNinh.xls"], - ["Thai Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/ThaiBinh.xls"], - ["Thai Nguyen", "https://cdnmedia.baotintuc.vn/2017/07/06/13/35/ThaiNguyen.xls"], - ["Thanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thanhoa.xls"], - ["Tra Vinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/TraVinh.xls"], - ["Thua Thien Hue", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thuathienhue.xls"], - ["Tien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/13/11/tiengiang.xls"], - ["Tuyen Quang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/39/TuyenQuang.xls"], - ["Vinh Phuc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/40/VinhPhuc.xls"], - ["Vinh Long", "https://cdnmedia.baotintuc.vn/2017/07/06/13/13/vinhlong.xls"], - ["Yen Bai", "https://cdnmedia.baotintuc.vn/2017/07/06/18/07/Yenbai.xls"], -]; - -function slug(name) { - return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, ""); -} - -async function downloadOne(name, url) { - const outPath = path.join(OUT_DIR, `${slug(name)}.xls`); - if (fs.existsSync(outPath) && fs.statSync(outPath).size > 0) { - return { name, url, outPath, status: "skip", size: fs.statSync(outPath).size }; - } - const res = await fetch(url, { - headers: { - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36", - Referer: - "https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm", - }, - }); - if (!res.ok) { - return { name, url, outPath, status: "fail", httpStatus: res.status }; - } - const buf = Buffer.from(await res.arrayBuffer()); - fs.writeFileSync(outPath, buf); - return { name, url, outPath, status: "ok", size: buf.length }; -} - -async function main() { - fs.mkdirSync(OUT_DIR, { recursive: true }); - console.log(`Downloading ${LINKS.length} files to ${OUT_DIR}...`); - - const CONCURRENCY = 6; - const results = []; - let i = 0; - async function worker() { - while (i < LINKS.length) { - const idx = i++; - const [name, url] = LINKS[idx]; - try { - const r = await downloadOne(name, url); - results.push(r); - const tag = r.status === "ok" ? "✓" : r.status === "skip" ? "·" : "✗"; - const sz = r.size ? `${(r.size / 1024).toFixed(0)} KB` : ""; - console.log(` ${tag} [${idx + 1}/${LINKS.length}] ${name.padEnd(20)} ${sz} ${r.status === "fail" ? "HTTP " + r.httpStatus : ""}`); - } catch (err) { - console.log(` ✗ [${idx + 1}/${LINKS.length}] ${name}: ${err.message}`); - results.push({ name, url, status: "error", message: err.message }); - } - } - } - await Promise.all(Array.from({ length: CONCURRENCY }, worker)); - - const ok = results.filter((r) => r.status === "ok").length; - const skip = results.filter((r) => r.status === "skip").length; - const fail = results.filter((r) => r.status === "fail" || r.status === "error"); - console.log(`\nDone. ok=${ok} skip=${skip} fail=${fail.length}`); - if (fail.length) { - console.log("Failed:", fail.map((f) => f.name).join(", ")); - process.exit(1); - } -} - -main(); diff --git a/go-parser/testdata/reader-fidelity-hashes.tsv b/go-parser/testdata/reader-fidelity-hashes.tsv index 8c9ce2f..76bb266 100644 --- a/go-parser/testdata/reader-fidelity-hashes.tsv +++ b/go-parser/testdata/reader-fidelity-hashes.tsv @@ -190,120 +190,3 @@ data/2017/tuyen-quang.xls 053d293e4acd9dfa6ce9532a047cc64b4d2edee1ff13b387723586 data/2017/vinh-long.xls 6e7584a3047b0ba7487dbb8c71f5b594db220f11b747149d41d31ea97e1e4eab data/2017/vinh-phuc.xls 9753e2d864584e76aef1e690e4e2de98b28240fbba810e614eab9926668d66d2 data/2017/yen-bai.xls fe6824a1254116bc6ed372e0a1fd635a816a09e010b7efde9659baca2e143138 -data/2017-old/10_BinhThuan_RIIW.xls.xlsx 2833e7366ea1a21ea869978a6bb920b033160bb8042fe0bfe9b7573dcfe85409 -data/2017-old/10_Ca_Mau_BKXT.xls.xlsx 53cb69b13325cb9cc9004b8c89f0cd1c85b5f943e7c78c8b010ddabad3f816c9 -data/2017-old/10_LamDong_GNFT.xls.xlsx f05b4cdc8a9a8413b29801fbddeb48a7c60d3c6bb2228bbe238113495e89bc98 -data/2017-old/10_Soc_Trang_XCGJ.xls.xlsx 8e540cef6f2ff387f3e825a5aa3190af394c6f9c7e9dfc66d5f7032ee47f7f14 -data/2017-old/11_BinhDuong_RYQL.xls.xlsx 11953116054a2889929c3fe05b68e8932c7b79ffd1b85d697afaac63fcdd9adf -data/2017-old/11_LaoCai_GMSU.xls.xlsx 5a73becf1bdb8027102faff74035d1df71e13c1cac39cc38ec6efaab6699ca52 -data/2017-old/12_BenTre_DKWF.xls.xlsx 579c0d94b20440f1c60a94fb0c762b71dd1c7719c8aa4db0b9418ecaba9b605e -data/2017-old/12_LongAn_ZZUK.xls.xlsx c4cf569706bfc30582d774ba9e699dcbc4bdc734f46bf6493df58eaecc37a755 -data/2017-old/13_NamDinh_ESEL.xls.xlsx 7e5256cb095262fef5399092f8f843c1ee82e117bca8e6571e363646bb1170a6 -data/2017-old/13_TraVinh_LKUJ.xls.xlsx 2b7321d68a4a910c4a506d7ca207531cfbdd9619f1d8ba0ce475baf1b91043aa -data/2017-old/14_NgheAn_BSLY.xls.xlsx 034234a8a5461910f8beff3977aafa29a6541c8a56c007816118e5c4eb2b7218 -data/2017-old/15_PhuTho_ABWQ.xls.xlsx 9891c17001cfcc49ccd587419339698ef59b2dd43b5ca7f2765b09a17c26659d -data/2017-old/16_QuangBinh_KGEU.xls.xlsx 01e249d45c1269d62ae1977bef0bb192f833941510471b723e57c32adb0eadaa -data/2017-old/17_QuangNam_AMTK.xls.xlsx 4c9f05d24d861ec3c82a1c564e4baa1541bd4882785f143459a1a81a6b8359c7 -data/2017-old/18_QuangNgai_KOFP.xls.xlsx ee9c26f2c16d36dabfde79901a054d2266c28c4acbfd19b628cfe50f8eb03957 -data/2017-old/19_QuangTri_OMZF.xls.xlsx 6c961bb461274c73b4589de4087624a9ef5edd43c04ea54675148df00f6420e5 -data/2017-old/1_BaRia_VungTau_HJKG.xls.xlsx cb03f933442a8cea6b1622da4e5f804ab476dff9f3462b404e957b6a2e87b2cd -data/2017-old/1_Da_Nang_AHWJ.xls.xlsx 59892fc01b6cb10497679f2bc69f59023215755d2f00fd764b23b895c98be2be -data/2017-old/1_Ha_Noi_CVXG.xls.xlsx 2718fe9c2b58843621c0082da5880a4b957266525b83123af02c78fd51d4f4a5 -data/2017-old/1_Son_La_JIDP.xls.xlsx 0a0e273c5def980406b6316cceb63e5cee53aae56e112dfae7e795261eb1cd06 -data/2017-old/1_TuyenQuang_JBYF.xls.xlsx 45826ed10248268ac3344d0b592cf2dba6f86278b7402dc60626a76b18fa4d6b -data/2017-old/20_TayNinh_ILFA.xls.xlsx 684a1820d02f658904c87c26aa195faa6aceb3d7ca00c674df11a63f9274ab14 -data/2017-old/21_ThaiBinh_FTVG.xls.xlsx 1a12e43d4ea368b8e3a2e4d881b5d66214a16140219e80e2e72fcb269899b285 -data/2017-old/22_ThaiNguyen_TLTW.xls.xlsx 29227bdc3204591327e803680525d8621228fc371ce7f802dc667ec70eb8d7b3 -data/2017-old/23_HaiPhong_HXBV.xls.xlsx e450e2344fbd724555de9892412f521414fc1fcc53e69887c6381dd266b4e92c -data/2017-old/24_HCM_XULN.xls.xlsx 49ed22e823607b78920a24f5999eaf1ef8634a141108dfd4bf970ee000aecbf9 -data/2017-old/2_BacKan_GFVQ.xls.xlsx 77a08fc4edb41d639fcdd798980903519181639db00a360ef0c6c6c2bd88f529 -data/2017-old/2_Ha_Giang_QNCM.xls.xlsx 6bf4d62413c1b92f88f980e5e3e414a7e25971c7546c6390c7e67a6b0f742b68 -data/2017-old/2_Ninh_Thuan_VHLY.xls.xlsx 6dfb6517f9ccc3420c222b6a88aabac54f6f25f6b743d993d6a5f28b4fa78ff2 -data/2017-old/2_Thanh_Hoa_AUFV.xls.xlsx 2f2ce90425c19a9089024061f328f37b9ef580ddadc076ddcf452f549690f4b3 -data/2017-old/2_VinhPhuc_GUDK.xls.xlsx fc0e4952445ebda1af28f7330aa808c032f2e3a7a3c03dfbb9692bf6a248867a -data/2017-old/3_BacGiang_TOIF.xls.xlsx 8043510ff89f11ba063f898dc1d2c8879a8da9063a420308647e20a2f39d3092 -data/2017-old/3_BinhPhuoc_YFMU.xls.xlsx 8869970f4e6005381264f3f4bb508c5181f15ae990a75fe904a65b305c0a123a -data/2017-old/3_Cao_Bang_CIEY.xls.xlsx 94a06bdb2333a9f44216001ba8d8463700b8cf3e51944b94db11eef7eac8dbd3 -data/2017-old/3_Dong_Thap_GSXQ.xls.xlsx 0cdd0763fc4f20f1d990621ca9e8dab107a3d4eb7c0ec3e53a4d8c240fbc569f -data/2017-old/3_Thua_Thien_Hue_HMDB.xls.xlsx 62e54eebda7bbd542ca63443bb923a81b472e7892e55dce39fd0611fc0f00abd -data/2017-old/4_An_Giang_JNOS.xls.xlsx 778ad205bbabb7c4aea3df34325770aa4a1daec0a1683f73f0405ac1cc5aaf9a -data/2017-old/4_BacNinh_STLR.xls.xlsx 009c3b7821f55210a86f9ace74c31eb0d5c16aafdb0cac8f2b71cac616abdfec -data/2017-old/4_Binh_Dinh_WWZW.xls.xlsx fbacd256d58627377d698462d69aac0e83d124c8c8ff1f5b63ba0bfb5442dd39 -data/2017-old/4_DienBien_SOJG.xls.xlsx 4931e8d6604013df48b2b469fda3eb05fdf1203b67bbc2ae4ff6d7949a115a24 -data/2017-old/4_Lang_Son_NYQL.xls.xlsx cc7d2454caadb18740ed6baf6edc62baa47e98b3a907b3ce06f72b9a16b7838a -data/2017-old/5_Bac_Lieu_XEKH.xls.xlsx dc1dca45fd5357b45abc930b0ebb95ab589ef80a09bef8b5ad4dab9e00be87e5 -data/2017-old/5_Gia_Lai_ABZI.xls.xlsx 5b7be25274d628c284c37efb3df0587171cc9273dc3bd8dc824c7bd1b8bccf1a -data/2017-old/5_HaiDuong_WWWG.xls.xlsx 1d800d0aa7338fb507c4eeb3e752620d2608c30ac820895ca086325dbaf3643d -data/2017-old/5_Hanam_QGJS.xls.xlsx cfc233ac37ab8e263b6cc569815ea2c687da30364f139109032c8f2cac1ad98e -data/2017-old/5_Yen_Bai_FAQR.xls.xlsx 4c8915417d8747c291f643ea4b378c87982c397460e0e9e3cc6df19670d6720f -data/2017-old/6_Dong_Nai_WOTM.xls.xlsx 9b582a8421c419156c2eab6e532d854736385735f8208f9d390c251f22ac764e -data/2017-old/6_Hau_Giang_KWDM.xls.xlsx c11473e3e1f964302c5378c90dd6b9d478eb11397351b7ef74df9ae709c9e908 -data/2017-old/6_HoaBinh_TPZY.xls.xlsx e1210961dc821642f6951cd6e4a71b5fbc81d5070ee3816b7c8b5dc56bf2b234 -data/2017-old/6_NinhBinh_IGFT.xls.xlsx 9399b8ab58e2a7a1c0ebf5425359319e567f8a28602349aa587efced72142fc8 -data/2017-old/6_Quang_Ninh_DQCJ.xls.xlsx 8cf95259b11d48267f710f151c3f1419ccac49d823b78b70a2708c59ab25a4f7 -data/2017-old/7_HaTinh_DDHD.xls.xlsx d96df9f2c8365dfc73be3e10c268367b93fd00fe92bb39c745d8e353ba75c05b -data/2017-old/7_HungYen_LTIK.xls.xlsx e80eb38af8307c8e193a91bdda5368c437693d7669e281b543bbe891a46e6f36 -data/2017-old/7_Kon_Tum_RSLR.xls.xlsx 41d2adea059449b6e214241414748f10776861d261cbaa1fd213629757deb846 -data/2017-old/7_Tien_Giang_EFHX.xls.xlsx 00d9bbc70be89f8f7dad6aff990ad36236c6b7244ddeb2e3d9b3d8ed1a6e21d0 -data/2017-old/8_Can_Tho_RQZM.xls.xlsx 3a1182b26d34fbdfabf4326098ff265609d8f1fe2b766b6eef897db9685d4cd4 -data/2017-old/8_Dak_Lak_YKPR.xls.xlsx 379b45f8a347bf4842daf9402d84be47426ed7107a64b2e7710bdc8e71eb57be -data/2017-old/8_KienGiang_OTOB.xls.xlsx 3e1789b8565a5ae7c187598cb7e9c3261029c06bc7f0cccc4eb503e8e302cb18 -data/2017-old/8_PhuYen_OGIM.xls.xlsx 91d1f202cba3bf4c57a7f26e7611d3d29faf8dd13de30552cbfc96eab7a451e9 -data/2017-old/9_DakNong_FBOP.xls.xlsx c611bf7f1d1c95cbfc78bdab3ea99bf8278e94c8db09a9fd3ccac55a538fe0b2 -data/2017-old/9_Khanh_Hoa_KPKQ.xls.xlsx 0e90c87dd74a5eb293a0a71b93da45653a19ed2e3a2600d3b880a867a695f31d -data/2017-old/9_LaiChau_ALKN.xls.xlsx 3b7a022f7727134620c6756925bd5adb041e2d80cabd4aab13a4063501739514 -data/2017-old/9_Vinh_Long_OLWA.xls.xlsx 43d7abb9c6b8d3205ce90f9cceaf9623875c87998d9e030c6dc8da9bc18bfa09 -data/2017-old2/10.BinhThuan_MVVG.xlsx 2833e7366ea1a21ea869978a6bb920b033160bb8042fe0bfe9b7573dcfe85409 -data/2017-old2/10.LamDong_YUQA.xlsx f05b4cdc8a9a8413b29801fbddeb48a7c60d3c6bb2228bbe238113495e89bc98 -data/2017-old2/10.Soc Trang_LQWU.xlsx 8e540cef6f2ff387f3e825a5aa3190af394c6f9c7e9dfc66d5f7032ee47f7f14 -data/2017-old2/11.BinhDuong_HVAH.xlsx 11953116054a2889929c3fe05b68e8932c7b79ffd1b85d697afaac63fcdd9adf -data/2017-old2/11.LaoCai_ZTBP.xlsx 5a73becf1bdb8027102faff74035d1df71e13c1cac39cc38ec6efaab6699ca52 -data/2017-old2/12.BenTre_NQTU.xlsx 579c0d94b20440f1c60a94fb0c762b71dd1c7719c8aa4db0b9418ecaba9b605e -data/2017-old2/12.LongAn_PDRH.xlsx d1c65184b475fc29ae273cc89b8574b7e273bc3dab83cd39489d72a5de1bb1e4 -data/2017-old2/13.NamDinh_NAYR.xlsx 7e5256cb095262fef5399092f8f843c1ee82e117bca8e6571e363646bb1170a6 -data/2017-old2/13.TraVinh_FODZ.xlsx 2b7321d68a4a910c4a506d7ca207531cfbdd9619f1d8ba0ce475baf1b91043aa -data/2017-old2/14.NgheAn_HTKD.xlsx 034234a8a5461910f8beff3977aafa29a6541c8a56c007816118e5c4eb2b7218 -data/2017-old2/15.PhuTho_IJZW.xlsx 9891c17001cfcc49ccd587419339698ef59b2dd43b5ca7f2765b09a17c26659d -data/2017-old2/17.QuangNam_NQMG.xlsx 4c9f05d24d861ec3c82a1c564e4baa1541bd4882785f143459a1a81a6b8359c7 -data/2017-old2/18.QuangNgai_IUPY.xlsx ee9c26f2c16d36dabfde79901a054d2266c28c4acbfd19b628cfe50f8eb03957 -data/2017-old2/19.QuangTri_MKNN.xlsx 6c961bb461274c73b4589de4087624a9ef5edd43c04ea54675148df00f6420e5 -data/2017-old2/1.BaRia-VungTau_PGZT.xlsx cb03f933442a8cea6b1622da4e5f804ab476dff9f3462b404e957b6a2e87b2cd -data/2017-old2/1.Da Nang_ABWU.xlsx 59892fc01b6cb10497679f2bc69f59023215755d2f00fd764b23b895c98be2be -data/2017-old2/1.Son La_XLFN.xlsx 0a0e273c5def980406b6316cceb63e5cee53aae56e112dfae7e795261eb1cd06 -data/2017-old2/1.TuyenQuang_PTMR.xlsx 45826ed10248268ac3344d0b592cf2dba6f86278b7402dc60626a76b18fa4d6b -data/2017-old2/20.TayNinh_KJAQ.xlsx 684a1820d02f658904c87c26aa195faa6aceb3d7ca00c674df11a63f9274ab14 -data/2017-old2/21.ThaiBinh_KTQN.xlsx 1a12e43d4ea368b8e3a2e4d881b5d66214a16140219e80e2e72fcb269899b285 -data/2017-old2/22.ThaiNguyen_BKIF.xlsx 29227bdc3204591327e803680525d8621228fc371ce7f802dc667ec70eb8d7b3 -data/2017-old2/24.HCM_UTLQ.xlsx 49ed22e823607b78920a24f5999eaf1ef8634a141108dfd4bf970ee000aecbf9 -data/2017-old2/2.BacKan_YQNX.xlsx 77a08fc4edb41d639fcdd798980903519181639db00a360ef0c6c6c2bd88f529 -data/2017-old2/2.Ha Giang_PIYK.xlsx 6bf4d62413c1b92f88f980e5e3e414a7e25971c7546c6390c7e67a6b0f742b68 -data/2017-old2/2.Ninh Thuan_BAGG.xlsx 6dfb6517f9ccc3420c222b6a88aabac54f6f25f6b743d993d6a5f28b4fa78ff2 -data/2017-old2/2.Thanh Hoa_UOPE.xlsx 2f2ce90425c19a9089024061f328f37b9ef580ddadc076ddcf452f549690f4b3 -data/2017-old2/2.VinhPhuc_QZJK.xlsx fc0e4952445ebda1af28f7330aa808c032f2e3a7a3c03dfbb9692bf6a248867a -data/2017-old2/3.BacGiang_SAVS.xlsx 8043510ff89f11ba063f898dc1d2c8879a8da9063a420308647e20a2f39d3092 -data/2017-old2/3.BinhPhuoc_IPHL.xlsx 8869970f4e6005381264f3f4bb508c5181f15ae990a75fe904a65b305c0a123a -data/2017-old2/3.Cao Bang_WMUU.xlsx 94a06bdb2333a9f44216001ba8d8463700b8cf3e51944b94db11eef7eac8dbd3 -data/2017-old2/3.Dong Thap_HKJX.xlsx 0cdd0763fc4f20f1d990621ca9e8dab107a3d4eb7c0ec3e53a4d8c240fbc569f -data/2017-old2/3.Thua Thien -Hue_MAET.xlsx 62e54eebda7bbd542ca63443bb923a81b472e7892e55dce39fd0611fc0f00abd -data/2017-old2/4.An Giang_PMJD.xlsx 778ad205bbabb7c4aea3df34325770aa4a1daec0a1683f73f0405ac1cc5aaf9a -data/2017-old2/4.BacNinh_NNIS.xlsx 009c3b7821f55210a86f9ace74c31eb0d5c16aafdb0cac8f2b71cac616abdfec -data/2017-old2/4.Binh Dinh_VOMJ.xlsx fbacd256d58627377d698462d69aac0e83d124c8c8ff1f5b63ba0bfb5442dd39 -data/2017-old2/4.DienBien_FYGN.xlsx 4931e8d6604013df48b2b469fda3eb05fdf1203b67bbc2ae4ff6d7949a115a24 -data/2017-old2/4.Lang Son_QWOG.xlsx cc7d2454caadb18740ed6baf6edc62baa47e98b3a907b3ce06f72b9a16b7838a -data/2017-old2/5.Bac Lieu_VIVY.xlsx dc1dca45fd5357b45abc930b0ebb95ab589ef80a09bef8b5ad4dab9e00be87e5 -data/2017-old2/5.Gia Lai_TAAS.xlsx 5b7be25274d628c284c37efb3df0587171cc9273dc3bd8dc824c7bd1b8bccf1a -data/2017-old2/5.HaiDuong_WNHD.xlsx 1d800d0aa7338fb507c4eeb3e752620d2608c30ac820895ca086325dbaf3643d -data/2017-old2/5.Hanam_SDKN.xlsx cfc233ac37ab8e263b6cc569815ea2c687da30364f139109032c8f2cac1ad98e -data/2017-old2/5.Yen Bai_BSLV.xlsx 4c8915417d8747c291f643ea4b378c87982c397460e0e9e3cc6df19670d6720f -data/2017-old2/6.Hau Giang_SIAJ.xlsx c11473e3e1f964302c5378c90dd6b9d478eb11397351b7ef74df9ae709c9e908 -data/2017-old2/6.HoaBinh_HLYQ.xlsx e1210961dc821642f6951cd6e4a71b5fbc81d5070ee3816b7c8b5dc56bf2b234 -data/2017-old2/6.NinhBinh_PKMQ.xlsx 9399b8ab58e2a7a1c0ebf5425359319e567f8a28602349aa587efced72142fc8 -data/2017-old2/6.Quang Ninh_YAKJ.xlsx 8cf95259b11d48267f710f151c3f1419ccac49d823b78b70a2708c59ab25a4f7 -data/2017-old2/7.HaTinh_XMFQ.xlsx d96df9f2c8365dfc73be3e10c268367b93fd00fe92bb39c745d8e353ba75c05b -data/2017-old2/7.HungYen_TCBE.xlsx e80eb38af8307c8e193a91bdda5368c437693d7669e281b543bbe891a46e6f36 -data/2017-old2/7.Kon Tum_CAQU.xlsx 41d2adea059449b6e214241414748f10776861d261cbaa1fd213629757deb846 -data/2017-old2/7.Tien Giang_AOWZ.xlsx 00d9bbc70be89f8f7dad6aff990ad36236c6b7244ddeb2e3d9b3d8ed1a6e21d0 -data/2017-old2/8.PhuYen_MLTQ.xlsx 9dd53293f99d86801692a1995a5cf7a08af8394b820c76e68c88c2b7644b9b67 -data/2017-old2/9.DakNong_ZWJT.xlsx c611bf7f1d1c95cbfc78bdab3ea99bf8278e94c8db09a9fd3ccac55a538fe0b2 -data/2017-old2/9.Khanh Hoa_IBTX.xlsx 0e90c87dd74a5eb293a0a71b93da45653a19ed2e3a2600d3b880a867a695f31d -data/2017-old2/9.Vinh Long_WVKI.xlsx 43d7abb9c6b8d3205ce90f9cceaf9623875c87998d9e030c6dc8da9bc18bfa09 diff --git a/package-lock.json b/package-lock.json index cc1c89f..e755a6a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,21 +8,15 @@ "name": "thptqg", "version": "1.0.0", "license": "ISC", - "dependencies": { - "react": "^19.2.4", - "react-dom": "^19.2.4", - "sql.js": "^1.14.1" - }, + "workspaces": [ + "web" + ], "devDependencies": { "@eslint/js": "^9.39.4", - "@types/react": "^19.2.14", - "@types/react-dom": "^19.2.3", - "@vitejs/plugin-react": "^6.0.1", "eslint": "^9.39.4", "eslint-plugin-react-hooks": "^7.0.1", "eslint-plugin-react-refresh": "^0.5.2", - "globals": "^17.4.0", - "vite": "^8.0.16" + "globals": "^17.4.0" } }, "node_modules/@babel/code-frame": { @@ -2347,6 +2341,10 @@ "node": ">=8" } }, + "node_modules/thptqg-web": { + "resolved": "web", + "link": true + }, "node_modules/tinyglobby": { "version": "0.2.17", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", @@ -2564,6 +2562,21 @@ "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" } + }, + "web": { + "name": "thptqg-web", + "version": "1.0.0", + "dependencies": { + "react": "^19.2.4", + "react-dom": "^19.2.4", + "sql.js": "^1.14.1" + }, + "devDependencies": { + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^6.0.1", + "vite": "^8.0.16" + } } } } diff --git a/package.json b/package.json index 3ba77e3..55325af 100644 --- a/package.json +++ b/package.json @@ -4,36 +4,33 @@ "version": "1.0.0", "type": "module", "description": "Tra cứu điểm thi THPT Quốc gia — 2016 và 2017", + "workspaces": [ + "web" + ], "scripts": { + "crawl:2016": "go -C crawler run ./cmd/crawl 2016", + "crawl:2017": "go -C crawler run ./cmd/crawl 2017", + "test:crawler": "go -C crawler test ./...", "build:go": "go -C go-parser build -o bin/xlsxread ./cmd/xlsxread", "build:db": "node go-parser/scripts/build-db.js", - "dev": "vite", - "build": "vite build", - "assemble": "node scripts/assemble-site.js", + "test:go": "go -C go-parser test ./...", + "dev": "npm run dev -w web", + "build": "npm run build -w web", + "preview": "npm run preview -w web", + "assemble": "node web/scripts/assemble-site.js", "build:site": "npm run build && npm run assemble", - "preview": "vite preview", - "lint": "eslint .", - "test:go": "go -C go-parser test ./..." + "lint": "eslint ." }, "repository": { "type": "git", "url": "git+https://github.com/tiennm99/thptqg.git" }, "license": "ISC", - "dependencies": { - "react": "^19.2.4", - "react-dom": "^19.2.4", - "sql.js": "^1.14.1" - }, "devDependencies": { "@eslint/js": "^9.39.4", - "@types/react": "^19.2.14", - "@types/react-dom": "^19.2.3", - "@vitejs/plugin-react": "^6.0.1", "eslint": "^9.39.4", "eslint-plugin-react-hooks": "^7.0.1", "eslint-plugin-react-refresh": "^0.5.2", - "globals": "^17.4.0", - "vite": "^8.0.16" + "globals": "^17.4.0" } } diff --git a/index.html b/web/index.html similarity index 100% rename from index.html rename to web/index.html diff --git a/web/package.json b/web/package.json new file mode 100644 index 0000000..3459e34 --- /dev/null +++ b/web/package.json @@ -0,0 +1,23 @@ +{ + "name": "thptqg-web", + "private": true, + "version": "1.0.0", + "type": "module", + "description": "Frontend — one Vite app serving all four datasets and the hub", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^19.2.4", + "react-dom": "^19.2.4", + "sql.js": "^1.14.1" + }, + "devDependencies": { + "@types/react": "^19.2.14", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^6.0.1", + "vite": "^8.0.16" + } +} diff --git a/scripts/assemble-site.js b/web/scripts/assemble-site.js similarity index 78% rename from scripts/assemble-site.js rename to web/scripts/assemble-site.js index 919aa0b..6907d30 100644 --- a/scripts/assemble-site.js +++ b/web/scripts/assemble-site.js @@ -12,7 +12,7 @@ * rewrites the URL and would interfere with the ?q= deep links the app relies * on. * - * Usage: node scripts/assemble-site.js + * Usage: node web/scripts/assemble-site.js */ import { cpSync, mkdirSync, rmSync, existsSync, readdirSync, statSync } from "node:fs"; @@ -21,12 +21,15 @@ import { fileURLToPath } from "node:url"; import { DATASET_IDS } from "../src/datasets.js"; -const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const DIST = join(ROOT, "dist"); -const SITE = join(ROOT, "_site"); +// The build output belongs to this workspace; the Pages artifact does not. The +// deploy action uploads _site from the repository root, so that one climbs out. +const WEB = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const DIST = join(WEB, "dist"); +const SITE = join(WEB, "..", "_site"); -/** URLs published before the switch to flat paths; the router rewrites these. */ -const LEGACY_PATHS = ["2017/old", "2017/old2"]; +// The two legacy nested URLs (2017/old, 2017/old2) are gone with the datasets +// they addressed. Unknown paths render the hub via 404.html, so old bookmarks +// land somewhere useful rather than on a Pages error page. if (!existsSync(join(DIST, "index.html"))) { console.error(`no build found at ${DIST} — run: npm run build`); @@ -42,8 +45,8 @@ cpSync(DIST, SITE, { recursive: true }); // Unknown paths render the hub rather than the default Pages 404. cpSync(join(DIST, "index.html"), join(SITE, "404.html")); -// One entry point per dataset, plus the legacy nested URLs. -for (const path of [...DATASET_IDS, ...LEGACY_PATHS]) { +// One entry point per dataset. +for (const path of DATASET_IDS) { mkdirSync(join(SITE, path), { recursive: true }); cpSync(join(DIST, "index.html"), join(SITE, path, "index.html")); } @@ -72,6 +75,6 @@ if (stray.length) { } console.log(`assembled ${SITE}`); -for (const path of ["", "404.html", ...DATASET_IDS, ...LEGACY_PATHS]) { +for (const path of ["", "404.html", ...DATASET_IDS]) { console.log(` /thptqg/${path}`); } diff --git a/src/App.css b/web/src/App.css similarity index 100% rename from src/App.css rename to web/src/App.css diff --git a/src/App.jsx b/web/src/App.jsx similarity index 100% rename from src/App.jsx rename to web/src/App.jsx diff --git a/src/assets/vite.svg b/web/src/assets/vite.svg similarity index 100% rename from src/assets/vite.svg rename to web/src/assets/vite.svg diff --git a/src/components/custom-query.jsx b/web/src/components/custom-query.jsx similarity index 100% rename from src/components/custom-query.jsx rename to web/src/components/custom-query.jsx diff --git a/src/components/hub.jsx b/web/src/components/hub.jsx similarity index 100% rename from src/components/hub.jsx rename to web/src/components/hub.jsx diff --git a/src/components/score-table.jsx b/web/src/components/score-table.jsx similarity index 100% rename from src/components/score-table.jsx rename to web/src/components/score-table.jsx diff --git a/src/components/search-form.jsx b/web/src/components/search-form.jsx similarity index 100% rename from src/components/search-form.jsx rename to web/src/components/search-form.jsx diff --git a/src/components/student-detail.jsx b/web/src/components/student-detail.jsx similarity index 100% rename from src/components/student-detail.jsx rename to web/src/components/student-detail.jsx diff --git a/src/datasets.js b/web/src/datasets.js similarity index 71% rename from src/datasets.js rename to web/src/datasets.js index 6f5509e..d370b23 100644 --- a/src/datasets.js +++ b/web/src/datasets.js @@ -11,6 +11,10 @@ * Imported by the Vite app *and* by go-parser/scripts/build-db.js under plain * Node, so this module must stay free of `import.meta.env` and any Vite-only * syntax. Callers pass the base URL in explicitly for that reason. + * + * That cross-package import is why this file cannot move into a Vite-only + * corner of the app: go-parser reads DATASET_IDS and dbSizeMb straight from + * here, and reaches across the workspace boundary to do it. */ // Extension is required: this module is also imported by plain Node @@ -44,30 +48,6 @@ export const DATASETS = [ examples: ["49008235", "Nguyễn Minh Tiến"], presets: PRESETS_2017, }, - { - id: "2017-old", - year: 2017, - label: "2017 — bản cũ", - blurb: "847.348 thí sinh · bản trước khi cập nhật", - title: "Tra cứu điểm thi THPT Quốc gia 2017 (bản cũ)", - subtitle: SUBTITLE, - source: "bản lưu trước đợt cập nhật baotintuc.vn", - dbSizeMb: 47, - examples: ["49008235", "Nguyễn Minh Tiến"], - presets: PRESETS_2017, - }, - { - id: "2017-old2", - year: 2017, - label: "2017 — bản cũ 2", - blurb: "679.764 thí sinh · bản xuất lại", - title: "Tra cứu điểm thi THPT Quốc gia 2017 (bản cũ 2)", - subtitle: SUBTITLE, - source: "bản xuất lại đã hiệu chỉnh", - dbSizeMb: 38, - examples: ["49008235", "Nguyễn Minh Tiến"], - presets: PRESETS_2017, - }, ]; /** Dataset IDs in build order. */ diff --git a/src/hooks/use-sqlite.js b/web/src/hooks/use-sqlite.js similarity index 100% rename from src/hooks/use-sqlite.js rename to web/src/hooks/use-sqlite.js diff --git a/src/index.css b/web/src/index.css similarity index 100% rename from src/index.css rename to web/src/index.css diff --git a/src/lib/admission-blocks.js b/web/src/lib/admission-blocks.js similarity index 100% rename from src/lib/admission-blocks.js rename to web/src/lib/admission-blocks.js diff --git a/src/lib/query-mode.js b/web/src/lib/query-mode.js similarity index 100% rename from src/lib/query-mode.js rename to web/src/lib/query-mode.js diff --git a/src/lib/sql-presets.js b/web/src/lib/sql-presets.js similarity index 100% rename from src/lib/sql-presets.js rename to web/src/lib/sql-presets.js diff --git a/src/lib/subjects.js b/web/src/lib/subjects.js similarity index 100% rename from src/lib/subjects.js rename to web/src/lib/subjects.js diff --git a/src/main.jsx b/web/src/main.jsx similarity index 100% rename from src/main.jsx rename to web/src/main.jsx diff --git a/src/router.js b/web/src/router.js similarity index 50% rename from src/router.js rename to web/src/router.js index d278465..f17117b 100644 --- a/src/router.js +++ b/web/src/router.js @@ -3,27 +3,23 @@ * * URLs are flat, one segment per dataset, and the segment IS the dataset id: * - * /thptqg/ → null (hub) - * /thptqg/2016/ → 2016 - * /thptqg/2017-old/ → 2017-old + * /thptqg/ → null (hub) + * /thptqg/2016/ → 2016 + * /thptqg/2017/ → 2017 * * That makes this an exact match on a single segment. The nested form this * replaced (/thptqg/2017/old/) needed longest-prefix matching, because * "/thptqg/2017/old/" also starts with "/thptqg/2017/" — an entire class of * routing bug the flat scheme designs out. + * + * Those nested URLs used to be rewritten here so old bookmarks kept working. + * Both of them addressed the 2017-old and 2017-old2 datasets, which no longer + * exist, so there is nothing left to rewrite them to: they now fall through to + * the hub like any other unknown path. */ import { DATASETS } from "./datasets"; -/** - * URLs published before the switch to flat paths. Kept working so existing - * links and bookmarks resolve instead of 404ing. - */ -const LEGACY_PATHS = { - "2017/old": "2017-old", - "2017/old2": "2017-old2", -}; - const BASE = import.meta.env.BASE_URL; /** Strip the site base and any trailing slash, leaving a bare route segment. */ @@ -32,25 +28,8 @@ function toSegment(pathname) { return path.replace(/^\/+/, "").replace(/\/+$/, ""); } -/** - * Resolve the current location to a dataset, or null for the hub. - * - * Rewrites legacy nested URLs to their flat equivalent, preserving the query - * string — a `?q=` deep link must survive the redirect, which is exactly what - * the usual SPA 404-fallback hack would break. - */ +/** Resolve the current location to a dataset, or null for the hub. */ export function resolveRoute(pathname = window.location.pathname) { - let segment = toSegment(pathname); - - const flat = LEGACY_PATHS[segment]; - if (flat) { - window.history.replaceState( - {}, - "", - `${BASE}${flat}/${window.location.search}${window.location.hash}`, - ); - segment = flat; - } - + const segment = toSegment(pathname); return DATASETS.find((d) => d.id === segment) ?? null; } diff --git a/vite.config.js b/web/vite.config.js similarity index 62% rename from vite.config.js rename to web/vite.config.js index e047942..10b02f6 100644 --- a/vite.config.js +++ b/web/vite.config.js @@ -12,8 +12,17 @@ import react from "@vitejs/plugin-react"; // // publicDir holds only the gzipped databases, staged there by // go-parser/scripts/build-db.js. Nothing uncompressed is ever placed in it. +// +// It sits at the repository root rather than inside this workspace because +// go-parser writes it, so the path has to climb out of web/. Vite resolves +// publicDir against the project root, which is this directory. +// +// outDir is left at its default, so the build lands in web/dist and this +// workspace stays self-contained. The Pages artifact (_site) is assembled at +// the repository root by web/scripts/assemble-site.js, since that is where the +// deploy action uploads from. export default defineConfig({ plugins: [react()], base: "/thptqg/", - publicDir: ".build/public", + publicDir: "../.build/public", });