diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index 80bb772..052f987 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -33,7 +33,9 @@ jobs: - uses: actions/setup-go@v5 with: go-version: '1.26' - cache-dependency-path: go-parser/go.sum + cache-dependency-path: | + go-parser/go.sum + crawler/go.sum - uses: actions/setup-node@v4 with: diff --git a/README.md b/README.md index 192fdf8..7b82751 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,8 @@ web/ the frontend — one Vite app serving both datasets and th src/router.js pathname → dataset scripts/ site assembly crawler/ Go — re-fetches the source spreadsheets - internal/sources/ one file per data source: its links and local filenames + internal/sources/ per dataset: which article to read, how to name its files + internal/article/ pulls the download links out of that article internal/fetch/ concurrent, resumable downloading go-parser/ Go — Excel to SQLite internal/schema/ canonical 22-column table: DDL, INSERT, subject regexes @@ -58,11 +59,12 @@ them: ```bash npm run crawl:2016 # re-fetch data/2016/ -npm run crawl:2017 # re-fetch data/2017/ from the baotintuc.vn CDN +npm run crawl:2017 # re-fetch data/2017/ ``` -Crawling is idempotent — files already present are skipped — and is never part -of the build. +Each reads the download links out of the article that published the dataset, so +no link list is kept in the repository. Crawling is idempotent — files already +present are skipped — and is never part of the build. Pushing to `main` runs the same steps in `.github/workflows/deploy-pages.yml` and publishes to GitHub Pages. diff --git a/crawler/cmd/crawl/main.go b/crawler/cmd/crawl/main.go index ac95d6d..f05dc9b 100644 --- a/crawler/cmd/crawl/main.go +++ b/crawler/cmd/crawl/main.go @@ -3,24 +3,27 @@ // The argument is the dataset id, the same one used by go-parser's configs and // the published site paths. // -// crawl 2017 # from the baotintuc.vn CDN -// crawl 2016 # source not yet configured +// crawl 2016 # 119 exam-cluster files +// crawl 2017 # 63 province files // crawl 2017 --list # show what would be downloaded // -// Runs are idempotent: a file already present and non-empty is skipped, so an -// interrupted crawl can simply be re-run. +// Each run reads the download links out of the article that published them, so +// --list needs network access too. Runs are idempotent: a file already present +// and non-empty is skipped, so an interrupted crawl can simply be re-run. package main import ( "context" "flag" "fmt" + "net/http" "os" "os/signal" "path/filepath" "syscall" "time" + "github.com/tiennm99/thptqg/crawler/internal/article" "github.com/tiennm99/thptqg/crawler/internal/fetch" "github.com/tiennm99/thptqg/crawler/internal/sources" ) @@ -70,12 +73,22 @@ func run(args []string) error { return err } - files, err := src.Files() + // Ctrl-C cancels the article fetch and any in-flight download; each worker + // deletes its partial file on the way out, so an interrupted run leaves + // nothing half-written. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + client := &http.Client{Timeout: *timeout} + + fmt.Printf("Reading %s\n", src.Article) + links, err := article.Fetch(ctx, client, src.Article, src.Headers, src.Exts...) if err != nil { return err } - if len(files) == 0 { - return fmt.Errorf("source %q produced no files", src.ID) + files, err := src.Resolve(links) + if err != nil { + return err } outDir, err := filepath.Abs(*out) @@ -99,11 +112,6 @@ func run(args []string) error { return nil } - // Ctrl-C cancels in-flight requests; each worker deletes its partial file - // on the way out, so an interrupted run leaves nothing half-written. - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - fmt.Printf("Downloading %d files to %s...\n", len(items), outDir) results, runErr := fetch.Run(ctx, items, fetch.Options{ diff --git a/crawler/go.mod b/crawler/go.mod index dd80379..218e045 100644 --- a/crawler/go.mod +++ b/crawler/go.mod @@ -1,3 +1,8 @@ module github.com/tiennm99/thptqg/crawler go 1.26.5 + +require ( + golang.org/x/net v0.58.0 + golang.org/x/text v0.41.0 +) diff --git a/crawler/go.sum b/crawler/go.sum new file mode 100644 index 0000000..cc9727d --- /dev/null +++ b/crawler/go.sum @@ -0,0 +1,4 @@ +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= diff --git a/crawler/internal/article/article.go b/crawler/internal/article/article.go new file mode 100644 index 0000000..c808d2b --- /dev/null +++ b/crawler/internal/article/article.go @@ -0,0 +1,144 @@ +// Package article reads download links out of a published web page. +// +// This is what makes the crawler a crawler: the file lists are not carried in +// the repository, they are read from the articles that published them, at run +// time. A source declares which page to read and how to name what it finds. +package article + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + + "golang.org/x/net/html" +) + +// Link is one downloadable file found on a page. +type Link struct { + // URL is absolute, resolved against the page it was found on. + URL string + // Text is the anchor's visible text, whitespace-collapsed. Some pages use + // it for the thing being linked ("Bạc Liêu"), others for boilerplate + // ("xem TẠI ĐÂY"), so a source decides whether it is worth anything. + Text string + // File is the last segment of the URL path, query string excluded. + File string +} + +// Extract returns every anchor on the page whose target ends in one of exts. +// +// Pure: no network, so the parsing rules can be tested against a saved copy of +// a real page. Order is document order, and duplicates are kept — deciding +// whether two links to the same file is a problem belongs to the caller, which +// knows what it would name them. +func Extract(pageURL string, body io.Reader, exts ...string) ([]Link, error) { + base, err := url.Parse(pageURL) + if err != nil { + return nil, fmt.Errorf("bad page URL %q: %w", pageURL, err) + } + + doc, err := html.Parse(body) + if err != nil { + return nil, fmt.Errorf("parse %s: %w", pageURL, err) + } + + var out []Link + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + if link, ok := linkOf(base, n, exts); ok { + out = append(out, link) + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + + return out, nil +} + +// linkOf turns one element into a Link, or reports that it is not one of +// the files being looked for. +func linkOf(base *url.URL, n *html.Node, exts []string) (Link, bool) { + var href string + for _, a := range n.Attr { + if strings.EqualFold(a.Key, "href") { + href = strings.TrimSpace(a.Val) + break + } + } + if href == "" { + return Link{}, false + } + + // Relative hrefs are the norm on the pages this reads; resolving against + // the page URL is what lets a source name a mirror and get its files. + ref, err := url.Parse(href) + if err != nil { + return Link{}, false + } + abs := base.ResolveReference(ref) + + file := path.Base(abs.Path) + if !hasExt(file, exts) { + return Link{}, false + } + + return Link{URL: abs.String(), Text: textOf(n), File: file}, true +} + +func hasExt(name string, exts []string) bool { + lower := strings.ToLower(name) + for _, e := range exts { + if strings.HasSuffix(lower, strings.ToLower(e)) { + return true + } + } + return false +} + +// textOf collects an element's visible text, collapsing whitespace. Anchors on +// these pages wrap the label in styling tags, so the text is rarely a single +// child node. +func textOf(n *html.Node) string { + var b strings.Builder + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.TextNode { + b.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return strings.Join(strings.Fields(b.String()), " ") +} + +// Fetch downloads a page and runs Extract on it. +func Fetch(ctx context.Context, client *http.Client, pageURL string, headers map[string]string, exts ...string) ([]Link, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) + if err != nil { + return nil, err + } + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch %s: %w", pageURL, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetch %s: HTTP %d", pageURL, resp.StatusCode) + } + + return Extract(pageURL, resp.Body, exts...) +} diff --git a/crawler/internal/article/article_test.go b/crawler/internal/article/article_test.go new file mode 100644 index 0000000..254bae3 --- /dev/null +++ b/crawler/internal/article/article_test.go @@ -0,0 +1,152 @@ +package article + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +const page = "https://example.test/news/scores.html" + +func extract(t *testing.T, body string, exts ...string) []Link { + t.Helper() + links, err := Extract(page, strings.NewReader(body), exts...) + if err != nil { + t.Fatal(err) + } + return links +} + +// TestResolvesRelativeHrefs is the behaviour the 2016 source depends on: its +// article links files as /upload/..., so they only become fetchable once +// resolved against the page they were found on. +func TestResolvesRelativeHrefs(t *testing.T) { + got := extract(t, ` + root-relative + page-relative + already absolute + protocol-relative + `, ".xlsx") + + want := []string{ + "https://example.test/upload/s/a.xlsx", + "https://example.test/news/sub/b.xlsx", + "https://cdn.other/c.xlsx", + "https://cdn.other/d.xlsx", + } + if len(got) != len(want) { + t.Fatalf("got %d links, want %d", len(got), len(want)) + } + for i, w := range want { + if got[i].URL != w { + t.Errorf("link %d = %q, want %q", i, got[i].URL, w) + } + } +} + +func TestFiltersByExtension(t *testing.T) { + got := extract(t, ` + keep + keep + drop + drop + keep, case-insensitive + no href at all + `, ".xls", ".xlsx") + + if len(got) != 3 { + t.Fatalf("got %d links, want 3: %+v", len(got), got) + } + // ".xls" must not swallow ".xlsx" or vice versa when only one is asked for. + only := extract(t, `xy`, ".xlsx") + if len(only) != 1 || only[0].File != "b.xlsx" { + t.Errorf("extension filter is too loose: %+v", only) + } +} + +// TestFileIgnoresQueryString: a query string is not part of the filename, and +// writing one to disk would produce a name go-parser never sees. +func TestFileIgnoresQueryString(t *testing.T) { + got := extract(t, `x`, ".xlsx") + if len(got) != 1 { + t.Fatalf("got %d links", len(got)) + } + if got[0].File != "report.xlsx" { + t.Errorf("File = %q, want report.xlsx", got[0].File) + } + if !strings.Contains(got[0].URL, "v=2") { + t.Errorf("the query string must survive in the URL: %q", got[0].URL) + } +} + +// TestTextCollectsNestedMarkup: both real articles wrap the label in styling +// tags, so the anchor text is never a single child node. 2017 names its files +// from this text. +func TestTextCollectsNestedMarkup(t *testing.T) { + got := extract(t, `Bà Rịa -Vũng Tàu`, ".xls") + if len(got) != 1 { + t.Fatalf("got %d links", len(got)) + } + if got[0].Text != "Bà Rịa -Vũng Tàu" { + t.Errorf("Text = %q, want %q", got[0].Text, "Bà Rịa -Vũng Tàu") + } +} + +func TestEntitiesInHrefAreDecoded(t *testing.T) { + got := extract(t, `x`, ".xlsx") + if len(got) != 1 || !strings.Contains(got[0].URL, "x=1&y=2") { + t.Errorf("href entity not decoded: %+v", got) + } +} + +// TestMalformedHTMLStillParses: these are hand-edited CMS pages, and the parser +// must not give up on unclosed tags. +func TestMalformedHTMLStillParses(t *testing.T) { + got := extract(t, `

listonetwo`, ".xls") + if len(got) != 2 { + t.Errorf("got %d links, want 2: %+v", len(got), got) + } +} + +func TestExtractRejectsBadPageURL(t *testing.T) { + if _, err := Extract("://nonsense", strings.NewReader("x"), ".xls"); err == nil { + t.Error("expected an error for an unparseable page URL") + } +} + +func TestFetchSendsHeadersAndParses(t *testing.T) { + var ua, ref string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ua, ref = r.Header.Get("User-Agent"), r.Header.Get("Referer") + w.Write([]byte(`An Giang`)) + })) + defer srv.Close() + + links, err := Fetch(t.Context(), srv.Client(), srv.URL, + map[string]string{"User-Agent": "test-agent", "Referer": "https://ref.test/"}, ".xls") + if err != nil { + t.Fatal(err) + } + if ua != "test-agent" || ref != "https://ref.test/" { + t.Errorf("headers not sent: ua=%q referer=%q", ua, ref) + } + if len(links) != 1 || links[0].Text != "An Giang" || links[0].File != "a.xls" { + t.Errorf("unexpected links: %+v", links) + } +} + +// TestFetchFailsLoudly: a CMS that answers 404 or 403 with an HTML error page +// would otherwise yield zero links and read as "nothing to download". +func TestFetchFailsLoudly(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "not found", http.StatusNotFound) + })) + defer srv.Close() + + if _, err := Fetch(t.Context(), srv.Client(), srv.URL, nil, ".xls"); err == nil { + t.Fatal("expected an error for a non-200 response") + } else if !strings.Contains(err.Error(), "404") { + t.Errorf("error should name the status, got: %v", err) + } +} diff --git a/crawler/internal/fetch/fetch.go b/crawler/internal/fetch/fetch.go index 5afff19..5e34d7e 100644 --- a/crawler/internal/fetch/fetch.go +++ b/crawler/internal/fetch/fetch.go @@ -138,6 +138,10 @@ func Run(ctx context.Context, items []Item, opts Options) ([]Result, error) { // complete. Without that, an interrupted run leaves a truncated file at the // final path — and because the skip check only tests for a non-empty file, // every later run would skip it and the corruption would persist silently. +// +// A .part left behind by an abrupt kill needs no cleanup: it does not satisfy +// the skip check, os.Create truncates it, and the next run re-fetches the file. +// go-parser ignores it in the meantime, since it reads only .xls and .xlsx. func download(ctx context.Context, client *http.Client, it Item, headers map[string]string) Result { if st, err := os.Stat(it.Path); err == nil && st.Size() > 0 { return Result{Item: it, Status: StatusSkip, Size: st.Size()} diff --git a/crawler/internal/sources/source_2016.go b/crawler/internal/sources/source_2016.go index 58f1764..680b31e 100644 --- a/crawler/internal/sources/source_2016.go +++ b/crawler/internal/sources/source_2016.go @@ -1,182 +1,40 @@ package sources +import ( + "fmt" + + "github.com/tiennm99/thptqg/crawler/internal/article" +) + // source2016 fetches the 2016 dataset: one spreadsheet per exam cluster // (cụm thi), 4 .xls and 115 .xlsx. // -// # Provenance +// The article is a mirror. The site that first published this list +// (dtntbacgiang.edu.vn) no longer resolves; this copy of the same article is +// still online, and its links are read at crawl time like any other source. // -// The list below was recovered from the aggregator article that originally -// published these files: +// Dest keeps the server's own filename verbatim — a 32-hex content hash, the +// cluster slug, then a millisecond timestamp. Two reasons not to prettify it: // -// cong-bo-diem-thi-thptqg-2016-toan-bo-120-cum-thi-da-co-diem.html +// - go-parser sorts inputs bytewise and inserts last-wins, so filenames decide +// which row survives a duplicate exam number. That is live here, not +// hypothetical: 877,464 source rows collapse to 877,461, so three rows' +// contents depend on this ordering. +// - go-parser/testdata/reader-fidelity-hashes.tsv is keyed by full path, and +// it is frozen — it was produced by the Rust reader, which no longer exists. // -// The site that first carried it (dtntbacgiang.edu.vn) no longer resolves, so -// the list was read from the Internet Archive's copy. It is verified, not -// guessed: all 119 filenames match data/2016/ exactly in both directions, which -// TestSource2016MatchesFilesOnDisk asserts. -// -// # Filenames are load-bearing — do not "tidy" them -// -// Dest is the server-assigned name, a 32-hex content hash followed by the -// cluster slug and a millisecond timestamp. It is kept verbatim because -// go-parser sorts input files bytewise and inserts last-wins, so filenames -// decide which row survives a duplicate exam number. That is live for 2016, not -// hypothetical: 877,464 source rows collapse to 877,461, so three rows' contents -// depend on this sort order. The hashes in -// go-parser/testdata/reader-fidelity-hashes.tsv are keyed by full path as well, -// and they are frozen — they were produced by the Rust reader, which no longer -// exists. -// -// # The host is not verified -// -// Only the paths are known to be right; the host that still serves them is not. -// The original is gone and the Internet Archive captured the article but none of -// the spreadsheets, so this could not be confirmed by fetching. baseURL points -// at a mirror of the same article that is still online. If it serves these files -// from a different directory, uploadPath is the single constant to change. -const ( - baseURL = "https://dtnt.bacninh.edu.vn" - uploadPath = "/upload/s/20180102/" -) - +// Unlike 2017 this needs no transliteration: the name comes from the URL, so it +// is exact by construction rather than derived from a label. var source2016 = Source{ - ID: "2016", - Summary: "119 exam-cluster files (4 .xls + 115 .xlsx)", - Files: func() ([]File, error) { - out := make([]File, 0, len(clusters2016)) - for _, c := range clusters2016 { - out = append(out, File{ - Name: c.name, - URL: baseURL + uploadPath + c.file, - Dest: c.file, - }) + ID: "2016", + Summary: "119 exam-cluster files (4 .xls + 115 .xlsx)", + Article: "https://dtnt.bacninh.edu.vn/tin-tuc/tin-tuc-su-kien/cong-bo-diem-thi-thptqg-2016-toan-bo-120-cum-thi-da-co-diem.html", + Exts: []string{".xls", ".xlsx"}, + WantFiles: 119, + Dest: func(l article.Link) (string, error) { + if l.File == "" { + return "", fmt.Errorf("link %s has no filename", l.URL) } - return out, nil + return l.File, nil }, } - -type cluster struct{ name, file string } - -// clusters2016 is in the article's publication order. That order is incidental — -// go-parser re-sorts by filename — but it is kept as published so the list can -// be diffed against the source article. -var clusters2016 = []cluster{ - {"Trường Đại học Bách khoa Hà Nội", "b34c777942ca8a4de91f23a35cce1c6bdhbachkhoahn-1468901491486.xlsx"}, - {"Trường Đại học Sư phạm Hà Nội", "76f19098881e9c79f66ac4f1b42a8a55dhsuphamhanoi-1468901144203.xlsx"}, - {"Trường Đại học Thuỷ lợi * Cơ sở 1 ở phía Bắc", "4a3fa95a83fe9b3b2fafab309323fd22dhthuyloi-1468901840919.xlsx"}, - {"Học viện Kỹ thuật Quân sự * Cơ sở 1 ở phía Bắc (Quân đội)", "4334ea476a2b8196d54ba40a341831echvkythuatquansu-1468920342994.xlsx"}, - {"Trường Đại học Lâm nghiệp", "c5ecc58a731fd1a54b7e6cc3bc99449ddhlamnghiep-1468939300790.xlsx"}, - {"Trường Đại học Bách khoa - Đại học Quốc gia Thành phố Hồ Chí Minh", "357c6aaf59aeaa6c11c3e83c595d38cfdhbachkhoa-tphcm-1468939678829.xlsx"}, - {"Trường Đại học Khoa học tự nhiên - Đại học Quốc gia Thành phố Hồ Chí Minh", "64e44466ffeff9a587260795644805d0dhkhtunhien-dhqgtphcm-1468921015146.xlsx"}, - {"Trường Đại học Khoa học xã hội và Nhân văn - Đại học Quốc gia Thành phố Hồ Chí Minh", "5fd13ae5287cbf0149bd733b507e0608dhkhxhnv-dhqgtphcm-1468920630193.xlsx"}, - {"Trường Đại học Sư phạm Tp.HCM", "a995de3f7a7055cf10b2f002f0e194f7dhsptphcm-1468920792291.xlsx"}, - {"Trường Đại học Hàng Hải", "c6ef16bc9f75ce0fd16c1229afad4e71dhhanghai-1468906457479.xlsx"}, - {"Trường Đại học Sư phạm - Đại học Thái Nguyên", "ee09c723da4e86cdbe26213413450af2dhsupham-dhthainguyen-1468939061872.xlsx"}, - {"Trường Đại học Kỹ thuật công nghiệp - Đại học Thái Nguyên", "8af8f1b29b6a40e26ccb4a2e19e8eededhkythuatcongnghiep-dhthainguyen-1468901919568.xlsx"}, - {"Trường Đại học Nông lâm - Đại học Thái Nguyên", "53e5e20ebd74d84741513b703fbd13bfdhnonglam-dhthainguyen-1468939023637.xlsx"}, - {"Học viện Ngân hàng", "f2846894bd5533e02b122a1e3b4333f6hvnganhang-1468939472850.xlsx"}, - {"Trường Đại học Luật Hà Nội", "9e0e564ae801e347bd7649da8b89ad90dhluathn-1468920377978.xlsx"}, - {"Trường Đại học Tân Trào", "20db5eaace927596f1806f151649ee6edhtantrao-1468901296040.xlsx"}, - {"Trường Đại học Xây dựng Hà Nội", "7b5d74564ab9f6e0e137f6d161fab6f0dhxaydung-1468940561682.xlsx"}, - {"Trường Đại học Khoa học - Đại học Thái Nguyên", "c2cfaaf287fb1e7120894e545771b027dhkhoahoc-dhthainguyen-1468900365545.xlsx"}, - {"Đại học Thái Nguyên", "3daa6ee33fdafadc3c112c18bff18c2ddhthainguyen-1468940268065.xlsx"}, - {"Học viện Tài chính", "96c95c85c5549a06f2e92ffcacf8ff21hvtaichinh-1468934470924.xlsx"}, - {"Trường Đại học Tây Bắc", "74e6cd0dc78a3fb7dcf56ae57283b9b9dhtaybac-1468940302052.xlsx"}, - {"Trường Đại học Hùng Vương", "778ab074ed6a95d209fa901fd5d68843dhhungvuong-1468901964127.xlsx"}, - {"Trường Đại học Sư phạm Hà Nội 2", "a11e707e6ed5fe4d29a8e32380d7558cdhsphanoi-2-1468902006284.xlsx"}, - {"Trường Đại học Ngoại thương * Cơ sở 1 ở phía Bắc", "ec84067268f47d523b1e80b493248377dhngoaithuong-1468920480753.xlsx"}, - {"Trường Đại học Kinh tế Quốc dân", "8f948c2a5ab49a3c5fd03b6ea04887a7dhkinhtequocdanfix-1468902083782.xlsx"}, - {"Trường Đại học Giao thông Vận tải", "51bb2f9b6e9f565c2d705606fd56958bdhgiaothongvantai-1468939107463.xlsx"}, - {"Học viện Nông Nghiệp Việt Nam", "5aede4951e7af8c1e1056c747a553386hvnongnghiepvn-1468900610577.xlsx"}, - {"Trường Đại học Sư phạm Kỹ thuật Hưng Yên", "f85f04bb2bbd84457b83ac846728a2a6dhspkythuathungyen-1468920667091.xlsx"}, - {"Trường Đại học Hải Phòng", "deca197a916a8f633f6894b4e13c3ca4dhhaiphong-1468901178368.xlsx"}, - {"Trường Đại học Thương mại", "b2552e20c9ff4b237244dfbf2e278a47dhthuongmai-1468902876264.xlsx"}, - {"Trường Đại học Công nghiệp Hà Nội", "82338a4a93ea1e0cde26e0061066ca8adhcongnghiephanoi-1468920154757.xlsx"}, - {"Y Dược Thái Bình", "22e8617886412abd90ae3d33cc6d7db4dhyduocthaibinh-1468941020145.xlsx"}, - {"Trường Đại học Mỏ Địa chất", "c174631d90303276ac63ff38b31157bbdhmodiachat-1468900697349.xlsx"}, - {"Trường Đại học Hồng Đức", "bd857add0cfe92c1c247469ff2104527dh-hong-duc-1468900427119.xlsx"}, - {"Trường Đại học Vinh", "64d2635efd30a0d0db4f7c30b9735261dhvinh-1468940230027.xlsx"}, - {"Trường ĐH Sư phạm - ĐH Huế", "ab2f259ef55cb08b3a436d3f15d42cc7dhsupham-dhhue-1468934431828.xlsx"}, - {"Trường ĐH Khoa học - ĐH Huế", "87ffaf3197ba20f0919f98a0b3f33914dhkhoahoc-dhhue-1468938729472.xlsx"}, - {"Trường ĐH Kinh tế - ĐH Huế", "8916da7b3c4d58cffc1dd29dc7a8aabfdhkinhte-dhhue-1468938560205.xlsx"}, - {"Đại học Huế", "919276a0348634459f8937dfcd6c7129dhhue-1468938792115.xlsx"}, - {"ĐH Đà Nẵng", "b42fd58ead4c54279d7df5f53160a851dhdanang-1468920187203.xlsx"}, - {"Trường ĐH Bách khoa - ĐH Đà Nẵng", "43213f61074068d9177685eb6c3f2d3bdhbachkhoa-dhdanang-1468938494437.xlsx"}, - {"Trường ĐH Sư phạm - ĐH Đà Nẵng", "54bbcf865c137d58cf62af15733bd376dhsupham-dhdanang-1468938526558.xlsx"}, - {"Trường Đại học Quy Nhơn", "70bf8cd2b6e87523219e55149a8524c5dhquynhon-1468938878328.xlsx"}, - {"Trường ĐH Xây dựng miền Trung", "4e9bb192d56c9fa92b3e0ec13d8236c8dhxaydungmientrung-1468940978346.xlsx"}, - {"Trường ĐH Nông Lâm TPHCM", "b2a739584a8861502573d1c6372fd6e9dhnonglamtphcm-1468939555326.xlsx"}, - {"Trường ĐH Ngoại ngữ - ĐH Đà Nẵng", "9ba098a7913ad837a66e173b3fc41fd7dhngoaingu-dhdanang-1468902750424.xlsx"}, - {"Trường ĐH Tây Nguyên", "216ee4e385dbc236767be899b068bf98dhtaynguyen-1468940490240.xlsx"}, - {"Trường ĐH Tài chính Marketing", "af6828f837a80e03d77a65f3d310ec60dh-tai-chinh-marketing-1468924711596.xlsx"}, - {"Trường ĐH Nha Trang", "6375fd13557bee08d20f1f3e46dac130dhnhatrang-coso1-1468901350425.xlsx"}, - {"Trường ĐH Giao thông vận tải TP.HCM", "c5c28959fcd0b4e39dd9fd538db5448adhgtvttphcm-1468939150821.xlsx"}, - {"Trường ĐH Sư phạm Kĩ thuật TP.HCM", "514877c1cfebdc9c2251f651b5f02c7edhspkythuattphcm-1468906568655.xlsx"}, - {"Trường ĐH Đà Lạt", "2b33640b0dd154490f2809d96be81797dhdalat-1468902130616.xlsx"}, - {"Trường ĐH Kinh tế TP.HCM", "f0aa5a19c2c8c1fde8211341f0e98e26dhkinhtetphcm-1468939229043.xlsx"}, - {"Trường ĐH Kinh tế - Luật - ĐHQG TP.HCM", "ff877788e43bd84f0119dae026996892dhkinhteluat-dhqgtphcm-1468939860894.xlsx"}, - {"Trường ĐH Công nghiệp Thực phẩm TP.HCM", "167116af6c8a096c2a9307edfa115f77dhcnthucpham-tphcm-1468910578187.xls"}, - {"Trường ĐH Công nghiệp TP.HCM", "487c7287c1c9735722b31d877942becddhcongnghieptphcm-1468983373303.xlsx"}, - {"Trường ĐH Sài Gòn", "d75282bff0a4d8cd1421c3fb74f789c3dhsaigon-1468900867561.xlsx"}, - {"Trường ĐH Đồng Tháp", "388fcdfdac18ef6eee6125a2a015714ddhdongthap-1468934545359.xlsx"}, - {"Trường Đại học An Giang", "fc511852659150f6305f6f624445300ddhangiang-1468940189321.xlsx"}, - {"Trường ĐH Tôn Đức Thắng", "9d78f6dce7965fb4b7a337279920d4b1dhtonducthang-1468902168623.xlsx"}, - {"Trường ĐH Tiền Giang", "bacd1b2133b37d0ea9fbf46445265036dhtiengiang-1468940391597.xlsx"}, - {"Trường ĐH Cần Thơ", "b35a3f26a162bec3740e2da5639327afdhcantho-1468907131146.xls"}, - {"Trường ĐH Cần Thơ - Hậu Giang", "7f1cbe7f35d1f539ca817a0ab8bd00c2dhcantho-haugiang-1468907167921.xls"}, - {"Trường ĐH Luật TP.HCM", "d652d24e8bd4fe6a48d634b116f37c27dhluattphcm-1468939398928.xlsx"}, - {"Trường ĐH Sư phạm Kỹ thuật Vĩnh Long", "4c2fd846ed2a0670ba6d1200d1d68e1dspkythuatvinhlong-1468902640438.xlsx"}, - {"Trường ĐH Trà Vinh", "7418e83ff9d07b146216a3beafa927bbdhtravinh-1468902716174.xlsx"}, - {"Trường ĐH Ngân hàng TP.HCM", "86a692bebf9b0b7c58adb9876b9befd5dhnganhangtphcm-1468902244566.xlsx"}, - {"Trường ĐH Cần Thơ - Bạc Liêu", "5ca241aa1de8d25a3e6ff6db75234025dhcantho-baclieu-1468907071460.xls"}, - {"Trường ĐH Kiên Giang", "cdc912664d7217c75eb9b67c6ef3797adhkiengiang-1468901222446.xlsx"}, - {"Trường ĐH Y Dược Cần Thơ", "023718c7d3cf7ace3a7116fabb12bd9cdhyduoccantho-1468920829104.xlsx"}, - {"Sở GDĐT Hà Nội", "1c66db0265bf45df237b37a7bcaeaef3hanoi-1468932790746.xlsx"}, - {"Sở GDĐT Hà Giang", "178814c33c8b85a77d0fc295fa6d2be1hagiang-1468932847230.xlsx"}, - {"Sở GDĐT Cao Bằng", "ddb631f833af894200ef10cd408d2ef0caobang-1468932877769.xlsx"}, - {"Sở GDĐT Lai Châu", "a3aa106104f114c6426deb2d5e5da5fdlaichau-1468932954079.xlsx"}, - {"Sở GDĐT Lào Cai", "d035f470c13adbf5785797c6ccdcb204laocai-1468919805337.xlsx"}, - {"xem TẠI ĐÂY ) 07 Sở GDĐT Tuyên Quang", "d5ecd2574b5ab36c62dffad435262cb8tuyenquang-1468902296807.xlsx"}, - {"Sở GDĐT Lạng Sơn", "7afd5d7fdff7c170a6342cd8f9474b22langson-1468932984587.xlsx"}, - {"Sở GDĐT Bắc Kạn", "17843bccde10f362c2b9600bc49cd3f8backan-1468933016076.xlsx"}, - {"Sở GDĐT Thái Nguyên", "5fb3a1b66aff077ee3da482be08cd391thainguyen-1468902797020.xlsx"}, - {"Sở GDĐT Yên Bái", "97aa741a4c2b1276559eb021779daa32yenbai-1468933116055.xlsx"}, - {"Sở GDĐT Sơn La", "4b98fdd7300b86a94636e4d15394fc19sonla-1468899319124.xlsx"}, - {"Sở GDĐT Phú Thọ", "5465cca59495dcd9a48f689e2d38b85ephutho-1468902409682.xlsx"}, - {"xem TẠI ĐÂY ) 14 Sở GDĐT Vĩnh Phúc", "4e0ccee19eb64513d23a6dff96ff7f9cvinhphuc-1468919874814.xlsx"}, - {"Sở GDĐT Quảng Ninh", "a31b3446bae5a17c517348d8c38c5b7equangninh-1468919920197.xlsx"}, - {"Sở GDĐT Bắc Giang", "b7a0c1ecce7c8445b3a1021091a5e654bacgiang-1468983153932.xlsx"}, - {"Sở GDĐT Bắc Ninh", "7e0d0b6fb981d407cffdb79d724e8aecbacninh-1468919984998.xlsx"}, - {"Sở GDĐT Hải Dương", "7cfc91bd6a1f5ae9d7917e920f469b90haiduong-1468899640007.xlsx"}, - {"Sở GDĐT Hưng Yên", "92045214159cb0335b2836aa1c4bb25ahungyen-1468899732002.xlsx"}, - {"Sở GDĐT Hoà Bình", "505a876dfd1ea321291e29192aebe95ehoabinh-1468983220125.xlsx"}, - {"Sở GDĐT Hà Nam", "e693bc9de67acc916032c6a3b2fc9eedhanam-1468933146526.xlsx"}, - {"Sở GDĐT Nam Định", "dfa63c5b68a8f638379ca0e9c5a1c556namdinh-1468933234061.xlsx"}, - {"Sở GDĐT Thái Bình", "6960b6fa493e1365a183af461e945389thaibinh-1468920019273.xlsx"}, - {"xem TẠI ĐÂY ) 24 Sở GDĐT Ninh Bình", "0b583c9875443e65ffa449d5ca76fae8ninhbinh-1468899778557.xlsx"}, - {"xem TẠI ĐÂY ) 25 Sở GDĐT Thanh Hoá", "238adc19e80c0daef089a307af23edcethanhhoa-1468933304999.xlsx"}, - {"Sở GDĐT Nghệ An", "7f12b90225e91f9d00aac3bcdf9974e1nghean-1468920043121.xlsx"}, - {"Sở GDĐT Quảng Bình", "9c730c43bf4309171528c3853ed53b19quangbinh-1468933376032.xlsx"}, - {"Sở GDĐT Quảng Trị", "2208903e1fa12220d100f93e40d5ff3bquangtri-1468933440097.xlsx"}, - {"Sở GDĐT Thừa Thiên -Huế", "6badc7cfa258b59ad67e2b64f713b95fthuathienhue-1468983258438.xlsx"}, - {"Sở GDĐT Quảng Nam", "ec765c32190773f5a18cd8202a5eaf0dquangnam-1468933501467.xlsx"}, - {"Sở GDĐT Quảng Ngãi", "e80dc028d53fa49ee247391a0653dee5quangngai-1468933592400.xlsx"}, - {"Sở GDĐT Kon Tum", "cc57dc986acb525086b86e85df54949dkontum-1468933632113.xlsx"}, - {"Sở GDĐT Bình Định", "412e62265f35ca0078a0d94af5d1ed8abinhdinh-1468899884192.xlsx"}, - {"Sở GDĐT Gia Lai", "698b930ab7697a0672bbc39168024c9cgialai-1468933753098.xlsx"}, - {"Sở GDĐT Đắk Lắk", "2004a8d225fd87524e324d2d519e87a4daclac-1468899960505.xlsx"}, - {"Sở GDĐT Khánh Hoà", "4ad416ccf014bd5d01f415238566b210khanhhoa-1468933789968.xlsx"}, - {"Sở GDĐT Lâm Đồng", "2fa6d9a308bd11bc09f02d787eef2c15lamdong-1468933821082.xlsx"}, - {"Sở GDĐT Ninh Thuận", "15c85d901a73dd49f2bae71aadcdbb5cninhthuan-1468920071763.xlsx"}, - {"Sở GDĐT Đồng Nai", "538c9676b70f2509573245d58fba93bcdongnai-1468938241935.xlsx"}, - {"Sở GDĐT Đồng Tháp", "575df8a4df74e87abb8f55d807324374dongthap-1468933902699.xlsx"}, - {"Sở GDĐT Kiên Giang", "7435bb054e8e4ec34c173af1b6ad8402kiengiang-1468900147395.xlsx"}, - {"Sở GDĐT Cần Thơ", "5b16d7156a03bb7ed0618796bae87bc3cantho-1468933954714.xlsx"}, - {"Sở GDĐT Bến Tre", "74145979ee30186bf4e40ee3e6a3ac74bentre-1468934010040.xlsx"}, - {"Sở GDĐT Vĩnh Long", "7f4362ef567cdf74495368e0fc11072avinhlong-1468934067981.xlsx"}, - {"Sở GDĐT Trà Vinh", "08cdeb3636dcb2adaef829d62968274atravinh-1468902443859.xlsx"}, - {"Sở GDĐT Sóc Trăng", "a8ff80d9478747e36d21ec95cc9ecb8fsoctrang-1468902686409.xlsx"}, - {"Sở GDĐT Bạc Liêu", "18055d17e46854ecc6c8be585bcb6e5cbaclieu-1468934144150.xlsx"}, - {"Sở GDĐT Điện Biên", "3c0e56abd8c5334f4e1ff7df3ac23a15dienbien-1468920113820.xlsx"}, - {"Sở GDĐT Đăk Nông", "bf259a908ff8a8dcc067a761204459f4daknong-1468934173803.xlsx"}, - {"Sở GDĐT Hậu Giang", "964c4368131f6be6058fabb1caefba1chaugiang-1468934247324.xlsx"}} diff --git a/crawler/internal/sources/source_2017.go b/crawler/internal/sources/source_2017.go index c3a8738..4f9d088 100644 --- a/crawler/internal/sources/source_2017.go +++ b/crawler/internal/sources/source_2017.go @@ -1,121 +1,70 @@ package sources import ( + "fmt" "regexp" "strings" + + "golang.org/x/text/unicode/norm" + + "github.com/tiennm99/thptqg/crawler/internal/article" ) -// article is the page the links were taken from. The CDN rejects requests that -// do not carry it as a Referer. -const article = "https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm" - -// browserUA is required too: the CDN 403s an unrecognised User-Agent. +// browserUA and the Referer are both required: the CDN 403s an unrecognised +// User-Agent, and rejects requests that do not carry the article as Referer. const browserUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" -// source2017 fetches the 2017 dataset from the baotintuc.vn CDN: one .xls per -// province. +const article2017 = "https://baotintuc.vn/tuyen-sinh/tra-cuu-diem-thi-thpt-2017-cua-63-tinh-thanh-pho-tren-baotintucvn-20170706073512672.htm" + +// source2017 fetches the 2017 dataset: one .xls per province, still served from +// the CDN that originally published it. // -// This is the only dataset that can be re-fetched from its original source. -// The CDN filenames are inconsistent (Angiang.xls, 1BaRiaVungTau.xls, -// 23HaiPhong.xls, gia-lai.xls), so local names are derived from the province -// name instead — which is what produced the data/2017/ files currently on disk, -// and what any re-crawl must keep producing. +// Dest cannot use the CDN's filenames the way 2016 does, because they are +// inconsistent — Angiang.xls, 1BaRiaVungTau.xls, 23HaiPhong.xls, Gia-Lai.xls — +// carrying upload-order prefixes and arbitrary capitalisation. The province name +// in the link text is the stable identifier, so the local name is derived from +// that instead, which is what produced the files in data/2017/. var source2017 = Source{ - ID: "2017", - Summary: "63 province .xls files from the baotintuc.vn CDN", + ID: "2017", + Summary: "63 province .xls files from the baotintuc.vn CDN", + Article: article2017, + Exts: []string{".xls"}, + WantFiles: 63, Headers: map[string]string{ "User-Agent": browserUA, - "Referer": article, + "Referer": article2017, }, - Files: func() ([]File, error) { - out := make([]File, 0, len(provinces2017)) - for _, p := range provinces2017 { - out = append(out, File{ - Name: p.name, - URL: p.url, - Dest: slug(p.name) + ".xls", - }) + Dest: func(l article.Link) (string, error) { + s := slug(l.Text) + if s == "" { + return "", fmt.Errorf("link %s has no usable label to name it by", l.URL) } - return out, nil + return s + ".xls", nil }, } var nonAlphanumeric = regexp.MustCompile(`[^a-z0-9]+`) -// slug converts a province name to its local filename stem: lowercase, with -// every run of non-alphanumeric characters collapsed to a single hyphen. +// slug turns a province name into a filename stem: transliterated to ASCII, +// lowercased, with every run of other characters collapsed to one hyphen. // -// "Ba Ria - Vung Tau" -> "ba-ria-vung-tau" +// "Bà Rịa -Vũng Tàu" -> "ba-ria-vung-tau" +// "Bắc Kạn" -> "bac-kan" // -// The names are already unaccented ASCII, so no transliteration is involved. +// The article writes the names with diacritics, so this has to strip them. +// Combining marks are removed by the literal range U+0300–U+036F rather than by +// the unicode.Mn category, matching what go-parser does to build ho_ten_ascii. func slug(name string) string { - return strings.Trim(nonAlphanumeric.ReplaceAllString(strings.ToLower(name), "-"), "-") -} - -type province struct{ name, url string } - -var provinces2017 = []province{ - {"An Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/17/57/Angiang.xls"}, - {"Bac Lieu", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Baclieu.xls"}, - {"Ba Ria - Vung Tau", "https://cdnmedia.baotintuc.vn/2017/07/06/08/17/1BaRiaVungTau.xls"}, - {"Bac Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacGiang.xls"}, - {"Bac Kan", "https://cdnmedia.baotintuc.vn/2017/07/06/08/27/BacKan.xls"}, - {"Bac Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/33/BacNinh.xls"}, - {"Ben Tre", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BenTre.xls"}, - {"Binh Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/34/BinhDuong.xls"}, - {"Binh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/BinhThuan.xls"}, - {"Binh Phuoc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/BinhPhuoc.xls"}, - {"Binh Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Binhdinh.xls"}, - {"Ca Mau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/05/Camau.xls"}, - {"Cao Bang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Caobang.xls"}, - {"Can Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/13/07/Cantho.xls"}, - {"Da Nang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/03/Danang.xls"}, - {"Dak Nong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/35/DakNong.xls"}, - {"Dak Lak", "https://cdnmedia.baotintuc.vn/2017/07/06/18/02/Daklak.xls"}, - {"Dong Nai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/08/dongnai.xls"}, - {"Dong Thap", "https://cdnmedia.baotintuc.vn/2017/07/06/17/59/Dongthap.xls"}, - {"Dien Bien", "https://cdnmedia.baotintuc.vn/2017/07/06/09/08/DienBien.xls"}, - {"Gia Lai", "https://cdnmedia.baotintuc.vn/2017/07/06/13/09/Gia-Lai.xls"}, - {"Ha Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/18/Hagiang.xls"}, - {"Ha Noi", "https://cdnmedia.baotintuc.vn/2017/07/07/08/16/HaNoi.xls"}, - {"Ha Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/07/Hanam.xls"}, - {"Ha Tinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaTinh.xls"}, - {"Hai Phong", "https://cdnmedia.baotintuc.vn/2017/07/06/08/18/23HaiPhong.xls"}, - {"Hai Duong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/36/HaiDuong.xls"}, - {"Hau Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Haugiang.xls"}, - {"Ho Chi Minh", "https://cdnmedia.baotintuc.vn/2017/07/06/08/26/HCM.xls"}, - {"Hoa Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/06/HoaBinh.xls"}, - {"Hung Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/08/25/HungYen.xls"}, - {"Khanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/18/04/Khanhhoa.xls"}, - {"Kien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/08/28/KienGiang.xls"}, - {"Kon Tum", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/KonTum.xls"}, - {"Nam Dinh", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/13NamDinh.xls"}, - {"Nghe An", "https://cdnmedia.baotintuc.vn/2017/07/06/07/55/14NgheAn.xls"}, - {"Ninh Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/37/NinhBinh.xls"}, - {"Ninh Thuan", "https://cdnmedia.baotintuc.vn/2017/07/06/18/00/Ninhthuan.xls"}, - {"Lao Cai", "https://cdnmedia.baotintuc.vn/2017/07/06/08/52/11LaoCai.xls"}, - {"Lai Chau", "https://cdnmedia.baotintuc.vn/2017/07/06/13/33/LaiChau.xls"}, - {"Lang Son", "https://cdnmedia.baotintuc.vn/2017/07/06/18/05/Langson.xls"}, - {"Lam Dong", "https://cdnmedia.baotintuc.vn/2017/07/06/09/00/LamDong.xls"}, - {"Long An", "https://cdnmedia.baotintuc.vn/2017/07/06/08/53/12LongAn.xls"}, - {"Quang Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/QuangBinh.xls"}, - {"Quang Nam", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNam.xls"}, - {"Quang Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Quangninh.xls"}, - {"Quang Ngai", "https://cdnmedia.baotintuc.vn/2017/07/06/09/24/QuangNgai.xls"}, - {"Quang Tri", "https://cdnmedia.baotintuc.vn/2017/07/06/09/25/QuangTri.xls"}, - {"Phu Tho", "https://cdnmedia.baotintuc.vn/2017/07/06/09/23/PhuTho.xls"}, - {"Phu Yen", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/PhuYen.xls"}, - {"Son La", "https://cdnmedia.baotintuc.vn/2017/07/06/18/01/Sonla.xls"}, - {"Soc Trang", "https://cdnmedia.baotintuc.vn/2017/07/06/18/06/Soctrang.xls"}, - {"Tay Ninh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/26/TayNinh.xls"}, - {"Thai Binh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/09/ThaiBinh.xls"}, - {"Thai Nguyen", "https://cdnmedia.baotintuc.vn/2017/07/06/13/35/ThaiNguyen.xls"}, - {"Thanh Hoa", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thanhoa.xls"}, - {"Tra Vinh", "https://cdnmedia.baotintuc.vn/2017/07/06/09/38/TraVinh.xls"}, - {"Thua Thien Hue", "https://cdnmedia.baotintuc.vn/2017/07/06/13/10/thuathienhue.xls"}, - {"Tien Giang", "https://cdnmedia.baotintuc.vn/2017/07/06/13/11/tiengiang.xls"}, - {"Tuyen Quang", "https://cdnmedia.baotintuc.vn/2017/07/06/09/39/TuyenQuang.xls"}, - {"Vinh Phuc", "https://cdnmedia.baotintuc.vn/2017/07/06/09/40/VinhPhuc.xls"}, - {"Vinh Long", "https://cdnmedia.baotintuc.vn/2017/07/06/13/13/vinhlong.xls"}, - {"Yen Bai", "https://cdnmedia.baotintuc.vn/2017/07/06/18/07/Yenbai.xls"}, + var b strings.Builder + for _, r := range norm.NFD.String(name) { + switch { + case r >= 0x0300 && r <= 0x036F: + // combining mark: drop + case r == 'đ' || r == 'Đ': + b.WriteByte('d') + default: + b.WriteRune(r) + } + } + return strings.Trim(nonAlphanumeric.ReplaceAllString(strings.ToLower(b.String()), "-"), "-") } diff --git a/crawler/internal/sources/sources.go b/crawler/internal/sources/sources.go index 0efafc7..c18da33 100644 --- a/crawler/internal/sources/sources.go +++ b/crawler/internal/sources/sources.go @@ -1,25 +1,16 @@ -// Package sources describes where each dataset's spreadsheets come from. +// Package sources says where each dataset's spreadsheets are published and +// what to call them locally. // -// A source is a dataset id plus the list of remote files that fill it. -// Downloading is internal/fetch's job; deciding what to download and what to -// call it locally is this package's. +// It carries no link lists. Each source names the article that published the +// files; internal/article reads the links out of that page at run time, and +// internal/fetch moves the bytes. package sources -import "fmt" +import ( + "fmt" -// File is one remote spreadsheet. -type File struct { - // Name is the human label used in progress output, e.g. "An Giang". - Name string - URL string - // Dest is the filename within the dataset directory. - // - // This is not cosmetic. go-parser sorts its input files and inserts with - // INSERT OR REPLACE, which is last-wins, so filenames decide which row - // survives a duplicate exam number. A re-crawl that names files differently - // can produce a database with the same row count and different content. - Dest string -} + "github.com/tiennm99/thptqg/crawler/internal/article" +) // Source is one crawlable dataset. type Source struct { @@ -28,12 +19,73 @@ type Source struct { // cannot be pointed at the wrong dataset's directory. ID string Summary string - // Headers are sent with every request for this source. + + // Article is the published page listing this dataset's files. + Article string + + // Exts are the file extensions to pick out of that page. + Exts []string + + // Headers are sent with every request for this source, for the article and + // the files alike. Headers map[string]string - // Files lists what to download. It returns an error rather than an empty - // list when a source is known but not yet configured, so "nothing to do" - // can never be mistaken for success. - Files func() ([]File, error) + + // WantFiles is how many links the article is expected to yield. A page that + // suddenly yields fewer has changed shape, and silently crawling a partial + // dataset is the failure this exists to prevent — go-parser would happily + // build a short database and only the row-count guard would catch it, after + // the fact. + WantFiles int + + // Dest names the local file for one discovered link. + // + // This is the load-bearing part. go-parser sorts input files bytewise and + // inserts last-wins, so the names chosen here decide which row survives a + // duplicate exam number. Two sources answer it differently and both have a + // reason: see source_2016.go and source_2017.go. + Dest func(article.Link) (string, error) +} + +// File is one spreadsheet to download. +type File struct { + // Name is the human label used in progress output. + Name string + URL string + // Dest is the filename within the dataset directory. + Dest string +} + +// Resolve turns the links found on a source's article into the files to fetch, +// enforcing the count and rejecting any two links that would write to the same +// name — which would silently cost the dataset a file. +func (s Source) Resolve(links []article.Link) ([]File, error) { + if len(links) != s.WantFiles { + return nil, fmt.Errorf( + "%s: found %d links on %s, expected %d — the page has changed shape; "+ + "check it before crawling, a partial dataset builds without complaint", + s.ID, len(links), s.Article, s.WantFiles) + } + + out := make([]File, 0, len(links)) + seen := make(map[string]string, len(links)) + for _, l := range links { + dest, err := s.Dest(l) + if err != nil { + return nil, fmt.Errorf("%s: %w", s.ID, err) + } + if prev, dup := seen[dest]; dup { + return nil, fmt.Errorf("%s: %q and %q both name the local file %q", + s.ID, prev, l.URL, dest) + } + seen[dest] = l.URL + + name := l.Text + if name == "" { + name = dest + } + out = append(out, File{Name: name, URL: l.URL, Dest: dest}) + } + return out, nil } // registry is ordered: it drives the help output. diff --git a/crawler/internal/sources/sources_test.go b/crawler/internal/sources/sources_test.go index dca1379..b4586f8 100644 --- a/crawler/internal/sources/sources_test.go +++ b/crawler/internal/sources/sources_test.go @@ -1,69 +1,62 @@ package sources import ( + "compress/gzip" "os" "path/filepath" "strings" "testing" + + "github.com/tiennm99/thptqg/crawler/internal/article" ) -func TestSlug(t *testing.T) { - for in, want := range map[string]string{ - "An Giang": "an-giang", - "Ba Ria - Vung Tau": "ba-ria-vung-tau", - "Ho Chi Minh": "ho-chi-minh", - "Thua Thien Hue": "thua-thien-hue", - } { - if got := slug(in); got != want { - t.Errorf("slug(%q) = %q, want %q", in, got, want) - } - } -} - -func TestSource2017IsComplete(t *testing.T) { - files, err := source2017.Files() +// The fixtures are saved copies of the two source articles, kept so the link +// extraction and the naming rules can be exercised without the network. +// +// article-2017 is the live page. article-2016 came from the Internet Archive's +// copy of the site that first published that list (dtntbacgiang.edu.vn, which +// no longer resolves); the mirror the crawler actually reads carries the same +// article. Its hrefs are relative, so they resolve against whichever host the +// source names, and the filenames are identical either way. +func fixture(t *testing.T, id string) *gzip.Reader { + t.Helper() + f, err := os.Open(filepath.Join("testdata", "article-"+id+".html.gz")) if err != nil { t.Fatal(err) } - if len(files) != 63 { - t.Errorf("got %d provinces, want 63", len(files)) - } - - // A duplicate Dest would silently overwrite one province with another and - // leave the dataset one file short, with no error anywhere. - seenDest := map[string]string{} - seenURL := map[string]string{} - for _, f := range files { - if prev, dup := seenDest[f.Dest]; dup { - t.Errorf("%q and %q both write to %s", prev, f.Name, f.Dest) - } - seenDest[f.Dest] = f.Name - if prev, dup := seenURL[f.URL]; dup { - t.Errorf("%q and %q share a URL: %s", prev, f.Name, f.URL) - } - seenURL[f.URL] = f.Name - - if !strings.HasPrefix(f.URL, "https://") { - t.Errorf("%s: URL is not https: %s", f.Name, f.URL) - } - if !strings.HasSuffix(f.Dest, ".xls") { - t.Errorf("%s: Dest %q does not end in .xls", f.Name, f.Dest) - } + t.Cleanup(func() { f.Close() }) + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) } + t.Cleanup(func() { gz.Close() }) + return gz } -// TestMatchesFilesOnDisk is the guard that matters. +// resolveFixture runs a source's full pipeline over its saved article. +func resolveFixture(t *testing.T, src Source) []File { + t.Helper() + links, err := article.Extract(src.Article, fixture(t, src.ID), src.Exts...) + if err != nil { + t.Fatal(err) + } + files, err := src.Resolve(links) + if err != nil { + t.Fatal(err) + } + return files +} + +// TestReproducesFilesOnDisk is the guard that matters. // // go-parser sorts its inputs and inserts last-wins, so filenames decide which -// row survives a duplicate exam number. If a re-crawl started naming files -// differently, the rebuilt database could differ in content while still passing -// the row-count guard in build-db.js. The committed data// is the oracle: -// whatever a source would write must match it exactly, in both directions. +// row survives a duplicate exam number. If extraction or naming drifted, a +// re-crawl could rebuild a database with the same row count and different +// content, which the row-count guard in build-db.js would not catch. // -// For 2016 this is also the only evidence that the recovered link list is the -// right one — the host that served those files is gone, so the list cannot be -// checked by fetching it. -func TestMatchesFilesOnDisk(t *testing.T) { +// The committed data// is the oracle: reading the source article and +// applying the source's naming rule must reproduce it exactly, both directions. +func TestReproducesFilesOnDisk(t *testing.T) { for _, src := range All() { t.Run(src.ID, func(t *testing.T) { dir := filepath.Join("..", "..", "..", "data", src.ID) @@ -74,7 +67,6 @@ func TestMatchesFilesOnDisk(t *testing.T) { if err != nil { t.Fatal(err) } - onDisk := map[string]bool{} for _, e := range entries { if !e.IsDir() { @@ -82,11 +74,7 @@ func TestMatchesFilesOnDisk(t *testing.T) { } } - files, err := src.Files() - if err != nil { - t.Fatal(err) - } - for _, f := range files { + for _, f := range resolveFixture(t, src) { if !onDisk[f.Dest] { t.Errorf("crawler would write %q, which is not in %s — "+ "a renamed input can change which duplicate row survives", f.Dest, dir) @@ -94,29 +82,106 @@ func TestMatchesFilesOnDisk(t *testing.T) { delete(onDisk, f.Dest) } for name := range onDisk { - t.Errorf("%s/%s exists but no source produces it", dir, name) + t.Errorf("%s/%s exists but the article produces no link for it", dir, name) } }) } } +// TestResolvedURLsAreAbsolute: 2016's article uses relative hrefs, so a +// mis-resolved base would produce URLs that cannot be fetched at all. +func TestResolvedURLsAreAbsolute(t *testing.T) { + for _, src := range All() { + t.Run(src.ID, func(t *testing.T) { + for _, f := range resolveFixture(t, src) { + if !strings.HasPrefix(f.URL, "https://") { + t.Errorf("%s: %q is not an absolute https URL", f.Dest, f.URL) + } + } + }) + } +} + +func TestExtensionsMatchWhatIsExpected(t *testing.T) { + counts := map[string]map[string]int{} + for _, src := range All() { + c := map[string]int{} + for _, f := range resolveFixture(t, src) { + c[filepath.Ext(f.Dest)]++ + } + counts[src.ID] = c + } + if got := counts["2016"]; got[".xls"] != 4 || got[".xlsx"] != 115 { + t.Errorf("2016: %d .xls and %d .xlsx, want 4 and 115", got[".xls"], got[".xlsx"]) + } + if got := counts["2017"]; got[".xls"] != 63 { + t.Errorf("2017: %d .xls, want 63", got[".xls"]) + } +} + +// TestResolveRejectsShortPage: a page that has changed shape must stop the +// crawl, not quietly produce a partial dataset. +func TestResolveRejectsShortPage(t *testing.T) { + src := source2017 + _, err := src.Resolve([]article.Link{{URL: "https://x/a.xls", Text: "An Giang", File: "a.xls"}}) + if err == nil { + t.Fatal("expected an error for a short link list") + } + if !strings.Contains(err.Error(), "expected 63") { + t.Errorf("error should name the expected count, got: %v", err) + } +} + +// TestResolveRejectsCollidingNames: two links naming the same local file would +// cost the dataset a file with no error anywhere downstream. +func TestResolveRejectsCollidingNames(t *testing.T) { + src := Source{ + ID: "t", WantFiles: 2, + Dest: func(article.Link) (string, error) { return "same.xls", nil }, + } + _, err := src.Resolve([]article.Link{ + {URL: "https://x/a.xls", File: "a.xls"}, + {URL: "https://x/b.xls", File: "b.xls"}, + }) + if err == nil || !strings.Contains(err.Error(), "same.xls") { + t.Errorf("expected a collision error naming the file, got: %v", err) + } +} + +func TestSlug(t *testing.T) { + // The article writes province names with diacritics; the committed + // filenames are ASCII. These are the awkward ones. + for in, want := range map[string]string{ + "An Giang": "an-giang", + "Bạc Liêu": "bac-lieu", + "Bắc Kạn": "bac-kan", + "Bà Rịa -Vũng Tàu": "ba-ria-vung-tau", + "Thừa Thiên - Huế": "thua-thien-hue", + "Đắk Lắk": "dak-lak", + } { + if got := slug(in); got != want { + t.Errorf("slug(%q) = %q, want %q", in, got, want) + } + } +} + func TestLookup(t *testing.T) { for _, s := range All() { got, err := Lookup(s.ID) if err != nil { t.Errorf("Lookup(%q): %v", s.ID, err) } - if got.ID != s.ID || got.Summary == "" { - t.Errorf("Lookup(%q) returned %+v", s.ID, got) + if got.ID != s.ID || got.Article == "" || got.WantFiles == 0 { + t.Errorf("Lookup(%q) returned an incomplete source: %+v", s.ID, got) } } if _, err := Lookup("nope"); err == nil { - t.Error("Lookup of an unknown source must fail") + t.Error("Lookup of an unknown dataset must fail") } } // TestIDsAreDatasetIDs: the ID doubles as the directory under data/, so a typo -// would send a crawl into a new directory the parser never reads. +// would send a crawl into a directory the parser never reads. func TestIDsAreDatasetIDs(t *testing.T) { for _, s := range All() { dir := filepath.Join("..", "..", "..", "data", s.ID) @@ -125,32 +190,3 @@ func TestIDsAreDatasetIDs(t *testing.T) { } } } - -// TestSource2016IsComplete: 4 .xls + 115 .xlsx, matching what the source -// article published and what data/2016/ holds. -func TestSource2016IsComplete(t *testing.T) { - files, err := source2016.Files() - if err != nil { - t.Fatal(err) - } - if len(files) != 119 { - t.Errorf("got %d clusters, want 119", len(files)) - } - var xls, xlsx int - for _, f := range files { - switch filepath.Ext(f.Dest) { - case ".xls": - xls++ - case ".xlsx": - xlsx++ - default: - t.Errorf("%s: unexpected extension in %q", f.Name, f.Dest) - } - if !strings.HasPrefix(f.URL, baseURL+uploadPath) { - t.Errorf("%s: URL does not use the configured base: %s", f.Name, f.URL) - } - } - if xls != 4 || xlsx != 115 { - t.Errorf("got %d .xls and %d .xlsx, want 4 and 115", xls, xlsx) - } -} diff --git a/crawler/internal/sources/testdata/article-2016.html.gz b/crawler/internal/sources/testdata/article-2016.html.gz new file mode 100644 index 0000000..3c60dc0 Binary files /dev/null and b/crawler/internal/sources/testdata/article-2016.html.gz differ diff --git a/crawler/internal/sources/testdata/article-2017.html.gz b/crawler/internal/sources/testdata/article-2017.html.gz new file mode 100644 index 0000000..b8b87d4 Binary files /dev/null and b/crawler/internal/sources/testdata/article-2017.html.gz differ diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index b88c711..c842ed8 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -21,7 +21,7 @@ runs are idempotent: files already present are skipped. ```bash npm run crawl:2016 # crawler/internal/sources/source_2016.go npm run crawl:2017 # crawler/internal/sources/source_2017.go -npm run crawl:2017 -- --list # show what would be downloaded, fetch nothing +npm run crawl:2017 -- --list # read the article, list the files, download none ``` **2017** comes from the article @@ -29,19 +29,36 @@ npm run crawl:2017 -- --list # show what would be downloaded, fetch nothing and its CDN is still serving the files. **2016** comes from the aggregator article -`cong-bo-diem-thi-thptqg-2016-toan-bo-120-cum-thi-da-co-diem.html`. The site -that originally published it (`dtntbacgiang.edu.vn`) no longer resolves, so the -link list was read from the Internet Archive's copy of the page. Two caveats -that matter: +`cong-bo-diem-thi-thptqg-2016-toan-bo-120-cum-thi-da-co-diem.html`, served from +a mirror — the site that first published it (`dtntbacgiang.edu.vn`) no longer +resolves. -- **The list is verified; the host is not.** All 119 filenames match `data/2016/` - exactly in both directions, which is asserted by a test. But the Archive - captured only the article, not the spreadsheets, so the URLs themselves could - not be confirmed by fetching. `baseURL` in `source_2016.go` points at a mirror - of the same article that is still online; if it serves the files from a - different directory, `uploadPath` is the one constant to change. -- **`data/2016/` is still the only confirmed copy.** Do not delete it on the - assumption that a crawl can restore it. +That mirror is **not reachable from every network.** It resolves to a Vietnamese +address that times out from at least some hosts abroad, in which case the crawl +stops with a connection error before downloading anything. `data/2016/` is +therefore still the only confirmed copy: do not delete it on the assumption that +a crawl can restore it. + +## How a source is defined + +A source carries no link list. It names the article that published the files and +says how to name what it finds there: + +| field | meaning | +| --- | --- | +| `Article` | the page to read links from | +| `Exts` | which file extensions to pick out of it | +| `WantFiles` | how many links to expect — fewer aborts the crawl | +| `Dest` | the local filename for one link | + +`internal/article` does the fetching and HTML parsing; `internal/fetch` does the +downloading. Because `Article` is read at run time, `--list` needs network access +too. + +`WantFiles` exists because a partial crawl is otherwise silent: go-parser will +build a short database from whatever files are present, and only the row-count +guard would notice, after the fact. A page that changes shape stops the crawl +instead. ### Local filenames are load-bearing @@ -56,13 +73,14 @@ Each source therefore pins its local names, and committed `data//` in both directions — every name it would write exists, and every file present is accounted for. -2017 derives its names from the province (the CDN's own names are inconsistent: +2017 derives its names from the province name in the link text, transliterated +to ASCII (the CDN's own names are inconsistent: `Angiang.xls`, `1BaRiaVungTau.xls`, `23HaiPhong.xls`). 2016 keeps the server-assigned names verbatim, since it did not choose them. ## Source Excel shapes -The 2017 datasets share one layout: +2017 has one layout: | Col | Name | Content | | --- | --- | --- | @@ -188,8 +206,12 @@ repo was unified (a hardcoded Windows path in one, an undeclared The latter two are superseded by `differential-parity.mjs`, which compares more and cannot silently skip a dataset. -`crawl-baotintuc.js` was not dropped but rewritten in Go as -`crawler/internal/sources/source_2017.go`, keeping the same link list and the same -local filenames. The rewrite added a per-file timeout and made downloads land on -a `.part` file first — writing straight to the destination meant an interrupted -run left a truncated file that the skip check would then skip forever. +`crawl-baotintuc.js` was not dropped but rewritten as the Go `crawler/` module, +producing the same local filenames. Two changes beyond the port: + +- It carried its 63 links as a hardcoded array. The Go version reads them from + the article instead, so the list cannot drift from what was published. +- Downloads land on a `.part` file and are renamed on completion. Writing + straight to the destination left a truncated file after an interrupted run, + and since the skip check only tests for a non-empty file, every later run + would skip it — the corruption was permanent and silent.