diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index b52db48..50eb934 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -96,12 +96,35 @@ jobs: GOVULNCHECK="$(go env GOPATH)/bin/govulncheck" for m in parser crawler assembler; do (cd "$m" && "$GOVULNCHECK" ./...); done - # One command runs the whole pipeline: compile the parser, build and - # verify each database against its registry row count and size, build the - # web app, and assemble _site — refusing to continue if a database is - # short, an artifact looks truncated, or one is missing entirely. + # Parsing 348 MB of spreadsheets is the slowest part of the job, and its + # output is a pure function of the inputs in the key: the source files, + # the parser (schema, transforms, configs) and the registry the row-count + # and size guards read. Anything else — web code, docs, workflow — cannot + # change a database, so those pushes restore one instead of rebuilding. + # + # Exact matches only, with no restore-keys: a near-miss here would mean + # publishing a database built from different inputs than the commit says. + - name: Cache the built databases + id: db-cache + uses: actions/cache@v4 + with: + path: .build/public/db + key: db-${{ hashFiles('data/**', 'parser/**', 'datasets.json') }} + + # The whole pipeline: compile the parser, build and verify each database + # against its registry row count and size, build the web app, and + # assemble _site — refusing to continue if a database is short, an + # artifact looks truncated, or one is missing entirely. On a cache hit + # the database step is skipped; the artifact it would have produced was + # verified by the run that built and stored it. - name: Build site - run: go -C assembler run ./cmd/assemble + run: | + if [ "${{ steps.db-cache.outputs.cache-hit }}" = "true" ]; then + echo "Databases restored from cache; building the site only." + go -C assembler run ./cmd/assemble site + else + go -C assembler run ./cmd/assemble + fi - uses: actions/upload-pages-artifact@v5 with: diff --git a/CLAUDE.md b/CLAUDE.md index 4c06bb4..e0043fd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,8 +64,13 @@ hashes every real input file. That is the point of it; do not skip it. dataset page is gated behind that download: `download-gate.svelte` states the transfer and memory cost and offers nothing but the download, because there is no answer to give without it. `sql.js` holds the file in WebAssembly - memory for as long as the tab is open, so it is RAM, not disk, and it is gone - on reload. + memory for as long as the tab is open, so the memory figure is RAM, not disk. +- **The download is kept in Cache Storage, versioned by ETag** (`db-cache.js`). + A later visit opens the stored copy without asking, since consent was given + once and reuse costs no network; a redeploy changes the ETag, so the new + version replaces the old rather than being served stale. When the server + cannot be reached at all, any stored version is used — which is what lets the + site answer offline. - **The schema carries no secondary indexes, deliberately.** An index saves a scan that already takes a few hundred milliseconds in memory, and costs every visitor megabytes of download. An earlier design read the file over HTTP diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 0d6412f..8b8b8f7 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -10,12 +10,17 @@ One-time setup: **Settings → Pages → Source: GitHub Actions**. 1. Checkout, Go toolchain, Node 24, `npm ci` in `web/` 2. Parser, crawler and assembler test suites, web lint, `govulncheck` over all three modules 3. `go -C assembler run ./cmd/assemble` — the whole pipeline: compile the - parser, build and verify each database, compress it into `.build/public/db/`, - run the web build, assemble `_site/` + parser, build and verify each database into `.build/public/db/`, run the web + build, assemble `_site/`. The databases are restored from the Actions cache + when nothing that determines them has changed, and only the site is built 4. `actions/upload-pages-artifact` + `actions/deploy-pages` -The database build dominates the runtime: roughly 348 MB of Excel is parsed on -every deploy. +The database build dominates the runtime: roughly 348 MB of Excel to parse. It +is cached in Actions, keyed on `data/**`, `parser/**` and `datasets.json`, so +only a change to the sources, the parser or the registry pays for it — a web or +docs change restores the databases instead. There are no `restore-keys`: a +near-miss would publish databases built from inputs the commit does not +describe. ## Resulting URLs diff --git a/web/src/lib/components/download-gate.svelte b/web/src/lib/components/download-gate.svelte index 686e6cb..5c6443d 100644 --- a/web/src/lib/components/download-gate.svelte +++ b/web/src/lib/components/download-gate.svelte @@ -11,7 +11,7 @@ * * It has no dismiss: without the database the page has no answers to give. */ - let { db, transferBytes = null, onDownload } = $props(); + let { db, onDownload } = $props(); const percent = $derived(Math.round(db.progress * 100)); @@ -22,7 +22,7 @@ aria-modal="true" aria-labelledby="gate-title" > -
+

Tải cơ sở dữ liệu

@@ -34,7 +34,7 @@

Dung lượng tải
- {transferBytes ? formatBytes(transferBytes) : "~31 MB"} + {db.transferBytes ? formatBytes(db.transferBytes) : "~31 MB"}
@@ -44,8 +44,8 @@

- Dữ liệu được giữ trong bộ nhớ của tab và sẽ mất khi bạn đóng trang, nên lần truy cập sau cần - tải lại. Không nên dùng trên thiết bị có ít bộ nhớ. + Sau khi tải, dữ liệu được lưu lại trên máy nên các lần sau không phải tải lại. Khi tra cứu, + dữ liệu nằm trong bộ nhớ (RAM) của tab, vì vậy không nên dùng trên thiết bị có ít bộ nhớ.

{#if db.error} diff --git a/web/src/lib/database.svelte.js b/web/src/lib/database.svelte.js index 325c6ca..476b725 100644 --- a/web/src/lib/database.svelte.js +++ b/web/src/lib/database.svelte.js @@ -1,3 +1,4 @@ +import { keepOnly, matchAny, matchVersion, openCache } from "./db-cache.js"; import initSqlJs from "sql.js"; import wasmUrl from "sql.js/dist/sql-wasm.wasm?url"; @@ -15,6 +16,10 @@ import wasmUrl from "sql.js/dist/sql-wasm.wasm?url"; * whole file sits in the tab's address space for as long as the page is open. * That is what the gate on the dataset page tells the visitor before they * commit to it. + * + * The transfer is paid once per device rather than once per visit: the + * response is kept in Cache Storage, versioned by ETag so that a redeploy + * replaces it instead of being served stale. See db-cache.js. */ /** Rows to return before a query is considered too broad to render. */ @@ -46,14 +51,47 @@ export class LocalDatabase { received = $state(0); /** How long the download and open took, once done. */ ms = $state(0); + /** True once the cache has been consulted, so the gate knows what to ask for. */ + checked = $state(false); + /** True when this copy came off the disk rather than the network. */ + fromCache = $state(false); + /** Compressed size of the download, as the server reports it. */ + transferBytes = $state(null); #db = null; + #etag = null; constructor(url, expectedBytes) { this.url = url; this.expectedBytes = expectedBytes; } + /** + * Ask the server what it holds, and the disk what we kept. + * + * A copy already on the device costs no network and needs no consent, so it + * opens straight away. Otherwise this only works out what a download would + * cost, and the gate waits for the visitor to accept it. + */ + async prepare() { + const head = await describe(this.url); + this.transferBytes = head?.bytes ?? null; + this.#etag = head?.etag ?? null; + + const cache = await openCache(); + // Without an ETag the server is unreachable; any stored version beats an + // error page, since these are frozen exam results. + const stored = this.#etag + ? await matchVersion(cache, this.url, this.#etag) + : await matchAny(cache, this.url); + + this.checked = true; + if (stored) { + this.fromCache = true; + await this.open(stored); + } + } + /** Fraction downloaded, 0 to 1, for the progress bar. */ get progress() { if (!this.expectedBytes) return 0; @@ -67,7 +105,7 @@ export class LocalDatabase { * sees movement: this is tens of megabytes on a phone connection, and a * progress bar is the difference between waiting and giving up. */ - async open() { + async open(stored = null) { if (this.ready || this.loading) return; this.loading = true; this.error = null; @@ -75,7 +113,10 @@ export class LocalDatabase { const started = performance.now(); try { - const [SQL, response] = await Promise.all([sqlEngine(), fetch(this.url)]); + const [SQL, response] = await Promise.all([ + sqlEngine(), + stored ?? fetchAndCache(this.url, this.#etag), + ]); if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); const reader = response.body.getReader(); @@ -133,22 +174,45 @@ export class LocalDatabase { } /** - * What the download will cost on the wire, from the server's own headers. + * What the file weighs on the wire, and which version it is. * - * The browser advertises gzip, and GitHub Pages compresses this file, so this - * reports the compressed size — the number that matters to someone on mobile - * data. Returns null when the server does not say. + * The browser advertises gzip and GitHub Pages compresses this file, so the + * length is the compressed one — the number that matters to someone on mobile + * data. The ETag is what keeps a stored copy honest across deploys. Returns + * null when the server cannot be reached, which is survivable: a stored copy + * still opens. */ -export async function weigh(url) { +async function describe(url) { try { const response = await fetch(url, { method: "HEAD" }); + if (!response.ok) return null; const length = Number(response.headers.get("Content-Length")); - return Number.isFinite(length) && length > 0 ? length : null; + return { + bytes: Number.isFinite(length) && length > 0 ? length : null, + etag: response.headers.get("ETag"), + }; } catch { return null; } } +/** + * Fetch the database, handing a second copy of the response to the cache. + * + * The clone is what gets stored, so the bytes stream to disk while the caller + * reads the original for the progress bar. Buffering the whole file a second + * time here would double the peak memory of the one thing already large enough + * to matter. + */ +async function fetchAndCache(url, etag) { + const response = await fetch(url); + if (response.ok && etag) { + const copy = response.clone(); + void openCache().then((cache) => keepOnly(cache, url, etag, copy)); + } + return response; +} + export function formatBytes(n) { return n < 1024 * 1024 ? `${Math.round(n / 1024)} KB` : `${(n / 1048576).toFixed(0)} MB`; } diff --git a/web/src/lib/db-cache.js b/web/src/lib/db-cache.js new file mode 100644 index 0000000..933f1d6 --- /dev/null +++ b/web/src/lib/db-cache.js @@ -0,0 +1,86 @@ +/** + * Where a downloaded database is kept between visits. + * + * Cache Storage rather than IndexedDB: the thing being stored is an HTTP + * response, the browser can stream it to disk instead of holding it as one + * buffer, and eviction under storage pressure is the browser's business rather + * than ours. + * + * Every entry is versioned by the server's ETag, because each deploy rebuilds + * the databases and a stale copy would quietly answer with last week's data. + * A new version is stored and the old ones for that database are dropped, so + * one dataset never occupies the disk twice. + */ + +const CACHE_NAME = "thptqg-db-v1"; + +/** The bucket, or null where Cache Storage is unavailable — SSR, or plain http. */ +export async function openCache() { + if (typeof caches === "undefined") return null; + try { + return await caches.open(CACHE_NAME); + } catch { + return null; + } +} + +/** Key for one version of one database. */ +export function cacheKey(url, etag) { + return `${url}?v=${encodeURIComponent(etag)}`; +} + +/** The stored copy of exactly this version, or null. */ +export async function matchVersion(cache, url, etag) { + if (!cache || !etag) return null; + return (await cache.match(cacheKey(url, etag))) ?? null; +} + +/** + * Any stored copy of this database, whichever version. + * + * The fallback for when the server cannot be reached at all: a slightly old + * copy of frozen exam results is worth more than an error page. + */ +export async function matchAny(cache, url) { + if (!cache) return null; + const prefix = `${url}?v=`; + for (const request of await cache.keys()) { + if (request.url.includes(prefix)) return (await cache.match(request)) ?? null; + } + return null; +} + +/** + * Store this version, then drop every other version of the same database. + * + * Best effort: a full disk means the visitor downloads again next time, which + * is worse than caching but no reason to fail the page they are on. + */ +export async function keepOnly(cache, url, etag, response) { + if (!cache || !etag) return false; + const key = cacheKey(url, etag); + try { + await cache.put(key, response); + } catch { + return false; + } + // Compared against the whole key, not the ETag alone: cache.keys() reports + // absolute URLs while the key is a path, and two ETags can share a suffix. + const prefix = `${url}?v=`; + for (const request of await cache.keys()) { + if (request.url.includes(prefix) && !request.url.endsWith(key)) { + await cache.delete(request); + } + } + return true; +} + +/** Forget every stored database. */ +export async function forgetAll() { + if (typeof caches === "undefined") return; + try { + await caches.delete(CACHE_NAME); + } catch { + // Nothing to do: the copy stays until the browser evicts it. + } +} diff --git a/web/src/lib/db-cache.test.js b/web/src/lib/db-cache.test.js new file mode 100644 index 0000000..5178625 --- /dev/null +++ b/web/src/lib/db-cache.test.js @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; +import { cacheKey, keepOnly, matchAny, matchVersion } from "./db-cache.js"; + +/** + * Enough of the Cache Storage contract for these functions to run against. + * + * keys() hands back request-shaped objects and match() accepts them, because + * that round trip — keys() then match(request) — is how matchAny works. + */ +function fakeCache(entries = {}) { + const store = new Map(Object.entries(entries)); + const asUrl = (key) => (typeof key === "string" ? key : key.url); + return { + store, + async match(key) { + return store.get(asUrl(key)) ?? undefined; + }, + async put(key, response) { + store.set(asUrl(key), response); + }, + async delete(key) { + return store.delete(asUrl(key)); + }, + async keys() { + return [...store.keys()].map((url) => ({ url })); + }, + }; +} + +const URL_2016 = "/thptqg/db/2016.sqlite3"; + +describe("cacheKey", () => { + it("versions the entry by ETag", () => { + expect(cacheKey(URL_2016, '"abc123"')).toBe(`${URL_2016}?v=%22abc123%22`); + }); +}); + +describe("matchVersion", () => { + it("returns the copy stored for that exact version", async () => { + const cache = fakeCache({ [cacheKey(URL_2016, "v1")]: "body" }); + await expect(matchVersion(cache, URL_2016, "v1")).resolves.toBe("body"); + }); + + it("misses when the server has a different version", async () => { + const cache = fakeCache({ [cacheKey(URL_2016, "v1")]: "body" }); + await expect(matchVersion(cache, URL_2016, "v2")).resolves.toBeNull(); + }); + + it("copes with no cache and no etag", async () => { + await expect(matchVersion(null, URL_2016, "v1")).resolves.toBeNull(); + await expect(matchVersion(fakeCache(), URL_2016, null)).resolves.toBeNull(); + }); +}); + +describe("matchAny", () => { + it("finds a stored copy whatever its version, for when the server is unreachable", async () => { + const cache = fakeCache({ [cacheKey(URL_2016, "old")]: "body" }); + await expect(matchAny(cache, URL_2016)).resolves.toBe("body"); + }); + + it("does not return another dataset's database", async () => { + const cache = fakeCache({ [cacheKey("/thptqg/db/2017.sqlite3", "v1")]: "body" }); + await expect(matchAny(cache, URL_2016)).resolves.toBeNull(); + }); +}); + +describe("keepOnly", () => { + it("stores the new version and drops the old one", async () => { + const cache = fakeCache({ [cacheKey(URL_2016, "old")]: "stale" }); + + await expect(keepOnly(cache, URL_2016, "new", "fresh")).resolves.toBe(true); + + expect([...cache.store.keys()]).toEqual([cacheKey(URL_2016, "new")]); + }); + + it("leaves the other dataset alone", async () => { + const other = cacheKey("/thptqg/db/2017.sqlite3", "v1"); + const cache = fakeCache({ [other]: "keep me" }); + + await keepOnly(cache, URL_2016, "new", "fresh"); + + expect(cache.store.has(other)).toBe(true); + }); + + it("reports failure rather than throwing when the disk is full", async () => { + const cache = fakeCache(); + cache.put = async () => { + throw new Error("QuotaExceededError"); + }; + await expect(keepOnly(cache, URL_2016, "new", "fresh")).resolves.toBe(false); + }); +}); diff --git a/web/src/routes/[dataset]/+page.svelte b/web/src/routes/[dataset]/+page.svelte index 1a7d1c2..1f1823b 100644 --- a/web/src/routes/[dataset]/+page.svelte +++ b/web/src/routes/[dataset]/+page.svelte @@ -7,7 +7,8 @@ import ScoreTable from "$lib/components/score-table.svelte"; import SearchForm from "$lib/components/search-form.svelte"; import StudentDetail from "$lib/components/student-detail.svelte"; - import { LocalDatabase, weigh } from "$lib/database.svelte"; + import { LocalDatabase, formatBytes } from "$lib/database.svelte"; + import { forgetAll } from "$lib/db-cache"; import { dbOf } from "$lib/datasets"; import { isExamId } from "$lib/query-mode"; import { MAX_RESULTS, lookupExamId, searchByName } from "$lib/search"; @@ -22,8 +23,6 @@ /** How long the last search took, in ms. */ let searchMs = $state(null); let activeTab = $state("search"); - /** Compressed size of the download, as the server reports it. */ - let transferBytes = $state(null); // Owned here, not in SearchForm, so it can be bound to the URL both ways. // The query string is unreadable while prerendering — there is no request — // so a deep link is picked up on the client only. @@ -34,16 +33,23 @@ // Created in the browser only: $effect does not run while prerendering. The // download itself waits for the visitor to accept it. $effect(() => { - const url = dbOf(dataset, base); - const opened = new LocalDatabase(url, dataset.dbSizeMb * 1024 * 1024); + const opened = new LocalDatabase(dbOf(dataset, base), dataset.dbSizeMb * 1024 * 1024); db = opened; - void weigh(url).then((bytes) => (transferBytes = bytes)); + // Consults the cache, and opens a stored copy without asking: consent was + // given the first time, and reusing it costs nothing. + void opened.prepare(); return () => { opened.close(); db = null; }; }); + /** Drop the stored copy, then reload so the gate asks again. */ + async function clearStored() { + await forgetAll(); + location.reload(); + } + // Hydrate a ?q= deep link as soon as the database is open. let hydrated = false; $effect(() => { @@ -135,8 +141,18 @@
- + + {#if db?.loading && db.fromCache} +

+ + Đang mở dữ liệu đã lưu trên máy… {Math.round(db.progress * 100)}% +

+ {/if}
+

+ {/if}
-{#if db && !db.ready} - db.open()} /> +{#if db && !db.ready && db.checked && !db.fromCache} + db.open()} /> {/if}