From d8b5b66fcf029fdde653cb5e6614814911962b9c Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Fri, 14 Aug 2026 17:52:09 +0700 Subject: [PATCH] feat: keep the downloaded database, and stop rebuilding it every deploy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two caches, one on each side. In the browser, the response is stored in Cache Storage keyed by the server's ETag, so the transfer is paid once per device rather than once per visit. A stored copy opens without the gate: consent was given the first time and reuse costs no network. A redeploy changes the ETag, so the new version replaces the old instead of answering with last week's data, and every older version of that database is dropped so one dataset never occupies the disk twice. When the server cannot be reached at all, any stored version is used, which incidentally makes the site work offline. Storing is best effort — a full disk means downloading again next time, which is no reason to fail the page. The response is cached from a clone while the original is read for the progress bar, rather than buffering the file a second time in the one place where memory is already the binding constraint. In CI, the built databases are cached on the inputs that determine them: data/**, parser/** and datasets.json. Parsing 348 MB of spreadsheets is the slow part of the job, and a web or docs change cannot alter a database, so those pushes restore instead of rebuilding. Exact matches only — no restore-keys, since a near-miss would publish databases built from inputs the commit does not describe. --- .github/workflows/deploy-pages.yml | 33 ++++++-- CLAUDE.md | 9 +- docs/deployment-guide.md | 13 ++- web/src/lib/components/download-gate.svelte | 10 +-- web/src/lib/database.svelte.js | 80 ++++++++++++++++-- web/src/lib/db-cache.js | 86 +++++++++++++++++++ web/src/lib/db-cache.test.js | 92 +++++++++++++++++++++ web/src/routes/[dataset]/+page.svelte | 48 ++++++++--- 8 files changed, 337 insertions(+), 34 deletions(-) create mode 100644 web/src/lib/db-cache.js create mode 100644 web/src/lib/db-cache.test.js diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index b52db48..50eb934 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -96,12 +96,35 @@ jobs: GOVULNCHECK="$(go env GOPATH)/bin/govulncheck" for m in parser crawler assembler; do (cd "$m" && "$GOVULNCHECK" ./...); done - # One command runs the whole pipeline: compile the parser, build and - # verify each database against its registry row count and size, build the - # web app, and assemble _site — refusing to continue if a database is - # short, an artifact looks truncated, or one is missing entirely. + # Parsing 348 MB of spreadsheets is the slowest part of the job, and its + # output is a pure function of the inputs in the key: the source files, + # the parser (schema, transforms, configs) and the registry the row-count + # and size guards read. Anything else — web code, docs, workflow — cannot + # change a database, so those pushes restore one instead of rebuilding. + # + # Exact matches only, with no restore-keys: a near-miss here would mean + # publishing a database built from different inputs than the commit says. + - name: Cache the built databases + id: db-cache + uses: actions/cache@v4 + with: + path: .build/public/db + key: db-${{ hashFiles('data/**', 'parser/**', 'datasets.json') }} + + # The whole pipeline: compile the parser, build and verify each database + # against its registry row count and size, build the web app, and + # assemble _site — refusing to continue if a database is short, an + # artifact looks truncated, or one is missing entirely. On a cache hit + # the database step is skipped; the artifact it would have produced was + # verified by the run that built and stored it. - name: Build site - run: go -C assembler run ./cmd/assemble + run: | + if [ "${{ steps.db-cache.outputs.cache-hit }}" = "true" ]; then + echo "Databases restored from cache; building the site only." + go -C assembler run ./cmd/assemble site + else + go -C assembler run ./cmd/assemble + fi - uses: actions/upload-pages-artifact@v5 with: diff --git a/CLAUDE.md b/CLAUDE.md index 4c06bb4..e0043fd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,8 +64,13 @@ hashes every real input file. That is the point of it; do not skip it. dataset page is gated behind that download: `download-gate.svelte` states the transfer and memory cost and offers nothing but the download, because there is no answer to give without it. `sql.js` holds the file in WebAssembly - memory for as long as the tab is open, so it is RAM, not disk, and it is gone - on reload. + memory for as long as the tab is open, so the memory figure is RAM, not disk. +- **The download is kept in Cache Storage, versioned by ETag** (`db-cache.js`). + A later visit opens the stored copy without asking, since consent was given + once and reuse costs no network; a redeploy changes the ETag, so the new + version replaces the old rather than being served stale. When the server + cannot be reached at all, any stored version is used — which is what lets the + site answer offline. - **The schema carries no secondary indexes, deliberately.** An index saves a scan that already takes a few hundred milliseconds in memory, and costs every visitor megabytes of download. An earlier design read the file over HTTP diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 0d6412f..8b8b8f7 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -10,12 +10,17 @@ One-time setup: **Settings → Pages → Source: GitHub Actions**. 1. Checkout, Go toolchain, Node 24, `npm ci` in `web/` 2. Parser, crawler and assembler test suites, web lint, `govulncheck` over all three modules 3. `go -C assembler run ./cmd/assemble` — the whole pipeline: compile the - parser, build and verify each database, compress it into `.build/public/db/`, - run the web build, assemble `_site/` + parser, build and verify each database into `.build/public/db/`, run the web + build, assemble `_site/`. The databases are restored from the Actions cache + when nothing that determines them has changed, and only the site is built 4. `actions/upload-pages-artifact` + `actions/deploy-pages` -The database build dominates the runtime: roughly 348 MB of Excel is parsed on -every deploy. +The database build dominates the runtime: roughly 348 MB of Excel to parse. It +is cached in Actions, keyed on `data/**`, `parser/**` and `datasets.json`, so +only a change to the sources, the parser or the registry pays for it — a web or +docs change restores the databases instead. There are no `restore-keys`: a +near-miss would publish databases built from inputs the commit does not +describe. ## Resulting URLs diff --git a/web/src/lib/components/download-gate.svelte b/web/src/lib/components/download-gate.svelte index 686e6cb..5c6443d 100644 --- a/web/src/lib/components/download-gate.svelte +++ b/web/src/lib/components/download-gate.svelte @@ -11,7 +11,7 @@ * * It has no dismiss: without the database the page has no answers to give. */ - let { db, transferBytes = null, onDownload } = $props(); + let { db, onDownload } = $props(); const percent = $derived(Math.round(db.progress * 100)); @@ -22,7 +22,7 @@ aria-modal="true" aria-labelledby="gate-title" > -
+

Tải cơ sở dữ liệu

@@ -34,7 +34,7 @@

Dung lượng tải
- {transferBytes ? formatBytes(transferBytes) : "~31 MB"} + {db.transferBytes ? formatBytes(db.transferBytes) : "~31 MB"}
@@ -44,8 +44,8 @@

- Dữ liệu được giữ trong bộ nhớ của tab và sẽ mất khi bạn đóng trang, nên lần truy cập sau cần - tải lại. Không nên dùng trên thiết bị có ít bộ nhớ. + Sau khi tải, dữ liệu được lưu lại trên máy nên các lần sau không phải tải lại. Khi tra cứu, + dữ liệu nằm trong bộ nhớ (RAM) của tab, vì vậy không nên dùng trên thiết bị có ít bộ nhớ.

{#if db.error} diff --git a/web/src/lib/database.svelte.js b/web/src/lib/database.svelte.js index 325c6ca..476b725 100644 --- a/web/src/lib/database.svelte.js +++ b/web/src/lib/database.svelte.js @@ -1,3 +1,4 @@ +import { keepOnly, matchAny, matchVersion, openCache } from "./db-cache.js"; import initSqlJs from "sql.js"; import wasmUrl from "sql.js/dist/sql-wasm.wasm?url"; @@ -15,6 +16,10 @@ import wasmUrl from "sql.js/dist/sql-wasm.wasm?url"; * whole file sits in the tab's address space for as long as the page is open. * That is what the gate on the dataset page tells the visitor before they * commit to it. + * + * The transfer is paid once per device rather than once per visit: the + * response is kept in Cache Storage, versioned by ETag so that a redeploy + * replaces it instead of being served stale. See db-cache.js. */ /** Rows to return before a query is considered too broad to render. */ @@ -46,14 +51,47 @@ export class LocalDatabase { received = $state(0); /** How long the download and open took, once done. */ ms = $state(0); + /** True once the cache has been consulted, so the gate knows what to ask for. */ + checked = $state(false); + /** True when this copy came off the disk rather than the network. */ + fromCache = $state(false); + /** Compressed size of the download, as the server reports it. */ + transferBytes = $state(null); #db = null; + #etag = null; constructor(url, expectedBytes) { this.url = url; this.expectedBytes = expectedBytes; } + /** + * Ask the server what it holds, and the disk what we kept. + * + * A copy already on the device costs no network and needs no consent, so it + * opens straight away. Otherwise this only works out what a download would + * cost, and the gate waits for the visitor to accept it. + */ + async prepare() { + const head = await describe(this.url); + this.transferBytes = head?.bytes ?? null; + this.#etag = head?.etag ?? null; + + const cache = await openCache(); + // Without an ETag the server is unreachable; any stored version beats an + // error page, since these are frozen exam results. + const stored = this.#etag + ? await matchVersion(cache, this.url, this.#etag) + : await matchAny(cache, this.url); + + this.checked = true; + if (stored) { + this.fromCache = true; + await this.open(stored); + } + } + /** Fraction downloaded, 0 to 1, for the progress bar. */ get progress() { if (!this.expectedBytes) return 0; @@ -67,7 +105,7 @@ export class LocalDatabase { * sees movement: this is tens of megabytes on a phone connection, and a * progress bar is the difference between waiting and giving up. */ - async open() { + async open(stored = null) { if (this.ready || this.loading) return; this.loading = true; this.error = null; @@ -75,7 +113,10 @@ export class LocalDatabase { const started = performance.now(); try { - const [SQL, response] = await Promise.all([sqlEngine(), fetch(this.url)]); + const [SQL, response] = await Promise.all([ + sqlEngine(), + stored ?? fetchAndCache(this.url, this.#etag), + ]); if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); const reader = response.body.getReader(); @@ -133,22 +174,45 @@ export class LocalDatabase { } /** - * What the download will cost on the wire, from the server's own headers. + * What the file weighs on the wire, and which version it is. * - * The browser advertises gzip, and GitHub Pages compresses this file, so this - * reports the compressed size — the number that matters to someone on mobile - * data. Returns null when the server does not say. + * The browser advertises gzip and GitHub Pages compresses this file, so the + * length is the compressed one — the number that matters to someone on mobile + * data. The ETag is what keeps a stored copy honest across deploys. Returns + * null when the server cannot be reached, which is survivable: a stored copy + * still opens. */ -export async function weigh(url) { +async function describe(url) { try { const response = await fetch(url, { method: "HEAD" }); + if (!response.ok) return null; const length = Number(response.headers.get("Content-Length")); - return Number.isFinite(length) && length > 0 ? length : null; + return { + bytes: Number.isFinite(length) && length > 0 ? length : null, + etag: response.headers.get("ETag"), + }; } catch { return null; } } +/** + * Fetch the database, handing a second copy of the response to the cache. + * + * The clone is what gets stored, so the bytes stream to disk while the caller + * reads the original for the progress bar. Buffering the whole file a second + * time here would double the peak memory of the one thing already large enough + * to matter. + */ +async function fetchAndCache(url, etag) { + const response = await fetch(url); + if (response.ok && etag) { + const copy = response.clone(); + void openCache().then((cache) => keepOnly(cache, url, etag, copy)); + } + return response; +} + export function formatBytes(n) { return n < 1024 * 1024 ? `${Math.round(n / 1024)} KB` : `${(n / 1048576).toFixed(0)} MB`; } diff --git a/web/src/lib/db-cache.js b/web/src/lib/db-cache.js new file mode 100644 index 0000000..933f1d6 --- /dev/null +++ b/web/src/lib/db-cache.js @@ -0,0 +1,86 @@ +/** + * Where a downloaded database is kept between visits. + * + * Cache Storage rather than IndexedDB: the thing being stored is an HTTP + * response, the browser can stream it to disk instead of holding it as one + * buffer, and eviction under storage pressure is the browser's business rather + * than ours. + * + * Every entry is versioned by the server's ETag, because each deploy rebuilds + * the databases and a stale copy would quietly answer with last week's data. + * A new version is stored and the old ones for that database are dropped, so + * one dataset never occupies the disk twice. + */ + +const CACHE_NAME = "thptqg-db-v1"; + +/** The bucket, or null where Cache Storage is unavailable — SSR, or plain http. */ +export async function openCache() { + if (typeof caches === "undefined") return null; + try { + return await caches.open(CACHE_NAME); + } catch { + return null; + } +} + +/** Key for one version of one database. */ +export function cacheKey(url, etag) { + return `${url}?v=${encodeURIComponent(etag)}`; +} + +/** The stored copy of exactly this version, or null. */ +export async function matchVersion(cache, url, etag) { + if (!cache || !etag) return null; + return (await cache.match(cacheKey(url, etag))) ?? null; +} + +/** + * Any stored copy of this database, whichever version. + * + * The fallback for when the server cannot be reached at all: a slightly old + * copy of frozen exam results is worth more than an error page. + */ +export async function matchAny(cache, url) { + if (!cache) return null; + const prefix = `${url}?v=`; + for (const request of await cache.keys()) { + if (request.url.includes(prefix)) return (await cache.match(request)) ?? null; + } + return null; +} + +/** + * Store this version, then drop every other version of the same database. + * + * Best effort: a full disk means the visitor downloads again next time, which + * is worse than caching but no reason to fail the page they are on. + */ +export async function keepOnly(cache, url, etag, response) { + if (!cache || !etag) return false; + const key = cacheKey(url, etag); + try { + await cache.put(key, response); + } catch { + return false; + } + // Compared against the whole key, not the ETag alone: cache.keys() reports + // absolute URLs while the key is a path, and two ETags can share a suffix. + const prefix = `${url}?v=`; + for (const request of await cache.keys()) { + if (request.url.includes(prefix) && !request.url.endsWith(key)) { + await cache.delete(request); + } + } + return true; +} + +/** Forget every stored database. */ +export async function forgetAll() { + if (typeof caches === "undefined") return; + try { + await caches.delete(CACHE_NAME); + } catch { + // Nothing to do: the copy stays until the browser evicts it. + } +} diff --git a/web/src/lib/db-cache.test.js b/web/src/lib/db-cache.test.js new file mode 100644 index 0000000..5178625 --- /dev/null +++ b/web/src/lib/db-cache.test.js @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; +import { cacheKey, keepOnly, matchAny, matchVersion } from "./db-cache.js"; + +/** + * Enough of the Cache Storage contract for these functions to run against. + * + * keys() hands back request-shaped objects and match() accepts them, because + * that round trip — keys() then match(request) — is how matchAny works. + */ +function fakeCache(entries = {}) { + const store = new Map(Object.entries(entries)); + const asUrl = (key) => (typeof key === "string" ? key : key.url); + return { + store, + async match(key) { + return store.get(asUrl(key)) ?? undefined; + }, + async put(key, response) { + store.set(asUrl(key), response); + }, + async delete(key) { + return store.delete(asUrl(key)); + }, + async keys() { + return [...store.keys()].map((url) => ({ url })); + }, + }; +} + +const URL_2016 = "/thptqg/db/2016.sqlite3"; + +describe("cacheKey", () => { + it("versions the entry by ETag", () => { + expect(cacheKey(URL_2016, '"abc123"')).toBe(`${URL_2016}?v=%22abc123%22`); + }); +}); + +describe("matchVersion", () => { + it("returns the copy stored for that exact version", async () => { + const cache = fakeCache({ [cacheKey(URL_2016, "v1")]: "body" }); + await expect(matchVersion(cache, URL_2016, "v1")).resolves.toBe("body"); + }); + + it("misses when the server has a different version", async () => { + const cache = fakeCache({ [cacheKey(URL_2016, "v1")]: "body" }); + await expect(matchVersion(cache, URL_2016, "v2")).resolves.toBeNull(); + }); + + it("copes with no cache and no etag", async () => { + await expect(matchVersion(null, URL_2016, "v1")).resolves.toBeNull(); + await expect(matchVersion(fakeCache(), URL_2016, null)).resolves.toBeNull(); + }); +}); + +describe("matchAny", () => { + it("finds a stored copy whatever its version, for when the server is unreachable", async () => { + const cache = fakeCache({ [cacheKey(URL_2016, "old")]: "body" }); + await expect(matchAny(cache, URL_2016)).resolves.toBe("body"); + }); + + it("does not return another dataset's database", async () => { + const cache = fakeCache({ [cacheKey("/thptqg/db/2017.sqlite3", "v1")]: "body" }); + await expect(matchAny(cache, URL_2016)).resolves.toBeNull(); + }); +}); + +describe("keepOnly", () => { + it("stores the new version and drops the old one", async () => { + const cache = fakeCache({ [cacheKey(URL_2016, "old")]: "stale" }); + + await expect(keepOnly(cache, URL_2016, "new", "fresh")).resolves.toBe(true); + + expect([...cache.store.keys()]).toEqual([cacheKey(URL_2016, "new")]); + }); + + it("leaves the other dataset alone", async () => { + const other = cacheKey("/thptqg/db/2017.sqlite3", "v1"); + const cache = fakeCache({ [other]: "keep me" }); + + await keepOnly(cache, URL_2016, "new", "fresh"); + + expect(cache.store.has(other)).toBe(true); + }); + + it("reports failure rather than throwing when the disk is full", async () => { + const cache = fakeCache(); + cache.put = async () => { + throw new Error("QuotaExceededError"); + }; + await expect(keepOnly(cache, URL_2016, "new", "fresh")).resolves.toBe(false); + }); +}); diff --git a/web/src/routes/[dataset]/+page.svelte b/web/src/routes/[dataset]/+page.svelte index 1a7d1c2..1f1823b 100644 --- a/web/src/routes/[dataset]/+page.svelte +++ b/web/src/routes/[dataset]/+page.svelte @@ -7,7 +7,8 @@ import ScoreTable from "$lib/components/score-table.svelte"; import SearchForm from "$lib/components/search-form.svelte"; import StudentDetail from "$lib/components/student-detail.svelte"; - import { LocalDatabase, weigh } from "$lib/database.svelte"; + import { LocalDatabase, formatBytes } from "$lib/database.svelte"; + import { forgetAll } from "$lib/db-cache"; import { dbOf } from "$lib/datasets"; import { isExamId } from "$lib/query-mode"; import { MAX_RESULTS, lookupExamId, searchByName } from "$lib/search"; @@ -22,8 +23,6 @@ /** How long the last search took, in ms. */ let searchMs = $state(null); let activeTab = $state("search"); - /** Compressed size of the download, as the server reports it. */ - let transferBytes = $state(null); // Owned here, not in SearchForm, so it can be bound to the URL both ways. // The query string is unreadable while prerendering — there is no request — // so a deep link is picked up on the client only. @@ -34,16 +33,23 @@ // Created in the browser only: $effect does not run while prerendering. The // download itself waits for the visitor to accept it. $effect(() => { - const url = dbOf(dataset, base); - const opened = new LocalDatabase(url, dataset.dbSizeMb * 1024 * 1024); + const opened = new LocalDatabase(dbOf(dataset, base), dataset.dbSizeMb * 1024 * 1024); db = opened; - void weigh(url).then((bytes) => (transferBytes = bytes)); + // Consults the cache, and opens a stored copy without asking: consent was + // given the first time, and reusing it costs nothing. + void opened.prepare(); return () => { opened.close(); db = null; }; }); + /** Drop the stored copy, then reload so the gate asks again. */ + async function clearStored() { + await forgetAll(); + location.reload(); + } + // Hydrate a ?q= deep link as soon as the database is open. let hydrated = false; $effect(() => { @@ -135,8 +141,18 @@
- + + {#if db?.loading && db.fromCache} +

+ + Đang mở dữ liệu đã lưu trên máy… {Math.round(db.progress * 100)}% +

+ {/if}
+

+ {/if}
-{#if db && !db.ready} - db.open()} /> +{#if db && !db.ready && db.checked && !db.fromCache} + db.open()} /> {/if}