Revert "Merge branch 'fix/httpvfs-chunked-length'"

This reverts commit d4f106f488, reversing
changes made to c5ef1f58ff.

web/src/lib/datasets.js

@@ -99,39 +99,12 @@ export function pathOf(dataset, base) {
 }
 
 /**
- * The chunk the whole database lives in. One chunk covers the file, so this is
- * the only index the library ever asks for, and the assembler publishes the
- * database under a name ending in it.
- */
-const CHUNK_INDEX = "0";
-
-/**
- * Everything a database URL has except the chunk index, e.g.
- * dbPrefixOf(d, "/thptqg") → "/thptqg/db/2017.sqlite3".
- *
- * sql.js-httpvfs reads the file in chunked mode and builds each URL as
- * prefix + chunk index. One chunk holds the whole database, so the index is
- * always 0 and the published file is "<id>.sqlite30".
- */
-export function dbPrefixOf(dataset, base) {
-  return `${base}/db/${dataset.id}.sqlite3`;
-}
-
-/**
- * The published database file, e.g. dbOf(d, "/thptqg") → "/thptqg/db/2017.sqlite30".
+ * The published database, e.g. dbOf(d, "/thptqg") → "/thptqg/db/2017.sqlite3".
  *
  * The browser downloads this whole file and queries it in memory. The server
  * compresses it on the wire, which is why the transfer is a third of the size
  * the registry records.
  */
 export function dbOf(dataset, base) {
-  return `${dbPrefixOf(dataset, base)}${CHUNK_INDEX}`;
-}
-
-/**
- * Both forms of the database's location, for RemoteDatabase: the file to read,
- * and the prefix the library appends the chunk index to.
- */
-export function dbSourceOf(dataset, base) {
-  return { url: dbOf(dataset, base), urlPrefix: dbPrefixOf(dataset, base) };
+  return `${base}/db/${dataset.id}.sqlite3`;
 }

web/src/lib/datasets.test.js

@@ -1,28 +1,14 @@
 import { describe, expect, it } from "vitest";
-import { DATASETS, dbOf, dbPrefixOf, dbSourceOf } from "./datasets.js";
+import { DATASETS, dbOf, pathOf } from "./datasets.js";
 
 const [dataset] = DATASETS;
 
-/**
- * sql.js-httpvfs builds every request URL as urlPrefix + chunk index, and one
- * chunk holds the whole database, so the index is always 0. If these two ever
- * stop agreeing, the library asks for a file the assembler never published and
- * every query 404s — which no other test would catch.
- */
-describe("database location", () => {
-  it("puts the chunk index where the published file name ends", () => {
-    expect(dbOf(dataset, "/thptqg")).toBe(`${dbPrefixOf(dataset, "/thptqg")}0`);
+describe("dataset locations", () => {
+  it("derives the site path from the id", () => {
+    expect(pathOf(dataset, "/thptqg")).toBe(`/thptqg/${dataset.id}/`);
   });
 
-  it("names the file the assembler publishes", () => {
-    expect(dbOf(dataset, "/thptqg")).toBe(`/thptqg/db/${dataset.id}.sqlite30`);
-  });
-
-  it("hands RemoteDatabase both forms of the same location", () => {
-    const source = dbSourceOf(dataset, "/thptqg");
-    expect(source).toEqual({
-      url: dbOf(dataset, "/thptqg"),
-      urlPrefix: dbPrefixOf(dataset, "/thptqg"),
-    });
+  it("names the database file the assembler publishes", () => {
+    expect(dbOf(dataset, "/thptqg")).toBe(`/thptqg/db/${dataset.id}.sqlite3`);
   });
 });

web/src/lib/db-probe.test.js

@@ -1,90 +0,0 @@
-import { describe, expect, it, vi } from "vitest";
-import { looksLikeSqlite, parseTotalBytes, probeDatabase, readPageSize } from "./db-probe.js";
-
-/** The first bytes of a real database: magic, NUL, then the page size at 16. */
-function header(pageSize = 1024, magic = "SQLite format 3") {
-  const bytes = new Uint8Array(100);
-  for (let i = 0; i < magic.length; i += 1) bytes[i] = magic.charCodeAt(i);
-  bytes[16] = pageSize >> 8;
-  bytes[17] = pageSize & 0xff;
-  return bytes;
-}
-
-function respond(bytes, { status = 206, contentRange = `bytes 0-99/${317096960}` } = {}) {
-  return {
-    status,
-    headers: { get: (name) => (name.toLowerCase() === "content-range" ? contentRange : null) },
-    arrayBuffer: async () => bytes.buffer,
-  };
-}
-
-describe("parseTotalBytes", () => {
-  it("takes the total from a Content-Range", () => {
-    expect(parseTotalBytes("bytes 0-99/317096960")).toBe(317096960);
-  });
-
-  it("refuses an unknown total", () => {
-    expect(() => parseTotalBytes("bytes 0-99/*")).toThrow(/how large/);
-    expect(() => parseTotalBytes(null)).toThrow(/absent/);
-  });
-});
-
-describe("readPageSize", () => {
-  it("reads the two big-endian bytes at offset 16", () => {
-    expect(readPageSize(header(1024))).toBe(1024);
-    expect(readPageSize(header(4096))).toBe(4096);
-  });
-
-  it("treats 1 as 65536, as the file format does", () => {
-    expect(readPageSize(header(1))).toBe(65536);
-  });
-});
-
-describe("looksLikeSqlite", () => {
-  it("accepts a real header", () => {
-    expect(looksLikeSqlite(header())).toBe(true);
-  });
-
-  it("rejects a gzip stream, which is what a compressing host returns", () => {
-    const gzip = new Uint8Array([0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00]);
-    expect(looksLikeSqlite(gzip)).toBe(false);
-  });
-
-  it("requires the NUL that terminates the magic", () => {
-    expect(looksLikeSqlite(header(1024, "SQLite format 3x"))).toBe(false);
-  });
-});
-
-describe("probeDatabase", () => {
-  it("asks for the header by range and returns the total length", async () => {
-    const fetchImpl = vi.fn(async () => respond(header()));
-    const total = await probeDatabase("/db/2016.sqlite30", 1024, fetchImpl);
-
-    expect(total).toBe(317096960);
-    expect(fetchImpl).toHaveBeenCalledWith("/db/2016.sqlite30", {
-      headers: { Range: "bytes=0-99" },
-    });
-  });
-
-  it("fails when the host ignores the range", async () => {
-    const fetchImpl = async () => respond(header(), { status: 200 });
-    await expect(probeDatabase("/db/2016.sqlite30", 1024, fetchImpl)).rejects.toThrow(/expected 206/);
-  });
-
-  it("fails when the bytes are not a database", async () => {
-    const gzip = new Uint8Array([0x1f, 0x8b, 0x08]);
-    const fetchImpl = async () => respond(gzip);
-    await expect(probeDatabase("/db/2016.sqlite30", 1024, fetchImpl)).rejects.toThrow(
-      /not a SQLite header/,
-    );
-  });
-
-  it("warns, but continues, when the page size is not the request size", async () => {
-    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const fetchImpl = async () => respond(header(4096));
-
-    await expect(probeDatabase("/db/2016.sqlite30", 1024, fetchImpl)).resolves.toBe(317096960);
-    expect(warn).toHaveBeenCalledWith(expect.stringMatching(/page size 4096/));
-    warn.mockRestore();
-  });
-});

web/src/lib/sqlite.svelte.js

@@ -1,191 +0,0 @@
-import { createDbWorker } from "sql.js-httpvfs";
-import workerUrl from "sql.js-httpvfs/dist/sqlite.worker.js?url";
-import wasmUrl from "sql.js-httpvfs/dist/sql-wasm.wasm?url";
-import { probeDatabase } from "./db-probe.js";
-
-/**
- * The database is read where it lies. SQLite asks for pages, the virtual file
- * system turns each into an HTTP range request, and only the pages a query
- * touches ever cross the network — a few hundred KB for a lookup, against the
- * 45 MB the whole file used to cost before the first query.
- *
- * That only holds while every query is index-driven. The schema exists for it:
- * name_word serves name search, and the score indexes serve the SQL presets.
- * An unindexed query walks the table and pulls all 100+ MB of it, which is what
- * the byte budget below is for.
- */
-
-// Must equal the page size the parser writes (PRAGMA page_size in
-// parser/internal/writer/writer.go), so one request is exactly one page. A
-// mismatch makes every logical page read span two requests.
-const CHUNK_BYTES = 1024;
-
-/** Generous for indexed work: a name search costs well under 1 MB. */
-export const SEARCH_BUDGET_BYTES = 25 * 1024 * 1024;
-
-/** What the SQL tab gets once the user has accepted the cost of a scan. */
-export const PLAYGROUND_BUDGET_BYTES = 250 * 1024 * 1024;
-
-/**
- * What one query cost over the network: `{ requests, bytes, ms }`.
- */
-
-/**
- * One remotely-paged database, with the load state the UI needs.
- *
- * `budgetBytes` is a hard ceiling for the worker's lifetime: past it a query
- * fails instead of quietly downloading the file. Raising it means a new worker,
- * which costs only the header pages.
- */
-export class RemoteDatabase {
-  ready = $state(false);
-  error = $state(null);
-  /** Bytes fetched by this database so far, prefetch included. */
-  bytesRead = $state(0);
-  /** HTTP range requests issued so far. */
-  requests = $state(0);
-  /** What the most recent query cost on its own. */
-  lastCost = $state(null);
-
-  #worker = null;
-  #opening;
-  #closed = false;
-  // Previous cumulative reading, so a query's own cost is a subtraction.
-  #seen = { requests: 0, bytes: 0 };
-
-  /**
-   * `source` carries both forms of the same location: `url` is the file, and
-   * `urlPrefix` is what the library appends the chunk index to. They must
-   * agree — see dbOf/dbPrefixOf, which derive one from the other.
-   */
-  constructor(source, budgetBytes = SEARCH_BUDGET_BYTES) {
-    this.url = source.url;
-    this.urlPrefix = source.urlPrefix;
-    this.budgetBytes = budgetBytes;
-    this.#opening = this.#open();
-  }
-
-  async #open() {
-    const opened = performance.now();
-    try {
-      // Chunked mode over a single chunk, which looks odd but is the only way
-      // to tell this library how long the file is: the worker reads
-      // databaseLengthBytes in chunked mode and hardcodes the length to
-      // undefined in full mode. Left to itself it sizes the file with a HEAD
-      // request, and GitHub Pages answers that with the gzipped length, which
-      // it then refuses to use.
-      //
-      // One chunk covers the whole database, so the chunk index is always 0
-      // and every request goes to urlPrefix + "0" — the file the assembler
-      // publishes as <id>.sqlite30.
-      const databaseLengthBytes = await probeDatabase(this.url, CHUNK_BYTES);
-      const worker = await createDbWorker(
-        [
-          {
-            from: "inline",
-            config: {
-              serverMode: "chunked",
-              urlPrefix: this.urlPrefix,
-              serverChunkSize: databaseLengthBytes,
-              databaseLengthBytes,
-              suffixLength: 1,
-              requestChunkSize: CHUNK_BYTES,
-            },
-          },
-        ],
-        workerUrl,
-        wasmUrl,
-        this.budgetBytes,
-      );
-      if (this.#closed) throw new Error("closed");
-      this.#worker = worker;
-      this.ready = true;
-      // Opening is not free either: the header and schema pages are read before
-      // any query runs, and that shows up in every later session total.
-      await this.#account(worker, `open ${this.url}`, performance.now() - opened);
-      return worker;
-    } catch (err) {
-      if (!this.#closed) {
-        this.error = message(err);
-        this.ready = false;
-      }
-      throw err;
-    }
-  }
-
-  /**
-   * Run a query and return its rows as objects.
-   *
-   * `label` names the query in the console trace — the only way to see what a
-   * search actually costs, since the byte count depends on how much the read
-   * heads prefetched, not just on the pages the plan needed.
-   */
-  async query(sql, params = [], label) {
-    const worker = this.#worker ?? (await this.#opening);
-    const started = performance.now();
-    try {
-      return await worker.db.query(sql, ...params);
-    } finally {
-      await this.#account(worker, label ?? firstLine(sql), performance.now() - started);
-    }
-  }
-
-  /**
-   * Read the cumulative counters and report the delta.
-   *
-   * getStats() rather than the worker's `bytesRead`: that one is the budget
-   * counter and resets itself to zero when a query trips the ceiling, so it
-   * would under-report exactly when the number matters most.
-   */
-  async #account(worker, label, ms) {
-    const stats = await worker.worker.getStats().catch(() => null);
-    if (!stats) return;
-
-    const cost = {
-      requests: stats.totalRequests - this.#seen.requests,
-      bytes: stats.totalFetchedBytes - this.#seen.bytes,
-      ms,
-    };
-    this.#seen = { requests: stats.totalRequests, bytes: stats.totalFetchedBytes };
-    this.requests = stats.totalRequests;
-    this.bytesRead = stats.totalFetchedBytes;
-    this.lastCost = cost;
-
-    console.info(
-      `[httpvfs] ${label} — ${cost.requests} request(s), ${formatBytes(cost.bytes)}, ${ms.toFixed(0)} ms` +
-        ` · session ${stats.totalRequests} request(s), ${formatBytes(stats.totalFetchedBytes)}` +
-        ` of ${formatBytes(stats.totalBytes)}`,
-    );
-  }
-
-  /**
-   * Drop this database. createDbWorker owns the Worker and exposes no handle to
-   * it, so the thread outlives this call; a page creates at most one per
-   * dataset and one more if the SQL budget is raised, which is why that is
-   * tolerable rather than a leak worth working around.
-   */
-  close() {
-    this.#closed = true;
-    this.#worker = null;
-    this.ready = false;
-  }
-}
-
-/** True when a query failed because it would have exceeded the byte budget. */
-export function isBudgetError(err) {
-  return /maxBytesToRead|too much data|exceeded/i.test(message(err));
-}
-
-function message(err) {
-  return err instanceof Error ? err.message : String(err);
-}
-
-/** Enough of a query to recognise it in the console. */
-function firstLine(sql) {
-  const line = sql.trim().split("\n")[0];
-  return line.length > 70 ? `${line.slice(0, 70)}…` : line;
-}
-
-export function formatBytes(n) {
-  return n < 1024 * 1024 ? `${Math.round(n / 1024)} KB` : `${(n / 1048576).toFixed(1)} MB`;
-}

web/src/routes/[dataset]/+page.svelte

@@ -7,7 +7,9 @@
   import ScoreTable from "$lib/components/score-table.svelte";
   import SearchForm from "$lib/components/search-form.svelte";
   import StudentDetail from "$lib/components/student-detail.svelte";
-  import { dbSourceOf } from "$lib/datasets";
+  import { LocalDatabase, formatBytes } from "$lib/database.svelte";
+  import { forgetAll } from "$lib/db-cache";
+  import { dbOf } from "$lib/datasets";
   import { isExamId } from "$lib/query-mode";
   import { MAX_RESULTS, lookupExamId, searchByName } from "$lib/search";
 
@@ -31,7 +33,7 @@
   // Created in the browser only: $effect does not run while prerendering. The
   // download itself waits for the visitor to accept it.
   $effect(() => {
-    const opened = new RemoteDatabase(dbSourceOf(dataset, base), budget);
+    const opened = new LocalDatabase(dbOf(dataset, base), dataset.dbSizeMb * 1024 * 1024);
     db = opened;
     // Consults the cache, and opens a stored copy without asking: consent was
     // given the first time, and reusing it costs nothing.