fix(web): take the database length from a range probe, not the host

GitHub Pages serves .sqlite3 as application/octet-stream, which mime-db
marks compressible, so an un-ranged response comes back gzipped with the
compressed length. sql.js-httpvfs sizes a file with a HEAD request, sees
that length is unusable and refuses to open the database:

  Length of the file not known. It must either be supplied in the config
  or given by the HTTP server.

Page reads were never affected. The Fetch standard requires browsers to
send Accept-Encoding: identity on any request carrying a Range header,
and the live site returns 206 with raw bytes to one. So the length is
probed the same way and passed as fileLength, which is the escape hatch
the library's own message points at.

The probe reads the first 100 bytes, so it also checks the file starts
with the SQLite magic and that its page size matches the request size —
a host that ever compresses a ranged response now fails with a clear
message rather than feeding the library the wrong bytes.

The post-deploy check the docs prescribed could not have caught this: a
bare `curl -sI` advertises no encoding, so it reports success whatever
the host does. It is replaced, in the docs and in CI, by a ranged read
that verifies the bytes.

web/src/lib/db-probe.js

@@ -0,0 +1,88 @@
+/**
+ * How long the database is, asked in the one way that survives a CDN.
+ *
+ * `sql.js-httpvfs` sizes a file with a HEAD request. That request carries no
+ * Range header, so the browser advertises gzip, and GitHub Pages answers with
+ * `Content-Encoding: gzip` and the length of the *compressed* body — 66 MB for
+ * a 302 MB database. The library rightly refuses to believe it and gives up
+ * with "Length of the file not known. It must either be supplied in the config
+ * or given by the HTTP server."
+ *
+ * Range requests do not have that problem: the Fetch standard requires
+ * `Accept-Encoding: identity` on any request carrying a Range header, so the
+ * page reads that do the actual work always come back uncompressed. Asking for
+ * the first hundred bytes therefore yields both a trustworthy total, from
+ * Content-Range, and the file header itself to check it against.
+ */
+
+/** A SQLite file opens with these characters and then a NUL byte. */
+const MAGIC = "SQLite format 3";
+
+/** Enough for the whole SQLite header. */
+const PROBE_BYTES = 100;
+
+/** Page size lives at offset 16, big-endian; the value 1 encodes 65536. */
+const PAGE_SIZE_OFFSET = 16;
+
+/** Total size of the representation, from `bytes <from>-<to>/<total>`. */
+export function parseTotalBytes(contentRange) {
+  const match = /\/\s*(\d+)\s*$/.exec(contentRange ?? "");
+  if (!match) {
+    throw new Error(
+      `the server did not say how large the database is (Content-Range: ${contentRange ?? "absent"})`,
+    );
+  }
+  return Number(match[1]);
+}
+
+/** Page size the file was written with, from its header. */
+export function readPageSize(header) {
+  const raw = (header[PAGE_SIZE_OFFSET] << 8) | header[PAGE_SIZE_OFFSET + 1];
+  return raw === 1 ? 65536 : raw;
+}
+
+/** True when these bytes begin a SQLite database. */
+export function looksLikeSqlite(header) {
+  const text = Array.from(MAGIC).every((ch, i) => header[i] === ch.charCodeAt(0));
+  return text && header[MAGIC.length] === 0;
+}
+
+/**
+ * Read the file header over a range request and return the database's length.
+ *
+ * Doubles as the check that the host is serving raw database bytes: a body that
+ * does not start with the SQLite magic means something rewrote it in transit —
+ * compression being the way that happens — and every later page read would be
+ * reading the wrong bytes.
+ */
+export async function probeDatabase(url, expectedPageSize, fetchImpl = fetch) {
+  const response = await fetchImpl(url, { headers: { Range: `bytes=0-${PROBE_BYTES - 1}` } });
+
+  if (response.status !== 206) {
+    throw new Error(
+      `${url}: expected 206 for a range request, got ${response.status}. ` +
+        "The host must serve byte ranges of the database.",
+    );
+  }
+
+  const total = parseTotalBytes(response.headers.get("Content-Range"));
+  const header = new Uint8Array(await response.arrayBuffer());
+
+  if (!looksLikeSqlite(header)) {
+    throw new Error(
+      `${url}: the first bytes are not a SQLite header, so the host is not ` +
+        "serving the database as stored — check for Content-Encoding on ranged responses.",
+    );
+  }
+
+  const pageSize = readPageSize(header);
+  if (expectedPageSize && pageSize !== expectedPageSize) {
+    // Not fatal: it still reads correctly, just at more requests per page.
+    console.warn(
+      `[httpvfs] ${url} has page size ${pageSize}, but requests are ${expectedPageSize} bytes. ` +
+        "Every page read now spans more than one request.",
+    );
+  }
+
+  return total;
+}

web/src/lib/db-probe.test.js

@@ -0,0 +1,90 @@
+import { describe, expect, it, vi } from "vitest";
+import { looksLikeSqlite, parseTotalBytes, probeDatabase, readPageSize } from "./db-probe.js";
+
+/** The first bytes of a real database: magic, NUL, then the page size at 16. */
+function header(pageSize = 1024, magic = "SQLite format 3") {
+  const bytes = new Uint8Array(100);
+  for (let i = 0; i < magic.length; i += 1) bytes[i] = magic.charCodeAt(i);
+  bytes[16] = pageSize >> 8;
+  bytes[17] = pageSize & 0xff;
+  return bytes;
+}
+
+function respond(bytes, { status = 206, contentRange = `bytes 0-99/${317096960}` } = {}) {
+  return {
+    status,
+    headers: { get: (name) => (name.toLowerCase() === "content-range" ? contentRange : null) },
+    arrayBuffer: async () => bytes.buffer,
+  };
+}
+
+describe("parseTotalBytes", () => {
+  it("takes the total from a Content-Range", () => {
+    expect(parseTotalBytes("bytes 0-99/317096960")).toBe(317096960);
+  });
+
+  it("refuses an unknown total", () => {
+    expect(() => parseTotalBytes("bytes 0-99/*")).toThrow(/how large/);
+    expect(() => parseTotalBytes(null)).toThrow(/absent/);
+  });
+});
+
+describe("readPageSize", () => {
+  it("reads the two big-endian bytes at offset 16", () => {
+    expect(readPageSize(header(1024))).toBe(1024);
+    expect(readPageSize(header(4096))).toBe(4096);
+  });
+
+  it("treats 1 as 65536, as the file format does", () => {
+    expect(readPageSize(header(1))).toBe(65536);
+  });
+});
+
+describe("looksLikeSqlite", () => {
+  it("accepts a real header", () => {
+    expect(looksLikeSqlite(header())).toBe(true);
+  });
+
+  it("rejects a gzip stream, which is what a compressing host returns", () => {
+    const gzip = new Uint8Array([0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00]);
+    expect(looksLikeSqlite(gzip)).toBe(false);
+  });
+
+  it("requires the NUL that terminates the magic", () => {
+    expect(looksLikeSqlite(header(1024, "SQLite format 3x"))).toBe(false);
+  });
+});
+
+describe("probeDatabase", () => {
+  it("asks for the header by range and returns the total length", async () => {
+    const fetchImpl = vi.fn(async () => respond(header()));
+    const total = await probeDatabase("/db/2016.sqlite3", 1024, fetchImpl);
+
+    expect(total).toBe(317096960);
+    expect(fetchImpl).toHaveBeenCalledWith("/db/2016.sqlite3", {
+      headers: { Range: "bytes=0-99" },
+    });
+  });
+
+  it("fails when the host ignores the range", async () => {
+    const fetchImpl = async () => respond(header(), { status: 200 });
+    await expect(probeDatabase("/db/2016.sqlite3", 1024, fetchImpl)).rejects.toThrow(/expected 206/);
+  });
+
+  it("fails when the bytes are not a database", async () => {
+    const gzip = new Uint8Array([0x1f, 0x8b, 0x08]);
+    const fetchImpl = async () => respond(gzip);
+    await expect(probeDatabase("/db/2016.sqlite3", 1024, fetchImpl)).rejects.toThrow(
+      /not a SQLite header/,
+    );
+  });
+
+  it("warns, but continues, when the page size is not the request size", async () => {
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+    const fetchImpl = async () => respond(header(4096));
+
+    await expect(probeDatabase("/db/2016.sqlite3", 1024, fetchImpl)).resolves.toBe(317096960);
+    expect(warn).toHaveBeenCalledWith(expect.stringMatching(/page size 4096/));
+    warn.mockRestore();
+  });
+});

web/src/lib/sqlite.svelte.js

@@ -1,6 +1,7 @@
 import { createDbWorker } from "sql.js-httpvfs";
 import workerUrl from "sql.js-httpvfs/dist/sqlite.worker.js?url";
 import wasmUrl from "sql.js-httpvfs/dist/sql-wasm.wasm?url";
+import { probeDatabase } from "./db-probe.js";
 
 /**
  * The database is read where it lies. SQLite asks for pages, the virtual file
@@ -61,8 +62,17 @@ export class RemoteDatabase {
   async #open() {
     const opened = performance.now();
     try {
+      // Supplied rather than left to the library, which would size the file
+      // with a HEAD request. GitHub Pages compresses that response and reports
+      // the compressed length, which the library refuses to use. See db-probe.
+      const fileLength = await probeDatabase(this.url, CHUNK_BYTES);
       const worker = await createDbWorker(
-        [{ from: "inline", config: { serverMode: "full", url: this.url, requestChunkSize: CHUNK_BYTES } }],
+        [
+          {
+            from: "inline",
+            config: { serverMode: "full", url: this.url, requestChunkSize: CHUNK_BYTES, fileLength },
+          },
+        ],
         workerUrl,
         wasmUrl,
         this.budgetBytes,