import { createDbWorker } from "sql.js-httpvfs"; import workerUrl from "sql.js-httpvfs/dist/sqlite.worker.js?url"; import wasmUrl from "sql.js-httpvfs/dist/sql-wasm.wasm?url"; import { probeDatabase } from "./db-probe.js"; /** * The database is read where it lies. SQLite asks for pages, the virtual file * system turns each into an HTTP range request, and only the pages a query * touches ever cross the network — a few hundred KB for a lookup, against the * 45 MB the whole file used to cost before the first query. * * That only holds while every query is index-driven. The schema exists for it: * name_word serves name search, and the score indexes serve the SQL presets. * An unindexed query walks the table and pulls all 100+ MB of it, which is what * the byte budget below is for. */ // Must equal the page size the parser writes (PRAGMA page_size in // parser/internal/writer/writer.go), so one request is exactly one page. A // mismatch makes every logical page read span two requests. const CHUNK_BYTES = 1024; /** Generous for indexed work: a name search costs well under 1 MB. */ export const SEARCH_BUDGET_BYTES = 25 * 1024 * 1024; /** What the SQL tab gets once the user has accepted the cost of a scan. */ export const PLAYGROUND_BUDGET_BYTES = 250 * 1024 * 1024; /** * What one query cost over the network: `{ requests, bytes, ms }`. */ /** * One remotely-paged database, with the load state the UI needs. * * `budgetBytes` is a hard ceiling for the worker's lifetime: past it a query * fails instead of quietly downloading the file. Raising it means a new worker, * which costs only the header pages. */ export class RemoteDatabase { ready = $state(false); error = $state(null); /** Bytes fetched by this database so far, prefetch included. */ bytesRead = $state(0); /** HTTP range requests issued so far. */ requests = $state(0); /** What the most recent query cost on its own. */ lastCost = $state(null); #worker = null; #opening; #closed = false; // Previous cumulative reading, so a query's own cost is a subtraction. #seen = { requests: 0, bytes: 0 }; constructor(url, budgetBytes = SEARCH_BUDGET_BYTES) { this.url = url; this.budgetBytes = budgetBytes; this.#opening = this.#open(); } async #open() { const opened = performance.now(); try { // Supplied rather than left to the library, which would size the file // with a HEAD request. GitHub Pages compresses that response and reports // the compressed length, which the library refuses to use. See db-probe. const fileLength = await probeDatabase(this.url, CHUNK_BYTES); const worker = await createDbWorker( [ { from: "inline", config: { serverMode: "full", url: this.url, requestChunkSize: CHUNK_BYTES, fileLength }, }, ], workerUrl, wasmUrl, this.budgetBytes, ); if (this.#closed) throw new Error("closed"); this.#worker = worker; this.ready = true; // Opening is not free either: the header and schema pages are read before // any query runs, and that shows up in every later session total. await this.#account(worker, `open ${this.url}`, performance.now() - opened); return worker; } catch (err) { if (!this.#closed) { this.error = message(err); this.ready = false; } throw err; } } /** * Run a query and return its rows as objects. * * `label` names the query in the console trace — the only way to see what a * search actually costs, since the byte count depends on how much the read * heads prefetched, not just on the pages the plan needed. */ async query(sql, params = [], label) { const worker = this.#worker ?? (await this.#opening); const started = performance.now(); try { return await worker.db.query(sql, ...params); } finally { await this.#account(worker, label ?? firstLine(sql), performance.now() - started); } } /** * Read the cumulative counters and report the delta. * * getStats() rather than the worker's `bytesRead`: that one is the budget * counter and resets itself to zero when a query trips the ceiling, so it * would under-report exactly when the number matters most. */ async #account(worker, label, ms) { const stats = await worker.worker.getStats().catch(() => null); if (!stats) return; const cost = { requests: stats.totalRequests - this.#seen.requests, bytes: stats.totalFetchedBytes - this.#seen.bytes, ms, }; this.#seen = { requests: stats.totalRequests, bytes: stats.totalFetchedBytes }; this.requests = stats.totalRequests; this.bytesRead = stats.totalFetchedBytes; this.lastCost = cost; console.info( `[httpvfs] ${label} — ${cost.requests} request(s), ${formatBytes(cost.bytes)}, ${ms.toFixed(0)} ms` + ` · session ${stats.totalRequests} request(s), ${formatBytes(stats.totalFetchedBytes)}` + ` of ${formatBytes(stats.totalBytes)}`, ); } /** * Drop this database. createDbWorker owns the Worker and exposes no handle to * it, so the thread outlives this call; a page creates at most one per * dataset and one more if the SQL budget is raised, which is why that is * tolerable rather than a leak worth working around. */ close() { this.#closed = true; this.#worker = null; this.ready = false; } } /** True when a query failed because it would have exceeded the byte budget. */ export function isBudgetError(err) { return /maxBytesToRead|too much data|exceeded/i.test(message(err)); } function message(err) { return err instanceof Error ? err.message : String(err); } /** Enough of a query to recognise it in the console. */ function firstLine(sql) { const line = sql.trim().split("\n")[0]; return line.length > 70 ? `${line.slice(0, 70)}…` : line; } export function formatBytes(n) { return n < 1024 * 1024 ? `${Math.round(n / 1024)} KB` : `${(n / 1048576).toFixed(1)} MB`; }