feat: read the databases over HTTP range requests

The browser downloaded 45 MB of gzipped SQLite before it could answer
anything. Now sql.js-httpvfs asks for the pages a query touches and the
databases ship uncompressed as <id>.sqlite3 — a byte range of a gzip
stream is not a byte range of a database.

That only works if every query the site issues is index-driven, and
measured against the real 2016 file, most were not:

  so_bao_danh = ?              SEARCH via PK          ~20 KB
  ho_ten_ascii LIKE '%x%'      SCAN                   127 MB
  ho_ten_ascii LIKE 'x%'       SCAN                   127 MB
  COUNT(*)                     covering index scan     20 MB
  ORDER BY toan DESC LIMIT 10  SCAN + temp b-tree     127 MB

Prefix LIKE scans because SQLite's LIKE optimisation needs a NOCASE
index; a range comparison does use the index. So the schema changed to
suit the access pattern rather than the search changing to suit the
schema.

name_word holds one row per word of each name, WITHOUT ROWID so the
table is the index, carrying ho_ten_ascii so a multi-word query is
resolved inside a single b-tree. name_word_freq says which word of a
query is rarest — the vocabulary is 4,397 words across 2.87M entries, so
"buu loc" seeks on 287 entries rather than walking the 300,000 that
"thi" would. Searching by any word of a name survives, at a few hundred
KB a query.

idx_ho_ten and idx_ho_ten_ascii are gone: no plan could use either.
Partial indexes on toan, khtn and khxh cost 12 MB and keep the SQL
presets off a full scan. The footer's candidate count now comes from
datasets.json instead of COUNT(*).

2016 grows 223.5 MB to 288.6 MB, 2017 162.7 MB to 237.7 MB, and the site
is 528 MB against the 1 GB GitHub Pages limit. Row counts are unchanged.

The SQL tab is the one place a user can still write a query that reads
the whole table, so it asks before it opens, runs under a byte budget
that stops a runaway query, and shows what each query actually fetched.

Verified: row counts through the assembler guards, every app query
index-driven under EXPLAIN QUERY PLAN, and GitHub Pages returning 206
with a correct Content-Range. Not verified in a browser — this machine
has none — and the library refuses to open a file the host compresses,
so the deployed response headers need a look.

assembler/internal/databases/databases.go

@@ -1,5 +1,4 @@
-// Package databases builds, verifies and compresses one SQLite file per
-// dataset.
+// Package databases builds and verifies one SQLite file per dataset.
 //
 // VERIFICATION IS THE POINT OF THIS PACKAGE, not an extra.
 //
@@ -11,13 +10,15 @@
 //
 // The guards below close that: a build whose row count does not match the
 // registry, or whose artifact is implausibly small, fails the pipeline.
+//
+// The databases ship uncompressed. The browser reads them a page at a time over
+// HTTP range requests, and a range of a gzip stream is not a range of the
+// database.
 package databases
 
 import (
-	"compress/gzip"
 	"database/sql"
 	"fmt"
-	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -30,10 +31,15 @@ import (
 // driverName is modernc.org/sqlite's registered name.
 const driverName = "sqlite"
 
-// minSizeRatio: a gzipped database far below its usual size means a truncated
-// build, even if the row count somehow passed.
+// minSizeRatio: a database far below its usual size means a truncated build,
+// even if the row count somehow passed.
 const minSizeRatio = 0.9
 
+// Extension is the published suffix. Not ".db": the sql.js-httpvfs ecosystem
+// uses ".sqlite3", and keeping ".db" free lets the site assembly treat any
+// stray .db or SQLite journal in the output as the leftover it is.
+const Extension = ".sqlite3"
+
 // Paths locates the pieces this package needs.
 type Paths struct {
 	// Root is the repository root.
@@ -75,7 +81,7 @@ func Build(p Paths, bin string, d registry.Dataset) error {
 	if err := os.MkdirAll(p.OutDir, 0o755); err != nil {
 		return err
 	}
-	db := filepath.Join(p.OutDir, d.ID+".db")
+	db := filepath.Join(p.OutDir, d.ID+Extension)
 
 	cmd := exec.Command(bin,
 		"build",
@@ -99,12 +105,11 @@ func Build(p Paths, bin string, d registry.Dataset) error {
 	}
 	fmt.Printf("  ✓ %s: %d rows (matches expected)\n", d.ID, rows)
 
-	gz, size, err := compress(db)
+	st, err := os.Stat(db)
 	if err != nil {
 		return fmt.Errorf("%s: %w", d.ID, err)
 	}
-
-	sizeMb := float64(size) / 1024 / 1024
+	sizeMb := float64(st.Size()) / 1024 / 1024
 	if min := d.DbSizeMb * minSizeRatio; sizeMb < min {
 		return fmt.Errorf(
 			"%s: %.1f MB is below %.1f MB (%.0f%% of the expected %.0f MB)\n"+
@@ -112,7 +117,7 @@ func Build(p Paths, bin string, d registry.Dataset) error {
 			d.ID, sizeMb, min, minSizeRatio*100, d.DbSizeMb)
 	}
 
-	fmt.Printf("  → %s (%.1f MB)\n\n", filepath.Base(gz), sizeMb)
+	fmt.Printf("  → %s (%.1f MB)\n\n", filepath.Base(db), sizeMb)
 	return nil
 }
 
@@ -131,61 +136,8 @@ func countRows(path string) (int64, error) {
 	return n, nil
 }
 
-// compress gzips path to path+".gz" and removes the original, returning the
-// compressed path and its size.
-//
-// The source is deleted only after the compressed file is closed successfully,
-// so a failure part-way through leaves the database rather than losing it.
-func compress(path string) (string, int64, error) {
-	in, err := os.Open(path)
-	if err != nil {
-		return "", 0, err
-	}
-	defer in.Close()
-
-	gzPath := path + ".gz"
-	out, err := os.Create(gzPath)
-	if err != nil {
-		return "", 0, err
-	}
-
-	zw, err := gzip.NewWriterLevel(out, gzip.BestCompression)
-	if err != nil {
-		out.Close()
-		return "", 0, err
-	}
-	if _, err := io.Copy(zw, in); err != nil {
-		zw.Close()
-		out.Close()
-		os.Remove(gzPath)
-		return "", 0, err
-	}
-	if err := zw.Close(); err != nil {
-		out.Close()
-		os.Remove(gzPath)
-		return "", 0, err
-	}
-	if err := out.Close(); err != nil {
-		os.Remove(gzPath)
-		return "", 0, err
-	}
-
-	if err := in.Close(); err != nil {
-		return "", 0, err
-	}
-	if err := os.Remove(path); err != nil {
-		return "", 0, fmt.Errorf("removing the uncompressed database: %w", err)
-	}
-
-	st, err := os.Stat(gzPath)
-	if err != nil {
-		return "", 0, err
-	}
-	return gzPath, st.Size(), nil
-}
-
 // Clean removes staged artifacts for datasets that are no longer in the
-// registry. Without this a removed dataset's .db.gz lingers in the staging
+// registry. Without this a removed dataset's file lingers in the staging
 // directory, and the site assembly copies that directory wholesale — so the
 // dead database would be published again.
 func Clean(p Paths, keep []registry.Dataset) error {
@@ -197,10 +149,9 @@ func Clean(p Paths, keep []registry.Dataset) error {
 		return err
 	}
 
-	wanted := make(map[string]bool, len(keep)*2)
+	wanted := make(map[string]bool, len(keep))
 	for _, d := range keep {
-		wanted[d.ID+".db"] = true
-		wanted[d.ID+".db.gz"] = true
+		wanted[d.ID+Extension] = true
 	}
 
 	for _, e := range entries {