mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-17 10:15:04 +00:00
feat: read the databases over HTTP range requests
The browser downloaded 45 MB of gzipped SQLite before it could answer anything. Now sql.js-httpvfs asks for the pages a query touches and the databases ship uncompressed as <id>.sqlite3 — a byte range of a gzip stream is not a byte range of a database. That only works if every query the site issues is index-driven, and measured against the real 2016 file, most were not: so_bao_danh = ? SEARCH via PK ~20 KB ho_ten_ascii LIKE '%x%' SCAN 127 MB ho_ten_ascii LIKE 'x%' SCAN 127 MB COUNT(*) covering index scan 20 MB ORDER BY toan DESC LIMIT 10 SCAN + temp b-tree 127 MB Prefix LIKE scans because SQLite's LIKE optimisation needs a NOCASE index; a range comparison does use the index. So the schema changed to suit the access pattern rather than the search changing to suit the schema. name_word holds one row per word of each name, WITHOUT ROWID so the table is the index, carrying ho_ten_ascii so a multi-word query is resolved inside a single b-tree. name_word_freq says which word of a query is rarest — the vocabulary is 4,397 words across 2.87M entries, so "buu loc" seeks on 287 entries rather than walking the 300,000 that "thi" would. Searching by any word of a name survives, at a few hundred KB a query. idx_ho_ten and idx_ho_ten_ascii are gone: no plan could use either. Partial indexes on toan, khtn and khxh cost 12 MB and keep the SQL presets off a full scan. The footer's candidate count now comes from datasets.json instead of COUNT(*). 2016 grows 223.5 MB to 288.6 MB, 2017 162.7 MB to 237.7 MB, and the site is 528 MB against the 1 GB GitHub Pages limit. Row counts are unchanged. The SQL tab is the one place a user can still write a query that reads the whole table, so it asks before it opens, runs under a byte budget that stops a runaway query, and shows what each query actually fetched. Verified: row counts through the assembler guards, every app query index-driven under EXPLAIN QUERY PLAN, and GitHub Pages returning 206 with a correct Content-Range. Not verified in a browser — this machine has none — and the library refuses to open a file the host compresses, so the deployed response headers need a look.
This commit is contained in:
1 parent
873e96d482
commit
dbc23c25c5
24 files changed
+743
-408
No files matched your search
@@ -1,8 +1,6 @@
|
||||
package databases
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
@@ -11,52 +9,12 @@ import (
|
||||
"github.com/tiennm99/thptqg/assembler/internal/registry"
|
||||
)
|
||||
|
||||
// TestCompressRoundTripsAndRemovesTheSource: only the .gz may survive, so that
|
||||
// shipping a 100+ MB uncompressed database is structurally impossible rather
|
||||
// than left to a cleanup step.
|
||||
func TestCompressRoundTripsAndRemovesTheSource(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := filepath.Join(dir, "2016.db")
|
||||
body := []byte("pretend this is a SQLite file")
|
||||
if err := os.WriteFile(src, body, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
gzPath, size, err := compress(src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gzPath != src+".gz" || size <= 0 {
|
||||
t.Fatalf("gzPath=%q size=%d", gzPath, size)
|
||||
}
|
||||
if _, err := os.Stat(src); !os.IsNotExist(err) {
|
||||
t.Error("the uncompressed database must not survive")
|
||||
}
|
||||
|
||||
f, err := os.Open(gzPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
zr, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := io.ReadAll(zr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != string(body) {
|
||||
t.Errorf("round-trip gave %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, name := range []string{
|
||||
"2016.db.gz", "2017.db.gz",
|
||||
"2017-old.db.gz", // dropped from the registry
|
||||
"2017-old2.db.gz", // dropped from the registry
|
||||
"2016.sqlite3", "2017.sqlite3",
|
||||
"2017-old.sqlite3", // dropped from the registry
|
||||
"2017-old2.sqlite3", // dropped from the registry
|
||||
"2016.db-journal", // interrupted run
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil {
|
||||
@@ -80,7 +38,7 @@ func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) {
|
||||
}
|
||||
slices.Sort(left)
|
||||
|
||||
want := []string{"2016.db.gz", "2017.db.gz"}
|
||||
want := []string{"2016.sqlite3", "2017.sqlite3"}
|
||||
if !slices.Equal(left, want) {
|
||||
t.Errorf("left %v, want %v", left, want)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user