Revert "Merge branch 'fix/httpvfs-chunked-length'"

This reverts commit d4f106f488, reversing
changes made to c5ef1f58ff.
This commit is contained in:
tiennm99 committed 2026-10-03 11:59:22 +07:00
1 parent d4f106f488
commit fc01d4a9bd
16 files changed
+132 -471

No files matched your search

+4 -12
View File
@@ -35,18 +35,10 @@ const driverName = "sqlite"
// even if the row count somehow passed.
const minSizeRatio = 0.9
// Extension is the published suffix. The trailing 0 is a chunk index, not a
// typo: the browser reads the file through sql.js-httpvfs in chunked mode,
// which is the only mode whose config accepts the file's length. That mode
// builds each request's URL as urlPrefix + chunkIndex, and one chunk holds the
// whole database, so the index is always 0 and the prefix is "<id>.sqlite3".
//
// The length has to come from the config because the library otherwise takes
// it from a HEAD request, which GitHub Pages answers with the gzipped size.
//
// Not ".db" either: keeping that name free lets the site assembly treat any
// stray .db or SQLite journal in the output as the leftover it is.
const Extension = ".sqlite30"
// Extension is the published suffix. Not ".db": keeping that name free lets
// the site assembly treat any stray .db or SQLite journal in the output as the
// leftover it is.
const Extension = ".sqlite3"
// Paths locates the pieces this package needs.
type Paths struct {
@@ -12,9 +12,9 @@ import (
func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) {
dir := t.TempDir()
for _, name := range []string{
"2016.sqlite30", "2017.sqlite30",
"2017-old.sqlite30", // dropped from the registry
"2017-old2.sqlite30", // dropped from the registry
"2016.sqlite3", "2017.sqlite3",
"2017-old.sqlite3", // dropped from the registry
"2017-old2.sqlite3", // dropped from the registry
"2016.db-journal", // interrupted run
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil {
@@ -38,7 +38,7 @@ func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) {
}
slices.Sort(left)
want := []string{"2016.sqlite30", "2017.sqlite30"}
want := []string{"2016.sqlite3", "2017.sqlite3"}
if !slices.Equal(left, want) {
t.Errorf("left %v, want %v", left, want)
}
+4 -5
View File
@@ -147,11 +147,10 @@ func checkDatabasesPresent(siteDir string, datasets []registry.Dataset) error {
return nil
}
// strayArtifact matches what must never reach the output: a SQLite journal from
// an interrupted run, a database under either older name — .db, or .sqlite3
// without the chunk index the client asks for — or a gzipped database from
// before the switch to range requests.
var strayArtifact = regexp.MustCompile(`(\.db|\.sqlite30?)(-journal|-wal|-shm)$|\.db$|\.sqlite3$|\.gz$`)
// strayArtifact matches what must never reach the output: a SQLite journal
// from an interrupted run, a database under the old .db name, or one under the
// .sqlite30 name a former range-request client asked for. Each is 100+ MB.
var strayArtifact = regexp.MustCompile(`(\.db|\.sqlite30?)(-journal|-wal|-shm)$|\.db$|\.sqlite30$|\.gz$`)
// checkNoStrayArtifacts rejects leftovers that would be published.
//
+15 -17
View File
@@ -39,7 +39,7 @@ func write(t *testing.T, path, body string) {
}
func TestAssembleProducesAPageForEveryDataset(t *testing.T) {
p := fakeBuild(t, "2016.sqlite30", "2017.sqlite30")
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
if err := Assemble(p, datasets); err != nil {
t.Fatal(err)
}
@@ -49,7 +49,7 @@ func TestAssembleProducesAPageForEveryDataset(t *testing.T) {
filepath.Join("2016", "index.html"),
filepath.Join("2017", "index.html"),
filepath.Join("_app", "immutable", "entry.js"),
filepath.Join("db", "2016.sqlite30"),
filepath.Join("db", "2016.sqlite3"),
} {
if _, err := os.Stat(filepath.Join(p.Site, want)); err != nil {
t.Errorf("missing from the artifact: %s", want)
@@ -61,7 +61,7 @@ func TestAssembleProducesAPageForEveryDataset(t *testing.T) {
// entry generator, which reads the same registry this does. If the two fall out
// of step, that dataset's URL 404s — so the build stops instead.
func TestMissingDatasetPageFailsTheBuild(t *testing.T) {
p := fakeBuild(t, "2016.sqlite30", "2017.sqlite30")
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
if err := os.RemoveAll(filepath.Join(p.Dist, "2017")); err != nil {
t.Fatal(err)
}
@@ -80,12 +80,12 @@ func TestMissingDatasetPageFailsTheBuild(t *testing.T) {
// renders, every query 404s, and CI stays green. The row-count and size guards
// cannot catch this — they only run when a database was built at all.
func TestMissingDatabaseFailsTheBuild(t *testing.T) {
p := fakeBuild(t, "2016.sqlite30") // 2017 never built
p := fakeBuild(t, "2016.sqlite3") // 2017 never built
err := Assemble(p, datasets)
if err == nil {
t.Fatal("expected an error when a database is missing")
}
if !strings.Contains(err.Error(), "2017.sqlite30") {
if !strings.Contains(err.Error(), "2017.sqlite3") {
t.Errorf("the error should name the missing database, got: %v", err)
}
}
@@ -93,25 +93,23 @@ func TestMissingDatabaseFailsTheBuild(t *testing.T) {
// TestEmptyDatabaseFailsTheBuild: a zero-byte file satisfies "exists" but is
// not a database.
func TestEmptyDatabaseFailsTheBuild(t *testing.T) {
p := fakeBuild(t, "2016.sqlite30", "2017.sqlite30")
write(t, filepath.Join(p.Dist, "db", "2017.sqlite30"), "")
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
write(t, filepath.Join(p.Dist, "db", "2017.sqlite3"), "")
if err := Assemble(p, datasets); err == nil {
t.Fatal("expected an error for a zero-byte database")
}
}
// TestStrayArtifactFailsTheBuild: a journal means an interrupted run, a .db or
// a chunk-index-less .sqlite3 means an older naming the client no longer asks
// for, a .gz means a database the site could not read a range of — and each is
// 100+ MB.
// a .sqlite30 means a name from an earlier design that no client asks for now,
// a .gz means a database left compressed — and each is 100+ MB.
func TestStrayArtifactFailsTheBuild(t *testing.T) {
for _, name := range []string{
"2016.db", "2016.sqlite3",
"2016.db", "2016.sqlite30",
"2016.sqlite3-journal", "2016.sqlite3-wal", "2016.sqlite3-shm", "2016.sqlite3.gz",
"2016.sqlite30-journal", "2016.sqlite30-wal", "2016.sqlite30-shm",
} {
t.Run(name, func(t *testing.T) {
p := fakeBuild(t, "2016.sqlite30", "2017.sqlite30")
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
write(t, filepath.Join(p.Dist, "db", name), "raw sqlite")
err := Assemble(p, datasets)
if err == nil {
@@ -127,12 +125,12 @@ func TestStrayArtifactFailsTheBuild(t *testing.T) {
// TestPublishedDatabaseIsNotMistakenForStray: the pattern must pass the one
// file the site is built to serve. Getting this wrong would fail every build.
func TestPublishedDatabaseIsNotMistakenForStray(t *testing.T) {
if strayArtifact.MatchString("2016.sqlite30") {
if strayArtifact.MatchString("2016.sqlite3") {
t.Error("the published database must not be treated as a stray artifact")
}
for _, name := range []string{
"2016.db", "x.sqlite3", "x.sqlite3-journal", "x.sqlite3-wal", "x.sqlite3-shm",
"x.sqlite30-journal", "x.sqlite30-wal", "x.sqlite30-shm", "x.sqlite3.gz", "x.db.gz",
"2016.db", "x.sqlite30", "x.sqlite3-journal", "x.sqlite3-wal", "x.sqlite3-shm",
"x.sqlite30-journal", "x.sqlite3.gz", "x.db.gz",
} {
if !strayArtifact.MatchString(name) {
t.Errorf("%s should be rejected", name)
@@ -151,7 +149,7 @@ func TestAssembleRejectsAMissingBuild(t *testing.T) {
// TestAssembleIsIdempotent: the site directory is rebuilt from scratch, so a
// previous run's leftovers cannot survive into the artifact.
func TestAssembleIsIdempotent(t *testing.T) {
p := fakeBuild(t, "2016.sqlite30", "2017.sqlite30")
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
if err := Assemble(p, datasets); err != nil {
t.Fatal(err)
}
+3 -3
View File
@@ -135,7 +135,7 @@ func compareOne(id, dirA, dirB string) (Result, error) {
return res, nil
}
// open finds <id>.sqlite30 in dir and returns a read-only handle.
// open finds <id>.sqlite3 in dir and returns a read-only handle.
//
// The older names are still accepted, and a gzipped database is expanded to a
// temporary file rather than rejected: the two sides of a comparison are often
@@ -143,7 +143,7 @@ func compareOne(id, dirA, dirB string) (Result, error) {
func open(dir, id string) (*sql.DB, func(), error) {
noop := func() {}
for _, name := range []string{id + ".sqlite30", id + ".sqlite3", id + ".db"} {
for _, name := range []string{id + ".sqlite3", id + ".sqlite30", id + ".db"} {
plain := filepath.Join(dir, name)
if _, err := os.Stat(plain); err == nil {
db, err := sql.Open(driverName, "file:"+plain+"?mode=ro")
@@ -154,7 +154,7 @@ func open(dir, id string) (*sql.DB, func(), error) {
gzPath := filepath.Join(dir, id+".db.gz")
f, err := os.Open(gzPath)
if err != nil {
return nil, noop, fmt.Errorf("no %s.sqlite30, %s.sqlite3, %s.db or %s.db.gz in %s", id, id, id, id, dir)
return nil, noop, fmt.Errorf("no %s.sqlite3, %s.sqlite30, %s.db or %s.db.gz in %s", id, id, id, id, dir)
}
defer f.Close()