mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-13 16:15:37 +00:00
Revert "Merge branch 'fix/httpvfs-chunked-length'"
This reverts commitd4f106f488, reversing changes made toc5ef1f58ff.
This commit is contained in:
1 parent
d4f106f488
commit
fc01d4a9bd
16 files changed
+132
-471
No files matched your search
@@ -35,18 +35,10 @@ const driverName = "sqlite"
|
||||
// even if the row count somehow passed.
|
||||
const minSizeRatio = 0.9
|
||||
|
||||
// Extension is the published suffix. The trailing 0 is a chunk index, not a
|
||||
// typo: the browser reads the file through sql.js-httpvfs in chunked mode,
|
||||
// which is the only mode whose config accepts the file's length. That mode
|
||||
// builds each request's URL as urlPrefix + chunkIndex, and one chunk holds the
|
||||
// whole database, so the index is always 0 and the prefix is "<id>.sqlite3".
|
||||
//
|
||||
// The length has to come from the config because the library otherwise takes
|
||||
// it from a HEAD request, which GitHub Pages answers with the gzipped size.
|
||||
//
|
||||
// Not ".db" either: keeping that name free lets the site assembly treat any
|
||||
// stray .db or SQLite journal in the output as the leftover it is.
|
||||
const Extension = ".sqlite30"
|
||||
// Extension is the published suffix. Not ".db": keeping that name free lets
|
||||
// the site assembly treat any stray .db or SQLite journal in the output as the
|
||||
// leftover it is.
|
||||
const Extension = ".sqlite3"
|
||||
|
||||
// Paths locates the pieces this package needs.
|
||||
type Paths struct {
|
||||
|
||||
@@ -12,9 +12,9 @@ import (
|
||||
func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, name := range []string{
|
||||
"2016.sqlite30", "2017.sqlite30",
|
||||
"2017-old.sqlite30", // dropped from the registry
|
||||
"2017-old2.sqlite30", // dropped from the registry
|
||||
"2016.sqlite3", "2017.sqlite3",
|
||||
"2017-old.sqlite3", // dropped from the registry
|
||||
"2017-old2.sqlite3", // dropped from the registry
|
||||
"2016.db-journal", // interrupted run
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o644); err != nil {
|
||||
@@ -38,7 +38,7 @@ func TestCleanRemovesOnlyDroppedDatasets(t *testing.T) {
|
||||
}
|
||||
slices.Sort(left)
|
||||
|
||||
want := []string{"2016.sqlite30", "2017.sqlite30"}
|
||||
want := []string{"2016.sqlite3", "2017.sqlite3"}
|
||||
if !slices.Equal(left, want) {
|
||||
t.Errorf("left %v, want %v", left, want)
|
||||
}
|
||||
|
||||
@@ -147,11 +147,10 @@ func checkDatabasesPresent(siteDir string, datasets []registry.Dataset) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// strayArtifact matches what must never reach the output: a SQLite journal from
|
||||
// an interrupted run, a database under either older name — .db, or .sqlite3
|
||||
// without the chunk index the client asks for — or a gzipped database from
|
||||
// before the switch to range requests.
|
||||
var strayArtifact = regexp.MustCompile(`(\.db|\.sqlite30?)(-journal|-wal|-shm)$|\.db$|\.sqlite3$|\.gz$`)
|
||||
// strayArtifact matches what must never reach the output: a SQLite journal
|
||||
// from an interrupted run, a database under the old .db name, or one under the
|
||||
// .sqlite30 name a former range-request client asked for. Each is 100+ MB.
|
||||
var strayArtifact = regexp.MustCompile(`(\.db|\.sqlite30?)(-journal|-wal|-shm)$|\.db$|\.sqlite30$|\.gz$`)
|
||||
|
||||
// checkNoStrayArtifacts rejects leftovers that would be published.
|
||||
//
|
||||
|
||||
@@ -39,7 +39,7 @@ func write(t *testing.T, path, body string) {
|
||||
}
|
||||
|
||||
func TestAssembleProducesAPageForEveryDataset(t *testing.T) {
|
||||
p := fakeBuild(t, "2016.sqlite30", "2017.sqlite30")
|
||||
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
|
||||
if err := Assemble(p, datasets); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -49,7 +49,7 @@ func TestAssembleProducesAPageForEveryDataset(t *testing.T) {
|
||||
filepath.Join("2016", "index.html"),
|
||||
filepath.Join("2017", "index.html"),
|
||||
filepath.Join("_app", "immutable", "entry.js"),
|
||||
filepath.Join("db", "2016.sqlite30"),
|
||||
filepath.Join("db", "2016.sqlite3"),
|
||||
} {
|
||||
if _, err := os.Stat(filepath.Join(p.Site, want)); err != nil {
|
||||
t.Errorf("missing from the artifact: %s", want)
|
||||
@@ -61,7 +61,7 @@ func TestAssembleProducesAPageForEveryDataset(t *testing.T) {
|
||||
// entry generator, which reads the same registry this does. If the two fall out
|
||||
// of step, that dataset's URL 404s — so the build stops instead.
|
||||
func TestMissingDatasetPageFailsTheBuild(t *testing.T) {
|
||||
p := fakeBuild(t, "2016.sqlite30", "2017.sqlite30")
|
||||
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
|
||||
if err := os.RemoveAll(filepath.Join(p.Dist, "2017")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -80,12 +80,12 @@ func TestMissingDatasetPageFailsTheBuild(t *testing.T) {
|
||||
// renders, every query 404s, and CI stays green. The row-count and size guards
|
||||
// cannot catch this — they only run when a database was built at all.
|
||||
func TestMissingDatabaseFailsTheBuild(t *testing.T) {
|
||||
p := fakeBuild(t, "2016.sqlite30") // 2017 never built
|
||||
p := fakeBuild(t, "2016.sqlite3") // 2017 never built
|
||||
err := Assemble(p, datasets)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error when a database is missing")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "2017.sqlite30") {
|
||||
if !strings.Contains(err.Error(), "2017.sqlite3") {
|
||||
t.Errorf("the error should name the missing database, got: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -93,25 +93,23 @@ func TestMissingDatabaseFailsTheBuild(t *testing.T) {
|
||||
// TestEmptyDatabaseFailsTheBuild: a zero-byte file satisfies "exists" but is
|
||||
// not a database.
|
||||
func TestEmptyDatabaseFailsTheBuild(t *testing.T) {
|
||||
p := fakeBuild(t, "2016.sqlite30", "2017.sqlite30")
|
||||
write(t, filepath.Join(p.Dist, "db", "2017.sqlite30"), "")
|
||||
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
|
||||
write(t, filepath.Join(p.Dist, "db", "2017.sqlite3"), "")
|
||||
if err := Assemble(p, datasets); err == nil {
|
||||
t.Fatal("expected an error for a zero-byte database")
|
||||
}
|
||||
}
|
||||
|
||||
// TestStrayArtifactFailsTheBuild: a journal means an interrupted run, a .db or
|
||||
// a chunk-index-less .sqlite3 means an older naming the client no longer asks
|
||||
// for, a .gz means a database the site could not read a range of — and each is
|
||||
// 100+ MB.
|
||||
// a .sqlite30 means a name from an earlier design that no client asks for now,
|
||||
// a .gz means a database left compressed — and each is 100+ MB.
|
||||
func TestStrayArtifactFailsTheBuild(t *testing.T) {
|
||||
for _, name := range []string{
|
||||
"2016.db", "2016.sqlite3",
|
||||
"2016.db", "2016.sqlite30",
|
||||
"2016.sqlite3-journal", "2016.sqlite3-wal", "2016.sqlite3-shm", "2016.sqlite3.gz",
|
||||
"2016.sqlite30-journal", "2016.sqlite30-wal", "2016.sqlite30-shm",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
p := fakeBuild(t, "2016.sqlite30", "2017.sqlite30")
|
||||
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
|
||||
write(t, filepath.Join(p.Dist, "db", name), "raw sqlite")
|
||||
err := Assemble(p, datasets)
|
||||
if err == nil {
|
||||
@@ -127,12 +125,12 @@ func TestStrayArtifactFailsTheBuild(t *testing.T) {
|
||||
// TestPublishedDatabaseIsNotMistakenForStray: the pattern must pass the one
|
||||
// file the site is built to serve. Getting this wrong would fail every build.
|
||||
func TestPublishedDatabaseIsNotMistakenForStray(t *testing.T) {
|
||||
if strayArtifact.MatchString("2016.sqlite30") {
|
||||
if strayArtifact.MatchString("2016.sqlite3") {
|
||||
t.Error("the published database must not be treated as a stray artifact")
|
||||
}
|
||||
for _, name := range []string{
|
||||
"2016.db", "x.sqlite3", "x.sqlite3-journal", "x.sqlite3-wal", "x.sqlite3-shm",
|
||||
"x.sqlite30-journal", "x.sqlite30-wal", "x.sqlite30-shm", "x.sqlite3.gz", "x.db.gz",
|
||||
"2016.db", "x.sqlite30", "x.sqlite3-journal", "x.sqlite3-wal", "x.sqlite3-shm",
|
||||
"x.sqlite30-journal", "x.sqlite3.gz", "x.db.gz",
|
||||
} {
|
||||
if !strayArtifact.MatchString(name) {
|
||||
t.Errorf("%s should be rejected", name)
|
||||
@@ -151,7 +149,7 @@ func TestAssembleRejectsAMissingBuild(t *testing.T) {
|
||||
// TestAssembleIsIdempotent: the site directory is rebuilt from scratch, so a
|
||||
// previous run's leftovers cannot survive into the artifact.
|
||||
func TestAssembleIsIdempotent(t *testing.T) {
|
||||
p := fakeBuild(t, "2016.sqlite30", "2017.sqlite30")
|
||||
p := fakeBuild(t, "2016.sqlite3", "2017.sqlite3")
|
||||
if err := Assemble(p, datasets); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -135,7 +135,7 @@ func compareOne(id, dirA, dirB string) (Result, error) {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// open finds <id>.sqlite30 in dir and returns a read-only handle.
|
||||
// open finds <id>.sqlite3 in dir and returns a read-only handle.
|
||||
//
|
||||
// The older names are still accepted, and a gzipped database is expanded to a
|
||||
// temporary file rather than rejected: the two sides of a comparison are often
|
||||
@@ -143,7 +143,7 @@ func compareOne(id, dirA, dirB string) (Result, error) {
|
||||
func open(dir, id string) (*sql.DB, func(), error) {
|
||||
noop := func() {}
|
||||
|
||||
for _, name := range []string{id + ".sqlite30", id + ".sqlite3", id + ".db"} {
|
||||
for _, name := range []string{id + ".sqlite3", id + ".sqlite30", id + ".db"} {
|
||||
plain := filepath.Join(dir, name)
|
||||
if _, err := os.Stat(plain); err == nil {
|
||||
db, err := sql.Open(driverName, "file:"+plain+"?mode=ro")
|
||||
@@ -154,7 +154,7 @@ func open(dir, id string) (*sql.DB, func(), error) {
|
||||
gzPath := filepath.Join(dir, id+".db.gz")
|
||||
f, err := os.Open(gzPath)
|
||||
if err != nil {
|
||||
return nil, noop, fmt.Errorf("no %s.sqlite30, %s.sqlite3, %s.db or %s.db.gz in %s", id, id, id, id, dir)
|
||||
return nil, noop, fmt.Errorf("no %s.sqlite3, %s.sqlite30, %s.db or %s.db.gz in %s", id, id, id, id, dir)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
|
||||
Reference in new issue
Block a user