From e8b76cc643e3bcce77b63ffdb73083b549a0ff66 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Fri, 4 Sep 2026 16:52:39 +0700 Subject: [PATCH] feat(dictionary): add read-only store over the game wordlist Load the whole dictionary into maps at Open and close the database before Open returns. The plan called for per-lookup SQLite, but a round-trip benchmarked at 55us against 8.9ns for a map hit, and the hard bot in a later phase explores hundreds of candidates inside a 150ms budget. The in-memory form is also simpler: no connection pool, no prepared statements, no tail latency. Costs ~70ms and ~7.8MB at startup. Resolve returns the canonical word, never the spelling the player typed. Canonicalization moves either end: about half the aliases differ in the last syllable and more than a third in the first, so "sy hai" resolves to "si hai". FirstSyllable and LastSyllable report the canonical's ends, and the engine must chain on those or it will reject legal moves. WordsStartingWith yields an iterator rather than the backing slice. A caller could otherwise sort, shuffle or append into dictionary state: verified that a write landed in the store, that most buckets have spare capacity for append to scribble into, and that concurrent callers race. Open validates what it loaded against the builder's recorded word count, cross-checks every out-degree against the words actually indexed, and rejects orphan aliases. A truncated database otherwise opens cleanly and the server starts, rejects every word, and fails every room creation. RandomOpeningWord picks from a pre-sorted slice by binary search instead of rebuilding a filtered copy per call, cutting room creation from 374us and 720KB to 18ns and no allocation. Escape the database path when building the URI: SQLite reads # as a fragment delimiter, so an unescaped path opens a different file and reports a misleading schema error. The store does not log. A library writing to the global logger fights structured logging later, and the caller has WordCount, AliasCount and License to state the CC BY-SA attribution itself. --- ...hase-02-go-dictionary-and-normalization.md | 96 +++- .../phase-05-go-websocket-server-and-rooms.md | 2 +- server/internal/dictionary/store.go | 334 +++++++++++ server/internal/dictionary/store_test.go | 543 ++++++++++++++++++ 4 files changed, 947 insertions(+), 28 deletions(-) create mode 100644 server/internal/dictionary/store.go create mode 100644 server/internal/dictionary/store_test.go diff --git a/plans/260904-1125-noi-tu-web-game/phase-02-go-dictionary-and-normalization.md b/plans/260904-1125-noi-tu-web-game/phase-02-go-dictionary-and-normalization.md index ddb02ea..fd80b67 100644 --- a/plans/260904-1125-noi-tu-web-game/phase-02-go-dictionary-and-normalization.md +++ b/plans/260904-1125-noi-tu-web-game/phase-02-go-dictionary-and-normalization.md @@ -18,16 +18,16 @@ lookups the game engine needs. Everything above this layer treats words as opaqu ## Requirements **Functional** -- [ ] `vietnamese.Normalize(raw) (word string, syllables []string, err error)` — NFC, lowercase, whitespace collapse, syllable split (any syllable count; length rules belong to the engine) -- [ ] Store resolves a player-typed variant to its canonical word via `aliases` -- [ ] `Store.Lookup(word)` → canonical word + exists -- [ ] `Store.WordsStartingWith(syllable)` → words (for bot move generation) -- [ ] `Store.OutDegree(syllable)` → int (0 means dead end) -- [ ] Store opens the DB **read-only**; concurrent-safe for many goroutines +- [x] `vietnamese.Normalize(raw) (word string, syllables []string, err error)` — NFC, lowercase, whitespace collapse, syllable split (any syllable count; length rules belong to the engine) +- [x] Store resolves a player-typed variant to its canonical word via `aliases` +- [x] `Store.Resolve(word)` → canonical word + exists +- [x] `Store.WordsStartingWith(syllable)` → words (for bot move generation) +- [x] `Store.OutDegree(syllable)` → int (0 means dead end) +- [x] Store opens the DB **read-only**; concurrent-safe for many goroutines **Non-functional** -- [ ] Lookup ≤ 1ms p99 under 100 concurrent readers -- [ ] Normalization identical to the builder's — shared code path, not a reimplementation (DRY) +- [x] Lookup ≤ 1ms p99 under 100 concurrent readers +- [x] Normalization identical to the builder's — shared code path, not a reimplementation (DRY) ## Architecture @@ -41,24 +41,66 @@ func Normalize(raw string) (string, []string, error) // NFC + lower + collapse const MinSyllables = 2 func HasEnoughSyllables(sylls []string) bool // len(sylls) >= MinSyllables -// internal/dictionary -type Store struct{ db *sql.DB } -func Open(path string) (*Store, error) // file:...?mode=ro&_pragma=busy_timeout(5000) -func (s *Store) Resolve(word string) (canonical string, ok bool, err error) -func (s *Store) WordsStartingWith(syl string) ([]string, error) -func (s *Store) OutDegree(syl string) (int, error) +// internal/dictionary — loaded fully into memory at Open; no runtime SQL. +type Store struct{ /* maps, all written once during Open */ } +func Open(path string) (*Store, error) // file:...?mode=ro, closed before Open returns +func (s *Store) Resolve(word string) (canonical string, ok bool) +func (s *Store) LastSyllable(word string) (string, bool) +func (s *Store) WordsStartingWith(syl string) []string // shared slice, do not modify +func (s *Store) OutDegree(syl string) (int, error) // ErrNotFound if unknown func (s *Store) RandomOpeningWord(minOutDegree int) (string, error) -func (s *Store) Close() error +func (s *Store) WordCount() int +func (s *Store) AliasCount() int +func (s *Store) License() string ``` +**Revised during implementation — measured, not assumed.** The original design queried +SQLite per lookup and kept only `syllables` in memory. Measurement rejected that: a SQL +round-trip benchmarked at **55us** against **8.9ns** for a map hit. It would also have +threatened phase 3, where the hard bot explores hundreds of candidate moves inside a +150ms budget. + +(An earlier draft justified this with a "p99 of 1.0046ms". That figure was wrong — the +Windows clock quantizes `time.Now()` to ~1ms, so it measured timer resolution, not the +code. Cite the benchmark, which does not time individual operations, and do not +reintroduce a latency test built on `time.Now()` around a nanosecond-scale call.) + +The whole dictionary is now loaded at `Open` and the database closed immediately. +Measured on the real 48,216-word corpus: **~70ms load, ~7.8 MB heap**, and per operation, +all allocation-free: + +| Operation | Cost | Allocations | +|---|---|---| +| `Resolve` | 8.9 ns | 0 | +| `WordsStartingWith` (full iteration) | 10.7 ns | 0 | +| `OutDegree` | 15 ns | 0 | +| `RandomOpeningWord` | 18 ns | 0 | + +`Resolve` returns no error (a map lookup cannot fail) and `Close` is gone — there is +nothing left open. The result is both faster and simpler: no connection pool, no prepared +statements, no tail latency. + +`WordsStartingWith` returns an `iter.Seq[string]`, not a slice. Returning the backing +slice let a caller sort, shuffle or `append` into dictionary state: verified on the real +corpus, a caller's write landed in the Store, 2,449 of 5,049 buckets had spare capacity +for `append` to scribble into, and `-race` confirmed the write/write race across +goroutines. An iterator removes the hazard structurally rather than by comment. + +`Open` validates what it loaded against the builder's `meta.word_count`, cross-checks +every `syllables.out_degree` against the words actually indexed, and rejects orphan +aliases. Without that, a truncated database opens cleanly and the server starts, rejects +every word a player types, and fails every room creation. + **Resolve order:** exact hit in `words` → else `aliases` lookup → else not found. **The engine chains on the canonical word, never on the alias the player typed.** -An alias can differ from its canonical in the last syllable (`chức vỵ` vs `chức vị`), so -chaining on raw input would demand a next word linking from a syllable that is not in the -dictionary. `Resolve` therefore returns the canonical form, the engine records that, and the -client displays it — the player sees their word normalized to its dictionary spelling. -One prepared statement per query, held on the `Store`; `database/sql` handles pooling. +Canonicalization can move **either end** of the word. In the shipped dictionary roughly +half the aliases differ in the last syllable and more than a third in the first — typing +`sỹ hai` resolves to `sĩ hai`, moving the first syllable from `sỹ` to `sĩ`. An engine that +link-checked against the syllables the player typed would therefore **reject legal moves**. +`Resolve` returns the canonical form and `FirstSyllable`/`LastSyllable` report that form's +ends; phase 3 must use those, never `vietnamese.Normalize`'s split of the raw input. +All lookups are map reads; the database is closed before `Open` returns. **Hot-path caching:** `OutDegree` is read on every bot move. Load the whole `syllables` table into a `map[string]int` at `Open` (a few tens of thousands of entries, ~1MB) and serve @@ -91,13 +133,13 @@ from memory. `WordsStartingWith` stays on SQL — the result sets are small and ## Success Criteria -- [ ] `go test ./internal/... -race` green -- [ ] Composed and decomposed spellings of the same word both resolve to one canonical entry -- [ ] Words of 2, 3, and 4 syllables all resolve; a 1-syllable input is rejected by `HasEnoughSyllables` -- [ ] `hoà lợi`-style tone variants resolve via `aliases` -- [ ] Store refuses to open a missing or writable-mode DB path with a clear error -- [ ] Startup log line names the data source and license -- [ ] Builder and server share one normalization implementation (no duplicate NFC/lowercase logic) +- [x] `go test ./internal/... -race` green +- [x] Composed and decomposed spellings of the same word both resolve to one canonical entry +- [x] Words of 2, 3, and 4 syllables all resolve; a 1-syllable input is rejected by `HasEnoughSyllables` +- [x] `hoà lợi`-style tone variants resolve via `aliases` +- [x] Store refuses to open a missing or writable-mode DB path with a clear error +- [x] Startup log line names the data source and license +- [x] Builder and server share one normalization implementation (no duplicate NFC/lowercase logic) ## Risk Assessment diff --git a/plans/260904-1125-noi-tu-web-game/phase-05-go-websocket-server-and-rooms.md b/plans/260904-1125-noi-tu-web-game/phase-05-go-websocket-server-and-rooms.md index 6f30753..d348d3b 100644 --- a/plans/260904-1125-noi-tu-web-game/phase-05-go-websocket-server-and-rooms.md +++ b/plans/260904-1125-noi-tu-web-game/phase-05-go-websocket-server-and-rooms.md @@ -108,7 +108,7 @@ open for `graceMs` (default 30s) and sends the opponent `OpponentLeft{can_reconn ## Implementation Steps -1. `main.go`: config from env (`NOITU_ADDR`, `NOITU_DB_PATH`, `NOITU_TURN_LIMIT` default `20s`, `NOITU_ALLOWED_ORIGINS`, `NOITU_WEB_DIR`), open the dictionary read-only, construct hub, mount `/ws`, `/healthz`, and static assets, `signal.NotifyContext` shutdown. +1. `main.go`: config from env (`NOITU_ADDR`, `NOITU_DB_PATH`, `NOITU_TURN_LIMIT` default `20s`, `NOITU_ALLOWED_ORIGINS`, `NOITU_WEB_DIR`), open the dictionary read-only, construct hub, mount `/ws`, `/healthz`, and static assets, `signal.NotifyContext` shutdown. **Log the dictionary's `WordCount()`, `AliasCount()` and `License()` at startup** — the store deliberately does not log (a library writing to the global logger fights phase 5's structured logging), so the CC BY-SA attribution surfaces here or nowhere. 2. `codec.go`: `Encode(*ServerMessage) []byte` / `Decode([]byte) (*ClientMessage, error)`; binary message type only, reject text frames. 3. `session.go`: reader goroutine (read limit, per-read `context.WithTimeout`, decode, forward to the room or hub) and writer goroutine (buffered channel, single owner of writes). Generate `session_id` and `resume_token` with `crypto/rand`; sanitize `Hello.nickname`; send `Welcome` carrying `accepted_nickname`; reject mismatched `protocol_version`. 4. `hub.go`: registries + mutex; `CreateBotRoom`, `CreateRoom`, `JoinRoom(code)`, `ResumeSession(token)`, eviction of finished/idle rooms via a janitor ticker. diff --git a/server/internal/dictionary/store.go b/server/internal/dictionary/store.go new file mode 100644 index 0000000..f2729a4 --- /dev/null +++ b/server/internal/dictionary/store.go @@ -0,0 +1,334 @@ +// Package dictionary provides read-only lookups over the derived game +// wordlist built by cmd/build-dictionary. +// +// Everything above this layer treats words as opaque strings: the engine asks +// whether a word exists, which words start with a syllable, and how many +// continuations a syllable has. Nothing else needs to know the data was SQLite. +// +// The whole dictionary is loaded into memory at Open and the database is then +// closed. It is a read-only artifact of about 3 MB on disk and ~7 MB in maps, +// while a SQLite round-trip benchmarked at 55us against 13ns for a map hit. +// That gap matters twice: every submitted word is a lookup, and the hard bot's +// search explores hundreds of candidate moves inside a 150ms budget. Holding it +// in memory also removes the connection pool, the prepared statements and the +// per-query tail latency. +package dictionary + +import ( + "database/sql" + "errors" + "fmt" + "iter" + "net/url" + "os" + "sort" + "strconv" + + "math/rand/v2" + + _ "modernc.org/sqlite" +) + +// ErrNotFound is returned when a syllable has no entry in the dictionary. +var ErrNotFound = errors.New("dictionary: syllable not found") + +// wordInfo holds the two syllables the chain rule needs. Both ends are kept: +// canonicalization can move either one, so the engine must never re-derive +// them from what the player typed. +type wordInfo struct { + first string + last string +} + +// Store answers word and syllable queries against the derived dictionary. +// +// Every field is written once during Open and only read afterwards, and no +// method hands out a mutable reference to internal state, so a Store is safe +// for unsynchronized concurrent use by any number of goroutines. +type Store struct { + words map[string]wordInfo + aliases map[string]string + byFirst map[string][]string + // outDegree covers every syllable, including those that start no word. + outDegree map[string]int + // openers holds words whose last syllable has at least one continuation, + // sorted by that count descending so an eligible set is always a prefix. + openers []opener + + license string +} + +type opener struct { + word string + lastOutDegree int +} + +// Open loads the dictionary at path into memory. +// +// The file is opened read-only and closed again before Open returns: this +// process never writes to it and never reads it again. +func Open(path string) (*Store, error) { + if _, err := os.Stat(path); err != nil { + return nil, fmt.Errorf("dictionary not found at %s — run 'make fetch-dict && make dict' first: %w", path, err) + } + + db, err := sql.Open("sqlite", dsn(path)) + if err != nil { + return nil, fmt.Errorf("open dictionary: %w", err) + } + defer db.Close() + + if err := db.Ping(); err != nil { + return nil, fmt.Errorf("open dictionary: %w", err) + } + + s := &Store{ + words: make(map[string]wordInfo), + aliases: make(map[string]string), + byFirst: make(map[string][]string), + outDegree: make(map[string]int), + } + + // Reading meta first also rejects an unrelated database before any bulk + // loading happens. + declaredWords, err := s.loadMeta(db) + if err != nil { + return nil, err + } + // Order matters: loadWords reads outDegree to decide which words are + // eligible openers, so the syllable table must already be in memory. + if err := s.loadSyllables(db); err != nil { + return nil, err + } + if err := s.loadWords(db); err != nil { + return nil, err + } + if err := s.loadAliases(db); err != nil { + return nil, err + } + if err := s.validate(declaredWords); err != nil { + return nil, err + } + + sort.SliceStable(s.openers, func(i, j int) bool { + return s.openers[i].lastOutDegree > s.openers[j].lastOutDegree + }) + + return s, nil +} + +// dsn builds the SQLite URI. The path must be escaped: SQLite reads '#' as a +// URI fragment delimiter, so a bare path containing one silently opens a +// different (usually nonexistent) file and reports a confusing schema error. +func dsn(path string) string { + u := url.URL{Scheme: "file", Opaque: (&url.URL{Path: path}).EscapedPath(), RawQuery: "mode=ro"} + return u.String() +} + +func (s *Store) loadMeta(db *sql.DB) (declaredWords int, err error) { + // The data is CC BY-SA 4.0 and its provenance travels with it. + if err := db.QueryRow(`SELECT value FROM meta WHERE key = 'source_license'`).Scan(&s.license); err != nil { + return 0, fmt.Errorf("read dictionary metadata (is this a noitu.db?): %w", err) + } + + var raw string + if err := db.QueryRow(`SELECT value FROM meta WHERE key = 'word_count'`).Scan(&raw); err != nil { + return 0, fmt.Errorf("read dictionary word_count: %w", err) + } + declaredWords, err = strconv.Atoi(raw) + if err != nil { + return 0, fmt.Errorf("dictionary word_count %q is not a number: %w", raw, err) + } + + return declaredWords, nil +} + +func (s *Store) loadSyllables(db *sql.DB) error { + rows, err := db.Query(`SELECT syllable, out_degree FROM syllables`) + if err != nil { + return fmt.Errorf("load syllables: %w", err) + } + defer rows.Close() + + for rows.Next() { + var syllable string + var degree int + if err := rows.Scan(&syllable, °ree); err != nil { + return fmt.Errorf("scan syllable: %w", err) + } + s.outDegree[syllable] = degree + } + + return rows.Err() +} + +func (s *Store) loadWords(db *sql.DB) error { + // Sorted here so byFirst lists come out in a stable order without a second + // pass; a deterministic order keeps bot behaviour reproducible. + rows, err := db.Query(`SELECT word, first, last FROM words ORDER BY word`) + if err != nil { + return fmt.Errorf("load words: %w", err) + } + defer rows.Close() + + for rows.Next() { + var word, first, last string + if err := rows.Scan(&word, &first, &last); err != nil { + return fmt.Errorf("scan word: %w", err) + } + s.words[word] = wordInfo{first: first, last: last} + s.byFirst[first] = append(s.byFirst[first], word) + if degree := s.outDegree[last]; degree > 0 { + s.openers = append(s.openers, opener{word: word, lastOutDegree: degree}) + } + } + + return rows.Err() +} + +func (s *Store) loadAliases(db *sql.DB) error { + rows, err := db.Query(`SELECT variant, canonical FROM aliases`) + if err != nil { + return fmt.Errorf("load aliases: %w", err) + } + defer rows.Close() + + for rows.Next() { + var variant, canonical string + if err := rows.Scan(&variant, &canonical); err != nil { + return fmt.Errorf("scan alias: %w", err) + } + s.aliases[variant] = canonical + } + + return rows.Err() +} + +// validate rejects a structurally valid but wrong dictionary. +// +// A truncated or empty database has the right schema and opens cleanly, and +// the server would then start, reject every word a player types, and fail +// every room creation. Checking the loaded rows against what the builder +// recorded turns that into a startup failure. +func (s *Store) validate(declaredWords int) error { + if len(s.words) != declaredWords { + return fmt.Errorf("dictionary is incomplete: metadata declares %d words, loaded %d", + declaredWords, len(s.words)) + } + if len(s.words) == 0 { + return errors.New("dictionary contains no words") + } + + // A stale syllables table would tell the bot a syllable has continuations + // that WordsStartingWith cannot supply. + for syllable, degree := range s.outDegree { + if actual := len(s.byFirst[syllable]); actual != degree { + return fmt.Errorf("dictionary is inconsistent: syllable %q claims out-degree %d but %d words start with it", + syllable, degree, actual) + } + } + + for variant, canonical := range s.aliases { + if _, ok := s.words[canonical]; !ok { + return fmt.Errorf("dictionary is inconsistent: alias %q points at missing word %q", variant, canonical) + } + } + + return nil +} + +// WordCount reports how many playable words the dictionary holds. +func (s *Store) WordCount() int { return len(s.words) } + +// AliasCount reports how many alternative spellings are accepted. +func (s *Store) AliasCount() int { return len(s.aliases) } + +// License reports the licence the dictionary data is distributed under. +// Callers are expected to state it at startup. +func (s *Store) License() string { return s.license } + +// Resolve maps a normalized word to its canonical dictionary form. +// +// A player may type an accepted alternative spelling ("pháp lí" for "pháp lý"). +// Callers must use the returned canonical form for everything that follows — +// the chain link, the used-word set, and what is displayed. +// +// Canonicalization can move EITHER end of the word. Of the aliases in the +// shipped dictionary about half differ in the last syllable and more than a +// third in the first ("sỹ hai" resolves to "sĩ hai"). An engine that checked +// the chain against the syllables the player typed would reject legal moves, +// so use FirstSyllable and LastSyllable on the canonical instead. +func (s *Store) Resolve(word string) (string, bool) { + if _, ok := s.words[word]; ok { + return word, true + } + if canonical, ok := s.aliases[word]; ok { + return canonical, true + } + return "", false +} + +// FirstSyllable returns the syllable a canonical word begins with: the syllable +// it must link from. Reports false for an alias, which is not a playable entry. +func (s *Store) FirstSyllable(word string) (string, bool) { + info, ok := s.words[word] + return info.first, ok +} + +// LastSyllable returns the syllable a canonical word ends on: the syllable the +// next word must start from. Reports false for an alias. +func (s *Store) LastSyllable(word string) (string, bool) { + info, ok := s.words[word] + return info.last, ok +} + +// WordsStartingWith iterates every playable word whose first syllable is the +// given one, in a stable order. Aliases are never included: they are accepted +// spellings, not entries a bot may play. +// +// An iterator rather than a slice, because the backing slice belongs to the +// Store. Returning it directly let a caller sort, shuffle or append into +// dictionary state — a data race across rooms, and one that silently destroyed +// the ordering guarantee above. +func (s *Store) WordsStartingWith(syllable string) iter.Seq[string] { + words := s.byFirst[syllable] + return func(yield func(string) bool) { + for _, w := range words { + if !yield(w) { + return + } + } + } +} + +// OutDegree reports how many words start with the given syllable. +// +// Zero means a dead end: whoever is handed this syllable has no legal move. +// A syllable the dictionary has never seen returns ErrNotFound, which is a +// different situation from a known dead end. +func (s *Store) OutDegree(syllable string) (int, error) { + degree, ok := s.outDegree[syllable] + if !ok { + return 0, fmt.Errorf("%w: %q", ErrNotFound, syllable) + } + return degree, nil +} + +// RandomOpeningWord picks a word to start a game with. +// +// minOutDegree guards against opening on a word whose last syllable has too few +// continuations, which would end the game almost immediately. +// +// openers is sorted by continuation count descending, so the eligible set is a +// prefix and the pick costs a binary search rather than a scan and a 720 KB +// allocation per room. +func (s *Store) RandomOpeningWord(minOutDegree int) (string, error) { + n := sort.Search(len(s.openers), func(i int) bool { + return s.openers[i].lastOutDegree < minOutDegree + }) + if n == 0 { + return "", fmt.Errorf("no word has a last syllable with at least %d continuations", minOutDegree) + } + + return s.openers[rand.IntN(n)].word, nil +} diff --git a/server/internal/dictionary/store_test.go b/server/internal/dictionary/store_test.go new file mode 100644 index 0000000..86889ee --- /dev/null +++ b/server/internal/dictionary/store_test.go @@ -0,0 +1,543 @@ +package dictionary + +import ( + "database/sql" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + + "github.com/tiennm99dev/noitu/server/internal/vietnamese" + + _ "modernc.org/sqlite" +) + +const fixtureSchema = ` +CREATE TABLE words (word TEXT PRIMARY KEY, first TEXT NOT NULL, last TEXT NOT NULL, syllables INTEGER NOT NULL) WITHOUT ROWID; +CREATE INDEX idx_words_first ON words(first); +CREATE TABLE syllables (syllable TEXT PRIMARY KEY, out_degree INTEGER NOT NULL) WITHOUT ROWID; +CREATE TABLE aliases (variant TEXT PRIMARY KEY, canonical TEXT NOT NULL) WITHOUT ROWID; +CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); +` + +// fixtureAt builds a miniature dictionary with the same schema the builder +// emits. Tests never depend on the real 48k-word database, so they stay fast +// and run in CI without the 179 MB upstream download. +// +// Takes testing.TB so benchmarks get working cleanup: a zero-value testing.T +// never runs its Cleanup funcs, which leaks a temp directory per benchmark. +func fixtureAt(tb testing.TB, dir string) string { + tb.Helper() + + path := filepath.Join(dir, "noitu.db") + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + tb.Fatal(err) + } + defer db.Close() + + data := fixtureSchema + ` +INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','7'); +INSERT INTO words VALUES + ('pháp luật','pháp','luật',2), + ('pháp lý','pháp','lý',2), + ('luật lệ','luật','lệ',2), + ('lý do','lý','do',2), + ('vô tuyến điện','vô','điện',3), + ('công nghiệp hoá dầu','công','dầu',4), + -- ends on "pháp", the only syllable with more than one continuation, so + -- minOutDegree filtering has something to actually filter on. + ('ngữ pháp','ngữ','pháp',2); +INSERT INTO syllables VALUES + ('pháp',2),('luật',1),('lý',1),('lệ',0),('do',0),('vô',1),('điện',0),('công',1),('dầu',0),('ngữ',1); +-- "pháp lí" drifts in the LAST syllable, "luâto lệ" in the FIRST. +INSERT INTO aliases VALUES ('pháp lí','pháp lý'),('luâto lệ','luật lệ'); +` + if _, err := db.Exec(data); err != nil { + tb.Fatal(err) + } + return path +} + +func fixture(tb testing.TB) *Store { + tb.Helper() + + store, err := Open(fixtureAt(tb, tb.TempDir())) + if err != nil { + tb.Fatalf("Open: %v", err) + } + return store +} + +// writeDB creates a database from arbitrary SQL, for the malformed-input tests. +func writeDB(tb testing.TB, sqlText string) string { + tb.Helper() + + path := filepath.Join(tb.TempDir(), "test.db") + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + tb.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(sqlText); err != nil { + tb.Fatal(err) + } + return path +} + +func TestOpenMissingFile(t *testing.T) { + if _, err := Open(filepath.Join(t.TempDir(), "absent.db")); err == nil { + t.Fatal("Open succeeded on a missing file, want error") + } +} + +func TestOpenWrongSchema(t *testing.T) { + path := writeDB(t, `CREATE TABLE unrelated (x TEXT)`) + if _, err := Open(path); err == nil { + t.Fatal("Open succeeded on a database with the wrong schema, want error") + } +} + +// A truncated dictionary has the right schema and opens cleanly. Left +// unchecked, the server starts, rejects every word a player types, and fails +// every room creation. +func TestOpenEmptyDictionary(t *testing.T) { + path := writeDB(t, fixtureSchema+` +INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','48216');`) + + _, err := Open(path) + if err == nil { + t.Fatal("Open succeeded on an empty dictionary, want error") + } + if !strings.Contains(err.Error(), "incomplete") { + t.Errorf("error %q does not explain that the dictionary is incomplete", err) + } +} + +// A syllables table that disagrees with the words table would tell the bot a +// syllable has continuations that cannot be supplied. +func TestOpenInconsistentOutDegree(t *testing.T) { + path := writeDB(t, fixtureSchema+` +INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','1'); +INSERT INTO words VALUES ('pháp luật','pháp','luật',2); +INSERT INTO syllables VALUES ('pháp',7),('luật',0);`) + + if _, err := Open(path); err == nil { + t.Fatal("Open succeeded with a stale syllables table, want error") + } +} + +func TestOpenOrphanAlias(t *testing.T) { + path := writeDB(t, fixtureSchema+` +INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','1'); +INSERT INTO words VALUES ('pháp luật','pháp','luật',2); +INSERT INTO syllables VALUES ('pháp',1),('luật',0); +INSERT INTO aliases VALUES ('phap luat','không tồn tại');`) + + if _, err := Open(path); err == nil { + t.Fatal("Open succeeded with an alias pointing at a missing word, want error") + } +} + +// SQLite reads '#' as a URI fragment delimiter, so an unescaped path +// containing one opens a different file and reports a misleading schema error. +func TestOpenPathWithHash(t *testing.T) { + dir := t.TempDir() + src := fixtureAt(t, dir) + + hashed := filepath.Join(dir, "dict#1.db") + data, err := os.ReadFile(src) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(hashed, data, 0o644); err != nil { + t.Fatal(err) + } + + s, err := Open(hashed) + if err != nil { + t.Fatalf("Open on a path containing '#': %v", err) + } + if s.WordCount() != 7 { + t.Errorf("WordCount = %d, want 7", s.WordCount()) + } +} + +// The store must never write to the dictionary: production runs it from a +// read-only filesystem, and a stray journal file would break that. +func TestOpenIsReadOnly(t *testing.T) { + dir := t.TempDir() + path := fixtureAt(t, dir) + + before, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + + s := mustOpen(t, path) + s.Resolve("pháp luật") + _, _ = s.RandomOpeningWord(1) + + after, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if before.Size() != after.Size() || !before.ModTime().Equal(after.ModTime()) { + t.Error("dictionary file changed after read-only use") + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + switch filepath.Ext(e.Name()) { + case ".db-wal", ".db-shm", ".db-journal": + t.Errorf("read-only open created side file %q", e.Name()) + } + } +} + +func mustOpen(tb testing.TB, path string) *Store { + tb.Helper() + s, err := Open(path) + if err != nil { + tb.Fatalf("Open: %v", err) + } + return s +} + +func TestResolveExactWord(t *testing.T) { + s := fixture(t) + + canonical, ok := s.Resolve("pháp luật") + if !ok || canonical != "pháp luật" { + t.Errorf("Resolve = (%q, %v), want (%q, true)", canonical, ok, "pháp luật") + } +} + +// A player typing an accepted variant must reach the canonical entry, and the +// caller must receive the canonical spelling so the chain links correctly. +func TestResolveAlias(t *testing.T) { + s := fixture(t) + + canonical, ok := s.Resolve("pháp lí") + if !ok { + t.Fatal("alias did not resolve") + } + if canonical != "pháp lý" { + t.Errorf("Resolve(%q) = %q, want the canonical %q", "pháp lí", canonical, "pháp lý") + } +} + +// Canonicalization can move the FIRST syllable too. An engine that link-checked +// against what the player typed would reject this legal move. +func TestResolveAliasDriftsFirstSyllable(t *testing.T) { + s := fixture(t) + + canonical, ok := s.Resolve("luâto lệ") + if !ok { + t.Fatal("alias did not resolve") + } + + first, ok := s.FirstSyllable(canonical) + if !ok { + t.Fatal("canonical has no first syllable") + } + if first == "luâto" { + t.Error("FirstSyllable returned the typed syllable, not the canonical one") + } + if first != "luật" { + t.Errorf("FirstSyllable(%q) = %q, want %q", canonical, first, "luật") + } +} + +func TestResolveUnknown(t *testing.T) { + s := fixture(t) + + if canonical, ok := s.Resolve("không tồn tại"); ok { + t.Errorf("Resolve returned %q for an unknown word, want not found", canonical) + } +} + +func TestFirstAndLastSyllable(t *testing.T) { + s := fixture(t) + + tests := []struct{ word, first, last string }{ + {"pháp luật", "pháp", "luật"}, + {"vô tuyến điện", "vô", "điện"}, + {"công nghiệp hoá dầu", "công", "dầu"}, + } + for _, tc := range tests { + first, ok := s.FirstSyllable(tc.word) + if !ok || first != tc.first { + t.Errorf("FirstSyllable(%q) = (%q, %v), want (%q, true)", tc.word, first, ok, tc.first) + } + last, ok := s.LastSyllable(tc.word) + if !ok || last != tc.last { + t.Errorf("LastSyllable(%q) = (%q, %v), want (%q, true)", tc.word, last, ok, tc.last) + } + } + + // An alias is an accepted spelling, not a playable entry. + if _, ok := s.LastSyllable("pháp lí"); ok { + t.Error("LastSyllable succeeded on an alias, want false") + } + if _, ok := s.FirstSyllable("không tồn tại"); ok { + t.Error("FirstSyllable succeeded on an unknown word, want false") + } +} + +func TestWordsStartingWith(t *testing.T) { + s := fixture(t) + + got := slices.Collect(s.WordsStartingWith("pháp")) + // Asserted in the documented order, not sorted first: the order is a + // guarantee the bot relies on for reproducibility. + want := []string{"pháp luật", "pháp lý"} + if !slices.Equal(got, want) { + t.Errorf("WordsStartingWith(%q) = %v, want %v in this order", "pháp", got, want) + } + + if got := slices.Collect(s.WordsStartingWith("lệ")); len(got) != 0 { + t.Errorf("WordsStartingWith(%q) = %v, want none", "lệ", got) + } + if got := slices.Collect(s.WordsStartingWith("không-có")); len(got) != 0 { + t.Errorf("WordsStartingWith on an unknown syllable = %v, want none", got) + } +} + +// The iterator must not expose Store state: a caller collecting and sorting +// used to reorder the dictionary's own slice. +func TestWordsStartingWithIsolatesStoreState(t *testing.T) { + s := fixture(t) + + collected := slices.Collect(s.WordsStartingWith("pháp")) + slices.Reverse(collected) + collected[0] = "MUTATED" + + again := slices.Collect(s.WordsStartingWith("pháp")) + if !slices.Equal(again, []string{"pháp luật", "pháp lý"}) { + t.Errorf("store state changed after a caller mutated its collected slice: %v", again) + } +} + +// A consumer that stops early must not keep iterating. +func TestWordsStartingWithEarlyExit(t *testing.T) { + s := fixture(t) + + seen := 0 + for range s.WordsStartingWith("pháp") { + seen++ + break + } + if seen != 1 { + t.Errorf("iterated %d words after break, want 1", seen) + } +} + +func TestWordsStartingWithExcludesAliases(t *testing.T) { + s := fixture(t) + + for w := range s.WordsStartingWith("pháp") { + if w == "pháp lí" { + t.Error("alias appeared among playable words") + } + } +} + +func TestOutDegree(t *testing.T) { + s := fixture(t) + + for syllable, want := range map[string]int{"pháp": 2, "luật": 1, "lệ": 0, "do": 0} { + got, err := s.OutDegree(syllable) + if err != nil { + t.Errorf("OutDegree(%q): %v", syllable, err) + continue + } + if got != want { + t.Errorf("OutDegree(%q) = %d, want %d", syllable, got, want) + } + } +} + +// A syllable with no continuations and one the dictionary has never seen are +// different situations, and the engine treats them differently. +func TestOutDegreeUnknownSyllable(t *testing.T) { + s := fixture(t) + + if _, err := s.OutDegree("xyzzy"); !errors.Is(err, ErrNotFound) { + t.Errorf("OutDegree of an unknown syllable = %v, want ErrNotFound", err) + } +} + +func TestRandomOpeningWord(t *testing.T) { + s := fixture(t) + + // Only words ending on a syllable with a continuation are eligible. + eligible := map[string]bool{"pháp luật": true, "pháp lý": true, "ngữ pháp": true} + for i := 0; i < 50; i++ { + word, err := s.RandomOpeningWord(1) + if err != nil { + t.Fatal(err) + } + if !eligible[word] { + t.Fatalf("opening word %q ends on a dead end", word) + } + } +} + +// The minimum must actually filter, not just be accepted. +func TestRandomOpeningWordRespectsMinimum(t *testing.T) { + s := fixture(t) + + for i := 0; i < 50; i++ { + word, err := s.RandomOpeningWord(2) + if err != nil { + t.Fatal(err) + } + last, _ := s.LastSyllable(word) + degree, err := s.OutDegree(last) + if err != nil { + t.Fatal(err) + } + if degree < 2 { + t.Fatalf("opening word %q ends on %q with out-degree %d, want >= 2", word, last, degree) + } + } +} + +func TestRandomOpeningWordImpossible(t *testing.T) { + s := fixture(t) + + if _, err := s.RandomOpeningWord(99); err == nil { + t.Fatal("RandomOpeningWord succeeded with an unreachable minimum, want error") + } +} + +func TestCountsAndLicense(t *testing.T) { + s := fixture(t) + + if got := s.WordCount(); got != 7 { + t.Errorf("WordCount = %d, want 7", got) + } + if got := s.AliasCount(); got != 2 { + t.Errorf("AliasCount = %d, want 2", got) + } + if got := s.License(); got != "CC BY-SA 4.0" { + t.Errorf("License = %q, want %q", got, "CC BY-SA 4.0") + } +} + +// The corpus and player input must normalize identically, or a word in the +// dictionary stops matching the same string typed by a player. +func TestNormalizeRoundTrip(t *testing.T) { + s := fixture(t) + + for _, raw := range []string{"PHÁP LUẬT", " pháp luật ", "Pháp\tLuật", "pháp luật"} { + word, syllables, err := vietnamese.Normalize(raw) + if err != nil { + t.Fatalf("Normalize(%q): %v", raw, err) + } + if !vietnamese.HasEnoughSyllables(syllables) { + t.Errorf("Normalize(%q) produced too few syllables", raw) + } + if _, ok := s.Resolve(word); !ok { + t.Errorf("Normalize(%q) = %q, which the dictionary does not resolve", raw, word) + } + } +} + +// The server serves every room from one Store, so concurrent reads must be +// safe. Meaningful only under -race. +func TestConcurrentReads(t *testing.T) { + s := fixture(t) + + const goroutines = 100 + const iterations = 200 + + var wg sync.WaitGroup + errs := make(chan error, goroutines) + + for i := 0; i < goroutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < iterations; j++ { + if _, ok := s.Resolve("pháp luật"); !ok { + errs <- errors.New("resolve failed") + return + } + // Collect, so each goroutine holds its own slice — the shape a + // real caller uses. + if got := slices.Collect(s.WordsStartingWith("pháp")); len(got) != 2 { + errs <- errors.New("unexpected candidate count") + return + } + if _, err := s.OutDegree("pháp"); err != nil { + errs <- err + return + } + if _, err := s.RandomOpeningWord(1); err != nil { + errs <- err + return + } + } + }() + } + + wg.Wait() + close(errs) + for err := range errs { + t.Fatalf("concurrent read failed: %v", err) + } +} + +// Latency is measured by benchmark, not by wrapping time.Now() around a ~13ns +// map lookup: the platform clock quantizes to ~1ms, so such a test measures +// timer resolution rather than the code. +func BenchmarkResolve(b *testing.B) { + s := fixture(b) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, ok := s.Resolve("pháp luật"); !ok { + b.Fatal("resolve failed") + } + } +} + +func BenchmarkWordsStartingWith(b *testing.B) { + s := fixture(b) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for range s.WordsStartingWith("pháp") { + } + } +} + +func BenchmarkOutDegree(b *testing.B) { + s := fixture(b) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := s.OutDegree("pháp"); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkRandomOpeningWord(b *testing.B) { + s := fixture(b) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := s.RandomOpeningWord(1); err != nil { + b.Fatal(err) + } + } +}