refactor: remove unused knobs, strings and stale text

Drop what nothing uses any more: the --max-syllables flag with its
sourceSpec field, meta row and reject reason; the store's clearRejection;
the --gap CSS token; three orphaned i18n strings; two exports with no
importer; the preview npm script; the proto.yml "baseline exists" step
that has been unconditionally true since the schema landed on main; and
two root-level .gitignore entries for paths Playwright never writes.

Correct text that outlived its subject: NOTICE named a tools/ directory
that never existed, the README still pointed at the first plan and
carried a migration note for retired sources, and a few comments still
said "release" for an upstream that is a weekly export. The pin test
header now says three copies and also holds README and ATTRIBUTION to
the same URL; the Make-targets table lists clean and help.

Wire fixtures regenerated so client_hello uses the ProtocolVersion
constant and server_game_started carries the 30s turn limit.
builder_version becomes 4 for the dropped meta row.
This commit is contained in:
tiennm99 committed 2026-09-08 18:01:37 +07:00
1 parent 805cbca39b
commit 557de1af94
22 files changed
+66 -111

No files matched your search

+2 -6
View File
@@ -17,7 +17,6 @@ type rejectReason string
const (
rejectEmpty rejectReason = "empty"
rejectTooShort rejectReason = "fewer than 2 syllables"
rejectTooLong rejectReason = "over max-syllables"
rejectDigit rejectReason = "contains a digit"
rejectPunct rejectReason = "contains punctuation"
rejectNonVietnam rejectReason = "no Vietnamese letters"
@@ -27,8 +26,8 @@ const (
)
// accept normalizes a raw source entry and decides whether it belongs in the
// game dictionary. maxSyllables of 0 means no upper bound.
func accept(raw string, maxSyllables int) (word string, syllables []string, reason rejectReason, ok bool) {
// game dictionary.
func accept(raw string) (word string, syllables []string, reason rejectReason, ok bool) {
word, syllables, err := vietnamese.Normalize(raw)
if err != nil {
return "", nil, rejectEmpty, false
@@ -37,9 +36,6 @@ func accept(raw string, maxSyllables int) (word string, syllables []string, reas
if !vietnamese.HasEnoughSyllables(syllables) {
return "", nil, rejectTooShort, false
}
if maxSyllables > 0 && len(syllables) > maxSyllables {
return "", nil, rejectTooLong, false
}
for _, r := range word {
switch {
+2 -20
View File
@@ -17,7 +17,7 @@ func TestAcceptKeepsValidWords(t *testing.T) {
}
for _, tc := range tests {
word, syllables, reason, ok := accept(tc.raw, 0)
word, syllables, reason, ok := accept(tc.raw)
if !ok {
t.Errorf("accept(%q) rejected: %s", tc.raw, reason)
continue
@@ -55,7 +55,7 @@ func TestAcceptRejects(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, _, reason, ok := accept(tc.raw, 0)
_, _, reason, ok := accept(tc.raw)
if ok {
t.Fatalf("accept(%q) succeeded, want rejection %s", tc.raw, tc.want)
}
@@ -65,21 +65,3 @@ func TestAcceptRejects(t *testing.T) {
})
}
}
func TestAcceptMaxSyllables(t *testing.T) {
const raw = "công nghiệp hóa"
if _, _, _, ok := accept(raw, 0); !ok {
t.Errorf("accept(%q, no limit) rejected, want accepted", raw)
}
if _, _, _, ok := accept(raw, 3); !ok {
t.Errorf("accept(%q, max 3) rejected, want accepted", raw)
}
_, _, reason, ok := accept(raw, 2)
if ok {
t.Fatalf("accept(%q, max 2) accepted, want rejection", raw)
}
if reason != rejectTooLong {
t.Errorf("reason = %s, want %s", reason, rejectTooLong)
}
}
+7 -8
View File
@@ -49,7 +49,7 @@ type kaikkiProvenance struct {
// Lines are read with bufio.Reader rather than bufio.Scanner because a row
// carries every sense and translation of its entry and can run to hundreds of
// kilobytes; a scanner's fixed cap would be a guess that eventually fails.
func readKaikkiList(path string, maxSyllables int) (map[string]entry, map[rejectReason]int, map[string]int, kaikkiProvenance, error) {
func readKaikkiList(path string) (map[string]entry, map[rejectReason]int, map[string]int, kaikkiProvenance, error) {
var prov kaikkiProvenance
f, err := os.Open(path)
@@ -92,7 +92,7 @@ func readKaikkiList(path string, maxSyllables int) (map[string]entry, map[reject
rejects[rejectNotVietnamese]++
} else {
pos[row.Pos]++
if word, syllables, reason, ok := accept(row.Word, maxSyllables); !ok {
if word, syllables, reason, ok := accept(row.Word); !ok {
rejects[reason]++
} else {
words[word] = entry{
@@ -153,13 +153,12 @@ func formatPosTally(pos map[string]int) string {
// kaikkiSourceSpec describes a kaikki build for the meta table. With no commit
// or checksum pinned upstream, the hash and row count of the bytes read are the
// provenance.
func kaikkiSourceSpec(path string, prov kaikkiProvenance, maxSyllables int) sourceSpec {
func kaikkiSourceSpec(path string, prov kaikkiProvenance) sourceSpec {
return sourceSpec{
table: "kaikki:" + filepath.Base(path),
url: kaikkiSourceURL,
license: "CC BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)",
attribution: "See data/ATTRIBUTION.md for required attribution and the list of modifications.",
maxSyllables: maxSyllables,
table: "kaikki:" + filepath.Base(path),
url: kaikkiSourceURL,
license: "CC BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)",
attribution: "See data/ATTRIBUTION.md for required attribution and the list of modifications.",
extra: [][2]string{
{"source_sha256", prov.sha256},
{"source_rows", fmt.Sprint(prov.rows)},
@@ -32,7 +32,7 @@ func defaultKaikkiLines() []string {
}
func TestKaikkiListKeepsVietnameseEntries(t *testing.T) {
words, rejects, pos, prov, err := readKaikkiList(fixtureKaikki(t, defaultKaikkiLines()...), 0)
words, rejects, pos, prov, err := readKaikkiList(fixtureKaikki(t, defaultKaikkiLines()...))
if err != nil {
t.Fatal(err)
}
@@ -59,7 +59,7 @@ func TestKaikkiListKeepsVietnameseEntries(t *testing.T) {
func TestKaikkiListHashesTheBytesItRead(t *testing.T) {
path := fixtureKaikki(t, defaultKaikkiLines()...)
_, _, _, prov, err := readKaikkiList(path, 0)
_, _, _, prov, err := readKaikkiList(path)
if err != nil {
t.Fatal(err)
}
@@ -104,7 +104,7 @@ func TestKaikkiListHandlesDownloadShapes(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
words, _, _, prov, err := readKaikkiList(fixtureKaikkiRaw(t, tc.raw), 0)
words, _, _, prov, err := readKaikkiList(fixtureKaikkiRaw(t, tc.raw))
if tc.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
t.Fatalf("err = %v, want one containing %q", err, tc.wantErr)
@@ -126,7 +126,7 @@ func TestKaikkiListNamesMalformedLine(t *testing.T) {
`{"word": "học sinh", "pos": "noun", "lang_code": "vi"}`,
`{"word": "broken"`,
)
_, _, _, _, err := readKaikkiList(path, 0)
_, _, _, _, err := readKaikkiList(path)
if err == nil {
t.Fatal("malformed line was skipped, want error")
}
@@ -140,7 +140,7 @@ func TestKaikkiListReadsLongLines(t *testing.T) {
// scanner buffer; the reader must not have a line cap.
padding := strings.Repeat("x", 2<<20)
path := fixtureKaikki(t, `{"word": "học sinh", "pos": "noun", "lang_code": "vi", "note": "`+padding+`"}`)
words, _, _, _, err := readKaikkiList(path, 0)
words, _, _, _, err := readKaikkiList(path)
if err != nil {
t.Fatal(err)
}
+9 -13
View File
@@ -33,17 +33,16 @@ import (
// builderVer changes whenever the meta table's contract does, so two databases
// with different provenance rows never claim the same builder.
const builderVer = "3"
const builderVer = "4"
type config struct {
// kaikki is the corpus: the upstream wiktextract JSONL export.
kaikki string
// words is an alternative source: a plain list, one word per line, used to
// build a small fixture database without the upstream download.
words string
out string
maxSyllables int
minWords int
words string
out string
minWords int
}
func main() {
@@ -53,7 +52,6 @@ func main() {
flag.StringVar(&cfg.kaikki, "kaikki", "", "upstream kaikki.org wiktextract JSONL export to read")
flag.StringVar(&cfg.words, "words", "", "read a plain word list instead of the upstream export (one word per line, # comments)")
flag.StringVar(&cfg.out, "out", "../data/noitu.db", "derived database to write")
flag.IntVar(&cfg.maxSyllables, "max-syllables", 0, "reject words longer than this (0 = no limit)")
flag.IntVar(&cfg.minWords, "min-words", 30000, "fail if fewer words survive filtering")
flag.Parse()
@@ -84,7 +82,7 @@ func runFromKaikkiList(cfg config) error {
return fmt.Errorf("kaikki export not found at %s — run 'make fetch-dict' first: %w", cfg.kaikki, err)
}
words, rejects, pos, prov, err := readKaikkiList(cfg.kaikki, cfg.maxSyllables)
words, rejects, pos, prov, err := readKaikkiList(cfg.kaikki)
if err != nil {
return err
}
@@ -92,7 +90,7 @@ func runFromKaikkiList(cfg config) error {
log.Printf("parts of speech: %s", formatPosTally(pos))
log.Printf("accepted %d distinct words from %s (%d rows, sha256 %s)", len(words), cfg.kaikki, prov.rows, prov.sha256)
return finish(cfg, words, kaikkiSourceSpec(cfg.kaikki, prov, cfg.maxSyllables))
return finish(cfg, words, kaikkiSourceSpec(cfg.kaikki, prov))
}
// finish is the tail every input mode shares: the size floor, alias
@@ -120,7 +118,7 @@ func finish(cfg config, words map[string]entry, src sourceSpec) error {
}
// runFromWordList derives a database from a plain list of words instead of the
// upstream release.
// upstream export.
//
// It exists so tests and CI have a real dictionary to play against without the
// upstream download. The filtering, alias generation, writing and verification
@@ -141,7 +139,7 @@ func runFromWordList(cfg config) error {
if line == "" || strings.HasPrefix(line, "#") {
continue
}
word, syllables, reason, ok := accept(line, cfg.maxSyllables)
word, syllables, reason, ok := accept(line)
if !ok {
rejects[reason]++
continue
@@ -222,8 +220,7 @@ type entry struct {
type sourceSpec struct {
// table names the input: "kaikki:<file>" for the corpus, "wordlist:<file>"
// for a fixture, so the output says which build produced it.
table string
maxSyllables int
table string
// url is the upstream artifact; empty for fixture builds.
url string
// license and attribution describe the data's licence obligations. The
@@ -407,7 +404,6 @@ CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
{"word_count", fmt.Sprint(len(words))},
{"alias_count", fmt.Sprint(len(aliases))},
{"source_table", src.table},
{"max_syllables", fmt.Sprint(src.maxSyllables)},
}
meta = append(meta, src.extra...)
for _, kv := range meta {
+9 -10
View File
@@ -39,15 +39,14 @@ func defaultRows() [][2]string {
}
}
func buildFixture(t *testing.T, rows [][2]string, maxSyllables int) string {
func buildFixture(t *testing.T, rows [][2]string) string {
t.Helper()
out := filepath.Join(t.TempDir(), "noitu.db")
cfg := config{
kaikki: fixtureSource(t, rows),
out: out,
maxSyllables: maxSyllables,
minWords: 1,
kaikki: fixtureSource(t, rows),
out: out,
minWords: 1,
}
if err := run(cfg); err != nil {
t.Fatalf("run: %v", err)
@@ -66,7 +65,7 @@ func openOut(t *testing.T, path string) *sql.DB {
}
func TestBuildProducesExpectedWords(t *testing.T) {
db := openOut(t, buildFixture(t, defaultRows(), 0))
db := openOut(t, buildFixture(t, defaultRows()))
var count int
if err := db.QueryRow(`SELECT COUNT(*) FROM words`).Scan(&count); err != nil {
@@ -99,7 +98,7 @@ func TestBuildProducesExpectedWords(t *testing.T) {
}
func TestBuildComputesOutDegree(t *testing.T) {
db := openOut(t, buildFixture(t, defaultRows(), 0))
db := openOut(t, buildFixture(t, defaultRows()))
// "pháp luật" and "pháp" (rejected) mean exactly one word starts with "pháp".
assertOutDegree(t, db, "pháp", 1)
@@ -121,7 +120,7 @@ func assertOutDegree(t *testing.T, db *sql.DB, syllable string, want int) {
}
func TestBuildWritesAliases(t *testing.T) {
db := openOut(t, buildFixture(t, defaultRows(), 0))
db := openOut(t, buildFixture(t, defaultRows()))
var canonical string
err := db.QueryRow(`SELECT canonical FROM aliases WHERE variant = ?`, "hoà bình").Scan(&canonical)
@@ -146,7 +145,7 @@ func TestBuildWritesAliases(t *testing.T) {
}
func TestBuildRecordsProvenance(t *testing.T) {
db := openOut(t, buildFixture(t, defaultRows(), 0))
db := openOut(t, buildFixture(t, defaultRows()))
for _, key := range []string{"source_url", "source_license", "attribution", "built_at", "word_count"} {
var value string
@@ -176,7 +175,7 @@ func TestBuildFailsBelowMinWords(t *testing.T) {
// A failed build must leave the previous good database untouched. Building in
// place would delete it and leave an empty file the server would happily open.
func TestFailedBuildPreservesPreviousOutput(t *testing.T) {
out := buildFixture(t, defaultRows(), 0)
out := buildFixture(t, defaultRows())
before, err := os.ReadFile(out)
if err != nil {
+4 -4
View File
@@ -28,7 +28,7 @@ const fixtureDir = "../../../proto/testdata"
func clientVariants() map[string]*noituv1.ClientMessage {
return map[string]*noituv1.ClientMessage{
"client_hello": {Payload: &noituv1.ClientMessage_Hello{Hello: &noituv1.Hello{
ProtocolVersion: 1,
ProtocolVersion: ProtocolVersion,
ResumeToken: "r-8f2c",
Nickname: "Người chơi ẩn danh",
}}},
@@ -50,11 +50,11 @@ func clientVariants() map[string]*noituv1.ClientMessage {
"client_set_ready": {Payload: &noituv1.ClientMessage_SetReady{SetReady: &noituv1.SetReady{
Ready: true,
}}},
"client_start_game": {Payload: &noituv1.ClientMessage_StartGame{StartGame: &noituv1.StartGame{}}},
"client_start_game": {Payload: &noituv1.ClientMessage_StartGame{StartGame: &noituv1.StartGame{}}},
"client_kick_player": {Payload: &noituv1.ClientMessage_KickPlayer{KickPlayer: &noituv1.KickPlayer{
PlayerId: "p3",
}}},
"client_leave_room": {Payload: &noituv1.ClientMessage_LeaveRoom{LeaveRoom: &noituv1.LeaveRoom{}}},
"client_leave_room": {Payload: &noituv1.ClientMessage_LeaveRoom{LeaveRoom: &noituv1.LeaveRoom{}}},
"client_send_chat": {Payload: &noituv1.ClientMessage_SendChat{SendChat: &noituv1.SendChat{
Text: "Chào bạn, ván này khó thật!",
}}},
@@ -76,7 +76,7 @@ func serverVariants() map[string]*noituv1.ServerMessage {
MyTurn: true,
DeadlineUnixMs: 1756998020000,
TurnSeq: 1,
TurnLimitMs: 20000,
TurnLimitMs: 30000,
Players: fixtureScores(),
TurnPlayerId: "p1",
}}},