mirror of
https://github.com/tiennm99/noitu.git
synced 2026-10-11 03:13:45 +00:00
refactor: remove unused knobs, strings and stale text
Drop what nothing uses any more: the --max-syllables flag with its sourceSpec field, meta row and reject reason; the store's clearRejection; the --gap CSS token; three orphaned i18n strings; two exports with no importer; the preview npm script; the proto.yml "baseline exists" step that has been unconditionally true since the schema landed on main; and two root-level .gitignore entries for paths Playwright never writes. Correct text that outlived its subject: NOTICE named a tools/ directory that never existed, the README still pointed at the first plan and carried a migration note for retired sources, and a few comments still said "release" for an upstream that is a weekly export. The pin test header now says three copies and also holds README and ATTRIBUTION to the same URL; the Make-targets table lists clean and help. Wire fixtures regenerated so client_hello uses the ProtocolVersion constant and server_game_started carries the 30s turn limit. builder_version becomes 4 for the dropped meta row.
This commit is contained in:
1 parent
805cbca39b
commit
557de1af94
22 files changed
+66
-111
No files matched your search
@@ -17,7 +17,6 @@ type rejectReason string
|
||||
const (
|
||||
rejectEmpty rejectReason = "empty"
|
||||
rejectTooShort rejectReason = "fewer than 2 syllables"
|
||||
rejectTooLong rejectReason = "over max-syllables"
|
||||
rejectDigit rejectReason = "contains a digit"
|
||||
rejectPunct rejectReason = "contains punctuation"
|
||||
rejectNonVietnam rejectReason = "no Vietnamese letters"
|
||||
@@ -27,8 +26,8 @@ const (
|
||||
)
|
||||
|
||||
// accept normalizes a raw source entry and decides whether it belongs in the
|
||||
// game dictionary. maxSyllables of 0 means no upper bound.
|
||||
func accept(raw string, maxSyllables int) (word string, syllables []string, reason rejectReason, ok bool) {
|
||||
// game dictionary.
|
||||
func accept(raw string) (word string, syllables []string, reason rejectReason, ok bool) {
|
||||
word, syllables, err := vietnamese.Normalize(raw)
|
||||
if err != nil {
|
||||
return "", nil, rejectEmpty, false
|
||||
@@ -37,9 +36,6 @@ func accept(raw string, maxSyllables int) (word string, syllables []string, reas
|
||||
if !vietnamese.HasEnoughSyllables(syllables) {
|
||||
return "", nil, rejectTooShort, false
|
||||
}
|
||||
if maxSyllables > 0 && len(syllables) > maxSyllables {
|
||||
return "", nil, rejectTooLong, false
|
||||
}
|
||||
|
||||
for _, r := range word {
|
||||
switch {
|
||||
|
||||
@@ -17,7 +17,7 @@ func TestAcceptKeepsValidWords(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
word, syllables, reason, ok := accept(tc.raw, 0)
|
||||
word, syllables, reason, ok := accept(tc.raw)
|
||||
if !ok {
|
||||
t.Errorf("accept(%q) rejected: %s", tc.raw, reason)
|
||||
continue
|
||||
@@ -55,7 +55,7 @@ func TestAcceptRejects(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, _, reason, ok := accept(tc.raw, 0)
|
||||
_, _, reason, ok := accept(tc.raw)
|
||||
if ok {
|
||||
t.Fatalf("accept(%q) succeeded, want rejection %s", tc.raw, tc.want)
|
||||
}
|
||||
@@ -65,21 +65,3 @@ func TestAcceptRejects(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptMaxSyllables(t *testing.T) {
|
||||
const raw = "công nghiệp hóa"
|
||||
|
||||
if _, _, _, ok := accept(raw, 0); !ok {
|
||||
t.Errorf("accept(%q, no limit) rejected, want accepted", raw)
|
||||
}
|
||||
if _, _, _, ok := accept(raw, 3); !ok {
|
||||
t.Errorf("accept(%q, max 3) rejected, want accepted", raw)
|
||||
}
|
||||
_, _, reason, ok := accept(raw, 2)
|
||||
if ok {
|
||||
t.Fatalf("accept(%q, max 2) accepted, want rejection", raw)
|
||||
}
|
||||
if reason != rejectTooLong {
|
||||
t.Errorf("reason = %s, want %s", reason, rejectTooLong)
|
||||
}
|
||||
}
|
||||
@@ -49,7 +49,7 @@ type kaikkiProvenance struct {
|
||||
// Lines are read with bufio.Reader rather than bufio.Scanner because a row
|
||||
// carries every sense and translation of its entry and can run to hundreds of
|
||||
// kilobytes; a scanner's fixed cap would be a guess that eventually fails.
|
||||
func readKaikkiList(path string, maxSyllables int) (map[string]entry, map[rejectReason]int, map[string]int, kaikkiProvenance, error) {
|
||||
func readKaikkiList(path string) (map[string]entry, map[rejectReason]int, map[string]int, kaikkiProvenance, error) {
|
||||
var prov kaikkiProvenance
|
||||
|
||||
f, err := os.Open(path)
|
||||
@@ -92,7 +92,7 @@ func readKaikkiList(path string, maxSyllables int) (map[string]entry, map[reject
|
||||
rejects[rejectNotVietnamese]++
|
||||
} else {
|
||||
pos[row.Pos]++
|
||||
if word, syllables, reason, ok := accept(row.Word, maxSyllables); !ok {
|
||||
if word, syllables, reason, ok := accept(row.Word); !ok {
|
||||
rejects[reason]++
|
||||
} else {
|
||||
words[word] = entry{
|
||||
@@ -153,13 +153,12 @@ func formatPosTally(pos map[string]int) string {
|
||||
// kaikkiSourceSpec describes a kaikki build for the meta table. With no commit
|
||||
// or checksum pinned upstream, the hash and row count of the bytes read are the
|
||||
// provenance.
|
||||
func kaikkiSourceSpec(path string, prov kaikkiProvenance, maxSyllables int) sourceSpec {
|
||||
func kaikkiSourceSpec(path string, prov kaikkiProvenance) sourceSpec {
|
||||
return sourceSpec{
|
||||
table: "kaikki:" + filepath.Base(path),
|
||||
url: kaikkiSourceURL,
|
||||
license: "CC BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)",
|
||||
attribution: "See data/ATTRIBUTION.md for required attribution and the list of modifications.",
|
||||
maxSyllables: maxSyllables,
|
||||
table: "kaikki:" + filepath.Base(path),
|
||||
url: kaikkiSourceURL,
|
||||
license: "CC BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)",
|
||||
attribution: "See data/ATTRIBUTION.md for required attribution and the list of modifications.",
|
||||
extra: [][2]string{
|
||||
{"source_sha256", prov.sha256},
|
||||
{"source_rows", fmt.Sprint(prov.rows)},
|
||||
|
||||
@@ -32,7 +32,7 @@ func defaultKaikkiLines() []string {
|
||||
}
|
||||
|
||||
func TestKaikkiListKeepsVietnameseEntries(t *testing.T) {
|
||||
words, rejects, pos, prov, err := readKaikkiList(fixtureKaikki(t, defaultKaikkiLines()...), 0)
|
||||
words, rejects, pos, prov, err := readKaikkiList(fixtureKaikki(t, defaultKaikkiLines()...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -59,7 +59,7 @@ func TestKaikkiListKeepsVietnameseEntries(t *testing.T) {
|
||||
|
||||
func TestKaikkiListHashesTheBytesItRead(t *testing.T) {
|
||||
path := fixtureKaikki(t, defaultKaikkiLines()...)
|
||||
_, _, _, prov, err := readKaikkiList(path, 0)
|
||||
_, _, _, prov, err := readKaikkiList(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -104,7 +104,7 @@ func TestKaikkiListHandlesDownloadShapes(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
words, _, _, prov, err := readKaikkiList(fixtureKaikkiRaw(t, tc.raw), 0)
|
||||
words, _, _, prov, err := readKaikkiList(fixtureKaikkiRaw(t, tc.raw))
|
||||
if tc.wantErr != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
|
||||
t.Fatalf("err = %v, want one containing %q", err, tc.wantErr)
|
||||
@@ -126,7 +126,7 @@ func TestKaikkiListNamesMalformedLine(t *testing.T) {
|
||||
`{"word": "học sinh", "pos": "noun", "lang_code": "vi"}`,
|
||||
`{"word": "broken"`,
|
||||
)
|
||||
_, _, _, _, err := readKaikkiList(path, 0)
|
||||
_, _, _, _, err := readKaikkiList(path)
|
||||
if err == nil {
|
||||
t.Fatal("malformed line was skipped, want error")
|
||||
}
|
||||
@@ -140,7 +140,7 @@ func TestKaikkiListReadsLongLines(t *testing.T) {
|
||||
// scanner buffer; the reader must not have a line cap.
|
||||
padding := strings.Repeat("x", 2<<20)
|
||||
path := fixtureKaikki(t, `{"word": "học sinh", "pos": "noun", "lang_code": "vi", "note": "`+padding+`"}`)
|
||||
words, _, _, _, err := readKaikkiList(path, 0)
|
||||
words, _, _, _, err := readKaikkiList(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -33,17 +33,16 @@ import (
|
||||
|
||||
// builderVer changes whenever the meta table's contract does, so two databases
|
||||
// with different provenance rows never claim the same builder.
|
||||
const builderVer = "3"
|
||||
const builderVer = "4"
|
||||
|
||||
type config struct {
|
||||
// kaikki is the corpus: the upstream wiktextract JSONL export.
|
||||
kaikki string
|
||||
// words is an alternative source: a plain list, one word per line, used to
|
||||
// build a small fixture database without the upstream download.
|
||||
words string
|
||||
out string
|
||||
maxSyllables int
|
||||
minWords int
|
||||
words string
|
||||
out string
|
||||
minWords int
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -53,7 +52,6 @@ func main() {
|
||||
flag.StringVar(&cfg.kaikki, "kaikki", "", "upstream kaikki.org wiktextract JSONL export to read")
|
||||
flag.StringVar(&cfg.words, "words", "", "read a plain word list instead of the upstream export (one word per line, # comments)")
|
||||
flag.StringVar(&cfg.out, "out", "../data/noitu.db", "derived database to write")
|
||||
flag.IntVar(&cfg.maxSyllables, "max-syllables", 0, "reject words longer than this (0 = no limit)")
|
||||
flag.IntVar(&cfg.minWords, "min-words", 30000, "fail if fewer words survive filtering")
|
||||
flag.Parse()
|
||||
|
||||
@@ -84,7 +82,7 @@ func runFromKaikkiList(cfg config) error {
|
||||
return fmt.Errorf("kaikki export not found at %s — run 'make fetch-dict' first: %w", cfg.kaikki, err)
|
||||
}
|
||||
|
||||
words, rejects, pos, prov, err := readKaikkiList(cfg.kaikki, cfg.maxSyllables)
|
||||
words, rejects, pos, prov, err := readKaikkiList(cfg.kaikki)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -92,7 +90,7 @@ func runFromKaikkiList(cfg config) error {
|
||||
log.Printf("parts of speech: %s", formatPosTally(pos))
|
||||
log.Printf("accepted %d distinct words from %s (%d rows, sha256 %s)", len(words), cfg.kaikki, prov.rows, prov.sha256)
|
||||
|
||||
return finish(cfg, words, kaikkiSourceSpec(cfg.kaikki, prov, cfg.maxSyllables))
|
||||
return finish(cfg, words, kaikkiSourceSpec(cfg.kaikki, prov))
|
||||
}
|
||||
|
||||
// finish is the tail every input mode shares: the size floor, alias
|
||||
@@ -120,7 +118,7 @@ func finish(cfg config, words map[string]entry, src sourceSpec) error {
|
||||
}
|
||||
|
||||
// runFromWordList derives a database from a plain list of words instead of the
|
||||
// upstream release.
|
||||
// upstream export.
|
||||
//
|
||||
// It exists so tests and CI have a real dictionary to play against without the
|
||||
// upstream download. The filtering, alias generation, writing and verification
|
||||
@@ -141,7 +139,7 @@ func runFromWordList(cfg config) error {
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
word, syllables, reason, ok := accept(line, cfg.maxSyllables)
|
||||
word, syllables, reason, ok := accept(line)
|
||||
if !ok {
|
||||
rejects[reason]++
|
||||
continue
|
||||
@@ -222,8 +220,7 @@ type entry struct {
|
||||
type sourceSpec struct {
|
||||
// table names the input: "kaikki:<file>" for the corpus, "wordlist:<file>"
|
||||
// for a fixture, so the output says which build produced it.
|
||||
table string
|
||||
maxSyllables int
|
||||
table string
|
||||
// url is the upstream artifact; empty for fixture builds.
|
||||
url string
|
||||
// license and attribution describe the data's licence obligations. The
|
||||
@@ -407,7 +404,6 @@ CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
||||
{"word_count", fmt.Sprint(len(words))},
|
||||
{"alias_count", fmt.Sprint(len(aliases))},
|
||||
{"source_table", src.table},
|
||||
{"max_syllables", fmt.Sprint(src.maxSyllables)},
|
||||
}
|
||||
meta = append(meta, src.extra...)
|
||||
for _, kv := range meta {
|
||||
|
||||
@@ -39,15 +39,14 @@ func defaultRows() [][2]string {
|
||||
}
|
||||
}
|
||||
|
||||
func buildFixture(t *testing.T, rows [][2]string, maxSyllables int) string {
|
||||
func buildFixture(t *testing.T, rows [][2]string) string {
|
||||
t.Helper()
|
||||
|
||||
out := filepath.Join(t.TempDir(), "noitu.db")
|
||||
cfg := config{
|
||||
kaikki: fixtureSource(t, rows),
|
||||
out: out,
|
||||
maxSyllables: maxSyllables,
|
||||
minWords: 1,
|
||||
kaikki: fixtureSource(t, rows),
|
||||
out: out,
|
||||
minWords: 1,
|
||||
}
|
||||
if err := run(cfg); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
@@ -66,7 +65,7 @@ func openOut(t *testing.T, path string) *sql.DB {
|
||||
}
|
||||
|
||||
func TestBuildProducesExpectedWords(t *testing.T) {
|
||||
db := openOut(t, buildFixture(t, defaultRows(), 0))
|
||||
db := openOut(t, buildFixture(t, defaultRows()))
|
||||
|
||||
var count int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM words`).Scan(&count); err != nil {
|
||||
@@ -99,7 +98,7 @@ func TestBuildProducesExpectedWords(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBuildComputesOutDegree(t *testing.T) {
|
||||
db := openOut(t, buildFixture(t, defaultRows(), 0))
|
||||
db := openOut(t, buildFixture(t, defaultRows()))
|
||||
|
||||
// "pháp luật" and "pháp" (rejected) mean exactly one word starts with "pháp".
|
||||
assertOutDegree(t, db, "pháp", 1)
|
||||
@@ -121,7 +120,7 @@ func assertOutDegree(t *testing.T, db *sql.DB, syllable string, want int) {
|
||||
}
|
||||
|
||||
func TestBuildWritesAliases(t *testing.T) {
|
||||
db := openOut(t, buildFixture(t, defaultRows(), 0))
|
||||
db := openOut(t, buildFixture(t, defaultRows()))
|
||||
|
||||
var canonical string
|
||||
err := db.QueryRow(`SELECT canonical FROM aliases WHERE variant = ?`, "hoà bình").Scan(&canonical)
|
||||
@@ -146,7 +145,7 @@ func TestBuildWritesAliases(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBuildRecordsProvenance(t *testing.T) {
|
||||
db := openOut(t, buildFixture(t, defaultRows(), 0))
|
||||
db := openOut(t, buildFixture(t, defaultRows()))
|
||||
|
||||
for _, key := range []string{"source_url", "source_license", "attribution", "built_at", "word_count"} {
|
||||
var value string
|
||||
@@ -176,7 +175,7 @@ func TestBuildFailsBelowMinWords(t *testing.T) {
|
||||
// A failed build must leave the previous good database untouched. Building in
|
||||
// place would delete it and leave an empty file the server would happily open.
|
||||
func TestFailedBuildPreservesPreviousOutput(t *testing.T) {
|
||||
out := buildFixture(t, defaultRows(), 0)
|
||||
out := buildFixture(t, defaultRows())
|
||||
|
||||
before, err := os.ReadFile(out)
|
||||
if err != nil {
|
||||
|
||||
@@ -28,7 +28,7 @@ const fixtureDir = "../../../proto/testdata"
|
||||
func clientVariants() map[string]*noituv1.ClientMessage {
|
||||
return map[string]*noituv1.ClientMessage{
|
||||
"client_hello": {Payload: &noituv1.ClientMessage_Hello{Hello: &noituv1.Hello{
|
||||
ProtocolVersion: 1,
|
||||
ProtocolVersion: ProtocolVersion,
|
||||
ResumeToken: "r-8f2c",
|
||||
Nickname: "Người chơi ẩn danh",
|
||||
}}},
|
||||
@@ -50,11 +50,11 @@ func clientVariants() map[string]*noituv1.ClientMessage {
|
||||
"client_set_ready": {Payload: &noituv1.ClientMessage_SetReady{SetReady: &noituv1.SetReady{
|
||||
Ready: true,
|
||||
}}},
|
||||
"client_start_game": {Payload: &noituv1.ClientMessage_StartGame{StartGame: &noituv1.StartGame{}}},
|
||||
"client_start_game": {Payload: &noituv1.ClientMessage_StartGame{StartGame: &noituv1.StartGame{}}},
|
||||
"client_kick_player": {Payload: &noituv1.ClientMessage_KickPlayer{KickPlayer: &noituv1.KickPlayer{
|
||||
PlayerId: "p3",
|
||||
}}},
|
||||
"client_leave_room": {Payload: &noituv1.ClientMessage_LeaveRoom{LeaveRoom: &noituv1.LeaveRoom{}}},
|
||||
"client_leave_room": {Payload: &noituv1.ClientMessage_LeaveRoom{LeaveRoom: &noituv1.LeaveRoom{}}},
|
||||
"client_send_chat": {Payload: &noituv1.ClientMessage_SendChat{SendChat: &noituv1.SendChat{
|
||||
Text: "Chào bạn, ván này khó thật!",
|
||||
}}},
|
||||
@@ -76,7 +76,7 @@ func serverVariants() map[string]*noituv1.ServerMessage {
|
||||
MyTurn: true,
|
||||
DeadlineUnixMs: 1756998020000,
|
||||
TurnSeq: 1,
|
||||
TurnLimitMs: 20000,
|
||||
TurnLimitMs: 30000,
|
||||
Players: fixtureScores(),
|
||||
TurnPlayerId: "p1",
|
||||
}}},
|
||||
|
||||
Reference in new issue
Block a user