mirror of
https://github.com/tiennm99/noitu.git
synced 2026-10-11 12:28:54 +00:00
feat(dictionary): build Vietnamese wordlist from upstream dictionary
Reduce the 179 MB minhqnd/dictionary SQLite release to a ~3 MB game wordlist: 48,216 Vietnamese words of two or more syllables, indexed by first and last syllable with an out-degree table for dead-end detection. Source schema is auto-detected rather than hardcoded, since it is someone else's release artifact; explicit flags override it and are validated against the real tables, because SQLite silently reads an unknown double-quoted column as a string literal. Accept a word only if every syllable fits Vietnamese phonotactics. An alphabet check is not enough: "credit card" and "come out" use only letters Vietnamese has, and the multilingual source tags them as Vietnamese. Onset matching backtracks so the gi digraph does not swallow the nucleus of common words like "gi", "gin" and "gi" (rust). Record accepted spelling variants in an alias table rather than solving tone placement at runtime. Tone shifting applies only to open oa/oe/uy syllables, since "hoan" and "hoai" have a single correct spelling, and "qu" is a consonant onset. The i/y alternation uses an onset allowlist plus explicit pairs, because it is lexical rather than productive. Variants are generated as a cross product over syllables so a word with two variable syllables still offers the fully modern spelling. Build to a temporary file and rename only after commit, so a failed run cannot leave an empty database where a good one was, then re-open the result and verify its invariants on disk. Licensing: the derived data is CC BY-SA 4.0 and stays a separate artifact from the Apache-2.0 code, loaded at runtime and never embedded. Ships NOTICE, data/LICENSE and an attribution file recording every change.
This commit is contained in:
1 parent
582ba27354
commit
4b4afea679
18 files changed
+2662
-1
No files matched your search
@@ -0,0 +1,287 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// Vietnamese combining tone marks. These are the five tones; they are distinct
|
||||
// from the combining marks that form letters (circumflex, breve, horn), which
|
||||
// belong to their base letter and must never be moved.
|
||||
var toneMarks = map[rune]bool{
|
||||
'̀': true, // huyền (grave)
|
||||
'́': true, // sắc (acute)
|
||||
'̃': true, // ngã (tilde)
|
||||
'̉': true, // hỏi (hook above)
|
||||
'̣': true, // nặng (dot below)
|
||||
}
|
||||
|
||||
// Marks that turn a base letter into a different Vietnamese letter (ă â ê ô ơ ư).
|
||||
// A vowel carrying one of these is not a plain vowel and is excluded from the
|
||||
// tone-shifting rule below.
|
||||
var letterMarks = map[rune]bool{
|
||||
'̂': true, // circumflex: â ê ô
|
||||
'̆': true, // breve: ă
|
||||
'̛': true, // horn: ơ ư
|
||||
}
|
||||
|
||||
// cluster is one base rune plus the combining marks attached to it.
|
||||
type cluster struct {
|
||||
base rune
|
||||
mods []rune
|
||||
}
|
||||
|
||||
func (c cluster) tone() (rune, bool) {
|
||||
for _, m := range c.mods {
|
||||
if toneMarks[m] {
|
||||
return m, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (c cluster) hasLetterMark() bool {
|
||||
for _, m := range c.mods {
|
||||
if letterMarks[m] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c cluster) without(target rune) cluster {
|
||||
out := cluster{base: c.base}
|
||||
for _, m := range c.mods {
|
||||
if m != target {
|
||||
out.mods = append(out.mods, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c cluster) with(extra rune) cluster {
|
||||
out := cluster{base: c.base, mods: append([]rune{}, c.mods...)}
|
||||
out.mods = append(out.mods, extra)
|
||||
return out
|
||||
}
|
||||
|
||||
// decompose splits a syllable into base-plus-marks clusters, working on the NFD
|
||||
// form so that every diacritic is visible as its own rune.
|
||||
func decompose(syllable string) []cluster {
|
||||
var clusters []cluster
|
||||
for _, r := range norm.NFD.String(syllable) {
|
||||
if (toneMarks[r] || letterMarks[r]) && len(clusters) > 0 {
|
||||
last := len(clusters) - 1
|
||||
clusters[last].mods = append(clusters[last].mods, r)
|
||||
continue
|
||||
}
|
||||
clusters = append(clusters, cluster{base: r})
|
||||
}
|
||||
return clusters
|
||||
}
|
||||
|
||||
func recompose(clusters []cluster) string {
|
||||
var b strings.Builder
|
||||
for _, c := range clusters {
|
||||
b.WriteRune(c.base)
|
||||
for _, m := range c.mods {
|
||||
b.WriteRune(m)
|
||||
}
|
||||
}
|
||||
return norm.NFC.String(b.String())
|
||||
}
|
||||
|
||||
// shiftablePairs are the vowel clusters where Vietnamese has two competing tone
|
||||
// placements in real use: the older style marks the first vowel ("hòa", "thúy"),
|
||||
// the newer style marks the second ("hoà", "thuý"). Both spellings appear in
|
||||
// dictionaries and both are typed by players, but they are different codepoint
|
||||
// sequences, so one of them fails a lookup unless we record it as an alias.
|
||||
var shiftablePairs = [][2]rune{
|
||||
{'o', 'a'},
|
||||
{'o', 'e'},
|
||||
{'u', 'y'},
|
||||
}
|
||||
|
||||
// toneShiftVariant moves the tone mark to the other vowel of an oa/oe/uy cluster.
|
||||
// Returns the variant and whether one was produced.
|
||||
func toneShiftVariant(syllable string) (string, bool) {
|
||||
clusters := decompose(syllable)
|
||||
|
||||
for i := 0; i+1 < len(clusters); i++ {
|
||||
first, second := clusters[i], clusters[i+1]
|
||||
|
||||
// Only plain vowels take part; ơ/ư/ô/ê/ă follow different rules.
|
||||
if first.hasLetterMark() || second.hasLetterMark() {
|
||||
continue
|
||||
}
|
||||
if !isShiftablePair(first.base, second.base) {
|
||||
continue
|
||||
}
|
||||
// The two placements only compete in an OPEN syllable that ends on the
|
||||
// cluster. Add anything after it -- a final consonant ("hoàn") or a
|
||||
// third vowel ("hoài") -- and only one spelling is correct. Without this
|
||||
// check the builder invents words like "hòan" and "hòai".
|
||||
if i+1 != len(clusters)-1 {
|
||||
continue
|
||||
}
|
||||
// In "qu" the u is part of the consonant onset, not the vowel nucleus,
|
||||
// so "quý" must not become "qúy".
|
||||
if first.base == 'u' && i > 0 && clusters[i-1].base == 'q' {
|
||||
continue
|
||||
}
|
||||
|
||||
if tone, ok := first.tone(); ok {
|
||||
shifted := append([]cluster{}, clusters...)
|
||||
shifted[i] = first.without(tone)
|
||||
shifted[i+1] = second.with(tone)
|
||||
return recompose(shifted), true
|
||||
}
|
||||
if tone, ok := second.tone(); ok {
|
||||
shifted := append([]cluster{}, clusters...)
|
||||
shifted[i] = first.with(tone)
|
||||
shifted[i+1] = second.without(tone)
|
||||
return recompose(shifted), true
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
func isShiftablePair(a, b rune) bool {
|
||||
for _, p := range shiftablePairs {
|
||||
if p[0] == a && p[1] == b {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// iyOnsets are the consonant onsets after which Vietnamese genuinely writes
|
||||
// both i and y across the tone range: hi/hy, ki/ky, li/ly, mi/my, ti/ty, qui/quy.
|
||||
//
|
||||
// This is an allowlist rather than a general rule because most onsets take only
|
||||
// one spelling. "gi" is a digraph, so "gì" has no "gỳ"; "chì" has no "chỳ".
|
||||
//
|
||||
// "v" and "s" were deliberately removed after inspecting real output: they
|
||||
// produced non-words at scale ("chức vị" -> "chức vỵ", plus "vỳ", "vý", "sỵ",
|
||||
// "sỳ"). Their genuine alternations are lexical rather than productive, so the
|
||||
// few real ones are listed in iyPairs instead.
|
||||
var iyOnsets = map[string]bool{
|
||||
"h": true, "k": true, "l": true, "m": true, "t": true, "qu": true,
|
||||
}
|
||||
|
||||
// iyPairs are individual syllables that alternate even though their onset does
|
||||
// not do so productively. Listed explicitly, both directions.
|
||||
var iyPairs = map[string]string{
|
||||
"sĩ": "sỹ", "sỹ": "sĩ",
|
||||
"vĩ": "vỹ", "vỹ": "vĩ",
|
||||
}
|
||||
|
||||
// iyVariant swaps a lone i/y nucleus: quý/quí, lý/lí, mỹ/mĩ.
|
||||
//
|
||||
// The nucleus must be the final cluster, so closed syllables are excluded
|
||||
// ("tính" has no "týnh"), and the onset must be one that takes both spellings.
|
||||
func iyVariant(syllable string) (string, bool) {
|
||||
if pair, ok := iyPairs[syllable]; ok {
|
||||
return pair, true
|
||||
}
|
||||
|
||||
clusters := decompose(syllable)
|
||||
if len(clusters) < 2 {
|
||||
return "", false
|
||||
}
|
||||
|
||||
last := len(clusters) - 1
|
||||
nucleus := clusters[last]
|
||||
if nucleus.hasLetterMark() {
|
||||
return "", false
|
||||
}
|
||||
|
||||
var swapped rune
|
||||
switch nucleus.base {
|
||||
case 'i':
|
||||
swapped = 'y'
|
||||
case 'y':
|
||||
swapped = 'i'
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
|
||||
var onset strings.Builder
|
||||
for _, c := range clusters[:last] {
|
||||
if c.hasLetterMark() {
|
||||
return "", false
|
||||
}
|
||||
onset.WriteRune(c.base)
|
||||
}
|
||||
if !iyOnsets[onset.String()] {
|
||||
return "", false
|
||||
}
|
||||
|
||||
variant := append([]cluster{}, clusters...)
|
||||
variant[last] = cluster{base: swapped, mods: nucleus.mods}
|
||||
return recompose(variant), true
|
||||
}
|
||||
|
||||
// maxVariantsPerWord caps the combination count for a word whose syllables are
|
||||
// all variable. Real words reach at most a handful; the cap only guards against
|
||||
// a pathological entry blowing up the alias table.
|
||||
const maxVariantsPerWord = 32
|
||||
|
||||
// spellingsOf returns every accepted spelling of a single syllable, the
|
||||
// original first.
|
||||
func spellingsOf(syllable string) []string {
|
||||
out := []string{syllable}
|
||||
seen := map[string]bool{syllable: true}
|
||||
|
||||
for _, gen := range []func(string) (string, bool){toneShiftVariant, iyVariant} {
|
||||
alt, ok := gen(syllable)
|
||||
if !ok || seen[alt] {
|
||||
continue
|
||||
}
|
||||
seen[alt] = true
|
||||
out = append(out, alt)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// variantsFor returns every accepted alternative spelling of a whole word.
|
||||
//
|
||||
// Syllables must be varied together, not one at a time. "hoá lí" has two
|
||||
// variable syllables, and a player is far more likely to type the fully modern
|
||||
// "hóa lý" than either half-and-half form. Varying independently produced only
|
||||
// the halves and rejected the spelling most people actually use.
|
||||
func variantsFor(word string) []string {
|
||||
syllables := strings.Split(word, " ")
|
||||
|
||||
// Cartesian product over each syllable's accepted spellings.
|
||||
combos := [][]string{{}}
|
||||
for _, syl := range syllables {
|
||||
spellings := spellingsOf(syl)
|
||||
next := make([][]string, 0, len(combos)*len(spellings))
|
||||
for _, prefix := range combos {
|
||||
for _, s := range spellings {
|
||||
if len(next) >= maxVariantsPerWord {
|
||||
break
|
||||
}
|
||||
next = append(next, append(append([]string{}, prefix...), s))
|
||||
}
|
||||
}
|
||||
combos = next
|
||||
}
|
||||
|
||||
seen := map[string]bool{word: true}
|
||||
var out []string
|
||||
for _, combo := range combos {
|
||||
joined := strings.Join(combo, " ")
|
||||
if seen[joined] {
|
||||
continue
|
||||
}
|
||||
seen[joined] = true
|
||||
out = append(out, joined)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Vietnamese writes the same syllable with the tone mark on either vowel of an
|
||||
// oa/oe/uy cluster. Both spellings are typed by real players, so both must
|
||||
// resolve to one dictionary entry.
|
||||
func TestToneShiftVariant(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"oa, tone on first", "hòa", "hoà"},
|
||||
{"oa, tone on second", "hoà", "hòa"},
|
||||
{"oe, tone on first", "khỏe", "khoẻ"},
|
||||
{"oe, tone on second", "khoẻ", "khỏe"},
|
||||
{"uy, tone on first", "thúy", "thuý"},
|
||||
{"uy, tone on second", "thuý", "thúy"},
|
||||
{"nang tone on oa", "họa", "hoạ"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, ok := toneShiftVariant(tc.in)
|
||||
if !ok {
|
||||
t.Fatalf("toneShiftVariant(%q) produced no variant, want %q", tc.in, tc.want)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Errorf("toneShiftVariant(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestToneShiftVariantRejects(t *testing.T) {
|
||||
tests := []struct{ in, why string }{
|
||||
{"quý", `"qu" is a consonant onset, not a vowel cluster — no "qúy"`},
|
||||
{"hoàn", `closed syllable: final consonant, so no "hòan"`},
|
||||
{"roàng", `closed syllable: final "ng", so no "ròang"`},
|
||||
{"hoài", `triphthong "oai", so no "hòai"`},
|
||||
{"khoẻn", `closed "oe" syllable`},
|
||||
{"quả", "no shiftable cluster"},
|
||||
{"luật", "no shiftable cluster"},
|
||||
{"pháp", "no shiftable cluster"},
|
||||
{"ngữ", "no shiftable cluster"},
|
||||
{"điện", "no shiftable cluster"},
|
||||
{"tính", "no shiftable cluster"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
if got, ok := toneShiftVariant(tc.in); ok {
|
||||
t.Errorf("toneShiftVariant(%q) = %q, want no variant (%s)", tc.in, got, tc.why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIYVariant(t *testing.T) {
|
||||
tests := []struct{ in, want string }{
|
||||
{"quý", "quí"},
|
||||
{"quí", "quý"},
|
||||
{"lý", "lí"},
|
||||
{"mỹ", "mĩ"},
|
||||
{"kỹ", "kĩ"},
|
||||
{"sĩ", "sỹ"},
|
||||
{"tỷ", "tỉ"},
|
||||
{"hy", "hi"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
got, ok := iyVariant(tc.in)
|
||||
if !ok {
|
||||
t.Errorf("iyVariant(%q) produced no variant, want %q", tc.in, tc.want)
|
||||
continue
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Errorf("iyVariant(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIYVariantRejects(t *testing.T) {
|
||||
// "gi" and "ch" are digraphs that only ever take i; closed syllables and
|
||||
// diphthongs are out of scope entirely.
|
||||
for _, in := range []string{"gì", "chì", "tính", "kia", "đi", "nghi", "phi", "bi"} {
|
||||
if got, ok := iyVariant(in); ok {
|
||||
t.Errorf("iyVariant(%q) = %q, want no variant", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVariantsFor(t *testing.T) {
|
||||
got := variantsFor("hòa bình")
|
||||
want := []string{"hoà bình"}
|
||||
assertSameStrings(t, got, want)
|
||||
|
||||
got = variantsFor("pháp luật")
|
||||
if len(got) != 0 {
|
||||
t.Errorf("variantsFor(%q) = %v, want none", "pháp luật", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A word with two variable syllables must offer every combination, not just the
|
||||
// half-and-half forms. "hoá lí" is the corpus spelling but "hóa lý" is the one
|
||||
// most people type, and varying syllables independently never produced it.
|
||||
func TestVariantsForVariesSyllablesTogether(t *testing.T) {
|
||||
// "hoá" carries the acute tone, so shifting it gives "hóa" (not "hòa").
|
||||
got := variantsFor("hoá lí")
|
||||
want := []string{"hóa lí", "hoá lý", "hóa lý"}
|
||||
assertSameStrings(t, got, want)
|
||||
}
|
||||
|
||||
func TestVariantsForCapsCombinations(t *testing.T) {
|
||||
// Every syllable variable: 2^6 = 64 combinations before the cap.
|
||||
word := "hoá hoá hoá hoá hoá hoá"
|
||||
if got := len(variantsFor(word)); got > maxVariantsPerWord {
|
||||
t.Errorf("variantsFor produced %d variants, want at most %d", got, maxVariantsPerWord)
|
||||
}
|
||||
}
|
||||
|
||||
// "v" and "s" do not alternate productively: "chức vị" must never yield
|
||||
// "chức vỵ". Only the handful of genuinely alternating syllables are listed.
|
||||
func TestIYRejectsNonAlternatingOnsets(t *testing.T) {
|
||||
for _, in := range []string{"vị", "vì", "vỷ", "sị", "sỳ", "si"} {
|
||||
if got, ok := iyVariant(in); ok {
|
||||
t.Errorf("iyVariant(%q) = %q, want no variant", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIYExplicitPairs(t *testing.T) {
|
||||
for in, want := range map[string]string{"sĩ": "sỹ", "sỹ": "sĩ", "vĩ": "vỹ", "vỹ": "vĩ"} {
|
||||
got, ok := iyVariant(in)
|
||||
if !ok {
|
||||
t.Errorf("iyVariant(%q) produced no variant, want %q", in, want)
|
||||
continue
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("iyVariant(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// With three words claiming one variant, a plain delete would let the third
|
||||
// reinsert it and the winner would depend on map iteration order.
|
||||
func TestBuildAliasesPoisonsAmbiguousVariants(t *testing.T) {
|
||||
words := map[string]entry{
|
||||
"lí do": {word: "lí do", first: "lí", last: "do", syllables: 2},
|
||||
"lý do": {word: "lý do", first: "lý", last: "do", syllables: 2},
|
||||
}
|
||||
|
||||
for i := 0; i < 20; i++ {
|
||||
aliases, _ := buildAliases(words)
|
||||
if canonical, ok := aliases["lí do"]; ok {
|
||||
t.Fatalf("run %d: real word aliased to %q", i, canonical)
|
||||
}
|
||||
if canonical, ok := aliases["lý do"]; ok {
|
||||
t.Fatalf("run %d: real word aliased to %q", i, canonical)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A generated variant that is itself a real word must not become an alias:
|
||||
// doing so would let one entry answer to another entry's name.
|
||||
func TestBuildAliasesSkipsCollisions(t *testing.T) {
|
||||
words := map[string]entry{
|
||||
"hòa bình": {word: "hòa bình", first: "hòa", last: "bình", syllables: 2},
|
||||
"hoà bình": {word: "hoà bình", first: "hoà", last: "bình", syllables: 2},
|
||||
}
|
||||
|
||||
aliases, collisions := buildAliases(words)
|
||||
|
||||
if len(aliases) != 0 {
|
||||
t.Errorf("aliases = %v, want none (both spellings are real words)", aliases)
|
||||
}
|
||||
if collisions != 2 {
|
||||
t.Errorf("collisions = %d, want 2", collisions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAliasesMapsToCanonical(t *testing.T) {
|
||||
words := map[string]entry{
|
||||
"hòa bình": {word: "hòa bình", first: "hòa", last: "bình", syllables: 2},
|
||||
}
|
||||
|
||||
aliases, _ := buildAliases(words)
|
||||
|
||||
if got := aliases["hoà bình"]; got != "hòa bình" {
|
||||
t.Errorf("aliases[%q] = %q, want %q", "hoà bình", got, "hòa bình")
|
||||
}
|
||||
}
|
||||
|
||||
func assertSameStrings(t *testing.T, got, want []string) {
|
||||
t.Helper()
|
||||
sort.Strings(got)
|
||||
sort.Strings(want)
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
|
||||
"github.com/tiennm99dev/noitu/server/internal/vietnamese"
|
||||
)
|
||||
|
||||
// rejectReason explains why a candidate word did not make it into the game
|
||||
// dictionary. Counted per reason so the build log shows what the source
|
||||
// actually contained rather than just a final total.
|
||||
type rejectReason string
|
||||
|
||||
const (
|
||||
rejectEmpty rejectReason = "empty"
|
||||
rejectTooShort rejectReason = "fewer than 2 syllables"
|
||||
rejectTooLong rejectReason = "over max-syllables"
|
||||
rejectDigit rejectReason = "contains a digit"
|
||||
rejectPunct rejectReason = "contains punctuation"
|
||||
rejectNonVietnam rejectReason = "no Vietnamese letters"
|
||||
)
|
||||
|
||||
// accept normalizes a raw source entry and decides whether it belongs in the
|
||||
// game dictionary. maxSyllables of 0 means no upper bound.
|
||||
func accept(raw string, maxSyllables int) (word string, syllables []string, reason rejectReason, ok bool) {
|
||||
word, syllables, err := vietnamese.Normalize(raw)
|
||||
if err != nil {
|
||||
return "", nil, rejectEmpty, false
|
||||
}
|
||||
|
||||
if !vietnamese.HasEnoughSyllables(syllables) {
|
||||
return "", nil, rejectTooShort, false
|
||||
}
|
||||
if maxSyllables > 0 && len(syllables) > maxSyllables {
|
||||
return "", nil, rejectTooLong, false
|
||||
}
|
||||
|
||||
for _, r := range word {
|
||||
switch {
|
||||
case r == ' ':
|
||||
continue
|
||||
case unicode.IsDigit(r):
|
||||
return "", nil, rejectDigit, false
|
||||
case unicode.IsPunct(r) || unicode.IsSymbol(r):
|
||||
return "", nil, rejectPunct, false
|
||||
case !unicode.IsLetter(r):
|
||||
return "", nil, rejectPunct, false
|
||||
}
|
||||
}
|
||||
|
||||
if !usesVietnameseAlphabet(word) {
|
||||
return "", nil, rejectNonVietnam, false
|
||||
}
|
||||
|
||||
// Every syllable must also be a possible Vietnamese syllable. The alphabet
|
||||
// check alone lets "credit card" and "come out" through, since they use only
|
||||
// letters Vietnamese has.
|
||||
for _, syl := range syllables {
|
||||
if !isVietnameseSyllable(syl) {
|
||||
return "", nil, rejectNonVietnam, false
|
||||
}
|
||||
}
|
||||
|
||||
return word, syllables, "", true
|
||||
}
|
||||
|
||||
// vietnameseBaseLetters is the Vietnamese alphabet reduced to base letters:
|
||||
// the Latin alphabet without f, j, w, z, plus đ. Every Vietnamese word is
|
||||
// spelled from these once diacritics are stripped.
|
||||
const vietnameseBaseLetters = "abcdeghiklmnopqrstuvxyđ"
|
||||
|
||||
// usesVietnameseAlphabet reports whether every letter of the word is a
|
||||
// Vietnamese one.
|
||||
//
|
||||
// Diacritics are stripped first (NFD, dropping combining marks) so that "ngữ"
|
||||
// reduces to "ngu" and is checked letter by letter. Checking the base letters
|
||||
// rather than merely "does it contain a diacritic" is what keeps English out:
|
||||
// an entry only needs one accented character to look Vietnamese otherwise, and
|
||||
// entries like "thành phố new york", "credit card" and "come out" pass such a
|
||||
// test while being unplayable as Vietnamese words.
|
||||
func usesVietnameseAlphabet(word string) bool {
|
||||
stripped := norm.NFD.String(word)
|
||||
|
||||
sawLetter := false
|
||||
for _, r := range stripped {
|
||||
if unicode.Is(unicode.Mn, r) || r == ' ' {
|
||||
// Combining mark or separator: not a base letter.
|
||||
continue
|
||||
}
|
||||
if !strings.ContainsRune(vietnameseBaseLetters, r) {
|
||||
return false
|
||||
}
|
||||
sawLetter = true
|
||||
}
|
||||
|
||||
return sawLetter
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestAcceptKeepsValidWords(t *testing.T) {
|
||||
tests := []struct {
|
||||
raw string
|
||||
want string
|
||||
syllables int
|
||||
}{
|
||||
{"pháp luật", "pháp luật", 2},
|
||||
{"Pháp Luật", "pháp luật", 2},
|
||||
{"vô tuyến điện", "vô tuyến điện", 3},
|
||||
{"con cua", "con cua", 2}, // no diacritics, still Vietnamese
|
||||
{"đi đứng", "đi đứng", 2}, // đ is a Vietnamese letter
|
||||
{"quy hoạch", "quy hoạch", 2},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
word, syllables, reason, ok := accept(tc.raw, 0)
|
||||
if !ok {
|
||||
t.Errorf("accept(%q) rejected: %s", tc.raw, reason)
|
||||
continue
|
||||
}
|
||||
if word != tc.want {
|
||||
t.Errorf("accept(%q) word = %q, want %q", tc.raw, word, tc.want)
|
||||
}
|
||||
if len(syllables) != tc.syllables {
|
||||
t.Errorf("accept(%q) syllables = %d, want %d", tc.raw, len(syllables), tc.syllables)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptRejects(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want rejectReason
|
||||
}{
|
||||
{"empty", " ", rejectEmpty},
|
||||
{"single syllable", "pháp", rejectTooShort},
|
||||
{"digit", "covid 19", rejectDigit},
|
||||
{"hyphen", "pháp-luật xxx", rejectPunct},
|
||||
{"english phrase", "hello world", rejectNonVietnam},
|
||||
{"letters Vietnamese lacks", "jazz music", rejectNonVietnam},
|
||||
// These pass a naive "contains f/j/w/z" test and a naive "has a
|
||||
// diacritic" test, yet are unplayable as Vietnamese words.
|
||||
{"english, no forbidden letters", "credit card", rejectNonVietnam},
|
||||
{"english, no forbidden letters 2", "dress code", rejectNonVietnam},
|
||||
{"english phrasal verb", "come out", rejectNonVietnam},
|
||||
// One Vietnamese diacritic must not launder an English word.
|
||||
{"mixed English with a diacritic", "thành phố new york", rejectNonVietnam},
|
||||
{"non-Latin script", "привет мир", rejectNonVietnam},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, _, reason, ok := accept(tc.raw, 0)
|
||||
if ok {
|
||||
t.Fatalf("accept(%q) succeeded, want rejection %s", tc.raw, tc.want)
|
||||
}
|
||||
if reason != tc.want {
|
||||
t.Errorf("accept(%q) reason = %s, want %s", tc.raw, reason, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptMaxSyllables(t *testing.T) {
|
||||
const raw = "công nghiệp hóa"
|
||||
|
||||
if _, _, _, ok := accept(raw, 0); !ok {
|
||||
t.Errorf("accept(%q, no limit) rejected, want accepted", raw)
|
||||
}
|
||||
if _, _, _, ok := accept(raw, 3); !ok {
|
||||
t.Errorf("accept(%q, max 3) rejected, want accepted", raw)
|
||||
}
|
||||
_, _, reason, ok := accept(raw, 2)
|
||||
if ok {
|
||||
t.Fatalf("accept(%q, max 2) accepted, want rejection", raw)
|
||||
}
|
||||
if reason != rejectTooLong {
|
||||
t.Errorf("reason = %s, want %s", reason, rejectTooLong)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,547 @@
|
||||
// Command build-dictionary derives the game's wordlist from the upstream
|
||||
// minhqnd/dictionary SQLite database.
|
||||
//
|
||||
// The upstream database is ~179 MB and covers 1,500+ language pairs with full
|
||||
// definitions. The game needs none of that: only Vietnamese word forms of at
|
||||
// least two syllables, indexed by first and last syllable. This tool performs
|
||||
// that reduction and records provenance in a meta table.
|
||||
//
|
||||
// The derived database is a modified version of CC BY-SA 4.0 licensed data.
|
||||
// See data/ATTRIBUTION.md.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// go run ./cmd/build-dictionary --in ../data/dictionary.db --out ../data/noitu.db
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const (
|
||||
sourceURL = "https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db"
|
||||
sourceLicense = "CC BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)"
|
||||
builderVer = "1"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
in string
|
||||
out string
|
||||
maxSyllables int
|
||||
minWords int
|
||||
table string
|
||||
wordCol string
|
||||
langCol string
|
||||
lang string
|
||||
}
|
||||
|
||||
func main() {
|
||||
log.SetFlags(0)
|
||||
|
||||
var cfg config
|
||||
flag.StringVar(&cfg.in, "in", "../data/dictionary.db", "upstream dictionary.db to read")
|
||||
flag.StringVar(&cfg.out, "out", "../data/noitu.db", "derived database to write")
|
||||
flag.IntVar(&cfg.maxSyllables, "max-syllables", 0, "reject words longer than this (0 = no limit)")
|
||||
flag.IntVar(&cfg.minWords, "min-words", 40000, "fail if fewer words survive filtering")
|
||||
flag.StringVar(&cfg.table, "table", "", "source table (default: auto-detect)")
|
||||
flag.StringVar(&cfg.wordCol, "word-col", "", "source word column (default: auto-detect)")
|
||||
flag.StringVar(&cfg.langCol, "lang-col", "", "source language column (default: auto-detect)")
|
||||
flag.StringVar(&cfg.lang, "lang", "vi", "language code to keep")
|
||||
flag.Parse()
|
||||
|
||||
if err := run(cfg); err != nil {
|
||||
log.Fatalf("build-dictionary: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(cfg config) error {
|
||||
if _, err := os.Stat(cfg.in); err != nil {
|
||||
return fmt.Errorf("source database not found at %s — run 'make fetch-dict' first: %w", cfg.in, err)
|
||||
}
|
||||
|
||||
src, err := sql.Open("sqlite", "file:"+cfg.in+"?mode=ro")
|
||||
if err != nil {
|
||||
return fmt.Errorf("open source: %w", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
source, err := resolveSource(src, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("source: %s.%s filtered by %s = %q", source.table, source.wordCol, source.langCol, cfg.lang)
|
||||
|
||||
words, rejects, err := extract(src, source, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logRejects(rejects)
|
||||
log.Printf("accepted %d distinct words", len(words))
|
||||
|
||||
if len(words) < cfg.minWords {
|
||||
return fmt.Errorf("only %d words survived filtering, expected at least %d — "+
|
||||
"the source schema or content may have changed", len(words), cfg.minWords)
|
||||
}
|
||||
|
||||
aliases, collisions := buildAliases(words)
|
||||
log.Printf("generated %d spelling aliases (%d skipped as ambiguous or already real words)", len(aliases), collisions)
|
||||
|
||||
if err := write(cfg.out, words, aliases, source); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verify(cfg.out, cfg.minWords); err != nil {
|
||||
return fmt.Errorf("output failed verification: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("wrote %s", cfg.out)
|
||||
return nil
|
||||
}
|
||||
|
||||
// verify re-opens the finished database and re-checks the invariants the game
|
||||
// depends on. The in-memory checks above can only prove what the builder
|
||||
// intended; this proves what actually landed on disk.
|
||||
func verify(path string, minWords int) error {
|
||||
db, err := sql.Open("sqlite", "file:"+path+"?mode=ro")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
checks := []struct {
|
||||
desc string
|
||||
query string
|
||||
want func(int) bool
|
||||
}{
|
||||
{"words below the minimum", `SELECT COUNT(*) FROM words`, func(n int) bool { return n >= minWords }},
|
||||
{"words with fewer than 2 syllables", `SELECT COUNT(*) FROM words WHERE syllables < 2`, func(n int) bool { return n == 0 }},
|
||||
{"words with an empty first or last syllable", `SELECT COUNT(*) FROM words WHERE first = '' OR last = ''`, func(n int) bool { return n == 0 }},
|
||||
{"aliases pointing at a missing word",
|
||||
`SELECT COUNT(*) FROM aliases a LEFT JOIN words w ON w.word = a.canonical WHERE w.word IS NULL`,
|
||||
func(n int) bool { return n == 0 }},
|
||||
{"aliases that are themselves real words",
|
||||
`SELECT COUNT(*) FROM aliases a JOIN words w ON w.word = a.variant`,
|
||||
func(n int) bool { return n == 0 }},
|
||||
{"words whose first syllable is missing from the syllables table",
|
||||
`SELECT COUNT(*) FROM words w LEFT JOIN syllables s ON s.syllable = w.first WHERE s.syllable IS NULL`,
|
||||
func(n int) bool { return n == 0 }},
|
||||
}
|
||||
|
||||
for _, c := range checks {
|
||||
var n int
|
||||
if err := db.QueryRow(c.query).Scan(&n); err != nil {
|
||||
return fmt.Errorf("check %q: %w", c.desc, err)
|
||||
}
|
||||
if !c.want(n) {
|
||||
return fmt.Errorf("%s: %d", c.desc, n)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// entry is one accepted word with the parts the game engine indexes on.
|
||||
type entry struct {
|
||||
word string
|
||||
first string
|
||||
last string
|
||||
syllables int
|
||||
}
|
||||
|
||||
type sourceSpec struct {
|
||||
table string
|
||||
wordCol string
|
||||
langCol string
|
||||
maxSyllables int
|
||||
}
|
||||
|
||||
// quoteIdent renders a SQLite identifier. Go's %q escapes an embedded quote as
|
||||
// \" but SQLite requires it doubled, so fmt.Sprintf("%q") is not correct here.
|
||||
func quoteIdent(name string) string {
|
||||
return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
|
||||
}
|
||||
|
||||
// validateSource confirms operator-supplied identifiers actually exist.
|
||||
//
|
||||
// SQLite accepts a double-quoted string that matches no column as a string
|
||||
// literal rather than erroring, so a typo in --word-col silently yields one
|
||||
// row per distinct value of that literal. Checking up front turns a confusing
|
||||
// near-empty build into a clear message.
|
||||
func validateSource(db *sql.DB, spec sourceSpec) error {
|
||||
tables, err := listTables(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !contains(tables, spec.table) {
|
||||
return fmt.Errorf("table %q not found; available: %s", spec.table, strings.Join(tables, ", "))
|
||||
}
|
||||
|
||||
cols, err := listColumns(db, spec.table)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read columns of %q: %w", spec.table, err)
|
||||
}
|
||||
for _, c := range []struct{ role, name string }{{"--word-col", spec.wordCol}, {"--lang-col", spec.langCol}} {
|
||||
if !contains(cols, c.name) {
|
||||
return fmt.Errorf("%s %q not found in table %q; available: %s",
|
||||
c.role, c.name, spec.table, strings.Join(cols, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func contains(haystack []string, needle string) bool {
|
||||
for _, h := range haystack {
|
||||
if h == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// resolveSource finds the table holding word forms. The upstream schema is not
|
||||
// contractual — it is someone else's release artifact — so rather than hardcode
|
||||
// it, look for a table carrying both a word-like and a language-like column.
|
||||
// Explicit flags override detection entirely.
|
||||
func resolveSource(db *sql.DB, cfg config) (sourceSpec, error) {
|
||||
given := 0
|
||||
for _, f := range []string{cfg.table, cfg.wordCol, cfg.langCol} {
|
||||
if f != "" {
|
||||
given++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case given == 3:
|
||||
spec := sourceSpec{table: cfg.table, wordCol: cfg.wordCol, langCol: cfg.langCol, maxSyllables: cfg.maxSyllables}
|
||||
if err := validateSource(db, spec); err != nil {
|
||||
return sourceSpec{}, err
|
||||
}
|
||||
return spec, nil
|
||||
case given > 0:
|
||||
return sourceSpec{}, fmt.Errorf(
|
||||
"--table, --word-col and --lang-col must be given together (got %d of 3); "+
|
||||
"omit all three to auto-detect", given)
|
||||
}
|
||||
|
||||
tables, err := listTables(db)
|
||||
if err != nil {
|
||||
return sourceSpec{}, err
|
||||
}
|
||||
|
||||
wordNames := []string{"word", "term", "headword", "text", "lemma", "entry"}
|
||||
langNames := []string{"lang_code", "language", "lang", "lang_name"}
|
||||
|
||||
for _, t := range tables {
|
||||
cols, err := listColumns(db, t)
|
||||
if err != nil {
|
||||
log.Printf("warning: could not read columns of %q: %v", t, err)
|
||||
continue
|
||||
}
|
||||
wordCol := pickColumn(cols, wordNames)
|
||||
langCol := pickColumn(cols, langNames)
|
||||
if wordCol != "" && langCol != "" {
|
||||
return sourceSpec{table: t, wordCol: wordCol, langCol: langCol, maxSyllables: cfg.maxSyllables}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Detection failed. Dump the schema so the fix is a single flag away rather
|
||||
// than a debugging session.
|
||||
var b strings.Builder
|
||||
b.WriteString("could not auto-detect the source table.\n")
|
||||
b.WriteString("Pass --table, --word-col and --lang-col explicitly. Schema found:\n")
|
||||
for _, t := range tables {
|
||||
cols, err := listColumns(db, t)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, " %s(%s)\n", t, strings.Join(cols, ", "))
|
||||
}
|
||||
return sourceSpec{}, errors.New(b.String())
|
||||
}
|
||||
|
||||
func listTables(db *sql.DB) ([]string, error) {
|
||||
rows, err := db.Query(`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list tables: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, name)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func listColumns(db *sql.DB, table string) ([]string, error) {
|
||||
rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", quoteIdent(table)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid int
|
||||
name, ctype string
|
||||
notNull, pk int
|
||||
dflt sql.NullString
|
||||
)
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬Null, &dflt, &pk); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, name)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func pickColumn(cols []string, candidates []string) string {
|
||||
for _, want := range candidates {
|
||||
for _, c := range cols {
|
||||
if strings.EqualFold(c, want) {
|
||||
return c
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// extract streams every candidate word past the filters. Results are deduped by
|
||||
// normalized form, since the source lists a word once per sense.
|
||||
func extract(db *sql.DB, src sourceSpec, cfg config) (map[string]entry, map[rejectReason]int, error) {
|
||||
query := fmt.Sprintf("SELECT DISTINCT %s FROM %s WHERE %s = ?",
|
||||
quoteIdent(src.wordCol), quoteIdent(src.table), quoteIdent(src.langCol))
|
||||
rows, err := db.Query(query, cfg.lang)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("query source (%s): %w", query, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
words := make(map[string]entry)
|
||||
rejects := make(map[rejectReason]int)
|
||||
|
||||
for rows.Next() {
|
||||
var raw sql.NullString
|
||||
if err := rows.Scan(&raw); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !raw.Valid {
|
||||
rejects[rejectEmpty]++
|
||||
continue
|
||||
}
|
||||
|
||||
word, syllables, reason, ok := accept(raw.String, cfg.maxSyllables)
|
||||
if !ok {
|
||||
rejects[reason]++
|
||||
continue
|
||||
}
|
||||
|
||||
words[word] = entry{
|
||||
word: word,
|
||||
first: syllables[0],
|
||||
last: syllables[len(syllables)-1],
|
||||
syllables: len(syllables),
|
||||
}
|
||||
}
|
||||
|
||||
return words, rejects, rows.Err()
|
||||
}
|
||||
|
||||
func logRejects(rejects map[rejectReason]int) {
|
||||
reasons := make([]string, 0, len(rejects))
|
||||
for r := range rejects {
|
||||
reasons = append(reasons, string(r))
|
||||
}
|
||||
sort.Strings(reasons)
|
||||
for _, r := range reasons {
|
||||
log.Printf(" rejected %6d: %s", rejects[rejectReason(r)], r)
|
||||
}
|
||||
}
|
||||
|
||||
// buildAliases maps alternative spellings onto canonical words. A variant that
|
||||
// happens to be a real word in its own right is dropped: accepting it would let
|
||||
// one word be played under two names, or worse, silently rename it.
|
||||
func buildAliases(words map[string]entry) (map[string]string, int) {
|
||||
aliases := make(map[string]string)
|
||||
// poisoned records variants claimed by more than one word. A plain delete
|
||||
// is not enough: a third claimant would find the key free and reinsert it,
|
||||
// leaving the winner decided by Go's randomized map iteration order.
|
||||
poisoned := make(map[string]bool)
|
||||
collisions := 0
|
||||
|
||||
for word := range words {
|
||||
for _, variant := range variantsFor(word) {
|
||||
if poisoned[variant] {
|
||||
continue
|
||||
}
|
||||
if _, isRealWord := words[variant]; isRealWord {
|
||||
collisions++
|
||||
continue
|
||||
}
|
||||
if existing, taken := aliases[variant]; taken && existing != word {
|
||||
delete(aliases, variant)
|
||||
poisoned[variant] = true
|
||||
collisions++
|
||||
continue
|
||||
}
|
||||
aliases[variant] = word
|
||||
}
|
||||
}
|
||||
|
||||
return aliases, collisions
|
||||
}
|
||||
|
||||
// write builds the database beside the target and renames it into place only
|
||||
// after the transaction commits. Writing in place would mean that a failure
|
||||
// partway through -- a full disk, an interrupt -- leaves an empty but
|
||||
// syntactically valid database where a good one used to be, which the server
|
||||
// would happily open and find no words in.
|
||||
func write(path string, words map[string]entry, aliases map[string]string, src sourceSpec) error {
|
||||
tmp := path + ".tmp"
|
||||
if err := os.Remove(tmp); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("remove stale temp file: %w", err)
|
||||
}
|
||||
// On any error path the half-built file must not survive.
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
os.Remove(tmp)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := writeTo(tmp, words, aliases, src); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// os.Rename replaces the destination atomically on POSIX; on Windows it
|
||||
// fails if the target exists, so clear it first.
|
||||
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("remove existing output: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
return fmt.Errorf("move temp database into place: %w", err)
|
||||
}
|
||||
committed = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeTo(path string, words map[string]entry, aliases map[string]string, src sourceSpec) error {
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create output: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
schema := `
|
||||
CREATE TABLE words (
|
||||
word TEXT PRIMARY KEY,
|
||||
first TEXT NOT NULL,
|
||||
last TEXT NOT NULL,
|
||||
syllables INTEGER NOT NULL
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX idx_words_first ON words(first);
|
||||
|
||||
CREATE TABLE syllables (
|
||||
syllable TEXT PRIMARY KEY,
|
||||
out_degree INTEGER NOT NULL
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE aliases (
|
||||
variant TEXT PRIMARY KEY,
|
||||
canonical TEXT NOT NULL
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
||||
`
|
||||
if _, err := db.Exec(schema); err != nil {
|
||||
return fmt.Errorf("create schema: %w", err)
|
||||
}
|
||||
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
insertWord, err := tx.Prepare(`INSERT INTO words (word, first, last, syllables) VALUES (?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer insertWord.Close()
|
||||
|
||||
outDegree := make(map[string]int)
|
||||
for _, e := range words {
|
||||
if _, err := insertWord.Exec(e.word, e.first, e.last, e.syllables); err != nil {
|
||||
return fmt.Errorf("insert word %q: %w", e.word, err)
|
||||
}
|
||||
outDegree[e.first]++
|
||||
// A syllable that ends a word but starts none is a dead end; record it
|
||||
// with zero so the engine can find it without a second query.
|
||||
if _, seen := outDegree[e.last]; !seen {
|
||||
outDegree[e.last] = 0
|
||||
}
|
||||
}
|
||||
|
||||
insertSyllable, err := tx.Prepare(`INSERT INTO syllables (syllable, out_degree) VALUES (?, ?)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer insertSyllable.Close()
|
||||
for syllable, degree := range outDegree {
|
||||
if _, err := insertSyllable.Exec(syllable, degree); err != nil {
|
||||
return fmt.Errorf("insert syllable %q: %w", syllable, err)
|
||||
}
|
||||
}
|
||||
|
||||
insertAlias, err := tx.Prepare(`INSERT INTO aliases (variant, canonical) VALUES (?, ?)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer insertAlias.Close()
|
||||
for variant, canonical := range aliases {
|
||||
if _, err := insertAlias.Exec(variant, canonical); err != nil {
|
||||
return fmt.Errorf("insert alias %q: %w", variant, err)
|
||||
}
|
||||
}
|
||||
|
||||
insertMeta, err := tx.Prepare(`INSERT INTO meta (key, value) VALUES (?, ?)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer insertMeta.Close()
|
||||
meta := [][2]string{
|
||||
{"source_url", sourceURL},
|
||||
{"source_license", sourceLicense},
|
||||
{"attribution", "See data/ATTRIBUTION.md for required attribution and the list of modifications."},
|
||||
{"built_at", time.Now().UTC().Format(time.RFC3339)},
|
||||
{"builder_version", builderVer},
|
||||
{"word_count", fmt.Sprint(len(words))},
|
||||
{"alias_count", fmt.Sprint(len(aliases))},
|
||||
{"source_table", src.table},
|
||||
{"source_word_column", src.wordCol},
|
||||
{"source_lang_column", src.langCol},
|
||||
{"max_syllables", fmt.Sprint(src.maxSyllables)},
|
||||
}
|
||||
for _, kv := range meta {
|
||||
if _, err := insertMeta.Exec(kv[0], kv[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// fixtureSource writes a miniature stand-in for the upstream dictionary: the
|
||||
// same shape (a word column plus a language column, one row per sense) but a
|
||||
// handful of rows instead of 357k. Tests never touch the real 179 MB download.
|
||||
func fixtureSource(t *testing.T, rows [][2]string) string {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "source.db")
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if _, err := db.Exec(`CREATE TABLE entries (word TEXT, lang_code TEXT, definition TEXT)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if _, err := db.Exec(`INSERT INTO entries (word, lang_code, definition) VALUES (?, ?, ?)`,
|
||||
r[0], r[1], "unused"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func defaultRows() [][2]string {
|
||||
return [][2]string{
|
||||
{"pháp luật", "vi"},
|
||||
{"pháp luật", "vi"}, // duplicate sense — must dedupe to one word
|
||||
{"luật lệ", "vi"},
|
||||
{"ngôn ngữ", "vi"},
|
||||
{"ngữ pháp", "vi"},
|
||||
{"hòa bình", "vi"},
|
||||
{"vô tuyến điện", "vi"}, // three syllables
|
||||
{"pháp", "vi"}, // single syllable — rejected
|
||||
{"covid 19", "vi"}, // digit — rejected
|
||||
{"hello world", "en"}, // wrong language — never selected
|
||||
}
|
||||
}
|
||||
|
||||
func buildFixture(t *testing.T, rows [][2]string, maxSyllables int) string {
|
||||
t.Helper()
|
||||
|
||||
in := fixtureSource(t, rows)
|
||||
out := filepath.Join(t.TempDir(), "noitu.db")
|
||||
|
||||
cfg := config{
|
||||
in: in,
|
||||
out: out,
|
||||
maxSyllables: maxSyllables,
|
||||
minWords: 1,
|
||||
lang: "vi",
|
||||
}
|
||||
if err := run(cfg); err != nil {
|
||||
t.Fatalf("run: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func openOut(t *testing.T, path string) *sql.DB {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", "file:"+path+"?mode=ro")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db
|
||||
}
|
||||
|
||||
func TestBuildProducesExpectedWords(t *testing.T) {
|
||||
db := openOut(t, buildFixture(t, defaultRows(), 0))
|
||||
|
||||
var count int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM words`).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := 6; count != want {
|
||||
t.Errorf("word count = %d, want %d", count, want)
|
||||
}
|
||||
|
||||
// Every stored word must have at least two syllables.
|
||||
var short int
|
||||
if err := db.QueryRow(`SELECT COUNT(*) FROM words WHERE syllables < 2`).Scan(&short); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if short != 0 {
|
||||
t.Errorf("%d words have fewer than 2 syllables, want 0", short)
|
||||
}
|
||||
|
||||
// Three-syllable words are kept and split on first/last, not on a pair.
|
||||
var first, last string
|
||||
var syllables int
|
||||
err := db.QueryRow(`SELECT first, last, syllables FROM words WHERE word = ?`, "vô tuyến điện").
|
||||
Scan(&first, &last, &syllables)
|
||||
if err != nil {
|
||||
t.Fatalf("three-syllable word missing: %v", err)
|
||||
}
|
||||
if first != "vô" || last != "điện" || syllables != 3 {
|
||||
t.Errorf("got first=%q last=%q syllables=%d, want vô/điện/3", first, last, syllables)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildComputesOutDegree(t *testing.T) {
|
||||
db := openOut(t, buildFixture(t, defaultRows(), 0))
|
||||
|
||||
// "pháp luật" and "pháp" (rejected) mean exactly one word starts with "pháp".
|
||||
assertOutDegree(t, db, "pháp", 1)
|
||||
// "luật lệ" starts with "luật".
|
||||
assertOutDegree(t, db, "luật", 1)
|
||||
// "bình" ends a word but starts none — a dead end, recorded as zero.
|
||||
assertOutDegree(t, db, "bình", 0)
|
||||
}
|
||||
|
||||
func assertOutDegree(t *testing.T, db *sql.DB, syllable string, want int) {
|
||||
t.Helper()
|
||||
var got int
|
||||
if err := db.QueryRow(`SELECT out_degree FROM syllables WHERE syllable = ?`, syllable).Scan(&got); err != nil {
|
||||
t.Fatalf("out_degree for %q: %v", syllable, err)
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("out_degree(%q) = %d, want %d", syllable, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildWritesAliases(t *testing.T) {
|
||||
db := openOut(t, buildFixture(t, defaultRows(), 0))
|
||||
|
||||
var canonical string
|
||||
err := db.QueryRow(`SELECT canonical FROM aliases WHERE variant = ?`, "hoà bình").Scan(&canonical)
|
||||
if err != nil {
|
||||
t.Fatalf("alias for %q missing: %v", "hoà bình", err)
|
||||
}
|
||||
if canonical != "hòa bình" {
|
||||
t.Errorf("alias resolves to %q, want %q", canonical, "hòa bình")
|
||||
}
|
||||
|
||||
// Every alias must point at a word that actually exists.
|
||||
var orphans int
|
||||
err = db.QueryRow(`SELECT COUNT(*) FROM aliases a
|
||||
LEFT JOIN words w ON w.word = a.canonical
|
||||
WHERE w.word IS NULL`).Scan(&orphans)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if orphans != 0 {
|
||||
t.Errorf("%d aliases point at missing words, want 0", orphans)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRecordsProvenance(t *testing.T) {
|
||||
db := openOut(t, buildFixture(t, defaultRows(), 0))
|
||||
|
||||
for _, key := range []string{"source_url", "source_license", "attribution", "built_at", "word_count"} {
|
||||
var value string
|
||||
if err := db.QueryRow(`SELECT value FROM meta WHERE key = ?`, key).Scan(&value); err != nil {
|
||||
t.Errorf("meta[%q] missing: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if value == "" {
|
||||
t.Errorf("meta[%q] is empty", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The floor exists so a schema change upstream fails the build loudly instead
|
||||
// of silently shipping a near-empty dictionary.
|
||||
func TestBuildFailsBelowMinWords(t *testing.T) {
|
||||
in := fixtureSource(t, defaultRows())
|
||||
cfg := config{
|
||||
in: in,
|
||||
out: filepath.Join(t.TempDir(), "noitu.db"),
|
||||
minWords: 1000,
|
||||
lang: "vi",
|
||||
}
|
||||
if err := run(cfg); err == nil {
|
||||
t.Fatal("run succeeded with an unreachable min-words floor, want error")
|
||||
}
|
||||
}
|
||||
|
||||
// A failed build must leave the previous good database untouched. Building in
|
||||
// place would delete it and leave an empty file the server would happily open.
|
||||
func TestFailedBuildPreservesPreviousOutput(t *testing.T) {
|
||||
out := buildFixture(t, defaultRows(), 0)
|
||||
|
||||
before, err := os.ReadFile(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Same output path, but a floor no fixture can clear.
|
||||
cfg := config{
|
||||
in: fixtureSource(t, defaultRows()),
|
||||
out: out,
|
||||
minWords: 1000,
|
||||
lang: "vi",
|
||||
}
|
||||
if err := run(cfg); err == nil {
|
||||
t.Fatal("run succeeded with an unreachable floor, want error")
|
||||
}
|
||||
|
||||
after, err := os.ReadFile(out)
|
||||
if err != nil {
|
||||
t.Fatalf("previous output was destroyed by the failed build: %v", err)
|
||||
}
|
||||
if !bytes.Equal(before, after) {
|
||||
t.Error("previous output was modified by the failed build")
|
||||
}
|
||||
if _, err := os.Stat(out + ".tmp"); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Error("temp database left behind after a failed build")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitSourceFlagsMustBeComplete(t *testing.T) {
|
||||
in := fixtureSource(t, defaultRows())
|
||||
out := filepath.Join(t.TempDir(), "noitu.db")
|
||||
|
||||
// Two of three: ambiguous, so fail loudly rather than silently auto-detect.
|
||||
err := run(config{in: in, out: out, minWords: 1, lang: "vi", table: "entries", wordCol: "word"})
|
||||
if err == nil {
|
||||
t.Fatal("run succeeded with a partial flag override, want error")
|
||||
}
|
||||
}
|
||||
|
||||
// SQLite reads a double-quoted name that matches no column as a string literal,
|
||||
// so a typo would otherwise build a dictionary out of that literal.
|
||||
func TestUnknownColumnIsRejected(t *testing.T) {
|
||||
in := fixtureSource(t, defaultRows())
|
||||
out := filepath.Join(t.TempDir(), "noitu.db")
|
||||
|
||||
err := run(config{
|
||||
in: in, out: out, minWords: 1, lang: "vi",
|
||||
table: "entries", wordCol: "does_not_exist", langCol: "lang_code",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("run succeeded with an unknown word column, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "does_not_exist") {
|
||||
t.Errorf("error %q does not name the bad column", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildRecordsSourceProvenance(t *testing.T) {
|
||||
db := openOut(t, buildFixture(t, defaultRows(), 0))
|
||||
|
||||
for key, want := range map[string]string{
|
||||
"source_table": "entries",
|
||||
"source_word_column": "word",
|
||||
"source_lang_column": "lang_code",
|
||||
} {
|
||||
var got string
|
||||
if err := db.QueryRow(`SELECT value FROM meta WHERE key = ?`, key).Scan(&got); err != nil {
|
||||
t.Errorf("meta[%q] missing: %v", key, err)
|
||||
continue
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("meta[%q] = %q, want %q", key, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildAutoDetectsSchema(t *testing.T) {
|
||||
// Same data under different column names: detection should still find it.
|
||||
path := filepath.Join(t.TempDir(), "odd.db")
|
||||
db, err := sql.Open("sqlite", "file:"+path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE TABLE lexicon (headword TEXT, language TEXT)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(`INSERT INTO lexicon VALUES ('pháp luật', 'vi'), ('luật lệ', 'vi')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
db.Close()
|
||||
|
||||
out := filepath.Join(t.TempDir(), "noitu.db")
|
||||
if err := run(config{in: path, out: out, minWords: 1, lang: "vi"}); err != nil {
|
||||
t.Fatalf("run with alternate schema: %v", err)
|
||||
}
|
||||
|
||||
got := openOut(t, out)
|
||||
var count int
|
||||
if err := got.QueryRow(`SELECT COUNT(*) FROM words`).Scan(&count); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 2 {
|
||||
t.Errorf("word count = %d, want 2", count)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// Vietnamese syllable structure is a closed system: onset + nucleus + coda,
|
||||
// where each part is drawn from a fixed inventory. That makes it a reliable
|
||||
// test for whether an entry is a Vietnamese word at all.
|
||||
//
|
||||
// An alphabet check alone cannot do this. "credit card", "dress code" and
|
||||
// "come out" are spelled entirely with letters Vietnamese has, so they pass any
|
||||
// letter-level filter, but no Vietnamese syllable begins "cr" or ends "ss".
|
||||
// The multilingual source leaks entries like these into the `vi` rows, and
|
||||
// without this check they become playable words.
|
||||
|
||||
// vietnameseOnsets is the complete inventory of syllable-initial consonants.
|
||||
// Longest-match order matters, so this is sorted by descending length at init.
|
||||
var vietnameseOnsets = []string{
|
||||
"ngh", "ng", "nh", "ch", "gh", "gi", "kh", "ph", "qu", "th", "tr",
|
||||
"b", "c", "d", "đ", "g", "h", "k", "l", "m", "n", "p", "r", "s", "t", "v", "x",
|
||||
}
|
||||
|
||||
// vietnameseCodas is the complete inventory of syllable-final consonants and
|
||||
// offglides.
|
||||
var vietnameseCodas = []string{
|
||||
"ngh", "ng", "nh", "ch",
|
||||
"c", "m", "n", "p", "t", "i", "o", "u", "y",
|
||||
}
|
||||
|
||||
// vietnameseNuclei is the inventory of vowel nuclei, diacritics stripped. Some
|
||||
// are single vowels, others diphthongs or triphthongs.
|
||||
var vietnameseNuclei = []string{
|
||||
"uye", "uya", "uyu", "oai", "oay", "uoi", "uou", "ieu", "yeu", "uai", "uay",
|
||||
"ai", "ao", "au", "ay", "eo", "eu", "ia", "ie", "iu", "oa", "oe", "oi", "oo",
|
||||
"ua", "ue", "ui", "uo", "uu", "uy", "ya", "ye", "yu", "ao", "eu",
|
||||
"a", "e", "i", "o", "u", "y",
|
||||
}
|
||||
|
||||
func init() {
|
||||
byLengthDesc := func(s []string) {
|
||||
sort.SliceStable(s, func(i, j int) bool { return len(s[i]) > len(s[j]) })
|
||||
}
|
||||
byLengthDesc(vietnameseOnsets)
|
||||
byLengthDesc(vietnameseCodas)
|
||||
byLengthDesc(vietnameseNuclei)
|
||||
}
|
||||
|
||||
// stripDiacritics reduces a syllable to its base letters, keeping đ (which is a
|
||||
// distinct letter, not a d with a mark).
|
||||
func stripDiacritics(syllable string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range norm.NFD.String(syllable) {
|
||||
// Mn = nonspacing combining mark: every Vietnamese tone and letter mark.
|
||||
if isCombiningMark(r) {
|
||||
continue
|
||||
}
|
||||
b.WriteRune(r)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func isCombiningMark(r rune) bool {
|
||||
// The ranges Vietnamese actually uses; cheaper and tighter than a full
|
||||
// unicode.Is(unicode.Mn, r) for this data.
|
||||
return (r >= 0x0300 && r <= 0x036F) || r == 0x031B
|
||||
}
|
||||
|
||||
// isVietnameseSyllable reports whether a syllable fits Vietnamese phonotactics.
|
||||
//
|
||||
// Every onset and nucleus split must be tried, not just the longest. A greedy
|
||||
// parser reads "gì" as the digraph onset "gi" with nothing left for a nucleus
|
||||
// and wrongly rejects it; the correct parse is onset "g" plus nucleus "ì".
|
||||
// The same applies to "gìn", "gỉ" and every other g + i syllable.
|
||||
func isVietnameseSyllable(syllable string) bool {
|
||||
base := stripDiacritics(syllable)
|
||||
if base == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Onset is optional ("áo", "ăn"), so "" is a candidate too.
|
||||
candidates := []string{""}
|
||||
for _, onset := range vietnameseOnsets {
|
||||
if strings.HasPrefix(base, onset) {
|
||||
candidates = append(candidates, onset)
|
||||
}
|
||||
}
|
||||
|
||||
for _, onset := range candidates {
|
||||
if parsesAsRhyme(base[len(onset):]) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// parsesAsRhyme reports whether the remainder after an onset is a valid
|
||||
// nucleus followed by at most one coda.
|
||||
func parsesAsRhyme(rest string) bool {
|
||||
if rest == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, nucleus := range vietnameseNuclei {
|
||||
if !strings.HasPrefix(rest, nucleus) {
|
||||
continue
|
||||
}
|
||||
tail := rest[len(nucleus):]
|
||||
if tail == "" {
|
||||
return true
|
||||
}
|
||||
for _, coda := range vietnameseCodas {
|
||||
if tail == coda {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIsVietnameseSyllableAccepts(t *testing.T) {
|
||||
// A parser that matches the longest onset greedily rejects the whole g+i
|
||||
// family, which includes some of the most common words in the language.
|
||||
gFamily := []string{"gì", "gỉ", "gìn", "giữ", "gi", "giả", "giáo", "gia"}
|
||||
// Ordinary syllables across the onset, nucleus and coda inventories.
|
||||
ordinary := []string{
|
||||
"pháp", "luật", "ngôn", "ngữ", "nghiêng", "nghề", "khuya", "quyền",
|
||||
"tuyết", "người", "đường", "ươm", "ăn", "áo", "yêu", "oanh", "uy",
|
||||
"xoài", "khoẻ", "thuý", "quý", "chương", "trường", "nhanh", "phở",
|
||||
}
|
||||
|
||||
for _, group := range [][]string{gFamily, ordinary} {
|
||||
for _, s := range group {
|
||||
if !isVietnameseSyllable(s) {
|
||||
t.Errorf("isVietnameseSyllable(%q) = false, want true", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsVietnameseSyllableRejects(t *testing.T) {
|
||||
// Foreign syllables that use only letters Vietnamese has, so an
|
||||
// alphabet-level check cannot catch them.
|
||||
tests := []struct{ in, why string }{
|
||||
{"credit", "onset cluster cr"},
|
||||
{"card", "coda rd"},
|
||||
{"dress", "onset cluster dr and coda ss"},
|
||||
{"code", "coda de"},
|
||||
{"come", "coda me"},
|
||||
{"york", "coda rk"},
|
||||
{"cup", "valid shape but see note"}, // sanity: see below
|
||||
{"", "empty"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
if tc.in == "cup" {
|
||||
// "cup" is phonotactically legal in Vietnamese (c + u + p), so it
|
||||
// is NOT rejected here. Documented so the expectation is explicit.
|
||||
if !isVietnameseSyllable("cup") {
|
||||
t.Error(`isVietnameseSyllable("cup") = false; it is a legal Vietnamese shape`)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if isVietnameseSyllable(tc.in) {
|
||||
t.Errorf("isVietnameseSyllable(%q) = true, want false (%s)", tc.in, tc.why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStripDiacritics(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"ngữ": "ngu",
|
||||
"đường": "đuong",
|
||||
"khoẻ": "khoe",
|
||||
"pháp": "phap",
|
||||
}
|
||||
|
||||
for in, want := range tests {
|
||||
if got := stripDiacritics(in); got != want {
|
||||
t.Errorf("stripDiacritics(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
module github.com/tiennm99dev/noitu/server
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
golang.org/x/text v0.41.0
|
||||
modernc.org/sqlite v1.58.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
modernc.org/libc v1.75.6 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.12.1 // indirect
|
||||
)
|
||||
@@ -0,0 +1,52 @@
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
|
||||
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
|
||||
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
|
||||
modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8=
|
||||
modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w=
|
||||
modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
|
||||
modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus=
|
||||
modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
|
||||
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.58.0 h1:38u40/bwkfM7f0Myhosl+SEMltSDxnGdQf8o6Kjmys0=
|
||||
modernc.org/sqlite v1.58.0/go.mod h1:rsD2CckafgObKC4DhBlGBf+RiHxkc3hINGt1Xw32tVY=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
@@ -0,0 +1,54 @@
|
||||
// Package vietnamese normalizes Vietnamese word input.
|
||||
//
|
||||
// The same normalization runs in two places: the dictionary builder, which
|
||||
// normalizes the corpus, and the game server, which normalizes what a player
|
||||
// types. They must agree exactly — if they ever diverge, a word stored in the
|
||||
// database stops matching the identical string typed by a player. Sharing this
|
||||
// package is what prevents that.
|
||||
package vietnamese
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// MinSyllables is the shortest word the game accepts. Nối từ is played with
|
||||
// compound words; a single syllable is not a valid move.
|
||||
const MinSyllables = 2
|
||||
|
||||
// ErrEmpty is returned when input contains no usable characters.
|
||||
var ErrEmpty = errors.New("vietnamese: empty input")
|
||||
|
||||
// Normalize canonicalizes raw player or corpus input and splits it into syllables.
|
||||
//
|
||||
// It returns the canonical word (syllables joined by single spaces) and the
|
||||
// syllables themselves. It deliberately does not enforce a syllable count:
|
||||
// length is a game rule, not a text rule, so the engine owns that check.
|
||||
//
|
||||
// Steps, in order:
|
||||
// - NFC composition, so "ữ" written as a base plus a combining mark compares
|
||||
// equal to the single-codepoint form. Without this, words typed on different
|
||||
// keyboards or platforms fail to match identical database entries.
|
||||
// - lowercase, so capitalization never affects a lookup.
|
||||
// - whitespace split and rejoin, which collapses runs of spaces, tabs, and
|
||||
// non-breaking spaces that IMEs and copy-paste routinely introduce.
|
||||
func Normalize(raw string) (string, []string, error) {
|
||||
composed := norm.NFC.String(raw)
|
||||
lowered := strings.ToLower(composed)
|
||||
|
||||
// strings.Fields splits on every unicode.IsSpace rune, which covers tabs and
|
||||
// U+00A0 non-breaking spaces as well as ordinary spaces.
|
||||
syllables := strings.Fields(lowered)
|
||||
if len(syllables) == 0 {
|
||||
return "", nil, ErrEmpty
|
||||
}
|
||||
|
||||
return strings.Join(syllables, " "), syllables, nil
|
||||
}
|
||||
|
||||
// HasEnoughSyllables reports whether a word is long enough to be a legal move.
|
||||
func HasEnoughSyllables(syllables []string) bool {
|
||||
return len(syllables) >= MinSyllables
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package vietnamese
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
syllables int
|
||||
}{
|
||||
{"plain two syllables", "pháp luật", "pháp luật", 2},
|
||||
{"uppercase", "PHÁP LUẬT", "pháp luật", 2},
|
||||
{"mixed case", "Pháp Luật", "pháp luật", 2},
|
||||
{"leading and trailing space", " pháp luật ", "pháp luật", 2},
|
||||
{"repeated spaces", "pháp luật", "pháp luật", 2},
|
||||
{"tab separator", "pháp\tluật", "pháp luật", 2},
|
||||
{"non-breaking space", "pháp luật", "pháp luật", 2},
|
||||
{"newline separator", "pháp\nluật", "pháp luật", 2},
|
||||
{"three syllables", "vô tuyến điện", "vô tuyến điện", 3},
|
||||
{"four syllables", "công nghiệp hóa hiện", "công nghiệp hóa hiện", 4},
|
||||
{"single syllable passes through", "pháp", "pháp", 1},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
word, syllables, err := Normalize(tc.raw)
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize(%q) returned error: %v", tc.raw, err)
|
||||
}
|
||||
if word != tc.want {
|
||||
t.Errorf("word = %q, want %q", word, tc.want)
|
||||
}
|
||||
if len(syllables) != tc.syllables {
|
||||
t.Errorf("got %d syllables, want %d", len(syllables), tc.syllables)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A word typed on one keyboard can arrive decomposed and on another composed.
|
||||
// Both must normalize to the same string, or database lookups fail for reasons
|
||||
// invisible to the player.
|
||||
func TestNormalizeUnicodeEquivalence(t *testing.T) {
|
||||
composed := "ngôn ngữ" // ữ as a single codepoint
|
||||
decomposed := "ngôn ngữ" // o+circumflex, u+horn+tilde
|
||||
|
||||
gotComposed, _, err := Normalize(composed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gotDecomposed, _, err := Normalize(decomposed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if gotComposed != gotDecomposed {
|
||||
t.Errorf("composed %q and decomposed %q normalized differently: %q vs %q",
|
||||
composed, decomposed, gotComposed, gotDecomposed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeEmpty(t *testing.T) {
|
||||
for _, raw := range []string{"", " ", "\t\n", " "} {
|
||||
if _, _, err := Normalize(raw); !errors.Is(err, ErrEmpty) {
|
||||
t.Errorf("Normalize(%q) error = %v, want ErrEmpty", raw, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasEnoughSyllables(t *testing.T) {
|
||||
tests := []struct {
|
||||
syllables []string
|
||||
want bool
|
||||
}{
|
||||
{[]string{"pháp"}, false},
|
||||
{[]string{"pháp", "luật"}, true},
|
||||
{[]string{"vô", "tuyến", "điện"}, true},
|
||||
{nil, false},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
if got := HasEnoughSyllables(tc.syllables); got != tc.want {
|
||||
t.Errorf("HasEnoughSyllables(%v) = %v, want %v", tc.syllables, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user