mirror of
https://github.com/tiennm99/noitu.git
synced 2026-10-14 04:16:27 +00:00
tiennm99
4b4afea679
Reduce the 179 MB minhqnd/dictionary SQLite release to a ~3 MB game wordlist: 48,216 Vietnamese words of two or more syllables, indexed by first and last syllable with an out-degree table for dead-end detection. Source schema is auto-detected rather than hardcoded, since it is someone else's release artifact; explicit flags override it and are validated against the real tables, because SQLite silently reads an unknown double-quoted column as a string literal. Accept a word only if every syllable fits Vietnamese phonotactics. An alphabet check is not enough: "credit card" and "come out" use only letters Vietnamese has, and the multilingual source tags them as Vietnamese. Onset matching backtracks so the gi digraph does not swallow the nucleus of common words like "gi", "gin" and "gi" (rust). Record accepted spelling variants in an alias table rather than solving tone placement at runtime. Tone shifting applies only to open oa/oe/uy syllables, since "hoan" and "hoai" have a single correct spelling, and "qu" is a consonant onset. The i/y alternation uses an onset allowlist plus explicit pairs, because it is lexical rather than productive. Variants are generated as a cross product over syllables so a word with two variable syllables still offers the fully modern spelling. Build to a temporary file and rename only after commit, so a failed run cannot leave an empty database where a good one was, then re-open the result and verify its invariants on disk. Licensing: the derived data is CC BY-SA 4.0 and stays a separate artifact from the Apache-2.0 code, loaded at runtime and never embedded. Ships NOTICE, data/LICENSE and an attribution file recording every change.
68 lines
2.0 KiB
Go
68 lines
2.0 KiB
Go
package main
|
|
|
|
import "testing"
|
|
|
|
func TestIsVietnameseSyllableAccepts(t *testing.T) {
|
|
// A parser that matches the longest onset greedily rejects the whole g+i
|
|
// family, which includes some of the most common words in the language.
|
|
gFamily := []string{"gì", "gỉ", "gìn", "giữ", "gi", "giả", "giáo", "gia"}
|
|
// Ordinary syllables across the onset, nucleus and coda inventories.
|
|
ordinary := []string{
|
|
"pháp", "luật", "ngôn", "ngữ", "nghiêng", "nghề", "khuya", "quyền",
|
|
"tuyết", "người", "đường", "ươm", "ăn", "áo", "yêu", "oanh", "uy",
|
|
"xoài", "khoẻ", "thuý", "quý", "chương", "trường", "nhanh", "phở",
|
|
}
|
|
|
|
for _, group := range [][]string{gFamily, ordinary} {
|
|
for _, s := range group {
|
|
if !isVietnameseSyllable(s) {
|
|
t.Errorf("isVietnameseSyllable(%q) = false, want true", s)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIsVietnameseSyllableRejects(t *testing.T) {
|
|
// Foreign syllables that use only letters Vietnamese has, so an
|
|
// alphabet-level check cannot catch them.
|
|
tests := []struct{ in, why string }{
|
|
{"credit", "onset cluster cr"},
|
|
{"card", "coda rd"},
|
|
{"dress", "onset cluster dr and coda ss"},
|
|
{"code", "coda de"},
|
|
{"come", "coda me"},
|
|
{"york", "coda rk"},
|
|
{"cup", "valid shape but see note"}, // sanity: see below
|
|
{"", "empty"},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
if tc.in == "cup" {
|
|
// "cup" is phonotactically legal in Vietnamese (c + u + p), so it
|
|
// is NOT rejected here. Documented so the expectation is explicit.
|
|
if !isVietnameseSyllable("cup") {
|
|
t.Error(`isVietnameseSyllable("cup") = false; it is a legal Vietnamese shape`)
|
|
}
|
|
continue
|
|
}
|
|
if isVietnameseSyllable(tc.in) {
|
|
t.Errorf("isVietnameseSyllable(%q) = true, want false (%s)", tc.in, tc.why)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestStripDiacritics(t *testing.T) {
|
|
tests := map[string]string{
|
|
"ngữ": "ngu",
|
|
"đường": "đuong",
|
|
"khoẻ": "khoe",
|
|
"pháp": "phap",
|
|
}
|
|
|
|
for in, want := range tests {
|
|
if got := stripDiacritics(in); got != want {
|
|
t.Errorf("stripDiacritics(%q) = %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|