From 4b4afea679b39dca779679102f2eb9c6ed19aeed Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Fri, 4 Sep 2026 16:25:43 +0700 Subject: [PATCH] feat(dictionary): build Vietnamese wordlist from upstream dictionary Reduce the 179 MB minhqnd/dictionary SQLite release to a ~3 MB game wordlist: 48,216 Vietnamese words of two or more syllables, indexed by first and last syllable with an out-degree table for dead-end detection. Source schema is auto-detected rather than hardcoded, since it is someone else's release artifact; explicit flags override it and are validated against the real tables, because SQLite silently reads an unknown double-quoted column as a string literal. Accept a word only if every syllable fits Vietnamese phonotactics. An alphabet check is not enough: "credit card" and "come out" use only letters Vietnamese has, and the multilingual source tags them as Vietnamese. Onset matching backtracks so the gi digraph does not swallow the nucleus of common words like "gi", "gin" and "gi" (rust). Record accepted spelling variants in an alias table rather than solving tone placement at runtime. Tone shifting applies only to open oa/oe/uy syllables, since "hoan" and "hoai" have a single correct spelling, and "qu" is a consonant onset. The i/y alternation uses an onset allowlist plus explicit pairs, because it is lexical rather than productive. Variants are generated as a cross product over syllables so a word with two variable syllables still offers the fully modern spelling. Build to a temporary file and rename only after commit, so a failed run cannot leave an empty database where a good one was, then re-open the result and verify its invariants on disk. Licensing: the derived data is CC BY-SA 4.0 and stays a separate artifact from the Apache-2.0 code, loaded at runtime and never embedded. Ships NOTICE, data/LICENSE and an attribution file recording every change. --- .gitignore | 31 ++ Makefile | 63 +++ NOTICE | 37 ++ README.md | 96 +++- data/ATTRIBUTION.md | 71 +++ data/LICENSE | 428 +++++++++++++++ server/cmd/build-dictionary/aliases.go | 287 ++++++++++ server/cmd/build-dictionary/aliases_test.go | 208 +++++++ server/cmd/build-dictionary/filter.go | 100 ++++ server/cmd/build-dictionary/filter_test.go | 85 +++ server/cmd/build-dictionary/main.go | 547 +++++++++++++++++++ server/cmd/build-dictionary/main_test.go | 303 ++++++++++ server/cmd/build-dictionary/syllable.go | 124 +++++ server/cmd/build-dictionary/syllable_test.go | 67 +++ server/go.mod | 20 + server/go.sum | 52 ++ server/internal/vietnamese/normalize.go | 54 ++ server/internal/vietnamese/normalize_test.go | 90 +++ 18 files changed, 2662 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 NOTICE create mode 100644 data/ATTRIBUTION.md create mode 100644 data/LICENSE create mode 100644 server/cmd/build-dictionary/aliases.go create mode 100644 server/cmd/build-dictionary/aliases_test.go create mode 100644 server/cmd/build-dictionary/filter.go create mode 100644 server/cmd/build-dictionary/filter_test.go create mode 100644 server/cmd/build-dictionary/main.go create mode 100644 server/cmd/build-dictionary/main_test.go create mode 100644 server/cmd/build-dictionary/syllable.go create mode 100644 server/cmd/build-dictionary/syllable_test.go create mode 100644 server/go.mod create mode 100644 server/go.sum create mode 100644 server/internal/vietnamese/normalize.go create mode 100644 server/internal/vietnamese/normalize_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1d0073f --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +# Dictionary data — never committed. +# data/dictionary.db is the 179 MB upstream asset; data/noitu.db is derived from it. +# Both are build artifacts produced by `make fetch-dict` and `make dict`. +data/*.db +data/*.db-journal +data/*.db-wal +data/*.db-shm + +# Go build output +/noitu-server +/noitu-server.exe +/build-dictionary +/build-dictionary.exe +*.test +*.out + +# Node / SvelteKit +web/node_modules/ +web/build/ +web/.svelte-kit/ +node_modules/ + +# Playwright +/test-results/ +/playwright-report/ + +# Editor / OS +.DS_Store +Thumbs.db +.idea/ +.vscode/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d7312c0 --- /dev/null +++ b/Makefile @@ -0,0 +1,63 @@ +# noitu — build targets. +# +# Every target has a raw equivalent documented in README.md, so contributors +# without `make` (notably on Windows) are never blocked. + +DICT_URL := https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db +DICT_SRC := data/dictionary.db +# Pinned to the v2.0.0 release asset. A mismatch means the upstream artifact +# changed under the same tag, or the download was truncated. +DICT_SHA256 := 9259403f0675b2991a1bd0ef6d0dbc5933afdb135632af095a60662f09bbf1d3 +DICT_OUT := data/noitu.db +SERVER_BIN := noitu-server + +.PHONY: help fetch-dict verify-dict dict server web test test-go test-web run clean + +help: + @echo "fetch-dict download + checksum the upstream dictionary (~179 MB) into data/" + @echo "verify-dict re-check the downloaded dictionary against its pinned SHA-256" + @echo "dict derive $(DICT_OUT) from $(DICT_SRC)" + @echo "server build the Go server binary" + @echo "web build the SvelteKit frontend" + @echo "test run all tests" + @echo "run build and run the server locally" + @echo "clean remove build artifacts (keeps downloaded dictionary)" + +# One-time download. Resumable (-C -) so an interrupted 179 MB fetch can continue. +fetch-dict: + @mkdir -p data + curl -L -C - -o $(DICT_SRC) $(DICT_URL) + @echo "$(DICT_SHA256) $(DICT_SRC)" | sha256sum -c - + @echo "downloaded and verified $(DICT_SRC)" + +# Verify an already-downloaded copy without re-fetching. +verify-dict: + @echo "$(DICT_SHA256) $(DICT_SRC)" | sha256sum -c - + +$(DICT_SRC): + @echo "$(DICT_SRC) not found — run 'make fetch-dict' first" >&2 + @exit 1 + +dict: $(DICT_SRC) + cd server && go run ./cmd/build-dictionary --in ../$(DICT_SRC) --out ../$(DICT_OUT) + +server: + cd server && CGO_ENABLED=0 go build -o ../$(SERVER_BIN) ./cmd/noitu-server + +web: + cd web && npm ci && npm run build + +test: test-go test-web + +test-go: + cd server && go vet ./... && go test ./... -race + +test-web: + @if [ -d web/node_modules ]; then cd web && npm test; else echo "web/ not set up yet, skipping"; fi + +run: server + ./$(SERVER_BIN) + +clean: + rm -f $(SERVER_BIN) $(SERVER_BIN).exe $(DICT_OUT) + rm -rf web/build web/.svelte-kit diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..3674af5 --- /dev/null +++ b/NOTICE @@ -0,0 +1,37 @@ +noitu — Vietnamese "nối từ" word-chain game +Copyright 2026 the noitu authors + +This product is distributed under two licenses, applying to different artifacts. + +------------------------------------------------------------------------------ +1. SOURCE CODE — Apache License 2.0 +------------------------------------------------------------------------------ + +All source code in this repository is licensed under the Apache License, +Version 2.0. See the LICENSE file at the repository root. + +This covers everything under server/, tools/, web/, and proto/. + +------------------------------------------------------------------------------ +2. DICTIONARY DATA — Creative Commons Attribution-ShareAlike 4.0 International +------------------------------------------------------------------------------ + +The Vietnamese dictionary database is NOT covered by the Apache License. It is +derived from a third-party dataset and remains under CC BY-SA 4.0: + + Affected artifacts: data/noitu.db (and data/dictionary.db, its source) + License text: data/LICENSE + Attribution and + list of changes: data/ATTRIBUTION.md + Upstream source: https://github.com/minhqnd/dictionary (release v2.0.0) + +CC BY-SA 4.0 is a share-alike license. Any distribution of the derived database +-- including inside a container image or other packaged build -- must carry the +same license, this attribution, and the record of modifications. + +The two regimes are kept on separate artifacts deliberately: the database is +loaded at runtime from a file and is never embedded, compiled, or linked into +the Go binary. + +Neither database file is committed to version control. Both are build artifacts +produced by `make fetch-dict` and `make dict`. diff --git a/README.md b/README.md index 84601c4..2a63db4 100644 --- a/README.md +++ b/README.md @@ -1 +1,95 @@ -# noitu \ No newline at end of file +# noitu + +Trò chơi **nối từ** tiếng Việt trên web — chơi với máy hoặc đấu 1v1 trực tuyến. + +A web implementation of the Vietnamese word-chain game *nối từ*: each player submits a +meaningful word of **at least 2 syllables** whose **first syllable matches the last syllable +of the previous word**. No word may be reused. Fail to answer in time and you lose. + +``` +ngôn ngữ → ngữ pháp → pháp luật → luật lệ → ... +``` + +## Status + +In development. See [`plans/260904-1125-noi-tu-web-game/plan.md`](./plans/260904-1125-noi-tu-web-game/plan.md) +for the implementation plan and phase breakdown. + +## Architecture + +The server is authoritative: the browser never holds the wordlist, so word validation +cannot be bypassed, and the bot and player-vs-player paths share one rule implementation. + +``` +SvelteKit SPA ──WebSocket + Protobuf──► Go server ──► SQLite dictionary +``` + +| Component | Choice | +|---|---| +| Frontend | SvelteKit 2 / Svelte 5 (JavaScript), `adapter-static` | +| Transport | WebSocket, Protobuf binary frames (`@bufbuild/protobuf` ↔ `protoc-gen-go`) | +| Backend | Go, [`coder/websocket`](https://github.com/coder/websocket) | +| Dictionary | SQLite via `modernc.org/sqlite` (CGo-free), read-only at runtime | + +## Setup + +Requires Go 1.25+, Node 20+, and optionally `make`. + +```sh +make fetch-dict # one-time: downloads the ~179 MB upstream dictionary into data/ +make dict # derives data/noitu.db (the game's wordlist) from it +make test # run all tests +make run # build and start the server +``` + +`make fetch-dict` is a one-time cost per machine. Neither database file is committed; +both are build artifacts. See [`data/ATTRIBUTION.md`](./data/ATTRIBUTION.md). + +## Make targets + +| Target | Does | +|---|---| +| `fetch-dict` | Download the upstream `dictionary.db` (~179 MB) into `data/` | +| `dict` | Derive `data/noitu.db` from the upstream database | +| `server` | Build the Go server binary | +| `web` | Build the SvelteKit frontend to static assets | +| `test` | Run Go and JavaScript tests | +| `run` | Build and run the server locally | +| `verify-dict` | Re-check the downloaded dictionary against its pinned SHA-256 | + +### Without `make` + +`make` is not installed everywhere (notably Windows). Every target is a thin wrapper: + +```sh +# fetch-dict +curl -L -C - -o data/dictionary.db https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db + +# dict +cd server && go run ./cmd/build-dictionary --in ../data/dictionary.db --out ../data/noitu.db + +# test +cd server && go vet ./... && go test ./... -race + +# server +cd server && CGO_ENABLED=0 go build -o ../noitu-server ./cmd/noitu-server +``` + +## License + +This project is distributed under **two licenses**, applying to different artifacts. +See [`NOTICE`](./NOTICE) for the full statement. + +| Artifact | License | +|---|---| +| All source code (`server/`, `web/`, `proto/`) | [Apache-2.0](./LICENSE) | +| Dictionary data (`data/noitu.db`) | [CC BY-SA 4.0](./data/LICENSE) | + +The dictionary is derived from [minhqnd/dictionary](https://github.com/minhqnd/dictionary) +(data licensed CC BY-SA 4.0), which itself aggregates Wiktionary and other Vietnamese +dictionary sources. CC BY-SA is a **share-alike** license: any redistribution of the derived +database — including inside a container image — must carry the same license, the attribution, +and the record of modifications recorded in [`data/ATTRIBUTION.md`](./data/ATTRIBUTION.md). + +The database is loaded at runtime from a file and is never embedded or linked into the Go +binary, keeping the two licensing regimes on separate artifacts. diff --git a/data/ATTRIBUTION.md b/data/ATTRIBUTION.md new file mode 100644 index 0000000..0d87958 --- /dev/null +++ b/data/ATTRIBUTION.md @@ -0,0 +1,71 @@ +# Dictionary Data Attribution + +The Vietnamese dictionary data used by this game is **not** original work of this project. +It is derived from a third-party dataset licensed under **CC BY-SA 4.0**, and this file +records the attribution and the modifications required by that license. + +## Source + +| Field | Value | +|---|---| +| Project | [minhqnd/dictionary](https://github.com/minhqnd/dictionary) | +| Asset | [`dictionary.db`, release v2.0.0](https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db) (~179 MB) | +| Author | [minhqnd](https://github.com/minhqnd) | +| Data license | [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/) — full text in [`LICENSE`](./LICENSE) | +| Code license (not used here) | MIT | + +That project is itself an aggregation. Its own upstream sources include +[Wiktionary](https://vi.wiktionary.org/) (CC BY-SA) and +[vntk/dictionary](https://github.com/vntk/dictionary), among other Vietnamese dictionary +projects. Those upstream attributions carry through this file. + +## Modifications made by this project + +`server/cmd/build-dictionary` transforms the upstream `dictionary.db` into `data/noitu.db`. +The derived database is a **modified version** of the source data. Changes: + +1. **Language filter** — kept only entries with `lang_code = 'vi'`; all other languages + (of 1,500+ language pairs in the source) were dropped. +2. **Length filter** — kept only words of **2 or more space-separated syllables**, as + required by the nối từ game rules. Single-syllable entries were dropped. +3. **Content filter** — dropped entries containing digits or punctuation, entries using + letters Vietnamese does not have (f, j, w, z), and entries whose syllables do not fit + Vietnamese phonotactics (a closed inventory of onsets, nuclei and codas). This removes + loanwords and foreign phrases such as "credit card" and "world cup" that the + multilingual source tags as Vietnamese. Diacritic-free Vietnamese words ("con cua") + are kept. +4. **Normalization** — all words Unicode NFC-normalized, lowercased, and whitespace-collapsed. +5. **Spelling aliases** — added an `aliases` table mapping alternative Vietnamese spellings + to canonical entries. Two kinds: competing tone placement in open oa/oe/uy syllables + (`hoà` → `hòa`, `thuý` → `thúy`), and i/y alternation in Sino-Vietnamese syllables + (`quí` → `quý`, `lí` → `lý`). The majority are the i/y kind. These aliases are generated + by this project and are not present upstream. +6. **Added columns and tables** — `first` and `last` syllable columns, a `syllables` count, + an index on `first`, a `syllables` out-degree table, and a `meta` table recording + provenance. All added for game lookups. +7. **Deduplication** — the source lists a word once per sense; entries were deduplicated by + normalized form. A generated spelling variant that is itself a real word, or that more + than one word would claim, is discarded rather than recorded as an alias. +8. **Dropped fields** — all definitions, translations, pronunciations, examples, part-of-speech + tags, and relations from the source were discarded. The derived database contains + **only word forms**, not meanings. + +## Share-alike obligation + +CC BY-SA 4.0 is a **share-alike** license. The derived database `data/noitu.db`, and any +distribution of it, remains licensed under **CC BY-SA 4.0** — including when it is shipped +inside a container image or any other packaged build of this project. + +This obligation applies to the **data only**. The source code of this project is licensed +separately under Apache-2.0 (see the repository root `LICENSE` and `NOTICE`). The derived +database is loaded at runtime from a file and is never compiled or linked into the binary, +keeping the two licensing regimes on separate artifacts. + +## How to reproduce the derived data + +```sh +make fetch-dict # downloads the ~179 MB upstream dictionary.db into data/ +make dict # derives data/noitu.db from it +``` + +Neither file is committed to version control; both are build artifacts. diff --git a/data/LICENSE b/data/LICENSE new file mode 100644 index 0000000..2d58298 --- /dev/null +++ b/data/LICENSE @@ -0,0 +1,428 @@ +Attribution-ShareAlike 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution-ShareAlike 4.0 International Public +License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-ShareAlike 4.0 International Public License ("Public +License"). To the extent this Public License may be interpreted as a +contract, You are granted the Licensed Rights in consideration of Your +acceptance of these terms and conditions, and the Licensor grants You +such rights in consideration of benefits the Licensor receives from +making the Licensed Material available under these terms and +conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. BY-SA Compatible License means a license listed at + creativecommons.org/compatiblelicenses, approved by Creative + Commons as essentially the equivalent of this Public License. + + d. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + e. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + f. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + g. License Elements means the license attributes listed in the name + of a Creative Commons Public License. The License Elements of this + Public License are Attribution and ShareAlike. + + h. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + i. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + j. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + k. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + l. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + m. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. Additional offer from the Licensor -- Adapted Material. + Every recipient of Adapted Material from You + automatically receives an offer from the Licensor to + exercise the Licensed Rights in the Adapted Material + under the conditions of the Adapter's License You apply. + + c. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + b. ShareAlike. + + In addition to the conditions in Section 3(a), if You Share + Adapted Material You produce, the following conditions also apply. + + 1. The Adapter's License You apply must be a Creative Commons + license with the same License Elements, this version or + later, or a BY-SA Compatible License. + + 2. You must include the text of, or the URI or hyperlink to, the + Adapter's License You apply. You may satisfy this condition + in any reasonable manner based on the medium, means, and + context in which You Share Adapted Material. + + 3. You may not offer or impose any additional or different terms + or conditions on, or apply any Effective Technological + Measures to, Adapted Material that restrict exercise of the + rights granted under the Adapter's License You apply. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material, + including for purposes of Section 3(b); and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. + diff --git a/server/cmd/build-dictionary/aliases.go b/server/cmd/build-dictionary/aliases.go new file mode 100644 index 0000000..25c7455 --- /dev/null +++ b/server/cmd/build-dictionary/aliases.go @@ -0,0 +1,287 @@ +package main + +import ( + "strings" + + "golang.org/x/text/unicode/norm" +) + +// Vietnamese combining tone marks. These are the five tones; they are distinct +// from the combining marks that form letters (circumflex, breve, horn), which +// belong to their base letter and must never be moved. +var toneMarks = map[rune]bool{ + '̀': true, // huyền (grave) + '́': true, // sắc (acute) + '̃': true, // ngã (tilde) + '̉': true, // hỏi (hook above) + '̣': true, // nặng (dot below) +} + +// Marks that turn a base letter into a different Vietnamese letter (ă â ê ô ơ ư). +// A vowel carrying one of these is not a plain vowel and is excluded from the +// tone-shifting rule below. +var letterMarks = map[rune]bool{ + '̂': true, // circumflex: â ê ô + '̆': true, // breve: ă + '̛': true, // horn: ơ ư +} + +// cluster is one base rune plus the combining marks attached to it. +type cluster struct { + base rune + mods []rune +} + +func (c cluster) tone() (rune, bool) { + for _, m := range c.mods { + if toneMarks[m] { + return m, true + } + } + return 0, false +} + +func (c cluster) hasLetterMark() bool { + for _, m := range c.mods { + if letterMarks[m] { + return true + } + } + return false +} + +func (c cluster) without(target rune) cluster { + out := cluster{base: c.base} + for _, m := range c.mods { + if m != target { + out.mods = append(out.mods, m) + } + } + return out +} + +func (c cluster) with(extra rune) cluster { + out := cluster{base: c.base, mods: append([]rune{}, c.mods...)} + out.mods = append(out.mods, extra) + return out +} + +// decompose splits a syllable into base-plus-marks clusters, working on the NFD +// form so that every diacritic is visible as its own rune. +func decompose(syllable string) []cluster { + var clusters []cluster + for _, r := range norm.NFD.String(syllable) { + if (toneMarks[r] || letterMarks[r]) && len(clusters) > 0 { + last := len(clusters) - 1 + clusters[last].mods = append(clusters[last].mods, r) + continue + } + clusters = append(clusters, cluster{base: r}) + } + return clusters +} + +func recompose(clusters []cluster) string { + var b strings.Builder + for _, c := range clusters { + b.WriteRune(c.base) + for _, m := range c.mods { + b.WriteRune(m) + } + } + return norm.NFC.String(b.String()) +} + +// shiftablePairs are the vowel clusters where Vietnamese has two competing tone +// placements in real use: the older style marks the first vowel ("hòa", "thúy"), +// the newer style marks the second ("hoà", "thuý"). Both spellings appear in +// dictionaries and both are typed by players, but they are different codepoint +// sequences, so one of them fails a lookup unless we record it as an alias. +var shiftablePairs = [][2]rune{ + {'o', 'a'}, + {'o', 'e'}, + {'u', 'y'}, +} + +// toneShiftVariant moves the tone mark to the other vowel of an oa/oe/uy cluster. +// Returns the variant and whether one was produced. +func toneShiftVariant(syllable string) (string, bool) { + clusters := decompose(syllable) + + for i := 0; i+1 < len(clusters); i++ { + first, second := clusters[i], clusters[i+1] + + // Only plain vowels take part; ơ/ư/ô/ê/ă follow different rules. + if first.hasLetterMark() || second.hasLetterMark() { + continue + } + if !isShiftablePair(first.base, second.base) { + continue + } + // The two placements only compete in an OPEN syllable that ends on the + // cluster. Add anything after it -- a final consonant ("hoàn") or a + // third vowel ("hoài") -- and only one spelling is correct. Without this + // check the builder invents words like "hòan" and "hòai". + if i+1 != len(clusters)-1 { + continue + } + // In "qu" the u is part of the consonant onset, not the vowel nucleus, + // so "quý" must not become "qúy". + if first.base == 'u' && i > 0 && clusters[i-1].base == 'q' { + continue + } + + if tone, ok := first.tone(); ok { + shifted := append([]cluster{}, clusters...) + shifted[i] = first.without(tone) + shifted[i+1] = second.with(tone) + return recompose(shifted), true + } + if tone, ok := second.tone(); ok { + shifted := append([]cluster{}, clusters...) + shifted[i] = first.with(tone) + shifted[i+1] = second.without(tone) + return recompose(shifted), true + } + } + + return "", false +} + +func isShiftablePair(a, b rune) bool { + for _, p := range shiftablePairs { + if p[0] == a && p[1] == b { + return true + } + } + return false +} + +// iyOnsets are the consonant onsets after which Vietnamese genuinely writes +// both i and y across the tone range: hi/hy, ki/ky, li/ly, mi/my, ti/ty, qui/quy. +// +// This is an allowlist rather than a general rule because most onsets take only +// one spelling. "gi" is a digraph, so "gì" has no "gỳ"; "chì" has no "chỳ". +// +// "v" and "s" were deliberately removed after inspecting real output: they +// produced non-words at scale ("chức vị" -> "chức vỵ", plus "vỳ", "vý", "sỵ", +// "sỳ"). Their genuine alternations are lexical rather than productive, so the +// few real ones are listed in iyPairs instead. +var iyOnsets = map[string]bool{ + "h": true, "k": true, "l": true, "m": true, "t": true, "qu": true, +} + +// iyPairs are individual syllables that alternate even though their onset does +// not do so productively. Listed explicitly, both directions. +var iyPairs = map[string]string{ + "sĩ": "sỹ", "sỹ": "sĩ", + "vĩ": "vỹ", "vỹ": "vĩ", +} + +// iyVariant swaps a lone i/y nucleus: quý/quí, lý/lí, mỹ/mĩ. +// +// The nucleus must be the final cluster, so closed syllables are excluded +// ("tính" has no "týnh"), and the onset must be one that takes both spellings. +func iyVariant(syllable string) (string, bool) { + if pair, ok := iyPairs[syllable]; ok { + return pair, true + } + + clusters := decompose(syllable) + if len(clusters) < 2 { + return "", false + } + + last := len(clusters) - 1 + nucleus := clusters[last] + if nucleus.hasLetterMark() { + return "", false + } + + var swapped rune + switch nucleus.base { + case 'i': + swapped = 'y' + case 'y': + swapped = 'i' + default: + return "", false + } + + var onset strings.Builder + for _, c := range clusters[:last] { + if c.hasLetterMark() { + return "", false + } + onset.WriteRune(c.base) + } + if !iyOnsets[onset.String()] { + return "", false + } + + variant := append([]cluster{}, clusters...) + variant[last] = cluster{base: swapped, mods: nucleus.mods} + return recompose(variant), true +} + +// maxVariantsPerWord caps the combination count for a word whose syllables are +// all variable. Real words reach at most a handful; the cap only guards against +// a pathological entry blowing up the alias table. +const maxVariantsPerWord = 32 + +// spellingsOf returns every accepted spelling of a single syllable, the +// original first. +func spellingsOf(syllable string) []string { + out := []string{syllable} + seen := map[string]bool{syllable: true} + + for _, gen := range []func(string) (string, bool){toneShiftVariant, iyVariant} { + alt, ok := gen(syllable) + if !ok || seen[alt] { + continue + } + seen[alt] = true + out = append(out, alt) + } + + return out +} + +// variantsFor returns every accepted alternative spelling of a whole word. +// +// Syllables must be varied together, not one at a time. "hoá lí" has two +// variable syllables, and a player is far more likely to type the fully modern +// "hóa lý" than either half-and-half form. Varying independently produced only +// the halves and rejected the spelling most people actually use. +func variantsFor(word string) []string { + syllables := strings.Split(word, " ") + + // Cartesian product over each syllable's accepted spellings. + combos := [][]string{{}} + for _, syl := range syllables { + spellings := spellingsOf(syl) + next := make([][]string, 0, len(combos)*len(spellings)) + for _, prefix := range combos { + for _, s := range spellings { + if len(next) >= maxVariantsPerWord { + break + } + next = append(next, append(append([]string{}, prefix...), s)) + } + } + combos = next + } + + seen := map[string]bool{word: true} + var out []string + for _, combo := range combos { + joined := strings.Join(combo, " ") + if seen[joined] { + continue + } + seen[joined] = true + out = append(out, joined) + } + + return out +} diff --git a/server/cmd/build-dictionary/aliases_test.go b/server/cmd/build-dictionary/aliases_test.go new file mode 100644 index 0000000..48e093b --- /dev/null +++ b/server/cmd/build-dictionary/aliases_test.go @@ -0,0 +1,208 @@ +package main + +import ( + "sort" + "testing" +) + +// Vietnamese writes the same syllable with the tone mark on either vowel of an +// oa/oe/uy cluster. Both spellings are typed by real players, so both must +// resolve to one dictionary entry. +func TestToneShiftVariant(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"oa, tone on first", "hòa", "hoà"}, + {"oa, tone on second", "hoà", "hòa"}, + {"oe, tone on first", "khỏe", "khoẻ"}, + {"oe, tone on second", "khoẻ", "khỏe"}, + {"uy, tone on first", "thúy", "thuý"}, + {"uy, tone on second", "thuý", "thúy"}, + {"nang tone on oa", "họa", "hoạ"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, ok := toneShiftVariant(tc.in) + if !ok { + t.Fatalf("toneShiftVariant(%q) produced no variant, want %q", tc.in, tc.want) + } + if got != tc.want { + t.Errorf("toneShiftVariant(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +func TestToneShiftVariantRejects(t *testing.T) { + tests := []struct{ in, why string }{ + {"quý", `"qu" is a consonant onset, not a vowel cluster — no "qúy"`}, + {"hoàn", `closed syllable: final consonant, so no "hòan"`}, + {"roàng", `closed syllable: final "ng", so no "ròang"`}, + {"hoài", `triphthong "oai", so no "hòai"`}, + {"khoẻn", `closed "oe" syllable`}, + {"quả", "no shiftable cluster"}, + {"luật", "no shiftable cluster"}, + {"pháp", "no shiftable cluster"}, + {"ngữ", "no shiftable cluster"}, + {"điện", "no shiftable cluster"}, + {"tính", "no shiftable cluster"}, + } + + for _, tc := range tests { + if got, ok := toneShiftVariant(tc.in); ok { + t.Errorf("toneShiftVariant(%q) = %q, want no variant (%s)", tc.in, got, tc.why) + } + } +} + +func TestIYVariant(t *testing.T) { + tests := []struct{ in, want string }{ + {"quý", "quí"}, + {"quí", "quý"}, + {"lý", "lí"}, + {"mỹ", "mĩ"}, + {"kỹ", "kĩ"}, + {"sĩ", "sỹ"}, + {"tỷ", "tỉ"}, + {"hy", "hi"}, + } + + for _, tc := range tests { + got, ok := iyVariant(tc.in) + if !ok { + t.Errorf("iyVariant(%q) produced no variant, want %q", tc.in, tc.want) + continue + } + if got != tc.want { + t.Errorf("iyVariant(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestIYVariantRejects(t *testing.T) { + // "gi" and "ch" are digraphs that only ever take i; closed syllables and + // diphthongs are out of scope entirely. + for _, in := range []string{"gì", "chì", "tính", "kia", "đi", "nghi", "phi", "bi"} { + if got, ok := iyVariant(in); ok { + t.Errorf("iyVariant(%q) = %q, want no variant", in, got) + } + } +} + +func TestVariantsFor(t *testing.T) { + got := variantsFor("hòa bình") + want := []string{"hoà bình"} + assertSameStrings(t, got, want) + + got = variantsFor("pháp luật") + if len(got) != 0 { + t.Errorf("variantsFor(%q) = %v, want none", "pháp luật", got) + } +} + +// A word with two variable syllables must offer every combination, not just the +// half-and-half forms. "hoá lí" is the corpus spelling but "hóa lý" is the one +// most people type, and varying syllables independently never produced it. +func TestVariantsForVariesSyllablesTogether(t *testing.T) { + // "hoá" carries the acute tone, so shifting it gives "hóa" (not "hòa"). + got := variantsFor("hoá lí") + want := []string{"hóa lí", "hoá lý", "hóa lý"} + assertSameStrings(t, got, want) +} + +func TestVariantsForCapsCombinations(t *testing.T) { + // Every syllable variable: 2^6 = 64 combinations before the cap. + word := "hoá hoá hoá hoá hoá hoá" + if got := len(variantsFor(word)); got > maxVariantsPerWord { + t.Errorf("variantsFor produced %d variants, want at most %d", got, maxVariantsPerWord) + } +} + +// "v" and "s" do not alternate productively: "chức vị" must never yield +// "chức vỵ". Only the handful of genuinely alternating syllables are listed. +func TestIYRejectsNonAlternatingOnsets(t *testing.T) { + for _, in := range []string{"vị", "vì", "vỷ", "sị", "sỳ", "si"} { + if got, ok := iyVariant(in); ok { + t.Errorf("iyVariant(%q) = %q, want no variant", in, got) + } + } +} + +func TestIYExplicitPairs(t *testing.T) { + for in, want := range map[string]string{"sĩ": "sỹ", "sỹ": "sĩ", "vĩ": "vỹ", "vỹ": "vĩ"} { + got, ok := iyVariant(in) + if !ok { + t.Errorf("iyVariant(%q) produced no variant, want %q", in, want) + continue + } + if got != want { + t.Errorf("iyVariant(%q) = %q, want %q", in, got, want) + } + } +} + +// With three words claiming one variant, a plain delete would let the third +// reinsert it and the winner would depend on map iteration order. +func TestBuildAliasesPoisonsAmbiguousVariants(t *testing.T) { + words := map[string]entry{ + "lí do": {word: "lí do", first: "lí", last: "do", syllables: 2}, + "lý do": {word: "lý do", first: "lý", last: "do", syllables: 2}, + } + + for i := 0; i < 20; i++ { + aliases, _ := buildAliases(words) + if canonical, ok := aliases["lí do"]; ok { + t.Fatalf("run %d: real word aliased to %q", i, canonical) + } + if canonical, ok := aliases["lý do"]; ok { + t.Fatalf("run %d: real word aliased to %q", i, canonical) + } + } +} + +// A generated variant that is itself a real word must not become an alias: +// doing so would let one entry answer to another entry's name. +func TestBuildAliasesSkipsCollisions(t *testing.T) { + words := map[string]entry{ + "hòa bình": {word: "hòa bình", first: "hòa", last: "bình", syllables: 2}, + "hoà bình": {word: "hoà bình", first: "hoà", last: "bình", syllables: 2}, + } + + aliases, collisions := buildAliases(words) + + if len(aliases) != 0 { + t.Errorf("aliases = %v, want none (both spellings are real words)", aliases) + } + if collisions != 2 { + t.Errorf("collisions = %d, want 2", collisions) + } +} + +func TestBuildAliasesMapsToCanonical(t *testing.T) { + words := map[string]entry{ + "hòa bình": {word: "hòa bình", first: "hòa", last: "bình", syllables: 2}, + } + + aliases, _ := buildAliases(words) + + if got := aliases["hoà bình"]; got != "hòa bình" { + t.Errorf("aliases[%q] = %q, want %q", "hoà bình", got, "hòa bình") + } +} + +func assertSameStrings(t *testing.T, got, want []string) { + t.Helper() + sort.Strings(got) + sort.Strings(want) + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for i := range got { + if got[i] != want[i] { + t.Fatalf("got %v, want %v", got, want) + } + } +} diff --git a/server/cmd/build-dictionary/filter.go b/server/cmd/build-dictionary/filter.go new file mode 100644 index 0000000..a680a11 --- /dev/null +++ b/server/cmd/build-dictionary/filter.go @@ -0,0 +1,100 @@ +package main + +import ( + "strings" + "unicode" + + "golang.org/x/text/unicode/norm" + + "github.com/tiennm99dev/noitu/server/internal/vietnamese" +) + +// rejectReason explains why a candidate word did not make it into the game +// dictionary. Counted per reason so the build log shows what the source +// actually contained rather than just a final total. +type rejectReason string + +const ( + rejectEmpty rejectReason = "empty" + rejectTooShort rejectReason = "fewer than 2 syllables" + rejectTooLong rejectReason = "over max-syllables" + rejectDigit rejectReason = "contains a digit" + rejectPunct rejectReason = "contains punctuation" + rejectNonVietnam rejectReason = "no Vietnamese letters" +) + +// accept normalizes a raw source entry and decides whether it belongs in the +// game dictionary. maxSyllables of 0 means no upper bound. +func accept(raw string, maxSyllables int) (word string, syllables []string, reason rejectReason, ok bool) { + word, syllables, err := vietnamese.Normalize(raw) + if err != nil { + return "", nil, rejectEmpty, false + } + + if !vietnamese.HasEnoughSyllables(syllables) { + return "", nil, rejectTooShort, false + } + if maxSyllables > 0 && len(syllables) > maxSyllables { + return "", nil, rejectTooLong, false + } + + for _, r := range word { + switch { + case r == ' ': + continue + case unicode.IsDigit(r): + return "", nil, rejectDigit, false + case unicode.IsPunct(r) || unicode.IsSymbol(r): + return "", nil, rejectPunct, false + case !unicode.IsLetter(r): + return "", nil, rejectPunct, false + } + } + + if !usesVietnameseAlphabet(word) { + return "", nil, rejectNonVietnam, false + } + + // Every syllable must also be a possible Vietnamese syllable. The alphabet + // check alone lets "credit card" and "come out" through, since they use only + // letters Vietnamese has. + for _, syl := range syllables { + if !isVietnameseSyllable(syl) { + return "", nil, rejectNonVietnam, false + } + } + + return word, syllables, "", true +} + +// vietnameseBaseLetters is the Vietnamese alphabet reduced to base letters: +// the Latin alphabet without f, j, w, z, plus đ. Every Vietnamese word is +// spelled from these once diacritics are stripped. +const vietnameseBaseLetters = "abcdeghiklmnopqrstuvxyđ" + +// usesVietnameseAlphabet reports whether every letter of the word is a +// Vietnamese one. +// +// Diacritics are stripped first (NFD, dropping combining marks) so that "ngữ" +// reduces to "ngu" and is checked letter by letter. Checking the base letters +// rather than merely "does it contain a diacritic" is what keeps English out: +// an entry only needs one accented character to look Vietnamese otherwise, and +// entries like "thành phố new york", "credit card" and "come out" pass such a +// test while being unplayable as Vietnamese words. +func usesVietnameseAlphabet(word string) bool { + stripped := norm.NFD.String(word) + + sawLetter := false + for _, r := range stripped { + if unicode.Is(unicode.Mn, r) || r == ' ' { + // Combining mark or separator: not a base letter. + continue + } + if !strings.ContainsRune(vietnameseBaseLetters, r) { + return false + } + sawLetter = true + } + + return sawLetter +} diff --git a/server/cmd/build-dictionary/filter_test.go b/server/cmd/build-dictionary/filter_test.go new file mode 100644 index 0000000..2024e6f --- /dev/null +++ b/server/cmd/build-dictionary/filter_test.go @@ -0,0 +1,85 @@ +package main + +import "testing" + +func TestAcceptKeepsValidWords(t *testing.T) { + tests := []struct { + raw string + want string + syllables int + }{ + {"pháp luật", "pháp luật", 2}, + {"Pháp Luật", "pháp luật", 2}, + {"vô tuyến điện", "vô tuyến điện", 3}, + {"con cua", "con cua", 2}, // no diacritics, still Vietnamese + {"đi đứng", "đi đứng", 2}, // đ is a Vietnamese letter + {"quy hoạch", "quy hoạch", 2}, + } + + for _, tc := range tests { + word, syllables, reason, ok := accept(tc.raw, 0) + if !ok { + t.Errorf("accept(%q) rejected: %s", tc.raw, reason) + continue + } + if word != tc.want { + t.Errorf("accept(%q) word = %q, want %q", tc.raw, word, tc.want) + } + if len(syllables) != tc.syllables { + t.Errorf("accept(%q) syllables = %d, want %d", tc.raw, len(syllables), tc.syllables) + } + } +} + +func TestAcceptRejects(t *testing.T) { + tests := []struct { + name string + raw string + want rejectReason + }{ + {"empty", " ", rejectEmpty}, + {"single syllable", "pháp", rejectTooShort}, + {"digit", "covid 19", rejectDigit}, + {"hyphen", "pháp-luật xxx", rejectPunct}, + {"english phrase", "hello world", rejectNonVietnam}, + {"letters Vietnamese lacks", "jazz music", rejectNonVietnam}, + // These pass a naive "contains f/j/w/z" test and a naive "has a + // diacritic" test, yet are unplayable as Vietnamese words. + {"english, no forbidden letters", "credit card", rejectNonVietnam}, + {"english, no forbidden letters 2", "dress code", rejectNonVietnam}, + {"english phrasal verb", "come out", rejectNonVietnam}, + // One Vietnamese diacritic must not launder an English word. + {"mixed English with a diacritic", "thành phố new york", rejectNonVietnam}, + {"non-Latin script", "привет мир", rejectNonVietnam}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, _, reason, ok := accept(tc.raw, 0) + if ok { + t.Fatalf("accept(%q) succeeded, want rejection %s", tc.raw, tc.want) + } + if reason != tc.want { + t.Errorf("accept(%q) reason = %s, want %s", tc.raw, reason, tc.want) + } + }) + } +} + +func TestAcceptMaxSyllables(t *testing.T) { + const raw = "công nghiệp hóa" + + if _, _, _, ok := accept(raw, 0); !ok { + t.Errorf("accept(%q, no limit) rejected, want accepted", raw) + } + if _, _, _, ok := accept(raw, 3); !ok { + t.Errorf("accept(%q, max 3) rejected, want accepted", raw) + } + _, _, reason, ok := accept(raw, 2) + if ok { + t.Fatalf("accept(%q, max 2) accepted, want rejection", raw) + } + if reason != rejectTooLong { + t.Errorf("reason = %s, want %s", reason, rejectTooLong) + } +} diff --git a/server/cmd/build-dictionary/main.go b/server/cmd/build-dictionary/main.go new file mode 100644 index 0000000..d0fbb51 --- /dev/null +++ b/server/cmd/build-dictionary/main.go @@ -0,0 +1,547 @@ +// Command build-dictionary derives the game's wordlist from the upstream +// minhqnd/dictionary SQLite database. +// +// The upstream database is ~179 MB and covers 1,500+ language pairs with full +// definitions. The game needs none of that: only Vietnamese word forms of at +// least two syllables, indexed by first and last syllable. This tool performs +// that reduction and records provenance in a meta table. +// +// The derived database is a modified version of CC BY-SA 4.0 licensed data. +// See data/ATTRIBUTION.md. +// +// Usage: +// +// go run ./cmd/build-dictionary --in ../data/dictionary.db --out ../data/noitu.db +package main + +import ( + "database/sql" + "errors" + "flag" + "fmt" + "log" + "os" + "sort" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +const ( + sourceURL = "https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db" + sourceLicense = "CC BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)" + builderVer = "1" +) + +type config struct { + in string + out string + maxSyllables int + minWords int + table string + wordCol string + langCol string + lang string +} + +func main() { + log.SetFlags(0) + + var cfg config + flag.StringVar(&cfg.in, "in", "../data/dictionary.db", "upstream dictionary.db to read") + flag.StringVar(&cfg.out, "out", "../data/noitu.db", "derived database to write") + flag.IntVar(&cfg.maxSyllables, "max-syllables", 0, "reject words longer than this (0 = no limit)") + flag.IntVar(&cfg.minWords, "min-words", 40000, "fail if fewer words survive filtering") + flag.StringVar(&cfg.table, "table", "", "source table (default: auto-detect)") + flag.StringVar(&cfg.wordCol, "word-col", "", "source word column (default: auto-detect)") + flag.StringVar(&cfg.langCol, "lang-col", "", "source language column (default: auto-detect)") + flag.StringVar(&cfg.lang, "lang", "vi", "language code to keep") + flag.Parse() + + if err := run(cfg); err != nil { + log.Fatalf("build-dictionary: %v", err) + } +} + +func run(cfg config) error { + if _, err := os.Stat(cfg.in); err != nil { + return fmt.Errorf("source database not found at %s — run 'make fetch-dict' first: %w", cfg.in, err) + } + + src, err := sql.Open("sqlite", "file:"+cfg.in+"?mode=ro") + if err != nil { + return fmt.Errorf("open source: %w", err) + } + defer src.Close() + + source, err := resolveSource(src, cfg) + if err != nil { + return err + } + log.Printf("source: %s.%s filtered by %s = %q", source.table, source.wordCol, source.langCol, cfg.lang) + + words, rejects, err := extract(src, source, cfg) + if err != nil { + return err + } + logRejects(rejects) + log.Printf("accepted %d distinct words", len(words)) + + if len(words) < cfg.minWords { + return fmt.Errorf("only %d words survived filtering, expected at least %d — "+ + "the source schema or content may have changed", len(words), cfg.minWords) + } + + aliases, collisions := buildAliases(words) + log.Printf("generated %d spelling aliases (%d skipped as ambiguous or already real words)", len(aliases), collisions) + + if err := write(cfg.out, words, aliases, source); err != nil { + return err + } + if err := verify(cfg.out, cfg.minWords); err != nil { + return fmt.Errorf("output failed verification: %w", err) + } + + log.Printf("wrote %s", cfg.out) + return nil +} + +// verify re-opens the finished database and re-checks the invariants the game +// depends on. The in-memory checks above can only prove what the builder +// intended; this proves what actually landed on disk. +func verify(path string, minWords int) error { + db, err := sql.Open("sqlite", "file:"+path+"?mode=ro") + if err != nil { + return err + } + defer db.Close() + + checks := []struct { + desc string + query string + want func(int) bool + }{ + {"words below the minimum", `SELECT COUNT(*) FROM words`, func(n int) bool { return n >= minWords }}, + {"words with fewer than 2 syllables", `SELECT COUNT(*) FROM words WHERE syllables < 2`, func(n int) bool { return n == 0 }}, + {"words with an empty first or last syllable", `SELECT COUNT(*) FROM words WHERE first = '' OR last = ''`, func(n int) bool { return n == 0 }}, + {"aliases pointing at a missing word", + `SELECT COUNT(*) FROM aliases a LEFT JOIN words w ON w.word = a.canonical WHERE w.word IS NULL`, + func(n int) bool { return n == 0 }}, + {"aliases that are themselves real words", + `SELECT COUNT(*) FROM aliases a JOIN words w ON w.word = a.variant`, + func(n int) bool { return n == 0 }}, + {"words whose first syllable is missing from the syllables table", + `SELECT COUNT(*) FROM words w LEFT JOIN syllables s ON s.syllable = w.first WHERE s.syllable IS NULL`, + func(n int) bool { return n == 0 }}, + } + + for _, c := range checks { + var n int + if err := db.QueryRow(c.query).Scan(&n); err != nil { + return fmt.Errorf("check %q: %w", c.desc, err) + } + if !c.want(n) { + return fmt.Errorf("%s: %d", c.desc, n) + } + } + + return nil +} + +// entry is one accepted word with the parts the game engine indexes on. +type entry struct { + word string + first string + last string + syllables int +} + +type sourceSpec struct { + table string + wordCol string + langCol string + maxSyllables int +} + +// quoteIdent renders a SQLite identifier. Go's %q escapes an embedded quote as +// \" but SQLite requires it doubled, so fmt.Sprintf("%q") is not correct here. +func quoteIdent(name string) string { + return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` +} + +// validateSource confirms operator-supplied identifiers actually exist. +// +// SQLite accepts a double-quoted string that matches no column as a string +// literal rather than erroring, so a typo in --word-col silently yields one +// row per distinct value of that literal. Checking up front turns a confusing +// near-empty build into a clear message. +func validateSource(db *sql.DB, spec sourceSpec) error { + tables, err := listTables(db) + if err != nil { + return err + } + if !contains(tables, spec.table) { + return fmt.Errorf("table %q not found; available: %s", spec.table, strings.Join(tables, ", ")) + } + + cols, err := listColumns(db, spec.table) + if err != nil { + return fmt.Errorf("read columns of %q: %w", spec.table, err) + } + for _, c := range []struct{ role, name string }{{"--word-col", spec.wordCol}, {"--lang-col", spec.langCol}} { + if !contains(cols, c.name) { + return fmt.Errorf("%s %q not found in table %q; available: %s", + c.role, c.name, spec.table, strings.Join(cols, ", ")) + } + } + + return nil +} + +func contains(haystack []string, needle string) bool { + for _, h := range haystack { + if h == needle { + return true + } + } + return false +} + +// resolveSource finds the table holding word forms. The upstream schema is not +// contractual — it is someone else's release artifact — so rather than hardcode +// it, look for a table carrying both a word-like and a language-like column. +// Explicit flags override detection entirely. +func resolveSource(db *sql.DB, cfg config) (sourceSpec, error) { + given := 0 + for _, f := range []string{cfg.table, cfg.wordCol, cfg.langCol} { + if f != "" { + given++ + } + } + switch { + case given == 3: + spec := sourceSpec{table: cfg.table, wordCol: cfg.wordCol, langCol: cfg.langCol, maxSyllables: cfg.maxSyllables} + if err := validateSource(db, spec); err != nil { + return sourceSpec{}, err + } + return spec, nil + case given > 0: + return sourceSpec{}, fmt.Errorf( + "--table, --word-col and --lang-col must be given together (got %d of 3); "+ + "omit all three to auto-detect", given) + } + + tables, err := listTables(db) + if err != nil { + return sourceSpec{}, err + } + + wordNames := []string{"word", "term", "headword", "text", "lemma", "entry"} + langNames := []string{"lang_code", "language", "lang", "lang_name"} + + for _, t := range tables { + cols, err := listColumns(db, t) + if err != nil { + log.Printf("warning: could not read columns of %q: %v", t, err) + continue + } + wordCol := pickColumn(cols, wordNames) + langCol := pickColumn(cols, langNames) + if wordCol != "" && langCol != "" { + return sourceSpec{table: t, wordCol: wordCol, langCol: langCol, maxSyllables: cfg.maxSyllables}, nil + } + } + + // Detection failed. Dump the schema so the fix is a single flag away rather + // than a debugging session. + var b strings.Builder + b.WriteString("could not auto-detect the source table.\n") + b.WriteString("Pass --table, --word-col and --lang-col explicitly. Schema found:\n") + for _, t := range tables { + cols, err := listColumns(db, t) + if err != nil { + continue + } + fmt.Fprintf(&b, " %s(%s)\n", t, strings.Join(cols, ", ")) + } + return sourceSpec{}, errors.New(b.String()) +} + +func listTables(db *sql.DB) ([]string, error) { + rows, err := db.Query(`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`) + if err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + defer rows.Close() + + var out []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + out = append(out, name) + } + return out, rows.Err() +} + +func listColumns(db *sql.DB, table string) ([]string, error) { + rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", quoteIdent(table))) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []string + for rows.Next() { + var ( + cid int + name, ctype string + notNull, pk int + dflt sql.NullString + ) + if err := rows.Scan(&cid, &name, &ctype, ¬Null, &dflt, &pk); err != nil { + return nil, err + } + out = append(out, name) + } + return out, rows.Err() +} + +func pickColumn(cols []string, candidates []string) string { + for _, want := range candidates { + for _, c := range cols { + if strings.EqualFold(c, want) { + return c + } + } + } + return "" +} + +// extract streams every candidate word past the filters. Results are deduped by +// normalized form, since the source lists a word once per sense. +func extract(db *sql.DB, src sourceSpec, cfg config) (map[string]entry, map[rejectReason]int, error) { + query := fmt.Sprintf("SELECT DISTINCT %s FROM %s WHERE %s = ?", + quoteIdent(src.wordCol), quoteIdent(src.table), quoteIdent(src.langCol)) + rows, err := db.Query(query, cfg.lang) + if err != nil { + return nil, nil, fmt.Errorf("query source (%s): %w", query, err) + } + defer rows.Close() + + words := make(map[string]entry) + rejects := make(map[rejectReason]int) + + for rows.Next() { + var raw sql.NullString + if err := rows.Scan(&raw); err != nil { + return nil, nil, err + } + if !raw.Valid { + rejects[rejectEmpty]++ + continue + } + + word, syllables, reason, ok := accept(raw.String, cfg.maxSyllables) + if !ok { + rejects[reason]++ + continue + } + + words[word] = entry{ + word: word, + first: syllables[0], + last: syllables[len(syllables)-1], + syllables: len(syllables), + } + } + + return words, rejects, rows.Err() +} + +func logRejects(rejects map[rejectReason]int) { + reasons := make([]string, 0, len(rejects)) + for r := range rejects { + reasons = append(reasons, string(r)) + } + sort.Strings(reasons) + for _, r := range reasons { + log.Printf(" rejected %6d: %s", rejects[rejectReason(r)], r) + } +} + +// buildAliases maps alternative spellings onto canonical words. A variant that +// happens to be a real word in its own right is dropped: accepting it would let +// one word be played under two names, or worse, silently rename it. +func buildAliases(words map[string]entry) (map[string]string, int) { + aliases := make(map[string]string) + // poisoned records variants claimed by more than one word. A plain delete + // is not enough: a third claimant would find the key free and reinsert it, + // leaving the winner decided by Go's randomized map iteration order. + poisoned := make(map[string]bool) + collisions := 0 + + for word := range words { + for _, variant := range variantsFor(word) { + if poisoned[variant] { + continue + } + if _, isRealWord := words[variant]; isRealWord { + collisions++ + continue + } + if existing, taken := aliases[variant]; taken && existing != word { + delete(aliases, variant) + poisoned[variant] = true + collisions++ + continue + } + aliases[variant] = word + } + } + + return aliases, collisions +} + +// write builds the database beside the target and renames it into place only +// after the transaction commits. Writing in place would mean that a failure +// partway through -- a full disk, an interrupt -- leaves an empty but +// syntactically valid database where a good one used to be, which the server +// would happily open and find no words in. +func write(path string, words map[string]entry, aliases map[string]string, src sourceSpec) error { + tmp := path + ".tmp" + if err := os.Remove(tmp); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove stale temp file: %w", err) + } + // On any error path the half-built file must not survive. + committed := false + defer func() { + if !committed { + os.Remove(tmp) + } + }() + + if err := writeTo(tmp, words, aliases, src); err != nil { + return err + } + + // os.Rename replaces the destination atomically on POSIX; on Windows it + // fails if the target exists, so clear it first. + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove existing output: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("move temp database into place: %w", err) + } + committed = true + + return nil +} + +func writeTo(path string, words map[string]entry, aliases map[string]string, src sourceSpec) error { + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + return fmt.Errorf("create output: %w", err) + } + defer db.Close() + + schema := ` +CREATE TABLE words ( + word TEXT PRIMARY KEY, + first TEXT NOT NULL, + last TEXT NOT NULL, + syllables INTEGER NOT NULL +) WITHOUT ROWID; +CREATE INDEX idx_words_first ON words(first); + +CREATE TABLE syllables ( + syllable TEXT PRIMARY KEY, + out_degree INTEGER NOT NULL +) WITHOUT ROWID; + +CREATE TABLE aliases ( + variant TEXT PRIMARY KEY, + canonical TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); +` + if _, err := db.Exec(schema); err != nil { + return fmt.Errorf("create schema: %w", err) + } + + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + insertWord, err := tx.Prepare(`INSERT INTO words (word, first, last, syllables) VALUES (?, ?, ?, ?)`) + if err != nil { + return err + } + defer insertWord.Close() + + outDegree := make(map[string]int) + for _, e := range words { + if _, err := insertWord.Exec(e.word, e.first, e.last, e.syllables); err != nil { + return fmt.Errorf("insert word %q: %w", e.word, err) + } + outDegree[e.first]++ + // A syllable that ends a word but starts none is a dead end; record it + // with zero so the engine can find it without a second query. + if _, seen := outDegree[e.last]; !seen { + outDegree[e.last] = 0 + } + } + + insertSyllable, err := tx.Prepare(`INSERT INTO syllables (syllable, out_degree) VALUES (?, ?)`) + if err != nil { + return err + } + defer insertSyllable.Close() + for syllable, degree := range outDegree { + if _, err := insertSyllable.Exec(syllable, degree); err != nil { + return fmt.Errorf("insert syllable %q: %w", syllable, err) + } + } + + insertAlias, err := tx.Prepare(`INSERT INTO aliases (variant, canonical) VALUES (?, ?)`) + if err != nil { + return err + } + defer insertAlias.Close() + for variant, canonical := range aliases { + if _, err := insertAlias.Exec(variant, canonical); err != nil { + return fmt.Errorf("insert alias %q: %w", variant, err) + } + } + + insertMeta, err := tx.Prepare(`INSERT INTO meta (key, value) VALUES (?, ?)`) + if err != nil { + return err + } + defer insertMeta.Close() + meta := [][2]string{ + {"source_url", sourceURL}, + {"source_license", sourceLicense}, + {"attribution", "See data/ATTRIBUTION.md for required attribution and the list of modifications."}, + {"built_at", time.Now().UTC().Format(time.RFC3339)}, + {"builder_version", builderVer}, + {"word_count", fmt.Sprint(len(words))}, + {"alias_count", fmt.Sprint(len(aliases))}, + {"source_table", src.table}, + {"source_word_column", src.wordCol}, + {"source_lang_column", src.langCol}, + {"max_syllables", fmt.Sprint(src.maxSyllables)}, + } + for _, kv := range meta { + if _, err := insertMeta.Exec(kv[0], kv[1]); err != nil { + return err + } + } + + return tx.Commit() +} diff --git a/server/cmd/build-dictionary/main_test.go b/server/cmd/build-dictionary/main_test.go new file mode 100644 index 0000000..da5732a --- /dev/null +++ b/server/cmd/build-dictionary/main_test.go @@ -0,0 +1,303 @@ +package main + +import ( + "bytes" + "database/sql" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + _ "modernc.org/sqlite" +) + +// fixtureSource writes a miniature stand-in for the upstream dictionary: the +// same shape (a word column plus a language column, one row per sense) but a +// handful of rows instead of 357k. Tests never touch the real 179 MB download. +func fixtureSource(t *testing.T, rows [][2]string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "source.db") + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + if _, err := db.Exec(`CREATE TABLE entries (word TEXT, lang_code TEXT, definition TEXT)`); err != nil { + t.Fatal(err) + } + for _, r := range rows { + if _, err := db.Exec(`INSERT INTO entries (word, lang_code, definition) VALUES (?, ?, ?)`, + r[0], r[1], "unused"); err != nil { + t.Fatal(err) + } + } + return path +} + +func defaultRows() [][2]string { + return [][2]string{ + {"pháp luật", "vi"}, + {"pháp luật", "vi"}, // duplicate sense — must dedupe to one word + {"luật lệ", "vi"}, + {"ngôn ngữ", "vi"}, + {"ngữ pháp", "vi"}, + {"hòa bình", "vi"}, + {"vô tuyến điện", "vi"}, // three syllables + {"pháp", "vi"}, // single syllable — rejected + {"covid 19", "vi"}, // digit — rejected + {"hello world", "en"}, // wrong language — never selected + } +} + +func buildFixture(t *testing.T, rows [][2]string, maxSyllables int) string { + t.Helper() + + in := fixtureSource(t, rows) + out := filepath.Join(t.TempDir(), "noitu.db") + + cfg := config{ + in: in, + out: out, + maxSyllables: maxSyllables, + minWords: 1, + lang: "vi", + } + if err := run(cfg); err != nil { + t.Fatalf("run: %v", err) + } + return out +} + +func openOut(t *testing.T, path string) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite", "file:"+path+"?mode=ro") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestBuildProducesExpectedWords(t *testing.T) { + db := openOut(t, buildFixture(t, defaultRows(), 0)) + + var count int + if err := db.QueryRow(`SELECT COUNT(*) FROM words`).Scan(&count); err != nil { + t.Fatal(err) + } + if want := 6; count != want { + t.Errorf("word count = %d, want %d", count, want) + } + + // Every stored word must have at least two syllables. + var short int + if err := db.QueryRow(`SELECT COUNT(*) FROM words WHERE syllables < 2`).Scan(&short); err != nil { + t.Fatal(err) + } + if short != 0 { + t.Errorf("%d words have fewer than 2 syllables, want 0", short) + } + + // Three-syllable words are kept and split on first/last, not on a pair. + var first, last string + var syllables int + err := db.QueryRow(`SELECT first, last, syllables FROM words WHERE word = ?`, "vô tuyến điện"). + Scan(&first, &last, &syllables) + if err != nil { + t.Fatalf("three-syllable word missing: %v", err) + } + if first != "vô" || last != "điện" || syllables != 3 { + t.Errorf("got first=%q last=%q syllables=%d, want vô/điện/3", first, last, syllables) + } +} + +func TestBuildComputesOutDegree(t *testing.T) { + db := openOut(t, buildFixture(t, defaultRows(), 0)) + + // "pháp luật" and "pháp" (rejected) mean exactly one word starts with "pháp". + assertOutDegree(t, db, "pháp", 1) + // "luật lệ" starts with "luật". + assertOutDegree(t, db, "luật", 1) + // "bình" ends a word but starts none — a dead end, recorded as zero. + assertOutDegree(t, db, "bình", 0) +} + +func assertOutDegree(t *testing.T, db *sql.DB, syllable string, want int) { + t.Helper() + var got int + if err := db.QueryRow(`SELECT out_degree FROM syllables WHERE syllable = ?`, syllable).Scan(&got); err != nil { + t.Fatalf("out_degree for %q: %v", syllable, err) + } + if got != want { + t.Errorf("out_degree(%q) = %d, want %d", syllable, got, want) + } +} + +func TestBuildWritesAliases(t *testing.T) { + db := openOut(t, buildFixture(t, defaultRows(), 0)) + + var canonical string + err := db.QueryRow(`SELECT canonical FROM aliases WHERE variant = ?`, "hoà bình").Scan(&canonical) + if err != nil { + t.Fatalf("alias for %q missing: %v", "hoà bình", err) + } + if canonical != "hòa bình" { + t.Errorf("alias resolves to %q, want %q", canonical, "hòa bình") + } + + // Every alias must point at a word that actually exists. + var orphans int + err = db.QueryRow(`SELECT COUNT(*) FROM aliases a + LEFT JOIN words w ON w.word = a.canonical + WHERE w.word IS NULL`).Scan(&orphans) + if err != nil { + t.Fatal(err) + } + if orphans != 0 { + t.Errorf("%d aliases point at missing words, want 0", orphans) + } +} + +func TestBuildRecordsProvenance(t *testing.T) { + db := openOut(t, buildFixture(t, defaultRows(), 0)) + + for _, key := range []string{"source_url", "source_license", "attribution", "built_at", "word_count"} { + var value string + if err := db.QueryRow(`SELECT value FROM meta WHERE key = ?`, key).Scan(&value); err != nil { + t.Errorf("meta[%q] missing: %v", key, err) + continue + } + if value == "" { + t.Errorf("meta[%q] is empty", key) + } + } +} + +// The floor exists so a schema change upstream fails the build loudly instead +// of silently shipping a near-empty dictionary. +func TestBuildFailsBelowMinWords(t *testing.T) { + in := fixtureSource(t, defaultRows()) + cfg := config{ + in: in, + out: filepath.Join(t.TempDir(), "noitu.db"), + minWords: 1000, + lang: "vi", + } + if err := run(cfg); err == nil { + t.Fatal("run succeeded with an unreachable min-words floor, want error") + } +} + +// A failed build must leave the previous good database untouched. Building in +// place would delete it and leave an empty file the server would happily open. +func TestFailedBuildPreservesPreviousOutput(t *testing.T) { + out := buildFixture(t, defaultRows(), 0) + + before, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + + // Same output path, but a floor no fixture can clear. + cfg := config{ + in: fixtureSource(t, defaultRows()), + out: out, + minWords: 1000, + lang: "vi", + } + if err := run(cfg); err == nil { + t.Fatal("run succeeded with an unreachable floor, want error") + } + + after, err := os.ReadFile(out) + if err != nil { + t.Fatalf("previous output was destroyed by the failed build: %v", err) + } + if !bytes.Equal(before, after) { + t.Error("previous output was modified by the failed build") + } + if _, err := os.Stat(out + ".tmp"); !errors.Is(err, os.ErrNotExist) { + t.Error("temp database left behind after a failed build") + } +} + +func TestExplicitSourceFlagsMustBeComplete(t *testing.T) { + in := fixtureSource(t, defaultRows()) + out := filepath.Join(t.TempDir(), "noitu.db") + + // Two of three: ambiguous, so fail loudly rather than silently auto-detect. + err := run(config{in: in, out: out, minWords: 1, lang: "vi", table: "entries", wordCol: "word"}) + if err == nil { + t.Fatal("run succeeded with a partial flag override, want error") + } +} + +// SQLite reads a double-quoted name that matches no column as a string literal, +// so a typo would otherwise build a dictionary out of that literal. +func TestUnknownColumnIsRejected(t *testing.T) { + in := fixtureSource(t, defaultRows()) + out := filepath.Join(t.TempDir(), "noitu.db") + + err := run(config{ + in: in, out: out, minWords: 1, lang: "vi", + table: "entries", wordCol: "does_not_exist", langCol: "lang_code", + }) + if err == nil { + t.Fatal("run succeeded with an unknown word column, want error") + } + if !strings.Contains(err.Error(), "does_not_exist") { + t.Errorf("error %q does not name the bad column", err) + } +} + +func TestBuildRecordsSourceProvenance(t *testing.T) { + db := openOut(t, buildFixture(t, defaultRows(), 0)) + + for key, want := range map[string]string{ + "source_table": "entries", + "source_word_column": "word", + "source_lang_column": "lang_code", + } { + var got string + if err := db.QueryRow(`SELECT value FROM meta WHERE key = ?`, key).Scan(&got); err != nil { + t.Errorf("meta[%q] missing: %v", key, err) + continue + } + if got != want { + t.Errorf("meta[%q] = %q, want %q", key, got, want) + } + } +} + +func TestBuildAutoDetectsSchema(t *testing.T) { + // Same data under different column names: detection should still find it. + path := filepath.Join(t.TempDir(), "odd.db") + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE lexicon (headword TEXT, language TEXT)`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO lexicon VALUES ('pháp luật', 'vi'), ('luật lệ', 'vi')`); err != nil { + t.Fatal(err) + } + db.Close() + + out := filepath.Join(t.TempDir(), "noitu.db") + if err := run(config{in: path, out: out, minWords: 1, lang: "vi"}); err != nil { + t.Fatalf("run with alternate schema: %v", err) + } + + got := openOut(t, out) + var count int + if err := got.QueryRow(`SELECT COUNT(*) FROM words`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Errorf("word count = %d, want 2", count) + } +} diff --git a/server/cmd/build-dictionary/syllable.go b/server/cmd/build-dictionary/syllable.go new file mode 100644 index 0000000..971a120 --- /dev/null +++ b/server/cmd/build-dictionary/syllable.go @@ -0,0 +1,124 @@ +package main + +import ( + "sort" + "strings" + + "golang.org/x/text/unicode/norm" +) + +// Vietnamese syllable structure is a closed system: onset + nucleus + coda, +// where each part is drawn from a fixed inventory. That makes it a reliable +// test for whether an entry is a Vietnamese word at all. +// +// An alphabet check alone cannot do this. "credit card", "dress code" and +// "come out" are spelled entirely with letters Vietnamese has, so they pass any +// letter-level filter, but no Vietnamese syllable begins "cr" or ends "ss". +// The multilingual source leaks entries like these into the `vi` rows, and +// without this check they become playable words. + +// vietnameseOnsets is the complete inventory of syllable-initial consonants. +// Longest-match order matters, so this is sorted by descending length at init. +var vietnameseOnsets = []string{ + "ngh", "ng", "nh", "ch", "gh", "gi", "kh", "ph", "qu", "th", "tr", + "b", "c", "d", "đ", "g", "h", "k", "l", "m", "n", "p", "r", "s", "t", "v", "x", +} + +// vietnameseCodas is the complete inventory of syllable-final consonants and +// offglides. +var vietnameseCodas = []string{ + "ngh", "ng", "nh", "ch", + "c", "m", "n", "p", "t", "i", "o", "u", "y", +} + +// vietnameseNuclei is the inventory of vowel nuclei, diacritics stripped. Some +// are single vowels, others diphthongs or triphthongs. +var vietnameseNuclei = []string{ + "uye", "uya", "uyu", "oai", "oay", "uoi", "uou", "ieu", "yeu", "uai", "uay", + "ai", "ao", "au", "ay", "eo", "eu", "ia", "ie", "iu", "oa", "oe", "oi", "oo", + "ua", "ue", "ui", "uo", "uu", "uy", "ya", "ye", "yu", "ao", "eu", + "a", "e", "i", "o", "u", "y", +} + +func init() { + byLengthDesc := func(s []string) { + sort.SliceStable(s, func(i, j int) bool { return len(s[i]) > len(s[j]) }) + } + byLengthDesc(vietnameseOnsets) + byLengthDesc(vietnameseCodas) + byLengthDesc(vietnameseNuclei) +} + +// stripDiacritics reduces a syllable to its base letters, keeping đ (which is a +// distinct letter, not a d with a mark). +func stripDiacritics(syllable string) string { + var b strings.Builder + for _, r := range norm.NFD.String(syllable) { + // Mn = nonspacing combining mark: every Vietnamese tone and letter mark. + if isCombiningMark(r) { + continue + } + b.WriteRune(r) + } + return b.String() +} + +func isCombiningMark(r rune) bool { + // The ranges Vietnamese actually uses; cheaper and tighter than a full + // unicode.Is(unicode.Mn, r) for this data. + return (r >= 0x0300 && r <= 0x036F) || r == 0x031B +} + +// isVietnameseSyllable reports whether a syllable fits Vietnamese phonotactics. +// +// Every onset and nucleus split must be tried, not just the longest. A greedy +// parser reads "gì" as the digraph onset "gi" with nothing left for a nucleus +// and wrongly rejects it; the correct parse is onset "g" plus nucleus "ì". +// The same applies to "gìn", "gỉ" and every other g + i syllable. +func isVietnameseSyllable(syllable string) bool { + base := stripDiacritics(syllable) + if base == "" { + return false + } + + // Onset is optional ("áo", "ăn"), so "" is a candidate too. + candidates := []string{""} + for _, onset := range vietnameseOnsets { + if strings.HasPrefix(base, onset) { + candidates = append(candidates, onset) + } + } + + for _, onset := range candidates { + if parsesAsRhyme(base[len(onset):]) { + return true + } + } + + return false +} + +// parsesAsRhyme reports whether the remainder after an onset is a valid +// nucleus followed by at most one coda. +func parsesAsRhyme(rest string) bool { + if rest == "" { + return false + } + + for _, nucleus := range vietnameseNuclei { + if !strings.HasPrefix(rest, nucleus) { + continue + } + tail := rest[len(nucleus):] + if tail == "" { + return true + } + for _, coda := range vietnameseCodas { + if tail == coda { + return true + } + } + } + + return false +} diff --git a/server/cmd/build-dictionary/syllable_test.go b/server/cmd/build-dictionary/syllable_test.go new file mode 100644 index 0000000..078378a --- /dev/null +++ b/server/cmd/build-dictionary/syllable_test.go @@ -0,0 +1,67 @@ +package main + +import "testing" + +func TestIsVietnameseSyllableAccepts(t *testing.T) { + // A parser that matches the longest onset greedily rejects the whole g+i + // family, which includes some of the most common words in the language. + gFamily := []string{"gì", "gỉ", "gìn", "giữ", "gi", "giả", "giáo", "gia"} + // Ordinary syllables across the onset, nucleus and coda inventories. + ordinary := []string{ + "pháp", "luật", "ngôn", "ngữ", "nghiêng", "nghề", "khuya", "quyền", + "tuyết", "người", "đường", "ươm", "ăn", "áo", "yêu", "oanh", "uy", + "xoài", "khoẻ", "thuý", "quý", "chương", "trường", "nhanh", "phở", + } + + for _, group := range [][]string{gFamily, ordinary} { + for _, s := range group { + if !isVietnameseSyllable(s) { + t.Errorf("isVietnameseSyllable(%q) = false, want true", s) + } + } + } +} + +func TestIsVietnameseSyllableRejects(t *testing.T) { + // Foreign syllables that use only letters Vietnamese has, so an + // alphabet-level check cannot catch them. + tests := []struct{ in, why string }{ + {"credit", "onset cluster cr"}, + {"card", "coda rd"}, + {"dress", "onset cluster dr and coda ss"}, + {"code", "coda de"}, + {"come", "coda me"}, + {"york", "coda rk"}, + {"cup", "valid shape but see note"}, // sanity: see below + {"", "empty"}, + } + + for _, tc := range tests { + if tc.in == "cup" { + // "cup" is phonotactically legal in Vietnamese (c + u + p), so it + // is NOT rejected here. Documented so the expectation is explicit. + if !isVietnameseSyllable("cup") { + t.Error(`isVietnameseSyllable("cup") = false; it is a legal Vietnamese shape`) + } + continue + } + if isVietnameseSyllable(tc.in) { + t.Errorf("isVietnameseSyllable(%q) = true, want false (%s)", tc.in, tc.why) + } + } +} + +func TestStripDiacritics(t *testing.T) { + tests := map[string]string{ + "ngữ": "ngu", + "đường": "đuong", + "khoẻ": "khoe", + "pháp": "phap", + } + + for in, want := range tests { + if got := stripDiacritics(in); got != want { + t.Errorf("stripDiacritics(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/server/go.mod b/server/go.mod new file mode 100644 index 0000000..07c7588 --- /dev/null +++ b/server/go.mod @@ -0,0 +1,20 @@ +module github.com/tiennm99dev/noitu/server + +go 1.25.0 + +require ( + golang.org/x/text v0.41.0 + modernc.org/sqlite v1.58.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.47.0 // indirect + modernc.org/libc v1.75.6 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect +) diff --git a/server/go.sum b/server/go.sum new file mode 100644 index 0000000..8d27bc6 --- /dev/null +++ b/server/go.sum @@ -0,0 +1,52 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus= +modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.58.0 h1:38u40/bwkfM7f0Myhosl+SEMltSDxnGdQf8o6Kjmys0= +modernc.org/sqlite v1.58.0/go.mod h1:rsD2CckafgObKC4DhBlGBf+RiHxkc3hINGt1Xw32tVY= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/server/internal/vietnamese/normalize.go b/server/internal/vietnamese/normalize.go new file mode 100644 index 0000000..708aea5 --- /dev/null +++ b/server/internal/vietnamese/normalize.go @@ -0,0 +1,54 @@ +// Package vietnamese normalizes Vietnamese word input. +// +// The same normalization runs in two places: the dictionary builder, which +// normalizes the corpus, and the game server, which normalizes what a player +// types. They must agree exactly — if they ever diverge, a word stored in the +// database stops matching the identical string typed by a player. Sharing this +// package is what prevents that. +package vietnamese + +import ( + "errors" + "strings" + + "golang.org/x/text/unicode/norm" +) + +// MinSyllables is the shortest word the game accepts. Nối từ is played with +// compound words; a single syllable is not a valid move. +const MinSyllables = 2 + +// ErrEmpty is returned when input contains no usable characters. +var ErrEmpty = errors.New("vietnamese: empty input") + +// Normalize canonicalizes raw player or corpus input and splits it into syllables. +// +// It returns the canonical word (syllables joined by single spaces) and the +// syllables themselves. It deliberately does not enforce a syllable count: +// length is a game rule, not a text rule, so the engine owns that check. +// +// Steps, in order: +// - NFC composition, so "ữ" written as a base plus a combining mark compares +// equal to the single-codepoint form. Without this, words typed on different +// keyboards or platforms fail to match identical database entries. +// - lowercase, so capitalization never affects a lookup. +// - whitespace split and rejoin, which collapses runs of spaces, tabs, and +// non-breaking spaces that IMEs and copy-paste routinely introduce. +func Normalize(raw string) (string, []string, error) { + composed := norm.NFC.String(raw) + lowered := strings.ToLower(composed) + + // strings.Fields splits on every unicode.IsSpace rune, which covers tabs and + // U+00A0 non-breaking spaces as well as ordinary spaces. + syllables := strings.Fields(lowered) + if len(syllables) == 0 { + return "", nil, ErrEmpty + } + + return strings.Join(syllables, " "), syllables, nil +} + +// HasEnoughSyllables reports whether a word is long enough to be a legal move. +func HasEnoughSyllables(syllables []string) bool { + return len(syllables) >= MinSyllables +} diff --git a/server/internal/vietnamese/normalize_test.go b/server/internal/vietnamese/normalize_test.go new file mode 100644 index 0000000..0addc03 --- /dev/null +++ b/server/internal/vietnamese/normalize_test.go @@ -0,0 +1,90 @@ +package vietnamese + +import ( + "errors" + "testing" +) + +func TestNormalize(t *testing.T) { + tests := []struct { + name string + raw string + want string + syllables int + }{ + {"plain two syllables", "pháp luật", "pháp luật", 2}, + {"uppercase", "PHÁP LUẬT", "pháp luật", 2}, + {"mixed case", "Pháp Luật", "pháp luật", 2}, + {"leading and trailing space", " pháp luật ", "pháp luật", 2}, + {"repeated spaces", "pháp luật", "pháp luật", 2}, + {"tab separator", "pháp\tluật", "pháp luật", 2}, + {"non-breaking space", "pháp luật", "pháp luật", 2}, + {"newline separator", "pháp\nluật", "pháp luật", 2}, + {"three syllables", "vô tuyến điện", "vô tuyến điện", 3}, + {"four syllables", "công nghiệp hóa hiện", "công nghiệp hóa hiện", 4}, + {"single syllable passes through", "pháp", "pháp", 1}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + word, syllables, err := Normalize(tc.raw) + if err != nil { + t.Fatalf("Normalize(%q) returned error: %v", tc.raw, err) + } + if word != tc.want { + t.Errorf("word = %q, want %q", word, tc.want) + } + if len(syllables) != tc.syllables { + t.Errorf("got %d syllables, want %d", len(syllables), tc.syllables) + } + }) + } +} + +// A word typed on one keyboard can arrive decomposed and on another composed. +// Both must normalize to the same string, or database lookups fail for reasons +// invisible to the player. +func TestNormalizeUnicodeEquivalence(t *testing.T) { + composed := "ngôn ngữ" // ữ as a single codepoint + decomposed := "ngôn ngữ" // o+circumflex, u+horn+tilde + + gotComposed, _, err := Normalize(composed) + if err != nil { + t.Fatal(err) + } + gotDecomposed, _, err := Normalize(decomposed) + if err != nil { + t.Fatal(err) + } + + if gotComposed != gotDecomposed { + t.Errorf("composed %q and decomposed %q normalized differently: %q vs %q", + composed, decomposed, gotComposed, gotDecomposed) + } +} + +func TestNormalizeEmpty(t *testing.T) { + for _, raw := range []string{"", " ", "\t\n", " "} { + if _, _, err := Normalize(raw); !errors.Is(err, ErrEmpty) { + t.Errorf("Normalize(%q) error = %v, want ErrEmpty", raw, err) + } + } +} + +func TestHasEnoughSyllables(t *testing.T) { + tests := []struct { + syllables []string + want bool + }{ + {[]string{"pháp"}, false}, + {[]string{"pháp", "luật"}, true}, + {[]string{"vô", "tuyến", "điện"}, true}, + {nil, false}, + } + + for _, tc := range tests { + if got := HasEnoughSyllables(tc.syllables); got != tc.want { + t.Errorf("HasEnoughSyllables(%v) = %v, want %v", tc.syllables, got, tc.want) + } + } +}