diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1d0073f --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +# Dictionary data — never committed. +# data/dictionary.db is the 179 MB upstream asset; data/noitu.db is derived from it. +# Both are build artifacts produced by `make fetch-dict` and `make dict`. +data/*.db +data/*.db-journal +data/*.db-wal +data/*.db-shm + +# Go build output +/noitu-server +/noitu-server.exe +/build-dictionary +/build-dictionary.exe +*.test +*.out + +# Node / SvelteKit +web/node_modules/ +web/build/ +web/.svelte-kit/ +node_modules/ + +# Playwright +/test-results/ +/playwright-report/ + +# Editor / OS +.DS_Store +Thumbs.db +.idea/ +.vscode/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d7312c0 --- /dev/null +++ b/Makefile @@ -0,0 +1,63 @@ +# noitu — build targets. +# +# Every target has a raw equivalent documented in README.md, so contributors +# without `make` (notably on Windows) are never blocked. + +DICT_URL := https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db +DICT_SRC := data/dictionary.db +# Pinned to the v2.0.0 release asset. A mismatch means the upstream artifact +# changed under the same tag, or the download was truncated. +DICT_SHA256 := 9259403f0675b2991a1bd0ef6d0dbc5933afdb135632af095a60662f09bbf1d3 +DICT_OUT := data/noitu.db +SERVER_BIN := noitu-server + +.PHONY: help fetch-dict verify-dict dict server web test test-go test-web run clean + +help: + @echo "fetch-dict download + checksum the upstream dictionary (~179 MB) into data/" + @echo "verify-dict re-check the downloaded dictionary against its pinned SHA-256" + @echo "dict derive $(DICT_OUT) from $(DICT_SRC)" + @echo "server build the Go server binary" + @echo "web build the SvelteKit frontend" + @echo "test run all tests" + @echo "run build and run the server locally" + @echo "clean remove build artifacts (keeps downloaded dictionary)" + +# One-time download. Resumable (-C -) so an interrupted 179 MB fetch can continue. +fetch-dict: + @mkdir -p data + curl -L -C - -o $(DICT_SRC) $(DICT_URL) + @echo "$(DICT_SHA256) $(DICT_SRC)" | sha256sum -c - + @echo "downloaded and verified $(DICT_SRC)" + +# Verify an already-downloaded copy without re-fetching. +verify-dict: + @echo "$(DICT_SHA256) $(DICT_SRC)" | sha256sum -c - + +$(DICT_SRC): + @echo "$(DICT_SRC) not found — run 'make fetch-dict' first" >&2 + @exit 1 + +dict: $(DICT_SRC) + cd server && go run ./cmd/build-dictionary --in ../$(DICT_SRC) --out ../$(DICT_OUT) + +server: + cd server && CGO_ENABLED=0 go build -o ../$(SERVER_BIN) ./cmd/noitu-server + +web: + cd web && npm ci && npm run build + +test: test-go test-web + +test-go: + cd server && go vet ./... && go test ./... -race + +test-web: + @if [ -d web/node_modules ]; then cd web && npm test; else echo "web/ not set up yet, skipping"; fi + +run: server + ./$(SERVER_BIN) + +clean: + rm -f $(SERVER_BIN) $(SERVER_BIN).exe $(DICT_OUT) + rm -rf web/build web/.svelte-kit diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..3674af5 --- /dev/null +++ b/NOTICE @@ -0,0 +1,37 @@ +noitu — Vietnamese "nối từ" word-chain game +Copyright 2026 the noitu authors + +This product is distributed under two licenses, applying to different artifacts. + +------------------------------------------------------------------------------ +1. SOURCE CODE — Apache License 2.0 +------------------------------------------------------------------------------ + +All source code in this repository is licensed under the Apache License, +Version 2.0. See the LICENSE file at the repository root. + +This covers everything under server/, tools/, web/, and proto/. + +------------------------------------------------------------------------------ +2. DICTIONARY DATA — Creative Commons Attribution-ShareAlike 4.0 International +------------------------------------------------------------------------------ + +The Vietnamese dictionary database is NOT covered by the Apache License. It is +derived from a third-party dataset and remains under CC BY-SA 4.0: + + Affected artifacts: data/noitu.db (and data/dictionary.db, its source) + License text: data/LICENSE + Attribution and + list of changes: data/ATTRIBUTION.md + Upstream source: https://github.com/minhqnd/dictionary (release v2.0.0) + +CC BY-SA 4.0 is a share-alike license. Any distribution of the derived database +-- including inside a container image or other packaged build -- must carry the +same license, this attribution, and the record of modifications. + +The two regimes are kept on separate artifacts deliberately: the database is +loaded at runtime from a file and is never embedded, compiled, or linked into +the Go binary. + +Neither database file is committed to version control. Both are build artifacts +produced by `make fetch-dict` and `make dict`. diff --git a/README.md b/README.md index 84601c4..2a63db4 100644 --- a/README.md +++ b/README.md @@ -1 +1,95 @@ -# noitu \ No newline at end of file +# noitu + +Trò chơi **nối từ** tiếng Việt trên web — chơi với máy hoặc đấu 1v1 trực tuyến. + +A web implementation of the Vietnamese word-chain game *nối từ*: each player submits a +meaningful word of **at least 2 syllables** whose **first syllable matches the last syllable +of the previous word**. No word may be reused. Fail to answer in time and you lose. + +``` +ngôn ngữ → ngữ pháp → pháp luật → luật lệ → ... +``` + +## Status + +In development. See [`plans/260904-1125-noi-tu-web-game/plan.md`](./plans/260904-1125-noi-tu-web-game/plan.md) +for the implementation plan and phase breakdown. + +## Architecture + +The server is authoritative: the browser never holds the wordlist, so word validation +cannot be bypassed, and the bot and player-vs-player paths share one rule implementation. + +``` +SvelteKit SPA ──WebSocket + Protobuf──► Go server ──► SQLite dictionary +``` + +| Component | Choice | +|---|---| +| Frontend | SvelteKit 2 / Svelte 5 (JavaScript), `adapter-static` | +| Transport | WebSocket, Protobuf binary frames (`@bufbuild/protobuf` ↔ `protoc-gen-go`) | +| Backend | Go, [`coder/websocket`](https://github.com/coder/websocket) | +| Dictionary | SQLite via `modernc.org/sqlite` (CGo-free), read-only at runtime | + +## Setup + +Requires Go 1.25+, Node 20+, and optionally `make`. + +```sh +make fetch-dict # one-time: downloads the ~179 MB upstream dictionary into data/ +make dict # derives data/noitu.db (the game's wordlist) from it +make test # run all tests +make run # build and start the server +``` + +`make fetch-dict` is a one-time cost per machine. Neither database file is committed; +both are build artifacts. See [`data/ATTRIBUTION.md`](./data/ATTRIBUTION.md). + +## Make targets + +| Target | Does | +|---|---| +| `fetch-dict` | Download the upstream `dictionary.db` (~179 MB) into `data/` | +| `dict` | Derive `data/noitu.db` from the upstream database | +| `server` | Build the Go server binary | +| `web` | Build the SvelteKit frontend to static assets | +| `test` | Run Go and JavaScript tests | +| `run` | Build and run the server locally | +| `verify-dict` | Re-check the downloaded dictionary against its pinned SHA-256 | + +### Without `make` + +`make` is not installed everywhere (notably Windows). Every target is a thin wrapper: + +```sh +# fetch-dict +curl -L -C - -o data/dictionary.db https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db + +# dict +cd server && go run ./cmd/build-dictionary --in ../data/dictionary.db --out ../data/noitu.db + +# test +cd server && go vet ./... && go test ./... -race + +# server +cd server && CGO_ENABLED=0 go build -o ../noitu-server ./cmd/noitu-server +``` + +## License + +This project is distributed under **two licenses**, applying to different artifacts. +See [`NOTICE`](./NOTICE) for the full statement. + +| Artifact | License | +|---|---| +| All source code (`server/`, `web/`, `proto/`) | [Apache-2.0](./LICENSE) | +| Dictionary data (`data/noitu.db`) | [CC BY-SA 4.0](./data/LICENSE) | + +The dictionary is derived from [minhqnd/dictionary](https://github.com/minhqnd/dictionary) +(data licensed CC BY-SA 4.0), which itself aggregates Wiktionary and other Vietnamese +dictionary sources. CC BY-SA is a **share-alike** license: any redistribution of the derived +database — including inside a container image — must carry the same license, the attribution, +and the record of modifications recorded in [`data/ATTRIBUTION.md`](./data/ATTRIBUTION.md). + +The database is loaded at runtime from a file and is never embedded or linked into the Go +binary, keeping the two licensing regimes on separate artifacts. diff --git a/data/ATTRIBUTION.md b/data/ATTRIBUTION.md new file mode 100644 index 0000000..0d87958 --- /dev/null +++ b/data/ATTRIBUTION.md @@ -0,0 +1,71 @@ +# Dictionary Data Attribution + +The Vietnamese dictionary data used by this game is **not** original work of this project. +It is derived from a third-party dataset licensed under **CC BY-SA 4.0**, and this file +records the attribution and the modifications required by that license. + +## Source + +| Field | Value | +|---|---| +| Project | [minhqnd/dictionary](https://github.com/minhqnd/dictionary) | +| Asset | [`dictionary.db`, release v2.0.0](https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db) (~179 MB) | +| Author | [minhqnd](https://github.com/minhqnd) | +| Data license | [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/) — full text in [`LICENSE`](./LICENSE) | +| Code license (not used here) | MIT | + +That project is itself an aggregation. Its own upstream sources include +[Wiktionary](https://vi.wiktionary.org/) (CC BY-SA) and +[vntk/dictionary](https://github.com/vntk/dictionary), among other Vietnamese dictionary +projects. Those upstream attributions carry through this file. + +## Modifications made by this project + +`server/cmd/build-dictionary` transforms the upstream `dictionary.db` into `data/noitu.db`. +The derived database is a **modified version** of the source data. Changes: + +1. **Language filter** — kept only entries with `lang_code = 'vi'`; all other languages + (of 1,500+ language pairs in the source) were dropped. +2. **Length filter** — kept only words of **2 or more space-separated syllables**, as + required by the nối từ game rules. Single-syllable entries were dropped. +3. **Content filter** — dropped entries containing digits or punctuation, entries using + letters Vietnamese does not have (f, j, w, z), and entries whose syllables do not fit + Vietnamese phonotactics (a closed inventory of onsets, nuclei and codas). This removes + loanwords and foreign phrases such as "credit card" and "world cup" that the + multilingual source tags as Vietnamese. Diacritic-free Vietnamese words ("con cua") + are kept. +4. **Normalization** — all words Unicode NFC-normalized, lowercased, and whitespace-collapsed. +5. **Spelling aliases** — added an `aliases` table mapping alternative Vietnamese spellings + to canonical entries. Two kinds: competing tone placement in open oa/oe/uy syllables + (`hoà` → `hòa`, `thuý` → `thúy`), and i/y alternation in Sino-Vietnamese syllables + (`quí` → `quý`, `lí` → `lý`). The majority are the i/y kind. These aliases are generated + by this project and are not present upstream. +6. **Added columns and tables** — `first` and `last` syllable columns, a `syllables` count, + an index on `first`, a `syllables` out-degree table, and a `meta` table recording + provenance. All added for game lookups. +7. **Deduplication** — the source lists a word once per sense; entries were deduplicated by + normalized form. A generated spelling variant that is itself a real word, or that more + than one word would claim, is discarded rather than recorded as an alias. +8. **Dropped fields** — all definitions, translations, pronunciations, examples, part-of-speech + tags, and relations from the source were discarded. The derived database contains + **only word forms**, not meanings. + +## Share-alike obligation + +CC BY-SA 4.0 is a **share-alike** license. The derived database `data/noitu.db`, and any +distribution of it, remains licensed under **CC BY-SA 4.0** — including when it is shipped +inside a container image or any other packaged build of this project. + +This obligation applies to the **data only**. The source code of this project is licensed +separately under Apache-2.0 (see the repository root `LICENSE` and `NOTICE`). The derived +database is loaded at runtime from a file and is never compiled or linked into the binary, +keeping the two licensing regimes on separate artifacts. + +## How to reproduce the derived data + +```sh +make fetch-dict # downloads the ~179 MB upstream dictionary.db into data/ +make dict # derives data/noitu.db from it +``` + +Neither file is committed to version control; both are build artifacts. diff --git a/data/LICENSE b/data/LICENSE new file mode 100644 index 0000000..2d58298 --- /dev/null +++ b/data/LICENSE @@ -0,0 +1,428 @@ +Attribution-ShareAlike 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution-ShareAlike 4.0 International Public +License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-ShareAlike 4.0 International Public License ("Public +License"). To the extent this Public License may be interpreted as a +contract, You are granted the Licensed Rights in consideration of Your +acceptance of these terms and conditions, and the Licensor grants You +such rights in consideration of benefits the Licensor receives from +making the Licensed Material available under these terms and +conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. BY-SA Compatible License means a license listed at + creativecommons.org/compatiblelicenses, approved by Creative + Commons as essentially the equivalent of this Public License. + + d. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + e. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + f. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + g. License Elements means the license attributes listed in the name + of a Creative Commons Public License. The License Elements of this + Public License are Attribution and ShareAlike. + + h. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + i. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + j. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + k. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + l. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + m. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. Additional offer from the Licensor -- Adapted Material. + Every recipient of Adapted Material from You + automatically receives an offer from the Licensor to + exercise the Licensed Rights in the Adapted Material + under the conditions of the Adapter's License You apply. + + c. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + b. ShareAlike. + + In addition to the conditions in Section 3(a), if You Share + Adapted Material You produce, the following conditions also apply. + + 1. The Adapter's License You apply must be a Creative Commons + license with the same License Elements, this version or + later, or a BY-SA Compatible License. + + 2. You must include the text of, or the URI or hyperlink to, the + Adapter's License You apply. You may satisfy this condition + in any reasonable manner based on the medium, means, and + context in which You Share Adapted Material. + + 3. You may not offer or impose any additional or different terms + or conditions on, or apply any Effective Technological + Measures to, Adapted Material that restrict exercise of the + rights granted under the Adapter's License You apply. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material, + including for purposes of Section 3(b); and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. + diff --git a/server/cmd/build-dictionary/aliases.go b/server/cmd/build-dictionary/aliases.go new file mode 100644 index 0000000..25c7455 --- /dev/null +++ b/server/cmd/build-dictionary/aliases.go @@ -0,0 +1,287 @@ +package main + +import ( + "strings" + + "golang.org/x/text/unicode/norm" +) + +// Vietnamese combining tone marks. These are the five tones; they are distinct +// from the combining marks that form letters (circumflex, breve, horn), which +// belong to their base letter and must never be moved. +var toneMarks = map[rune]bool{ + '̀': true, // huyền (grave) + '́': true, // sắc (acute) + '̃': true, // ngã (tilde) + '̉': true, // hỏi (hook above) + '̣': true, // nặng (dot below) +} + +// Marks that turn a base letter into a different Vietnamese letter (ă â ê ô ơ ư). +// A vowel carrying one of these is not a plain vowel and is excluded from the +// tone-shifting rule below. +var letterMarks = map[rune]bool{ + '̂': true, // circumflex: â ê ô + '̆': true, // breve: ă + '̛': true, // horn: ơ ư +} + +// cluster is one base rune plus the combining marks attached to it. +type cluster struct { + base rune + mods []rune +} + +func (c cluster) tone() (rune, bool) { + for _, m := range c.mods { + if toneMarks[m] { + return m, true + } + } + return 0, false +} + +func (c cluster) hasLetterMark() bool { + for _, m := range c.mods { + if letterMarks[m] { + return true + } + } + return false +} + +func (c cluster) without(target rune) cluster { + out := cluster{base: c.base} + for _, m := range c.mods { + if m != target { + out.mods = append(out.mods, m) + } + } + return out +} + +func (c cluster) with(extra rune) cluster { + out := cluster{base: c.base, mods: append([]rune{}, c.mods...)} + out.mods = append(out.mods, extra) + return out +} + +// decompose splits a syllable into base-plus-marks clusters, working on the NFD +// form so that every diacritic is visible as its own rune. +func decompose(syllable string) []cluster { + var clusters []cluster + for _, r := range norm.NFD.String(syllable) { + if (toneMarks[r] || letterMarks[r]) && len(clusters) > 0 { + last := len(clusters) - 1 + clusters[last].mods = append(clusters[last].mods, r) + continue + } + clusters = append(clusters, cluster{base: r}) + } + return clusters +} + +func recompose(clusters []cluster) string { + var b strings.Builder + for _, c := range clusters { + b.WriteRune(c.base) + for _, m := range c.mods { + b.WriteRune(m) + } + } + return norm.NFC.String(b.String()) +} + +// shiftablePairs are the vowel clusters where Vietnamese has two competing tone +// placements in real use: the older style marks the first vowel ("hòa", "thúy"), +// the newer style marks the second ("hoà", "thuý"). Both spellings appear in +// dictionaries and both are typed by players, but they are different codepoint +// sequences, so one of them fails a lookup unless we record it as an alias. +var shiftablePairs = [][2]rune{ + {'o', 'a'}, + {'o', 'e'}, + {'u', 'y'}, +} + +// toneShiftVariant moves the tone mark to the other vowel of an oa/oe/uy cluster. +// Returns the variant and whether one was produced. +func toneShiftVariant(syllable string) (string, bool) { + clusters := decompose(syllable) + + for i := 0; i+1 < len(clusters); i++ { + first, second := clusters[i], clusters[i+1] + + // Only plain vowels take part; ơ/ư/ô/ê/ă follow different rules. + if first.hasLetterMark() || second.hasLetterMark() { + continue + } + if !isShiftablePair(first.base, second.base) { + continue + } + // The two placements only compete in an OPEN syllable that ends on the + // cluster. Add anything after it -- a final consonant ("hoàn") or a + // third vowel ("hoài") -- and only one spelling is correct. Without this + // check the builder invents words like "hòan" and "hòai". + if i+1 != len(clusters)-1 { + continue + } + // In "qu" the u is part of the consonant onset, not the vowel nucleus, + // so "quý" must not become "qúy". + if first.base == 'u' && i > 0 && clusters[i-1].base == 'q' { + continue + } + + if tone, ok := first.tone(); ok { + shifted := append([]cluster{}, clusters...) + shifted[i] = first.without(tone) + shifted[i+1] = second.with(tone) + return recompose(shifted), true + } + if tone, ok := second.tone(); ok { + shifted := append([]cluster{}, clusters...) + shifted[i] = first.with(tone) + shifted[i+1] = second.without(tone) + return recompose(shifted), true + } + } + + return "", false +} + +func isShiftablePair(a, b rune) bool { + for _, p := range shiftablePairs { + if p[0] == a && p[1] == b { + return true + } + } + return false +} + +// iyOnsets are the consonant onsets after which Vietnamese genuinely writes +// both i and y across the tone range: hi/hy, ki/ky, li/ly, mi/my, ti/ty, qui/quy. +// +// This is an allowlist rather than a general rule because most onsets take only +// one spelling. "gi" is a digraph, so "gì" has no "gỳ"; "chì" has no "chỳ". +// +// "v" and "s" were deliberately removed after inspecting real output: they +// produced non-words at scale ("chức vị" -> "chức vỵ", plus "vỳ", "vý", "sỵ", +// "sỳ"). Their genuine alternations are lexical rather than productive, so the +// few real ones are listed in iyPairs instead. +var iyOnsets = map[string]bool{ + "h": true, "k": true, "l": true, "m": true, "t": true, "qu": true, +} + +// iyPairs are individual syllables that alternate even though their onset does +// not do so productively. Listed explicitly, both directions. +var iyPairs = map[string]string{ + "sĩ": "sỹ", "sỹ": "sĩ", + "vĩ": "vỹ", "vỹ": "vĩ", +} + +// iyVariant swaps a lone i/y nucleus: quý/quí, lý/lí, mỹ/mĩ. +// +// The nucleus must be the final cluster, so closed syllables are excluded +// ("tính" has no "týnh"), and the onset must be one that takes both spellings. +func iyVariant(syllable string) (string, bool) { + if pair, ok := iyPairs[syllable]; ok { + return pair, true + } + + clusters := decompose(syllable) + if len(clusters) < 2 { + return "", false + } + + last := len(clusters) - 1 + nucleus := clusters[last] + if nucleus.hasLetterMark() { + return "", false + } + + var swapped rune + switch nucleus.base { + case 'i': + swapped = 'y' + case 'y': + swapped = 'i' + default: + return "", false + } + + var onset strings.Builder + for _, c := range clusters[:last] { + if c.hasLetterMark() { + return "", false + } + onset.WriteRune(c.base) + } + if !iyOnsets[onset.String()] { + return "", false + } + + variant := append([]cluster{}, clusters...) + variant[last] = cluster{base: swapped, mods: nucleus.mods} + return recompose(variant), true +} + +// maxVariantsPerWord caps the combination count for a word whose syllables are +// all variable. Real words reach at most a handful; the cap only guards against +// a pathological entry blowing up the alias table. +const maxVariantsPerWord = 32 + +// spellingsOf returns every accepted spelling of a single syllable, the +// original first. +func spellingsOf(syllable string) []string { + out := []string{syllable} + seen := map[string]bool{syllable: true} + + for _, gen := range []func(string) (string, bool){toneShiftVariant, iyVariant} { + alt, ok := gen(syllable) + if !ok || seen[alt] { + continue + } + seen[alt] = true + out = append(out, alt) + } + + return out +} + +// variantsFor returns every accepted alternative spelling of a whole word. +// +// Syllables must be varied together, not one at a time. "hoá lí" has two +// variable syllables, and a player is far more likely to type the fully modern +// "hóa lý" than either half-and-half form. Varying independently produced only +// the halves and rejected the spelling most people actually use. +func variantsFor(word string) []string { + syllables := strings.Split(word, " ") + + // Cartesian product over each syllable's accepted spellings. + combos := [][]string{{}} + for _, syl := range syllables { + spellings := spellingsOf(syl) + next := make([][]string, 0, len(combos)*len(spellings)) + for _, prefix := range combos { + for _, s := range spellings { + if len(next) >= maxVariantsPerWord { + break + } + next = append(next, append(append([]string{}, prefix...), s)) + } + } + combos = next + } + + seen := map[string]bool{word: true} + var out []string + for _, combo := range combos { + joined := strings.Join(combo, " ") + if seen[joined] { + continue + } + seen[joined] = true + out = append(out, joined) + } + + return out +} diff --git a/server/cmd/build-dictionary/aliases_test.go b/server/cmd/build-dictionary/aliases_test.go new file mode 100644 index 0000000..48e093b --- /dev/null +++ b/server/cmd/build-dictionary/aliases_test.go @@ -0,0 +1,208 @@ +package main + +import ( + "sort" + "testing" +) + +// Vietnamese writes the same syllable with the tone mark on either vowel of an +// oa/oe/uy cluster. Both spellings are typed by real players, so both must +// resolve to one dictionary entry. +func TestToneShiftVariant(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"oa, tone on first", "hòa", "hoà"}, + {"oa, tone on second", "hoà", "hòa"}, + {"oe, tone on first", "khỏe", "khoẻ"}, + {"oe, tone on second", "khoẻ", "khỏe"}, + {"uy, tone on first", "thúy", "thuý"}, + {"uy, tone on second", "thuý", "thúy"}, + {"nang tone on oa", "họa", "hoạ"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, ok := toneShiftVariant(tc.in) + if !ok { + t.Fatalf("toneShiftVariant(%q) produced no variant, want %q", tc.in, tc.want) + } + if got != tc.want { + t.Errorf("toneShiftVariant(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +func TestToneShiftVariantRejects(t *testing.T) { + tests := []struct{ in, why string }{ + {"quý", `"qu" is a consonant onset, not a vowel cluster — no "qúy"`}, + {"hoàn", `closed syllable: final consonant, so no "hòan"`}, + {"roàng", `closed syllable: final "ng", so no "ròang"`}, + {"hoài", `triphthong "oai", so no "hòai"`}, + {"khoẻn", `closed "oe" syllable`}, + {"quả", "no shiftable cluster"}, + {"luật", "no shiftable cluster"}, + {"pháp", "no shiftable cluster"}, + {"ngữ", "no shiftable cluster"}, + {"điện", "no shiftable cluster"}, + {"tính", "no shiftable cluster"}, + } + + for _, tc := range tests { + if got, ok := toneShiftVariant(tc.in); ok { + t.Errorf("toneShiftVariant(%q) = %q, want no variant (%s)", tc.in, got, tc.why) + } + } +} + +func TestIYVariant(t *testing.T) { + tests := []struct{ in, want string }{ + {"quý", "quí"}, + {"quí", "quý"}, + {"lý", "lí"}, + {"mỹ", "mĩ"}, + {"kỹ", "kĩ"}, + {"sĩ", "sỹ"}, + {"tỷ", "tỉ"}, + {"hy", "hi"}, + } + + for _, tc := range tests { + got, ok := iyVariant(tc.in) + if !ok { + t.Errorf("iyVariant(%q) produced no variant, want %q", tc.in, tc.want) + continue + } + if got != tc.want { + t.Errorf("iyVariant(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestIYVariantRejects(t *testing.T) { + // "gi" and "ch" are digraphs that only ever take i; closed syllables and + // diphthongs are out of scope entirely. + for _, in := range []string{"gì", "chì", "tính", "kia", "đi", "nghi", "phi", "bi"} { + if got, ok := iyVariant(in); ok { + t.Errorf("iyVariant(%q) = %q, want no variant", in, got) + } + } +} + +func TestVariantsFor(t *testing.T) { + got := variantsFor("hòa bình") + want := []string{"hoà bình"} + assertSameStrings(t, got, want) + + got = variantsFor("pháp luật") + if len(got) != 0 { + t.Errorf("variantsFor(%q) = %v, want none", "pháp luật", got) + } +} + +// A word with two variable syllables must offer every combination, not just the +// half-and-half forms. "hoá lí" is the corpus spelling but "hóa lý" is the one +// most people type, and varying syllables independently never produced it. +func TestVariantsForVariesSyllablesTogether(t *testing.T) { + // "hoá" carries the acute tone, so shifting it gives "hóa" (not "hòa"). + got := variantsFor("hoá lí") + want := []string{"hóa lí", "hoá lý", "hóa lý"} + assertSameStrings(t, got, want) +} + +func TestVariantsForCapsCombinations(t *testing.T) { + // Every syllable variable: 2^6 = 64 combinations before the cap. + word := "hoá hoá hoá hoá hoá hoá" + if got := len(variantsFor(word)); got > maxVariantsPerWord { + t.Errorf("variantsFor produced %d variants, want at most %d", got, maxVariantsPerWord) + } +} + +// "v" and "s" do not alternate productively: "chức vị" must never yield +// "chức vỵ". Only the handful of genuinely alternating syllables are listed. +func TestIYRejectsNonAlternatingOnsets(t *testing.T) { + for _, in := range []string{"vị", "vì", "vỷ", "sị", "sỳ", "si"} { + if got, ok := iyVariant(in); ok { + t.Errorf("iyVariant(%q) = %q, want no variant", in, got) + } + } +} + +func TestIYExplicitPairs(t *testing.T) { + for in, want := range map[string]string{"sĩ": "sỹ", "sỹ": "sĩ", "vĩ": "vỹ", "vỹ": "vĩ"} { + got, ok := iyVariant(in) + if !ok { + t.Errorf("iyVariant(%q) produced no variant, want %q", in, want) + continue + } + if got != want { + t.Errorf("iyVariant(%q) = %q, want %q", in, got, want) + } + } +} + +// With three words claiming one variant, a plain delete would let the third +// reinsert it and the winner would depend on map iteration order. +func TestBuildAliasesPoisonsAmbiguousVariants(t *testing.T) { + words := map[string]entry{ + "lí do": {word: "lí do", first: "lí", last: "do", syllables: 2}, + "lý do": {word: "lý do", first: "lý", last: "do", syllables: 2}, + } + + for i := 0; i < 20; i++ { + aliases, _ := buildAliases(words) + if canonical, ok := aliases["lí do"]; ok { + t.Fatalf("run %d: real word aliased to %q", i, canonical) + } + if canonical, ok := aliases["lý do"]; ok { + t.Fatalf("run %d: real word aliased to %q", i, canonical) + } + } +} + +// A generated variant that is itself a real word must not become an alias: +// doing so would let one entry answer to another entry's name. +func TestBuildAliasesSkipsCollisions(t *testing.T) { + words := map[string]entry{ + "hòa bình": {word: "hòa bình", first: "hòa", last: "bình", syllables: 2}, + "hoà bình": {word: "hoà bình", first: "hoà", last: "bình", syllables: 2}, + } + + aliases, collisions := buildAliases(words) + + if len(aliases) != 0 { + t.Errorf("aliases = %v, want none (both spellings are real words)", aliases) + } + if collisions != 2 { + t.Errorf("collisions = %d, want 2", collisions) + } +} + +func TestBuildAliasesMapsToCanonical(t *testing.T) { + words := map[string]entry{ + "hòa bình": {word: "hòa bình", first: "hòa", last: "bình", syllables: 2}, + } + + aliases, _ := buildAliases(words) + + if got := aliases["hoà bình"]; got != "hòa bình" { + t.Errorf("aliases[%q] = %q, want %q", "hoà bình", got, "hòa bình") + } +} + +func assertSameStrings(t *testing.T, got, want []string) { + t.Helper() + sort.Strings(got) + sort.Strings(want) + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for i := range got { + if got[i] != want[i] { + t.Fatalf("got %v, want %v", got, want) + } + } +} diff --git a/server/cmd/build-dictionary/filter.go b/server/cmd/build-dictionary/filter.go new file mode 100644 index 0000000..a680a11 --- /dev/null +++ b/server/cmd/build-dictionary/filter.go @@ -0,0 +1,100 @@ +package main + +import ( + "strings" + "unicode" + + "golang.org/x/text/unicode/norm" + + "github.com/tiennm99dev/noitu/server/internal/vietnamese" +) + +// rejectReason explains why a candidate word did not make it into the game +// dictionary. Counted per reason so the build log shows what the source +// actually contained rather than just a final total. +type rejectReason string + +const ( + rejectEmpty rejectReason = "empty" + rejectTooShort rejectReason = "fewer than 2 syllables" + rejectTooLong rejectReason = "over max-syllables" + rejectDigit rejectReason = "contains a digit" + rejectPunct rejectReason = "contains punctuation" + rejectNonVietnam rejectReason = "no Vietnamese letters" +) + +// accept normalizes a raw source entry and decides whether it belongs in the +// game dictionary. maxSyllables of 0 means no upper bound. +func accept(raw string, maxSyllables int) (word string, syllables []string, reason rejectReason, ok bool) { + word, syllables, err := vietnamese.Normalize(raw) + if err != nil { + return "", nil, rejectEmpty, false + } + + if !vietnamese.HasEnoughSyllables(syllables) { + return "", nil, rejectTooShort, false + } + if maxSyllables > 0 && len(syllables) > maxSyllables { + return "", nil, rejectTooLong, false + } + + for _, r := range word { + switch { + case r == ' ': + continue + case unicode.IsDigit(r): + return "", nil, rejectDigit, false + case unicode.IsPunct(r) || unicode.IsSymbol(r): + return "", nil, rejectPunct, false + case !unicode.IsLetter(r): + return "", nil, rejectPunct, false + } + } + + if !usesVietnameseAlphabet(word) { + return "", nil, rejectNonVietnam, false + } + + // Every syllable must also be a possible Vietnamese syllable. The alphabet + // check alone lets "credit card" and "come out" through, since they use only + // letters Vietnamese has. + for _, syl := range syllables { + if !isVietnameseSyllable(syl) { + return "", nil, rejectNonVietnam, false + } + } + + return word, syllables, "", true +} + +// vietnameseBaseLetters is the Vietnamese alphabet reduced to base letters: +// the Latin alphabet without f, j, w, z, plus đ. Every Vietnamese word is +// spelled from these once diacritics are stripped. +const vietnameseBaseLetters = "abcdeghiklmnopqrstuvxyđ" + +// usesVietnameseAlphabet reports whether every letter of the word is a +// Vietnamese one. +// +// Diacritics are stripped first (NFD, dropping combining marks) so that "ngữ" +// reduces to "ngu" and is checked letter by letter. Checking the base letters +// rather than merely "does it contain a diacritic" is what keeps English out: +// an entry only needs one accented character to look Vietnamese otherwise, and +// entries like "thành phố new york", "credit card" and "come out" pass such a +// test while being unplayable as Vietnamese words. +func usesVietnameseAlphabet(word string) bool { + stripped := norm.NFD.String(word) + + sawLetter := false + for _, r := range stripped { + if unicode.Is(unicode.Mn, r) || r == ' ' { + // Combining mark or separator: not a base letter. + continue + } + if !strings.ContainsRune(vietnameseBaseLetters, r) { + return false + } + sawLetter = true + } + + return sawLetter +} diff --git a/server/cmd/build-dictionary/filter_test.go b/server/cmd/build-dictionary/filter_test.go new file mode 100644 index 0000000..2024e6f --- /dev/null +++ b/server/cmd/build-dictionary/filter_test.go @@ -0,0 +1,85 @@ +package main + +import "testing" + +func TestAcceptKeepsValidWords(t *testing.T) { + tests := []struct { + raw string + want string + syllables int + }{ + {"pháp luật", "pháp luật", 2}, + {"Pháp Luật", "pháp luật", 2}, + {"vô tuyến điện", "vô tuyến điện", 3}, + {"con cua", "con cua", 2}, // no diacritics, still Vietnamese + {"đi đứng", "đi đứng", 2}, // đ is a Vietnamese letter + {"quy hoạch", "quy hoạch", 2}, + } + + for _, tc := range tests { + word, syllables, reason, ok := accept(tc.raw, 0) + if !ok { + t.Errorf("accept(%q) rejected: %s", tc.raw, reason) + continue + } + if word != tc.want { + t.Errorf("accept(%q) word = %q, want %q", tc.raw, word, tc.want) + } + if len(syllables) != tc.syllables { + t.Errorf("accept(%q) syllables = %d, want %d", tc.raw, len(syllables), tc.syllables) + } + } +} + +func TestAcceptRejects(t *testing.T) { + tests := []struct { + name string + raw string + want rejectReason + }{ + {"empty", " ", rejectEmpty}, + {"single syllable", "pháp", rejectTooShort}, + {"digit", "covid 19", rejectDigit}, + {"hyphen", "pháp-luật xxx", rejectPunct}, + {"english phrase", "hello world", rejectNonVietnam}, + {"letters Vietnamese lacks", "jazz music", rejectNonVietnam}, + // These pass a naive "contains f/j/w/z" test and a naive "has a + // diacritic" test, yet are unplayable as Vietnamese words. + {"english, no forbidden letters", "credit card", rejectNonVietnam}, + {"english, no forbidden letters 2", "dress code", rejectNonVietnam}, + {"english phrasal verb", "come out", rejectNonVietnam}, + // One Vietnamese diacritic must not launder an English word. + {"mixed English with a diacritic", "thành phố new york", rejectNonVietnam}, + {"non-Latin script", "привет мир", rejectNonVietnam}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, _, reason, ok := accept(tc.raw, 0) + if ok { + t.Fatalf("accept(%q) succeeded, want rejection %s", tc.raw, tc.want) + } + if reason != tc.want { + t.Errorf("accept(%q) reason = %s, want %s", tc.raw, reason, tc.want) + } + }) + } +} + +func TestAcceptMaxSyllables(t *testing.T) { + const raw = "công nghiệp hóa" + + if _, _, _, ok := accept(raw, 0); !ok { + t.Errorf("accept(%q, no limit) rejected, want accepted", raw) + } + if _, _, _, ok := accept(raw, 3); !ok { + t.Errorf("accept(%q, max 3) rejected, want accepted", raw) + } + _, _, reason, ok := accept(raw, 2) + if ok { + t.Fatalf("accept(%q, max 2) accepted, want rejection", raw) + } + if reason != rejectTooLong { + t.Errorf("reason = %s, want %s", reason, rejectTooLong) + } +} diff --git a/server/cmd/build-dictionary/main.go b/server/cmd/build-dictionary/main.go new file mode 100644 index 0000000..d0fbb51 --- /dev/null +++ b/server/cmd/build-dictionary/main.go @@ -0,0 +1,547 @@ +// Command build-dictionary derives the game's wordlist from the upstream +// minhqnd/dictionary SQLite database. +// +// The upstream database is ~179 MB and covers 1,500+ language pairs with full +// definitions. The game needs none of that: only Vietnamese word forms of at +// least two syllables, indexed by first and last syllable. This tool performs +// that reduction and records provenance in a meta table. +// +// The derived database is a modified version of CC BY-SA 4.0 licensed data. +// See data/ATTRIBUTION.md. +// +// Usage: +// +// go run ./cmd/build-dictionary --in ../data/dictionary.db --out ../data/noitu.db +package main + +import ( + "database/sql" + "errors" + "flag" + "fmt" + "log" + "os" + "sort" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +const ( + sourceURL = "https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db" + sourceLicense = "CC BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)" + builderVer = "1" +) + +type config struct { + in string + out string + maxSyllables int + minWords int + table string + wordCol string + langCol string + lang string +} + +func main() { + log.SetFlags(0) + + var cfg config + flag.StringVar(&cfg.in, "in", "../data/dictionary.db", "upstream dictionary.db to read") + flag.StringVar(&cfg.out, "out", "../data/noitu.db", "derived database to write") + flag.IntVar(&cfg.maxSyllables, "max-syllables", 0, "reject words longer than this (0 = no limit)") + flag.IntVar(&cfg.minWords, "min-words", 40000, "fail if fewer words survive filtering") + flag.StringVar(&cfg.table, "table", "", "source table (default: auto-detect)") + flag.StringVar(&cfg.wordCol, "word-col", "", "source word column (default: auto-detect)") + flag.StringVar(&cfg.langCol, "lang-col", "", "source language column (default: auto-detect)") + flag.StringVar(&cfg.lang, "lang", "vi", "language code to keep") + flag.Parse() + + if err := run(cfg); err != nil { + log.Fatalf("build-dictionary: %v", err) + } +} + +func run(cfg config) error { + if _, err := os.Stat(cfg.in); err != nil { + return fmt.Errorf("source database not found at %s — run 'make fetch-dict' first: %w", cfg.in, err) + } + + src, err := sql.Open("sqlite", "file:"+cfg.in+"?mode=ro") + if err != nil { + return fmt.Errorf("open source: %w", err) + } + defer src.Close() + + source, err := resolveSource(src, cfg) + if err != nil { + return err + } + log.Printf("source: %s.%s filtered by %s = %q", source.table, source.wordCol, source.langCol, cfg.lang) + + words, rejects, err := extract(src, source, cfg) + if err != nil { + return err + } + logRejects(rejects) + log.Printf("accepted %d distinct words", len(words)) + + if len(words) < cfg.minWords { + return fmt.Errorf("only %d words survived filtering, expected at least %d — "+ + "the source schema or content may have changed", len(words), cfg.minWords) + } + + aliases, collisions := buildAliases(words) + log.Printf("generated %d spelling aliases (%d skipped as ambiguous or already real words)", len(aliases), collisions) + + if err := write(cfg.out, words, aliases, source); err != nil { + return err + } + if err := verify(cfg.out, cfg.minWords); err != nil { + return fmt.Errorf("output failed verification: %w", err) + } + + log.Printf("wrote %s", cfg.out) + return nil +} + +// verify re-opens the finished database and re-checks the invariants the game +// depends on. The in-memory checks above can only prove what the builder +// intended; this proves what actually landed on disk. +func verify(path string, minWords int) error { + db, err := sql.Open("sqlite", "file:"+path+"?mode=ro") + if err != nil { + return err + } + defer db.Close() + + checks := []struct { + desc string + query string + want func(int) bool + }{ + {"words below the minimum", `SELECT COUNT(*) FROM words`, func(n int) bool { return n >= minWords }}, + {"words with fewer than 2 syllables", `SELECT COUNT(*) FROM words WHERE syllables < 2`, func(n int) bool { return n == 0 }}, + {"words with an empty first or last syllable", `SELECT COUNT(*) FROM words WHERE first = '' OR last = ''`, func(n int) bool { return n == 0 }}, + {"aliases pointing at a missing word", + `SELECT COUNT(*) FROM aliases a LEFT JOIN words w ON w.word = a.canonical WHERE w.word IS NULL`, + func(n int) bool { return n == 0 }}, + {"aliases that are themselves real words", + `SELECT COUNT(*) FROM aliases a JOIN words w ON w.word = a.variant`, + func(n int) bool { return n == 0 }}, + {"words whose first syllable is missing from the syllables table", + `SELECT COUNT(*) FROM words w LEFT JOIN syllables s ON s.syllable = w.first WHERE s.syllable IS NULL`, + func(n int) bool { return n == 0 }}, + } + + for _, c := range checks { + var n int + if err := db.QueryRow(c.query).Scan(&n); err != nil { + return fmt.Errorf("check %q: %w", c.desc, err) + } + if !c.want(n) { + return fmt.Errorf("%s: %d", c.desc, n) + } + } + + return nil +} + +// entry is one accepted word with the parts the game engine indexes on. +type entry struct { + word string + first string + last string + syllables int +} + +type sourceSpec struct { + table string + wordCol string + langCol string + maxSyllables int +} + +// quoteIdent renders a SQLite identifier. Go's %q escapes an embedded quote as +// \" but SQLite requires it doubled, so fmt.Sprintf("%q") is not correct here. +func quoteIdent(name string) string { + return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` +} + +// validateSource confirms operator-supplied identifiers actually exist. +// +// SQLite accepts a double-quoted string that matches no column as a string +// literal rather than erroring, so a typo in --word-col silently yields one +// row per distinct value of that literal. Checking up front turns a confusing +// near-empty build into a clear message. +func validateSource(db *sql.DB, spec sourceSpec) error { + tables, err := listTables(db) + if err != nil { + return err + } + if !contains(tables, spec.table) { + return fmt.Errorf("table %q not found; available: %s", spec.table, strings.Join(tables, ", ")) + } + + cols, err := listColumns(db, spec.table) + if err != nil { + return fmt.Errorf("read columns of %q: %w", spec.table, err) + } + for _, c := range []struct{ role, name string }{{"--word-col", spec.wordCol}, {"--lang-col", spec.langCol}} { + if !contains(cols, c.name) { + return fmt.Errorf("%s %q not found in table %q; available: %s", + c.role, c.name, spec.table, strings.Join(cols, ", ")) + } + } + + return nil +} + +func contains(haystack []string, needle string) bool { + for _, h := range haystack { + if h == needle { + return true + } + } + return false +} + +// resolveSource finds the table holding word forms. The upstream schema is not +// contractual — it is someone else's release artifact — so rather than hardcode +// it, look for a table carrying both a word-like and a language-like column. +// Explicit flags override detection entirely. +func resolveSource(db *sql.DB, cfg config) (sourceSpec, error) { + given := 0 + for _, f := range []string{cfg.table, cfg.wordCol, cfg.langCol} { + if f != "" { + given++ + } + } + switch { + case given == 3: + spec := sourceSpec{table: cfg.table, wordCol: cfg.wordCol, langCol: cfg.langCol, maxSyllables: cfg.maxSyllables} + if err := validateSource(db, spec); err != nil { + return sourceSpec{}, err + } + return spec, nil + case given > 0: + return sourceSpec{}, fmt.Errorf( + "--table, --word-col and --lang-col must be given together (got %d of 3); "+ + "omit all three to auto-detect", given) + } + + tables, err := listTables(db) + if err != nil { + return sourceSpec{}, err + } + + wordNames := []string{"word", "term", "headword", "text", "lemma", "entry"} + langNames := []string{"lang_code", "language", "lang", "lang_name"} + + for _, t := range tables { + cols, err := listColumns(db, t) + if err != nil { + log.Printf("warning: could not read columns of %q: %v", t, err) + continue + } + wordCol := pickColumn(cols, wordNames) + langCol := pickColumn(cols, langNames) + if wordCol != "" && langCol != "" { + return sourceSpec{table: t, wordCol: wordCol, langCol: langCol, maxSyllables: cfg.maxSyllables}, nil + } + } + + // Detection failed. Dump the schema so the fix is a single flag away rather + // than a debugging session. + var b strings.Builder + b.WriteString("could not auto-detect the source table.\n") + b.WriteString("Pass --table, --word-col and --lang-col explicitly. Schema found:\n") + for _, t := range tables { + cols, err := listColumns(db, t) + if err != nil { + continue + } + fmt.Fprintf(&b, " %s(%s)\n", t, strings.Join(cols, ", ")) + } + return sourceSpec{}, errors.New(b.String()) +} + +func listTables(db *sql.DB) ([]string, error) { + rows, err := db.Query(`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`) + if err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + defer rows.Close() + + var out []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + out = append(out, name) + } + return out, rows.Err() +} + +func listColumns(db *sql.DB, table string) ([]string, error) { + rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", quoteIdent(table))) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []string + for rows.Next() { + var ( + cid int + name, ctype string + notNull, pk int + dflt sql.NullString + ) + if err := rows.Scan(&cid, &name, &ctype, ¬Null, &dflt, &pk); err != nil { + return nil, err + } + out = append(out, name) + } + return out, rows.Err() +} + +func pickColumn(cols []string, candidates []string) string { + for _, want := range candidates { + for _, c := range cols { + if strings.EqualFold(c, want) { + return c + } + } + } + return "" +} + +// extract streams every candidate word past the filters. Results are deduped by +// normalized form, since the source lists a word once per sense. +func extract(db *sql.DB, src sourceSpec, cfg config) (map[string]entry, map[rejectReason]int, error) { + query := fmt.Sprintf("SELECT DISTINCT %s FROM %s WHERE %s = ?", + quoteIdent(src.wordCol), quoteIdent(src.table), quoteIdent(src.langCol)) + rows, err := db.Query(query, cfg.lang) + if err != nil { + return nil, nil, fmt.Errorf("query source (%s): %w", query, err) + } + defer rows.Close() + + words := make(map[string]entry) + rejects := make(map[rejectReason]int) + + for rows.Next() { + var raw sql.NullString + if err := rows.Scan(&raw); err != nil { + return nil, nil, err + } + if !raw.Valid { + rejects[rejectEmpty]++ + continue + } + + word, syllables, reason, ok := accept(raw.String, cfg.maxSyllables) + if !ok { + rejects[reason]++ + continue + } + + words[word] = entry{ + word: word, + first: syllables[0], + last: syllables[len(syllables)-1], + syllables: len(syllables), + } + } + + return words, rejects, rows.Err() +} + +func logRejects(rejects map[rejectReason]int) { + reasons := make([]string, 0, len(rejects)) + for r := range rejects { + reasons = append(reasons, string(r)) + } + sort.Strings(reasons) + for _, r := range reasons { + log.Printf(" rejected %6d: %s", rejects[rejectReason(r)], r) + } +} + +// buildAliases maps alternative spellings onto canonical words. A variant that +// happens to be a real word in its own right is dropped: accepting it would let +// one word be played under two names, or worse, silently rename it. +func buildAliases(words map[string]entry) (map[string]string, int) { + aliases := make(map[string]string) + // poisoned records variants claimed by more than one word. A plain delete + // is not enough: a third claimant would find the key free and reinsert it, + // leaving the winner decided by Go's randomized map iteration order. + poisoned := make(map[string]bool) + collisions := 0 + + for word := range words { + for _, variant := range variantsFor(word) { + if poisoned[variant] { + continue + } + if _, isRealWord := words[variant]; isRealWord { + collisions++ + continue + } + if existing, taken := aliases[variant]; taken && existing != word { + delete(aliases, variant) + poisoned[variant] = true + collisions++ + continue + } + aliases[variant] = word + } + } + + return aliases, collisions +} + +// write builds the database beside the target and renames it into place only +// after the transaction commits. Writing in place would mean that a failure +// partway through -- a full disk, an interrupt -- leaves an empty but +// syntactically valid database where a good one used to be, which the server +// would happily open and find no words in. +func write(path string, words map[string]entry, aliases map[string]string, src sourceSpec) error { + tmp := path + ".tmp" + if err := os.Remove(tmp); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove stale temp file: %w", err) + } + // On any error path the half-built file must not survive. + committed := false + defer func() { + if !committed { + os.Remove(tmp) + } + }() + + if err := writeTo(tmp, words, aliases, src); err != nil { + return err + } + + // os.Rename replaces the destination atomically on POSIX; on Windows it + // fails if the target exists, so clear it first. + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove existing output: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + return fmt.Errorf("move temp database into place: %w", err) + } + committed = true + + return nil +} + +func writeTo(path string, words map[string]entry, aliases map[string]string, src sourceSpec) error { + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + return fmt.Errorf("create output: %w", err) + } + defer db.Close() + + schema := ` +CREATE TABLE words ( + word TEXT PRIMARY KEY, + first TEXT NOT NULL, + last TEXT NOT NULL, + syllables INTEGER NOT NULL +) WITHOUT ROWID; +CREATE INDEX idx_words_first ON words(first); + +CREATE TABLE syllables ( + syllable TEXT PRIMARY KEY, + out_degree INTEGER NOT NULL +) WITHOUT ROWID; + +CREATE TABLE aliases ( + variant TEXT PRIMARY KEY, + canonical TEXT NOT NULL +) WITHOUT ROWID; + +CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); +` + if _, err := db.Exec(schema); err != nil { + return fmt.Errorf("create schema: %w", err) + } + + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + insertWord, err := tx.Prepare(`INSERT INTO words (word, first, last, syllables) VALUES (?, ?, ?, ?)`) + if err != nil { + return err + } + defer insertWord.Close() + + outDegree := make(map[string]int) + for _, e := range words { + if _, err := insertWord.Exec(e.word, e.first, e.last, e.syllables); err != nil { + return fmt.Errorf("insert word %q: %w", e.word, err) + } + outDegree[e.first]++ + // A syllable that ends a word but starts none is a dead end; record it + // with zero so the engine can find it without a second query. + if _, seen := outDegree[e.last]; !seen { + outDegree[e.last] = 0 + } + } + + insertSyllable, err := tx.Prepare(`INSERT INTO syllables (syllable, out_degree) VALUES (?, ?)`) + if err != nil { + return err + } + defer insertSyllable.Close() + for syllable, degree := range outDegree { + if _, err := insertSyllable.Exec(syllable, degree); err != nil { + return fmt.Errorf("insert syllable %q: %w", syllable, err) + } + } + + insertAlias, err := tx.Prepare(`INSERT INTO aliases (variant, canonical) VALUES (?, ?)`) + if err != nil { + return err + } + defer insertAlias.Close() + for variant, canonical := range aliases { + if _, err := insertAlias.Exec(variant, canonical); err != nil { + return fmt.Errorf("insert alias %q: %w", variant, err) + } + } + + insertMeta, err := tx.Prepare(`INSERT INTO meta (key, value) VALUES (?, ?)`) + if err != nil { + return err + } + defer insertMeta.Close() + meta := [][2]string{ + {"source_url", sourceURL}, + {"source_license", sourceLicense}, + {"attribution", "See data/ATTRIBUTION.md for required attribution and the list of modifications."}, + {"built_at", time.Now().UTC().Format(time.RFC3339)}, + {"builder_version", builderVer}, + {"word_count", fmt.Sprint(len(words))}, + {"alias_count", fmt.Sprint(len(aliases))}, + {"source_table", src.table}, + {"source_word_column", src.wordCol}, + {"source_lang_column", src.langCol}, + {"max_syllables", fmt.Sprint(src.maxSyllables)}, + } + for _, kv := range meta { + if _, err := insertMeta.Exec(kv[0], kv[1]); err != nil { + return err + } + } + + return tx.Commit() +} diff --git a/server/cmd/build-dictionary/main_test.go b/server/cmd/build-dictionary/main_test.go new file mode 100644 index 0000000..da5732a --- /dev/null +++ b/server/cmd/build-dictionary/main_test.go @@ -0,0 +1,303 @@ +package main + +import ( + "bytes" + "database/sql" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + _ "modernc.org/sqlite" +) + +// fixtureSource writes a miniature stand-in for the upstream dictionary: the +// same shape (a word column plus a language column, one row per sense) but a +// handful of rows instead of 357k. Tests never touch the real 179 MB download. +func fixtureSource(t *testing.T, rows [][2]string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "source.db") + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + if _, err := db.Exec(`CREATE TABLE entries (word TEXT, lang_code TEXT, definition TEXT)`); err != nil { + t.Fatal(err) + } + for _, r := range rows { + if _, err := db.Exec(`INSERT INTO entries (word, lang_code, definition) VALUES (?, ?, ?)`, + r[0], r[1], "unused"); err != nil { + t.Fatal(err) + } + } + return path +} + +func defaultRows() [][2]string { + return [][2]string{ + {"pháp luật", "vi"}, + {"pháp luật", "vi"}, // duplicate sense — must dedupe to one word + {"luật lệ", "vi"}, + {"ngôn ngữ", "vi"}, + {"ngữ pháp", "vi"}, + {"hòa bình", "vi"}, + {"vô tuyến điện", "vi"}, // three syllables + {"pháp", "vi"}, // single syllable — rejected + {"covid 19", "vi"}, // digit — rejected + {"hello world", "en"}, // wrong language — never selected + } +} + +func buildFixture(t *testing.T, rows [][2]string, maxSyllables int) string { + t.Helper() + + in := fixtureSource(t, rows) + out := filepath.Join(t.TempDir(), "noitu.db") + + cfg := config{ + in: in, + out: out, + maxSyllables: maxSyllables, + minWords: 1, + lang: "vi", + } + if err := run(cfg); err != nil { + t.Fatalf("run: %v", err) + } + return out +} + +func openOut(t *testing.T, path string) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite", "file:"+path+"?mode=ro") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func TestBuildProducesExpectedWords(t *testing.T) { + db := openOut(t, buildFixture(t, defaultRows(), 0)) + + var count int + if err := db.QueryRow(`SELECT COUNT(*) FROM words`).Scan(&count); err != nil { + t.Fatal(err) + } + if want := 6; count != want { + t.Errorf("word count = %d, want %d", count, want) + } + + // Every stored word must have at least two syllables. + var short int + if err := db.QueryRow(`SELECT COUNT(*) FROM words WHERE syllables < 2`).Scan(&short); err != nil { + t.Fatal(err) + } + if short != 0 { + t.Errorf("%d words have fewer than 2 syllables, want 0", short) + } + + // Three-syllable words are kept and split on first/last, not on a pair. + var first, last string + var syllables int + err := db.QueryRow(`SELECT first, last, syllables FROM words WHERE word = ?`, "vô tuyến điện"). + Scan(&first, &last, &syllables) + if err != nil { + t.Fatalf("three-syllable word missing: %v", err) + } + if first != "vô" || last != "điện" || syllables != 3 { + t.Errorf("got first=%q last=%q syllables=%d, want vô/điện/3", first, last, syllables) + } +} + +func TestBuildComputesOutDegree(t *testing.T) { + db := openOut(t, buildFixture(t, defaultRows(), 0)) + + // "pháp luật" and "pháp" (rejected) mean exactly one word starts with "pháp". + assertOutDegree(t, db, "pháp", 1) + // "luật lệ" starts with "luật". + assertOutDegree(t, db, "luật", 1) + // "bình" ends a word but starts none — a dead end, recorded as zero. + assertOutDegree(t, db, "bình", 0) +} + +func assertOutDegree(t *testing.T, db *sql.DB, syllable string, want int) { + t.Helper() + var got int + if err := db.QueryRow(`SELECT out_degree FROM syllables WHERE syllable = ?`, syllable).Scan(&got); err != nil { + t.Fatalf("out_degree for %q: %v", syllable, err) + } + if got != want { + t.Errorf("out_degree(%q) = %d, want %d", syllable, got, want) + } +} + +func TestBuildWritesAliases(t *testing.T) { + db := openOut(t, buildFixture(t, defaultRows(), 0)) + + var canonical string + err := db.QueryRow(`SELECT canonical FROM aliases WHERE variant = ?`, "hoà bình").Scan(&canonical) + if err != nil { + t.Fatalf("alias for %q missing: %v", "hoà bình", err) + } + if canonical != "hòa bình" { + t.Errorf("alias resolves to %q, want %q", canonical, "hòa bình") + } + + // Every alias must point at a word that actually exists. + var orphans int + err = db.QueryRow(`SELECT COUNT(*) FROM aliases a + LEFT JOIN words w ON w.word = a.canonical + WHERE w.word IS NULL`).Scan(&orphans) + if err != nil { + t.Fatal(err) + } + if orphans != 0 { + t.Errorf("%d aliases point at missing words, want 0", orphans) + } +} + +func TestBuildRecordsProvenance(t *testing.T) { + db := openOut(t, buildFixture(t, defaultRows(), 0)) + + for _, key := range []string{"source_url", "source_license", "attribution", "built_at", "word_count"} { + var value string + if err := db.QueryRow(`SELECT value FROM meta WHERE key = ?`, key).Scan(&value); err != nil { + t.Errorf("meta[%q] missing: %v", key, err) + continue + } + if value == "" { + t.Errorf("meta[%q] is empty", key) + } + } +} + +// The floor exists so a schema change upstream fails the build loudly instead +// of silently shipping a near-empty dictionary. +func TestBuildFailsBelowMinWords(t *testing.T) { + in := fixtureSource(t, defaultRows()) + cfg := config{ + in: in, + out: filepath.Join(t.TempDir(), "noitu.db"), + minWords: 1000, + lang: "vi", + } + if err := run(cfg); err == nil { + t.Fatal("run succeeded with an unreachable min-words floor, want error") + } +} + +// A failed build must leave the previous good database untouched. Building in +// place would delete it and leave an empty file the server would happily open. +func TestFailedBuildPreservesPreviousOutput(t *testing.T) { + out := buildFixture(t, defaultRows(), 0) + + before, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + + // Same output path, but a floor no fixture can clear. + cfg := config{ + in: fixtureSource(t, defaultRows()), + out: out, + minWords: 1000, + lang: "vi", + } + if err := run(cfg); err == nil { + t.Fatal("run succeeded with an unreachable floor, want error") + } + + after, err := os.ReadFile(out) + if err != nil { + t.Fatalf("previous output was destroyed by the failed build: %v", err) + } + if !bytes.Equal(before, after) { + t.Error("previous output was modified by the failed build") + } + if _, err := os.Stat(out + ".tmp"); !errors.Is(err, os.ErrNotExist) { + t.Error("temp database left behind after a failed build") + } +} + +func TestExplicitSourceFlagsMustBeComplete(t *testing.T) { + in := fixtureSource(t, defaultRows()) + out := filepath.Join(t.TempDir(), "noitu.db") + + // Two of three: ambiguous, so fail loudly rather than silently auto-detect. + err := run(config{in: in, out: out, minWords: 1, lang: "vi", table: "entries", wordCol: "word"}) + if err == nil { + t.Fatal("run succeeded with a partial flag override, want error") + } +} + +// SQLite reads a double-quoted name that matches no column as a string literal, +// so a typo would otherwise build a dictionary out of that literal. +func TestUnknownColumnIsRejected(t *testing.T) { + in := fixtureSource(t, defaultRows()) + out := filepath.Join(t.TempDir(), "noitu.db") + + err := run(config{ + in: in, out: out, minWords: 1, lang: "vi", + table: "entries", wordCol: "does_not_exist", langCol: "lang_code", + }) + if err == nil { + t.Fatal("run succeeded with an unknown word column, want error") + } + if !strings.Contains(err.Error(), "does_not_exist") { + t.Errorf("error %q does not name the bad column", err) + } +} + +func TestBuildRecordsSourceProvenance(t *testing.T) { + db := openOut(t, buildFixture(t, defaultRows(), 0)) + + for key, want := range map[string]string{ + "source_table": "entries", + "source_word_column": "word", + "source_lang_column": "lang_code", + } { + var got string + if err := db.QueryRow(`SELECT value FROM meta WHERE key = ?`, key).Scan(&got); err != nil { + t.Errorf("meta[%q] missing: %v", key, err) + continue + } + if got != want { + t.Errorf("meta[%q] = %q, want %q", key, got, want) + } + } +} + +func TestBuildAutoDetectsSchema(t *testing.T) { + // Same data under different column names: detection should still find it. + path := filepath.Join(t.TempDir(), "odd.db") + db, err := sql.Open("sqlite", "file:"+path) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE lexicon (headword TEXT, language TEXT)`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO lexicon VALUES ('pháp luật', 'vi'), ('luật lệ', 'vi')`); err != nil { + t.Fatal(err) + } + db.Close() + + out := filepath.Join(t.TempDir(), "noitu.db") + if err := run(config{in: path, out: out, minWords: 1, lang: "vi"}); err != nil { + t.Fatalf("run with alternate schema: %v", err) + } + + got := openOut(t, out) + var count int + if err := got.QueryRow(`SELECT COUNT(*) FROM words`).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Errorf("word count = %d, want 2", count) + } +} diff --git a/server/cmd/build-dictionary/syllable.go b/server/cmd/build-dictionary/syllable.go new file mode 100644 index 0000000..971a120 --- /dev/null +++ b/server/cmd/build-dictionary/syllable.go @@ -0,0 +1,124 @@ +package main + +import ( + "sort" + "strings" + + "golang.org/x/text/unicode/norm" +) + +// Vietnamese syllable structure is a closed system: onset + nucleus + coda, +// where each part is drawn from a fixed inventory. That makes it a reliable +// test for whether an entry is a Vietnamese word at all. +// +// An alphabet check alone cannot do this. "credit card", "dress code" and +// "come out" are spelled entirely with letters Vietnamese has, so they pass any +// letter-level filter, but no Vietnamese syllable begins "cr" or ends "ss". +// The multilingual source leaks entries like these into the `vi` rows, and +// without this check they become playable words. + +// vietnameseOnsets is the complete inventory of syllable-initial consonants. +// Longest-match order matters, so this is sorted by descending length at init. +var vietnameseOnsets = []string{ + "ngh", "ng", "nh", "ch", "gh", "gi", "kh", "ph", "qu", "th", "tr", + "b", "c", "d", "đ", "g", "h", "k", "l", "m", "n", "p", "r", "s", "t", "v", "x", +} + +// vietnameseCodas is the complete inventory of syllable-final consonants and +// offglides. +var vietnameseCodas = []string{ + "ngh", "ng", "nh", "ch", + "c", "m", "n", "p", "t", "i", "o", "u", "y", +} + +// vietnameseNuclei is the inventory of vowel nuclei, diacritics stripped. Some +// are single vowels, others diphthongs or triphthongs. +var vietnameseNuclei = []string{ + "uye", "uya", "uyu", "oai", "oay", "uoi", "uou", "ieu", "yeu", "uai", "uay", + "ai", "ao", "au", "ay", "eo", "eu", "ia", "ie", "iu", "oa", "oe", "oi", "oo", + "ua", "ue", "ui", "uo", "uu", "uy", "ya", "ye", "yu", "ao", "eu", + "a", "e", "i", "o", "u", "y", +} + +func init() { + byLengthDesc := func(s []string) { + sort.SliceStable(s, func(i, j int) bool { return len(s[i]) > len(s[j]) }) + } + byLengthDesc(vietnameseOnsets) + byLengthDesc(vietnameseCodas) + byLengthDesc(vietnameseNuclei) +} + +// stripDiacritics reduces a syllable to its base letters, keeping đ (which is a +// distinct letter, not a d with a mark). +func stripDiacritics(syllable string) string { + var b strings.Builder + for _, r := range norm.NFD.String(syllable) { + // Mn = nonspacing combining mark: every Vietnamese tone and letter mark. + if isCombiningMark(r) { + continue + } + b.WriteRune(r) + } + return b.String() +} + +func isCombiningMark(r rune) bool { + // The ranges Vietnamese actually uses; cheaper and tighter than a full + // unicode.Is(unicode.Mn, r) for this data. + return (r >= 0x0300 && r <= 0x036F) || r == 0x031B +} + +// isVietnameseSyllable reports whether a syllable fits Vietnamese phonotactics. +// +// Every onset and nucleus split must be tried, not just the longest. A greedy +// parser reads "gì" as the digraph onset "gi" with nothing left for a nucleus +// and wrongly rejects it; the correct parse is onset "g" plus nucleus "ì". +// The same applies to "gìn", "gỉ" and every other g + i syllable. +func isVietnameseSyllable(syllable string) bool { + base := stripDiacritics(syllable) + if base == "" { + return false + } + + // Onset is optional ("áo", "ăn"), so "" is a candidate too. + candidates := []string{""} + for _, onset := range vietnameseOnsets { + if strings.HasPrefix(base, onset) { + candidates = append(candidates, onset) + } + } + + for _, onset := range candidates { + if parsesAsRhyme(base[len(onset):]) { + return true + } + } + + return false +} + +// parsesAsRhyme reports whether the remainder after an onset is a valid +// nucleus followed by at most one coda. +func parsesAsRhyme(rest string) bool { + if rest == "" { + return false + } + + for _, nucleus := range vietnameseNuclei { + if !strings.HasPrefix(rest, nucleus) { + continue + } + tail := rest[len(nucleus):] + if tail == "" { + return true + } + for _, coda := range vietnameseCodas { + if tail == coda { + return true + } + } + } + + return false +} diff --git a/server/cmd/build-dictionary/syllable_test.go b/server/cmd/build-dictionary/syllable_test.go new file mode 100644 index 0000000..078378a --- /dev/null +++ b/server/cmd/build-dictionary/syllable_test.go @@ -0,0 +1,67 @@ +package main + +import "testing" + +func TestIsVietnameseSyllableAccepts(t *testing.T) { + // A parser that matches the longest onset greedily rejects the whole g+i + // family, which includes some of the most common words in the language. + gFamily := []string{"gì", "gỉ", "gìn", "giữ", "gi", "giả", "giáo", "gia"} + // Ordinary syllables across the onset, nucleus and coda inventories. + ordinary := []string{ + "pháp", "luật", "ngôn", "ngữ", "nghiêng", "nghề", "khuya", "quyền", + "tuyết", "người", "đường", "ươm", "ăn", "áo", "yêu", "oanh", "uy", + "xoài", "khoẻ", "thuý", "quý", "chương", "trường", "nhanh", "phở", + } + + for _, group := range [][]string{gFamily, ordinary} { + for _, s := range group { + if !isVietnameseSyllable(s) { + t.Errorf("isVietnameseSyllable(%q) = false, want true", s) + } + } + } +} + +func TestIsVietnameseSyllableRejects(t *testing.T) { + // Foreign syllables that use only letters Vietnamese has, so an + // alphabet-level check cannot catch them. + tests := []struct{ in, why string }{ + {"credit", "onset cluster cr"}, + {"card", "coda rd"}, + {"dress", "onset cluster dr and coda ss"}, + {"code", "coda de"}, + {"come", "coda me"}, + {"york", "coda rk"}, + {"cup", "valid shape but see note"}, // sanity: see below + {"", "empty"}, + } + + for _, tc := range tests { + if tc.in == "cup" { + // "cup" is phonotactically legal in Vietnamese (c + u + p), so it + // is NOT rejected here. Documented so the expectation is explicit. + if !isVietnameseSyllable("cup") { + t.Error(`isVietnameseSyllable("cup") = false; it is a legal Vietnamese shape`) + } + continue + } + if isVietnameseSyllable(tc.in) { + t.Errorf("isVietnameseSyllable(%q) = true, want false (%s)", tc.in, tc.why) + } + } +} + +func TestStripDiacritics(t *testing.T) { + tests := map[string]string{ + "ngữ": "ngu", + "đường": "đuong", + "khoẻ": "khoe", + "pháp": "phap", + } + + for in, want := range tests { + if got := stripDiacritics(in); got != want { + t.Errorf("stripDiacritics(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/server/go.mod b/server/go.mod new file mode 100644 index 0000000..07c7588 --- /dev/null +++ b/server/go.mod @@ -0,0 +1,20 @@ +module github.com/tiennm99dev/noitu/server + +go 1.25.0 + +require ( + golang.org/x/text v0.41.0 + modernc.org/sqlite v1.58.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.47.0 // indirect + modernc.org/libc v1.75.6 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect +) diff --git a/server/go.sum b/server/go.sum new file mode 100644 index 0000000..8d27bc6 --- /dev/null +++ b/server/go.sum @@ -0,0 +1,52 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus= +modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.58.0 h1:38u40/bwkfM7f0Myhosl+SEMltSDxnGdQf8o6Kjmys0= +modernc.org/sqlite v1.58.0/go.mod h1:rsD2CckafgObKC4DhBlGBf+RiHxkc3hINGt1Xw32tVY= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/server/internal/vietnamese/normalize.go b/server/internal/vietnamese/normalize.go new file mode 100644 index 0000000..708aea5 --- /dev/null +++ b/server/internal/vietnamese/normalize.go @@ -0,0 +1,54 @@ +// Package vietnamese normalizes Vietnamese word input. +// +// The same normalization runs in two places: the dictionary builder, which +// normalizes the corpus, and the game server, which normalizes what a player +// types. They must agree exactly — if they ever diverge, a word stored in the +// database stops matching the identical string typed by a player. Sharing this +// package is what prevents that. +package vietnamese + +import ( + "errors" + "strings" + + "golang.org/x/text/unicode/norm" +) + +// MinSyllables is the shortest word the game accepts. Nối từ is played with +// compound words; a single syllable is not a valid move. +const MinSyllables = 2 + +// ErrEmpty is returned when input contains no usable characters. +var ErrEmpty = errors.New("vietnamese: empty input") + +// Normalize canonicalizes raw player or corpus input and splits it into syllables. +// +// It returns the canonical word (syllables joined by single spaces) and the +// syllables themselves. It deliberately does not enforce a syllable count: +// length is a game rule, not a text rule, so the engine owns that check. +// +// Steps, in order: +// - NFC composition, so "ữ" written as a base plus a combining mark compares +// equal to the single-codepoint form. Without this, words typed on different +// keyboards or platforms fail to match identical database entries. +// - lowercase, so capitalization never affects a lookup. +// - whitespace split and rejoin, which collapses runs of spaces, tabs, and +// non-breaking spaces that IMEs and copy-paste routinely introduce. +func Normalize(raw string) (string, []string, error) { + composed := norm.NFC.String(raw) + lowered := strings.ToLower(composed) + + // strings.Fields splits on every unicode.IsSpace rune, which covers tabs and + // U+00A0 non-breaking spaces as well as ordinary spaces. + syllables := strings.Fields(lowered) + if len(syllables) == 0 { + return "", nil, ErrEmpty + } + + return strings.Join(syllables, " "), syllables, nil +} + +// HasEnoughSyllables reports whether a word is long enough to be a legal move. +func HasEnoughSyllables(syllables []string) bool { + return len(syllables) >= MinSyllables +} diff --git a/server/internal/vietnamese/normalize_test.go b/server/internal/vietnamese/normalize_test.go new file mode 100644 index 0000000..0addc03 --- /dev/null +++ b/server/internal/vietnamese/normalize_test.go @@ -0,0 +1,90 @@ +package vietnamese + +import ( + "errors" + "testing" +) + +func TestNormalize(t *testing.T) { + tests := []struct { + name string + raw string + want string + syllables int + }{ + {"plain two syllables", "pháp luật", "pháp luật", 2}, + {"uppercase", "PHÁP LUẬT", "pháp luật", 2}, + {"mixed case", "Pháp Luật", "pháp luật", 2}, + {"leading and trailing space", " pháp luật ", "pháp luật", 2}, + {"repeated spaces", "pháp luật", "pháp luật", 2}, + {"tab separator", "pháp\tluật", "pháp luật", 2}, + {"non-breaking space", "pháp luật", "pháp luật", 2}, + {"newline separator", "pháp\nluật", "pháp luật", 2}, + {"three syllables", "vô tuyến điện", "vô tuyến điện", 3}, + {"four syllables", "công nghiệp hóa hiện", "công nghiệp hóa hiện", 4}, + {"single syllable passes through", "pháp", "pháp", 1}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + word, syllables, err := Normalize(tc.raw) + if err != nil { + t.Fatalf("Normalize(%q) returned error: %v", tc.raw, err) + } + if word != tc.want { + t.Errorf("word = %q, want %q", word, tc.want) + } + if len(syllables) != tc.syllables { + t.Errorf("got %d syllables, want %d", len(syllables), tc.syllables) + } + }) + } +} + +// A word typed on one keyboard can arrive decomposed and on another composed. +// Both must normalize to the same string, or database lookups fail for reasons +// invisible to the player. +func TestNormalizeUnicodeEquivalence(t *testing.T) { + composed := "ngôn ngữ" // ữ as a single codepoint + decomposed := "ngôn ngữ" // o+circumflex, u+horn+tilde + + gotComposed, _, err := Normalize(composed) + if err != nil { + t.Fatal(err) + } + gotDecomposed, _, err := Normalize(decomposed) + if err != nil { + t.Fatal(err) + } + + if gotComposed != gotDecomposed { + t.Errorf("composed %q and decomposed %q normalized differently: %q vs %q", + composed, decomposed, gotComposed, gotDecomposed) + } +} + +func TestNormalizeEmpty(t *testing.T) { + for _, raw := range []string{"", " ", "\t\n", " "} { + if _, _, err := Normalize(raw); !errors.Is(err, ErrEmpty) { + t.Errorf("Normalize(%q) error = %v, want ErrEmpty", raw, err) + } + } +} + +func TestHasEnoughSyllables(t *testing.T) { + tests := []struct { + syllables []string + want bool + }{ + {[]string{"pháp"}, false}, + {[]string{"pháp", "luật"}, true}, + {[]string{"vô", "tuyến", "điện"}, true}, + {nil, false}, + } + + for _, tc := range tests { + if got := HasEnoughSyllables(tc.syllables); got != tc.want { + t.Errorf("HasEnoughSyllables(%v) = %v, want %v", tc.syllables, got, tc.want) + } + } +}