diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1d7f26..dbffc52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,7 +123,7 @@ jobs: docker build --build-arg FIXTURE_DICT=1 -t noitu:ci . fi - # CC BY-SA 3.0 applies to the derived wordlist wherever it is + # CC BY-SA 4.0 applies to the derived wordlist wherever it is # distributed, and an image is distribution. This is the assertion that # the obligation actually shipped. - name: The licence travels with the data @@ -137,8 +137,8 @@ jobs: grep -qx "$required" files.txt || { echo "missing from the image: $required"; exit 1; } done - # The upstream wordlist must never reach the final image. - if grep -q 'words\.jsonl$' files.txt; then + # The upstream export must never reach the final image. + if grep -q '\.jsonl$' files.txt; then echo "the upstream wordlist leaked into the image" exit 1 fi diff --git a/.gitignore b/.gitignore index 5c51091..d9ec25d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ # Dictionary data — never committed. -# data/undertheseanlp-words.jsonl is the upstream wordlist; data/noitu.db is +# data/kaikki-viwiktionary-vi.jsonl is the upstream export; data/noitu.db is # derived from it. Both are build artifacts produced by `make fetch-dict` and # `make dict`. -data/undertheseanlp-words.jsonl +data/*.jsonl +data/*.jsonl.part data/*.db data/*.db-journal data/*.db-wal diff --git a/Dockerfile b/Dockerfile index d9edbc2..d7d6576 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # One image: the binary, the built frontend, and the derived dictionary. # # The upstream wordlist is downloaded in a builder stage and never reaches the -# final image — only the ~1.7 MB database derived from it does. That -# derived database is CC BY-SA 3.0 while the code is Apache-2.0, so it is +# final image — only the ~2 MB database derived from it does. That +# derived database is CC BY-SA 4.0 while the code is Apache-2.0, so it is # copied in as its own layer alongside its licence and attribution rather than # being embedded in the binary. @@ -30,11 +30,12 @@ RUN CGO_ENABLED=0 go build -trimpath -o /out/build-dictionary ./cmd/build-dictio # --- the dictionary --------------------------------------------------------- FROM alpine:3.22 AS dict -# Pinned by commit and checked by digest: the derived wordlist is the one thing -# in this image that cannot be rebuilt from the repository alone. The Makefile -# pins the same file for local builds, and a test asserts the two agree. -ARG DICT_URL=https://raw.githubusercontent.com/undertheseanlp/dictionary/2c078cfc373b06e2980d324ce1d7bd13740c3319/dictionary/words.txt -ARG DICT_SHA256=4c3e0e6117e4bdfa97731e135c3d4a05881889909267394a8de8d88ef79f13f0 +# Fetched fresh, not pinned: kaikki.org re-exports Wiktionary about weekly and +# keeps no dated snapshots. The derived wordlist is the one thing in this image +# that cannot be rebuilt from the repository alone, so the builder records the +# SHA-256 of the file it read in the database's meta table. The Makefile uses +# the same URL for local builds, and a test asserts the two agree. +ARG DICT_URL=https://kaikki.org/viwiktionary/Ti%E1%BA%BFng%20Vi%E1%BB%87t/kaikki.org-dictionary-Ti%E1%BA%BFngVi%E1%BB%87t.jsonl # Set to 1 to build from the checked-in word sample instead of downloading the # upstream wordlist. That produces a playable but tiny dictionary, and exists so @@ -51,9 +52,8 @@ RUN set -eu; \ if [ "$FIXTURE_DICT" = "1" ]; then \ build-dictionary --words ./fixture-words.txt --out /out/noitu.db --min-words 150; \ else \ - curl -fsSL -o undertheseanlp-words.jsonl "$DICT_URL"; \ - echo "$DICT_SHA256 undertheseanlp-words.jsonl" | sha256sum -c -; \ - build-dictionary --merged ./undertheseanlp-words.jsonl --out /out/noitu.db; \ + curl -fsSL -o kaikki-viwiktionary-vi.jsonl "$DICT_URL"; \ + build-dictionary --kaikki ./kaikki-viwiktionary-vi.jsonl --out /out/noitu.db; \ fi # --- the image -------------------------------------------------------------- @@ -64,7 +64,7 @@ COPY --from=build /out/noitu-server /app/noitu-server COPY --from=web /src/web/build /app/web # The share-alike half of the image. data/LICENSE and data/ATTRIBUTION.md ship -# with the derived wordlist because CC BY-SA 3.0 applies to it wherever it is +# with the derived wordlist because CC BY-SA 4.0 applies to it wherever it is # distributed, and a container image is distribution. COPY --from=dict /out/noitu.db /app/data/noitu.db COPY data/LICENSE /app/data/LICENSE diff --git a/Makefile b/Makefile index 34f7b29..e5f69e4 100644 --- a/Makefile +++ b/Makefile @@ -3,21 +3,21 @@ # Every target has a raw equivalent documented in README.md, so contributors # without `make` (notably on Windows) are never blocked. -DICT_URL := https://raw.githubusercontent.com/undertheseanlp/dictionary/2c078cfc373b06e2980d324ce1d7bd13740c3319/dictionary/words.txt -DICT_SRC := data/undertheseanlp-words.jsonl -# Pinned to a commit, not a branch, so the URL is immutable and cannot drift -# from the checksum. A mismatch means the download was truncated or tampered. -DICT_SHA256 := 4c3e0e6117e4bdfa97731e135c3d4a05881889909267394a8de8d88ef79f13f0 +# kaikki.org re-exports Wiktionary tiếng Việt about weekly and keeps no dated +# snapshots, so this is fetched fresh and unpinned by design: the builder +# records the SHA-256 of what it read in the database's meta table. The URL +# stays percent-encoded — the path has a space in it. +DICT_URL := https://kaikki.org/viwiktionary/Ti%E1%BA%BFng%20Vi%E1%BB%87t/kaikki.org-dictionary-Ti%E1%BA%BFngVi%E1%BB%87t.jsonl +DICT_SRC := data/kaikki-viwiktionary-vi.jsonl DICT_OUT := data/noitu.db FIXTURE_WORDS := testdata/fixture-words.txt FIXTURE_DB := data/fixture.db SERVER_BIN := noitu-server -.PHONY: help fetch-dict verify-dict dict fixture-dict proto proto-check server web web-dev test test-go test-web test-e2e run clean +.PHONY: help fetch-dict dict fixture-dict proto proto-check server web web-dev test test-go test-web test-e2e run clean help: - @echo "fetch-dict download + checksum the upstream wordlist (~4.8 MB) into data/" - @echo "verify-dict re-check the downloaded dictionary against its pinned SHA-256" + @echo "fetch-dict download the current upstream wordlist (~62 MB) into data/" @echo "dict derive $(DICT_OUT) from $(DICT_SRC)" @echo "fixture-dict build the small test dictionary — no download needed" @echo "proto regenerate the Go and JS wire types from proto/" @@ -30,24 +30,22 @@ help: @echo "run build and run the server locally" @echo "clean remove build artifacts (keeps downloaded dictionary)" -# One-time download. -f so an HTTP error page fails here, not as a confusing -# checksum mismatch. +# Fetches whatever kaikki currently serves. -f so an HTTP error fails here +# rather than as a JSON parse error later; no resume flag, because resuming a +# file that may have changed underneath would splice two exports together; +# downloaded to a .part name and renamed only on success, so an interrupted +# fetch never leaves a truncated file for the next `make dict` to consume. fetch-dict: @mkdir -p data - curl -fL -o $(DICT_SRC) $(DICT_URL) - @echo "$(DICT_SHA256) $(DICT_SRC)" | sha256sum -c - - @echo "downloaded and verified $(DICT_SRC)" - -# Verify an already-downloaded copy without re-fetching. -verify-dict: - @echo "$(DICT_SHA256) $(DICT_SRC)" | sha256sum -c - + curl -fL -o $(DICT_SRC).part $(DICT_URL) && mv $(DICT_SRC).part $(DICT_SRC) + @echo "downloaded $(DICT_SRC)" $(DICT_SRC): @echo "$(DICT_SRC) not found — run 'make fetch-dict' first" >&2 @exit 1 dict: $(DICT_SRC) - cd server && go run ./cmd/build-dictionary --merged ../$(DICT_SRC) --out ../$(DICT_OUT) + cd server && go run ./cmd/build-dictionary --kaikki ../$(DICT_SRC) --out ../$(DICT_OUT) # The dictionary tests, end-to-end runs and CI all play against. Built from a # checked-in word list through the same pipeline as the real one, so nothing diff --git a/NOTICE b/NOTICE index 9330b1b..33a7280 100644 --- a/NOTICE +++ b/NOTICE @@ -13,22 +13,23 @@ Version 2.0. See the LICENSE file at the repository root. This covers everything under server/, tools/, web/, and proto/. ------------------------------------------------------------------------------ -2. DICTIONARY DATA — Creative Commons Attribution-ShareAlike 3.0 Unported +2. DICTIONARY DATA — Creative Commons Attribution-ShareAlike 4.0 International ------------------------------------------------------------------------------ The Vietnamese dictionary database is NOT covered by the Apache License. It is -derived from Wiktionary text and remains under the licence that text carried, -CC BY-SA 3.0: +derived from Wiktionary text and remains under the licence that text carries, +CC BY-SA 4.0: - Affected artifacts: data/noitu.db (and data/undertheseanlp-words.jsonl, its source) + Affected artifacts: data/noitu.db (and data/kaikki-viwiktionary-vi.jsonl, its source) License text: data/LICENSE Attribution and list of changes: data/ATTRIBUTION.md Original work: Wiktionary tiếng Việt (https://vi.wiktionary.org/), by its contributors - Upstream source: https://github.com/undertheseanlp/dictionary (commit 2c078cf, - file dictionary/words.txt, rows tagged "wiktionary" only) + Extraction: wiktextract, published at https://kaikki.org/viwiktionary/ + (Tatu Ylonen); fetched fresh for each build, identified + by the SHA-256 recorded in the database's meta table -CC BY-SA 3.0 is a share-alike license. Any distribution of the derived database +CC BY-SA 4.0 is a share-alike license. Any distribution of the derived database -- including inside a container image or other packaged build -- must carry the same license, this attribution, and the record of modifications. diff --git a/README.md b/README.md index 9f18067..2a78ae8 100644 --- a/README.md +++ b/README.md @@ -159,16 +159,19 @@ is needed only to change the WebSocket schema — the generated code is committe building and running the project does not require it. ```sh -make fetch-dict # one-time: downloads the ~4.8 MB upstream wordlist into data/ +make fetch-dict # downloads the current ~62 MB Wiktionary export into data/ make dict # derives data/noitu.db (the game's wordlist) from it make test # run all tests make run # build and start the server ``` -`make fetch-dict` is a one-time cost per machine. Neither the wordlist nor the derived -database is committed; both are build artifacts. See -[`data/ATTRIBUTION.md`](./data/ATTRIBUTION.md). If you have a `data/dictionary.db` from -before the corpus switch, it is no longer read and can be deleted. +The export is fetched fresh, not pinned: kaikki.org re-exports Wiktionary about weekly and +keeps no dated snapshots, so two builds a week apart can differ slightly. The database +records the SHA-256 of the file it was built from in its `meta` table. Neither the export +nor the derived database is committed; both are build artifacts. See +[`data/ATTRIBUTION.md`](./data/ATTRIBUTION.md). Leftover `data/dictionary.db` or +`data/undertheseanlp-words.jsonl` files from earlier sources are no longer read and can be +deleted. ## Running the server @@ -220,8 +223,8 @@ dev-only URL to get wrong. | Target | Does | |---|---| -| `fetch-dict` | Download the upstream wordlist (~4.8 MB) into `data/` and verify its checksum | -| `dict` | Derive `data/noitu.db` from the upstream wordlist | +| `fetch-dict` | Download the current upstream Wiktionary export (~62 MB) into `data/` | +| `dict` | Derive `data/noitu.db` from the upstream export | | `server` | Build the Go server binary | | `web` | Build the SvelteKit frontend to static assets | | `web-dev` | Run the frontend dev server, proxying `/ws` to a local server | @@ -231,19 +234,17 @@ dev-only URL to get wrong. | `test` | Run Go and JavaScript tests | | `test-e2e` | Run the Playwright suite against the fixture dictionary | | `run` | Build and run the server locally | -| `verify-dict` | Re-check the downloaded dictionary against its pinned SHA-256 | ### Without `make` `make` is not installed everywhere (notably Windows). Every target is a thin wrapper: ```sh -# fetch-dict (URL and checksum are pinned in the Makefile) -curl -fL -o data/undertheseanlp-words.jsonl https://raw.githubusercontent.com/undertheseanlp/dictionary/2c078cfc373b06e2980d324ce1d7bd13740c3319/dictionary/words.txt -echo "4c3e0e6117e4bdfa97731e135c3d4a05881889909267394a8de8d88ef79f13f0 data/undertheseanlp-words.jsonl" | sha256sum -c - +# fetch-dict (the URL is in the Makefile; keep it percent-encoded, the path has a space) +curl -fL -o data/kaikki-viwiktionary-vi.jsonl "https://kaikki.org/viwiktionary/Ti%E1%BA%BFng%20Vi%E1%BB%87t/kaikki.org-dictionary-Ti%E1%BA%BFngVi%E1%BB%87t.jsonl" # dict -cd server && go run ./cmd/build-dictionary --merged ../data/undertheseanlp-words.jsonl --out ../data/noitu.db +cd server && go run ./cmd/build-dictionary --kaikki ../data/kaikki-viwiktionary-vi.jsonl --out ../data/noitu.db # test cd server && go vet ./... && go test ./... -race @@ -299,12 +300,12 @@ See [`NOTICE`](./NOTICE) for the full statement. | Artifact | License | |---|---| | All source code (`server/`, `web/`, `proto/`) | [Apache-2.0](./LICENSE) | -| Dictionary data (`data/noitu.db`) | [CC BY-SA 3.0](./data/LICENSE) | +| Dictionary data (`data/noitu.db`) | [CC BY-SA 4.0](./data/LICENSE) | The dictionary is derived from the [Wiktionary tiếng Việt](https://vi.wiktionary.org/) -entries (CC BY-SA, by their contributors) as scraped and redistributed by -[undertheseanlp/dictionary](https://github.com/undertheseanlp/dictionary); the other two -wordlists in that file are not used. CC BY-SA is a **share-alike** license: any redistribution of the derived +entries (CC BY-SA 4.0, by their contributors) as extracted by +[wiktextract](https://github.com/tatuylonen/wiktextract) and published on +[kaikki.org](https://kaikki.org/viwiktionary/). CC BY-SA is a **share-alike** license: any redistribution of the derived database — including inside a container image — must carry the same license, the attribution, and the record of modifications recorded in [`data/ATTRIBUTION.md`](./data/ATTRIBUTION.md). diff --git a/data/ATTRIBUTION.md b/data/ATTRIBUTION.md index 2e35011..2ec592d 100644 --- a/data/ATTRIBUTION.md +++ b/data/ATTRIBUTION.md @@ -1,48 +1,40 @@ # Dictionary Data Attribution The Vietnamese dictionary data used by this game is **not** original work of this project. -It is derived from Wiktionary text licensed under **CC BY-SA 3.0 Unported**, and this file -records the attribution and the modifications required by that license. +It is derived from Wiktionary text licensed under **CC BY-SA 4.0**, and this file records +the attribution and the modifications required by that license. ## Source | Field | Value | |---|---| | Original work | Entries of [Wiktionary tiếng Việt](https://vi.wiktionary.org/), written by its contributors | -| Original license | [CC BY-SA 3.0 Unported](https://creativecommons.org/licenses/by-sa/3.0/) at the time of the 2018 scrape (Wikimedia projects moved to 4.0 on 2023-06-29); Wiktionary text is dual-licensed CC BY-SA / GFDL | -| This database's license | **CC BY-SA 3.0 Unported**, the same licence the source text carried — full text in [`LICENSE`](./LICENSE) | -| Redistributed by | [undertheseanlp/dictionary](https://github.com/undertheseanlp/dictionary), a 2018-12-10 scrape of vi.wiktionary.org merged with two other wordlists | -| Asset | [`dictionary/words.txt` at commit `2c078cf`](https://raw.githubusercontent.com/undertheseanlp/dictionary/2c078cfc373b06e2980d324ce1d7bd13740c3319/dictionary/words.txt) — 4,813,111 bytes, 79,226 JSONL rows | -| SHA-256 | `4c3e0e6117e4bdfa97731e135c3d4a05881889909267394a8de8d88ef79f13f0` | -| Rows used | the 32,484 tagged `wiktionary` | -| Repository code license (not used here) | GPL-3.0 | +| Original license | [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/) (Wiktionary text is dual-licensed CC BY-SA / GFDL) — full text in [`LICENSE`](./LICENSE) | +| Extracted by | [wiktextract](https://github.com/tatuylonen/wiktextract), published on [kaikki.org](https://kaikki.org/viwiktionary/) by Tatu Ylonen; kaikki.org distributes the extracted data under the same CC BY-SA / GFDL terms as the underlying Wiktionary text | +| Asset | [`Tiếng Việt/kaikki.org-dictionary-TiếngViệt.jsonl`](https://kaikki.org/viwiktionary/Ti%E1%BA%BFng%20Vi%E1%BB%87t/kaikki.org-dictionary-Ti%E1%BA%BFngVi%E1%BB%87t.jsonl) — the Vietnamese-language entries of the Vietnamese Wiktionary edition, ~62 MB, ~44,000 entries | +| Refresh | kaikki re-extracts from the monthly Wikimedia dump about once a week | -The asset is pinned by commit, so the URL is immutable and the checksum above always -describes the same bytes. The attribution chain has two links — Wiktionary's contributors, -then undertheseanlp as the intermediary that scraped and redistributed — and both are named -here because CC BY-SA attribution belongs to the authors, not only to the last host. +**The asset is not pinned.** kaikki.org keeps no dated snapshots, so each build fetches the +current export. The exact bytes a given `data/noitu.db` was built from are recorded in its +`meta` table: `source_sha256` (SHA-256 of the file as read), `source_rows` (entries read) +and `source_fetched_at` (the file's modification time). Two builds a week apart may differ +by a few hundred words; the hash says which words a given image shipped. -### Branches deliberately not used - -The merged file tags every word with which of three wordlists contain it. Only -`wiktionary` rows are read; the other two are never consulted — not for words, not for -anything. - -- **`hongocduc`** — Hồ Ngọc Đức's Free Vietnamese Dictionary Project wordlist, distributed - under the GNU GPL according to the branch README (a mirror; the canonical site no longer - resolves). Using it would make this database GPL-derived and force a relicense to GPLv3. - Excluded so the data stays CC BY-SA. -- **`tudientv`** — its branch README declares copyright *"Chưa rõ"* (unknown) and names the - Soha/Vietlex dictionary (Hoàng Phê, editor) as its primary source: an unlicensed - derivative of a commercial work. Excluded, and not to be reopened. +The attribution chain has two links before this project — Wiktionary's contributors, who +wrote the entries, and wiktextract/kaikki.org, which turned the wiki markup into structured +data — and both are named here because CC BY-SA attribution belongs to the authors, not only +to the last host. kaikki.org asks users of its data to cite: +*Tatu Ylonen: Wiktextract: Wiktionary as Machine-Readable Structured Data, Proceedings of +the 13th Conference on Language Resources and Evaluation (LREC), pp. 1317–1325, Marseille, +20–25 June 2022.* ## Modifications made by this project -`server/cmd/build-dictionary` transforms the upstream wordlist into `data/noitu.db`. The +`server/cmd/build-dictionary` transforms the upstream export into `data/noitu.db`. The derived database is a **modified version** of the source data. Changes: -1. **Source selection** — kept only rows whose source list contains `wiktionary`; rows from - the two other wordlists were not read. +1. **Language selection** — kept only entries with `lang_code = "vi"`. The file is + Vietnamese-only today; any other language would be rejected and counted. 2. **Length filter** — kept only words of **2 or more space-separated syllables**, as required by the nối từ game rules. Single-syllable entries were dropped. 3. **Content filter** — dropped entries containing digits or punctuation, entries using @@ -51,7 +43,7 @@ derived database is a **modified version** of the source data. Changes: Vietnamese words ("con cua") are kept. 4. **Normalization** — all words Unicode NFC-normalized, lowercased, and whitespace-collapsed. Capitalized headwords (`Hà Nội`) become lowercase entries; nothing - is removed on the basis of capitalization. + is removed on the basis of capitalization or part of speech. 5. **Spelling aliases** — added an `aliases` table mapping alternative Vietnamese spellings to canonical entries. Two kinds: competing tone placement in open oa/oe/uy syllables (`hoà` → `hòa`, `thuý` → `thúy`), and i/y alternation in Sino-Vietnamese syllables @@ -59,19 +51,20 @@ derived database is a **modified version** of the source data. Changes: by this project and are not present upstream. 6. **Added columns and tables** — `first` and `last` syllable columns, a `syllables` count, an index on `first`, a `syllables` out-degree table, and a `meta` table recording - provenance (source URL, commit, sources kept and excluded). All added for game lookups. -7. **Deduplication** — entries were deduplicated by normalized form, so a word listed both - capitalized and lowercase upstream appears once. A generated spelling variant that is - itself a real word, or that more than one word would claim, is discarded rather than - recorded as an alias. -8. **Dropped fields** — the per-word source tags were discarded after selection. The - derived database contains **only word forms**, not meanings; the upstream wordlist - carries none either. + provenance (source URL, SHA-256, row count, fetch time, licence). All added for game + lookups. +7. **Deduplication** — an entry appears once per part of speech upstream; entries were + deduplicated by normalized word form. A generated spelling variant that is itself a real + word, or that more than one word would claim, is discarded rather than recorded as an + alias. +8. **Dropped fields** — all senses, glosses, examples, translations, pronunciations, + etymologies, categories and part-of-speech tags were discarded. The derived database + contains **only word forms**, not meanings. ## Share-alike obligation -CC BY-SA 3.0 is a **share-alike** license. The derived database `data/noitu.db`, and any -distribution of it, remains licensed under **CC BY-SA 3.0** — including when it is shipped +CC BY-SA 4.0 is a **share-alike** license. The derived database `data/noitu.db`, and any +distribution of it, remains licensed under **CC BY-SA 4.0** — including when it is shipped inside a container image or any other packaged build of this project. This obligation applies to the **data only**. The source code of this project is licensed @@ -82,8 +75,9 @@ keeping the two licensing regimes on separate artifacts. ## How to reproduce the derived data ```sh -make fetch-dict # downloads the ~4.8 MB upstream wordlist into data/ and verifies its checksum -make dict # derives data/noitu.db from it +make fetch-dict # downloads kaikki's current export (~62 MB) into data/ +make dict # derives data/noitu.db from it and records the file's SHA-256 in meta ``` -Neither file is committed to version control; both are build artifacts. +Neither file is committed to version control; both are build artifacts. Because the export +is refreshed upstream, a rebuild on a later day may not be byte-identical to an earlier one. diff --git a/data/LICENSE b/data/LICENSE index 604209a..2d58298 100644 --- a/data/LICENSE +++ b/data/LICENSE @@ -1,359 +1,428 @@ -Creative Commons Legal Code +Attribution-ShareAlike 4.0 International -Attribution-ShareAlike 3.0 Unported +======================================================================= - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR - DAMAGES RESULTING FROM ITS USE. +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution-ShareAlike 4.0 International Public License -THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE -COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY -COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS -AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. - -BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE -TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY -BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS -CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND -CONDITIONS. - -1. Definitions - - a. "Adaptation" means a work based upon the Work, or upon the Work and - other pre-existing works, such as a translation, adaptation, - derivative work, arrangement of music or other alterations of a - literary or artistic work, or phonogram or performance and includes - cinematographic adaptations or any other form in which the Work may be - recast, transformed, or adapted including in any form recognizably - derived from the original, except that a work that constitutes a - Collection will not be considered an Adaptation for the purpose of - this License. For the avoidance of doubt, where the Work is a musical - work, performance or phonogram, the synchronization of the Work in - timed-relation with a moving image ("synching") will be considered an - Adaptation for the purpose of this License. - b. "Collection" means a collection of literary or artistic works, such as - encyclopedias and anthologies, or performances, phonograms or - broadcasts, or other works or subject matter other than works listed - in Section 1(f) below, which, by reason of the selection and - arrangement of their contents, constitute intellectual creations, in - which the Work is included in its entirety in unmodified form along - with one or more other contributions, each constituting separate and - independent works in themselves, which together are assembled into a - collective whole. A work that constitutes a Collection will not be - considered an Adaptation (as defined below) for the purposes of this - License. - c. "Creative Commons Compatible License" means a license that is listed - at https://creativecommons.org/compatiblelicenses that has been - approved by Creative Commons as being essentially equivalent to this - License, including, at a minimum, because that license: (i) contains - terms that have the same purpose, meaning and effect as the License - Elements of this License; and, (ii) explicitly permits the relicensing - of adaptations of works made available under that license under this - License or a Creative Commons jurisdiction license with the same - License Elements as this License. - d. "Distribute" means to make available to the public the original and - copies of the Work or Adaptation, as appropriate, through sale or - other transfer of ownership. - e. "License Elements" means the following high-level license attributes - as selected by Licensor and indicated in the title of this License: - Attribution, ShareAlike. - f. "Licensor" means the individual, individuals, entity or entities that - offer(s) the Work under the terms of this License. - g. "Original Author" means, in the case of a literary or artistic work, - the individual, individuals, entity or entities who created the Work - or if no individual or entity can be identified, the publisher; and in - addition (i) in the case of a performance the actors, singers, - musicians, dancers, and other persons who act, sing, deliver, declaim, - play in, interpret or otherwise perform literary or artistic works or - expressions of folklore; (ii) in the case of a phonogram the producer - being the person or legal entity who first fixes the sounds of a - performance or other sounds; and, (iii) in the case of broadcasts, the - organization that transmits the broadcast. - h. "Work" means the literary and/or artistic work offered under the terms - of this License including without limitation any production in the - literary, scientific and artistic domain, whatever may be the mode or - form of its expression including digital form, such as a book, - pamphlet and other writing; a lecture, address, sermon or other work - of the same nature; a dramatic or dramatico-musical work; a - choreographic work or entertainment in dumb show; a musical - composition with or without words; a cinematographic work to which are - assimilated works expressed by a process analogous to cinematography; - a work of drawing, painting, architecture, sculpture, engraving or - lithography; a photographic work to which are assimilated works - expressed by a process analogous to photography; a work of applied - art; an illustration, map, plan, sketch or three-dimensional work - relative to geography, topography, architecture or science; a - performance; a broadcast; a phonogram; a compilation of data to the - extent it is protected as a copyrightable work; or a work performed by - a variety or circus performer to the extent it is not otherwise - considered a literary or artistic work. - i. "You" means an individual or entity exercising rights under this - License who has not previously violated the terms of this License with - respect to the Work, or who has received express permission from the - Licensor to exercise rights under this License despite a previous - violation. - j. "Publicly Perform" means to perform public recitations of the Work and - to communicate to the public those public recitations, by any means or - process, including by wire or wireless means or public digital - performances; to make available to the public Works in such a way that - members of the public may access these Works from a place and at a - place individually chosen by them; to perform the Work to the public - by any means or process and the communication to the public of the - performances of the Work, including by public digital performance; to - broadcast and rebroadcast the Work by any means including signs, - sounds or images. - k. "Reproduce" means to make copies of the Work by any means including - without limitation by sound or visual recordings and the right of - fixation and reproducing fixations of the Work, including storage of a - protected performance or phonogram in digital form or other electronic - medium. - -2. Fair Dealing Rights. Nothing in this License is intended to reduce, -limit, or restrict any uses free from copyright or rights arising from -limitations or exceptions that are provided for in connection with the -copyright protection under copyright law or other applicable laws. - -3. License Grant. Subject to the terms and conditions of this License, -Licensor hereby grants You a worldwide, royalty-free, non-exclusive, -perpetual (for the duration of the applicable copyright) license to -exercise the rights in the Work as stated below: - - a. to Reproduce the Work, to incorporate the Work into one or more - Collections, and to Reproduce the Work as incorporated in the - Collections; - b. to create and Reproduce Adaptations provided that any such Adaptation, - including any translation in any medium, takes reasonable steps to - clearly label, demarcate or otherwise identify that changes were made - to the original Work. For example, a translation could be marked "The - original work was translated from English to Spanish," or a - modification could indicate "The original work has been modified."; - c. to Distribute and Publicly Perform the Work including as incorporated - in Collections; and, - d. to Distribute and Publicly Perform Adaptations. - e. For the avoidance of doubt: - - i. Non-waivable Compulsory License Schemes. In those jurisdictions in - which the right to collect royalties through any statutory or - compulsory licensing scheme cannot be waived, the Licensor - reserves the exclusive right to collect such royalties for any - exercise by You of the rights granted under this License; - ii. Waivable Compulsory License Schemes. In those jurisdictions in - which the right to collect royalties through any statutory or - compulsory licensing scheme can be waived, the Licensor waives the - exclusive right to collect such royalties for any exercise by You - of the rights granted under this License; and, - iii. Voluntary License Schemes. The Licensor waives the right to - collect royalties, whether individually or, in the event that the - Licensor is a member of a collecting society that administers - voluntary licensing schemes, via that society, from any exercise - by You of the rights granted under this License. - -The above rights may be exercised in all media and formats whether now -known or hereafter devised. The above rights include the right to make -such modifications as are technically necessary to exercise the rights in -other media and formats. Subject to Section 8(f), all rights not expressly -granted by Licensor are hereby reserved. - -4. Restrictions. The license granted in Section 3 above is expressly made -subject to and limited by the following restrictions: - - a. You may Distribute or Publicly Perform the Work only under the terms - of this License. You must include a copy of, or the Uniform Resource - Identifier (URI) for, this License with every copy of the Work You - Distribute or Publicly Perform. You may not offer or impose any terms - on the Work that restrict the terms of this License or the ability of - the recipient of the Work to exercise the rights granted to that - recipient under the terms of the License. You may not sublicense the - Work. You must keep intact all notices that refer to this License and - to the disclaimer of warranties with every copy of the Work You - Distribute or Publicly Perform. When You Distribute or Publicly - Perform the Work, You may not impose any effective technological - measures on the Work that restrict the ability of a recipient of the - Work from You to exercise the rights granted to that recipient under - the terms of the License. This Section 4(a) applies to the Work as - incorporated in a Collection, but this does not require the Collection - apart from the Work itself to be made subject to the terms of this - License. If You create a Collection, upon notice from any Licensor You - must, to the extent practicable, remove from the Collection any credit - as required by Section 4(c), as requested. If You create an - Adaptation, upon notice from any Licensor You must, to the extent - practicable, remove from the Adaptation any credit as required by - Section 4(c), as requested. - b. You may Distribute or Publicly Perform an Adaptation only under the - terms of: (i) this License; (ii) a later version of this License with - the same License Elements as this License; (iii) a Creative Commons - jurisdiction license (either this or a later license version) that - contains the same License Elements as this License (e.g., - Attribution-ShareAlike 3.0 US)); (iv) a Creative Commons Compatible - License. If you license the Adaptation under one of the licenses - mentioned in (iv), you must comply with the terms of that license. If - you license the Adaptation under the terms of any of the licenses - mentioned in (i), (ii) or (iii) (the "Applicable License"), you must - comply with the terms of the Applicable License generally and the - following provisions: (I) You must include a copy of, or the URI for, - the Applicable License with every copy of each Adaptation You - Distribute or Publicly Perform; (II) You may not offer or impose any - terms on the Adaptation that restrict the terms of the Applicable - License or the ability of the recipient of the Adaptation to exercise - the rights granted to that recipient under the terms of the Applicable - License; (III) You must keep intact all notices that refer to the - Applicable License and to the disclaimer of warranties with every copy - of the Work as included in the Adaptation You Distribute or Publicly - Perform; (IV) when You Distribute or Publicly Perform the Adaptation, - You may not impose any effective technological measures on the - Adaptation that restrict the ability of a recipient of the Adaptation - from You to exercise the rights granted to that recipient under the - terms of the Applicable License. This Section 4(b) applies to the - Adaptation as incorporated in a Collection, but this does not require - the Collection apart from the Adaptation itself to be made subject to - the terms of the Applicable License. - c. If You Distribute, or Publicly Perform the Work or any Adaptations or - Collections, You must, unless a request has been made pursuant to - Section 4(a), keep intact all copyright notices for the Work and - provide, reasonable to the medium or means You are utilizing: (i) the - name of the Original Author (or pseudonym, if applicable) if supplied, - and/or if the Original Author and/or Licensor designate another party - or parties (e.g., a sponsor institute, publishing entity, journal) for - attribution ("Attribution Parties") in Licensor's copyright notice, - terms of service or by other reasonable means, the name of such party - or parties; (ii) the title of the Work if supplied; (iii) to the - extent reasonably practicable, the URI, if any, that Licensor - specifies to be associated with the Work, unless such URI does not - refer to the copyright notice or licensing information for the Work; - and (iv) , consistent with Ssection 3(b), in the case of an - Adaptation, a credit identifying the use of the Work in the Adaptation - (e.g., "French translation of the Work by Original Author," or - "Screenplay based on original Work by Original Author"). The credit - required by this Section 4(c) may be implemented in any reasonable - manner; provided, however, that in the case of a Adaptation or - Collection, at a minimum such credit will appear, if a credit for all - contributing authors of the Adaptation or Collection appears, then as - part of these credits and in a manner at least as prominent as the - credits for the other contributing authors. For the avoidance of - doubt, You may only use the credit required by this Section for the - purpose of attribution in the manner set out above and, by exercising - Your rights under this License, You may not implicitly or explicitly - assert or imply any connection with, sponsorship or endorsement by the - Original Author, Licensor and/or Attribution Parties, as appropriate, - of You or Your use of the Work, without the separate, express prior - written permission of the Original Author, Licensor and/or Attribution - Parties. - d. Except as otherwise agreed in writing by the Licensor or as may be - otherwise permitted by applicable law, if You Reproduce, Distribute or - Publicly Perform the Work either by itself or as part of any - Adaptations or Collections, You must not distort, mutilate, modify or - take other derogatory action in relation to the Work which would be - prejudicial to the Original Author's honor or reputation. Licensor - agrees that in those jurisdictions (e.g. Japan), in which any exercise - of the right granted in Section 3(b) of this License (the right to - make Adaptations) would be deemed to be a distortion, mutilation, - modification or other derogatory action prejudicial to the Original - Author's honor and reputation, the Licensor will waive or not assert, - as appropriate, this Section, to the fullest extent permitted by the - applicable national law, to enable You to reasonably exercise Your - right under Section 3(b) of this License (right to make Adaptations) - but not otherwise. - -5. Representations, Warranties and Disclaimer - -UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR -OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY -KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, -INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, -FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF -LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, -WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION -OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. - -6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE -LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR -ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES -ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS -BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. Termination - - a. This License and the rights granted hereunder will terminate - automatically upon any breach by You of the terms of this License. - Individuals or entities who have received Adaptations or Collections - from You under this License, however, will not have their licenses - terminated provided such individuals or entities remain in full - compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will - survive any termination of this License. - b. Subject to the above terms and conditions, the license granted here is - perpetual (for the duration of the applicable copyright in the Work). - Notwithstanding the above, Licensor reserves the right to release the - Work under different license terms or to stop distributing the Work at - any time; provided, however that any such election will not serve to - withdraw this License (or any other license that has been, or is - required to be, granted under the terms of this License), and this - License will continue in full force and effect unless terminated as - stated above. - -8. Miscellaneous - - a. Each time You Distribute or Publicly Perform the Work or a Collection, - the Licensor offers to the recipient a license to the Work on the same - terms and conditions as the license granted to You under this License. - b. Each time You Distribute or Publicly Perform an Adaptation, Licensor - offers to the recipient a license to the original Work on the same - terms and conditions as the license granted to You under this License. - c. If any provision of this License is invalid or unenforceable under - applicable law, it shall not affect the validity or enforceability of - the remainder of the terms of this License, and without further action - by the parties to this agreement, such provision shall be reformed to - the minimum extent necessary to make such provision valid and - enforceable. - d. No term or provision of this License shall be deemed waived and no - breach consented to unless such waiver or consent shall be in writing - and signed by the party to be charged with such waiver or consent. - e. This License constitutes the entire agreement between the parties with - respect to the Work licensed here. There are no understandings, - agreements or representations with respect to the Work not specified - here. Licensor shall not be bound by any additional provisions that - may appear in any communication from You. This License may not be - modified without the mutual written agreement of the Licensor and You. - f. The rights granted under, and the subject matter referenced, in this - License were drafted utilizing the terminology of the Berne Convention - for the Protection of Literary and Artistic Works (as amended on - September 28, 1979), the Rome Convention of 1961, the WIPO Copyright - Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 - and the Universal Copyright Convention (as revised on July 24, 1971). - These rights and subject matter take effect in the relevant - jurisdiction in which the License terms are sought to be enforced - according to the corresponding provisions of the implementation of - those treaty provisions in the applicable national law. If the - standard suite of rights granted under applicable copyright law - includes additional rights not granted under this License, such - additional rights are deemed to be included in the License; this - License is not intended to restrict the license of any rights under - applicable law. +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-ShareAlike 4.0 International Public License ("Public +License"). To the extent this Public License may be interpreted as a +contract, You are granted the Licensed Rights in consideration of Your +acceptance of these terms and conditions, and the Licensor grants You +such rights in consideration of benefits the Licensor receives from +making the Licensed Material available under these terms and +conditions. -Creative Commons Notice +Section 1 -- Definitions. - Creative Commons is not a party to this License, and makes no warranty - whatsoever in connection with the Work. Creative Commons will not be - liable to You or any party on any legal theory for any damages - whatsoever, including without limitation any general, special, - incidental or consequential damages arising in connection to this - license. Notwithstanding the foregoing two (2) sentences, if Creative - Commons has expressly identified itself as the Licensor hereunder, it - shall have all rights and obligations of Licensor. + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. - Except for the limited purpose of indicating to the public that the - Work is licensed under the CCPL, Creative Commons does not authorize - the use by either party of the trademark "Creative Commons" or any - related trademark or logo of Creative Commons without the prior - written consent of Creative Commons. Any permitted use will be in - compliance with Creative Commons' then-current trademark usage - guidelines, as may be published on its website or otherwise made - available upon request from time to time. For the avoidance of doubt, - this trademark restriction does not form part of the License. + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. BY-SA Compatible License means a license listed at + creativecommons.org/compatiblelicenses, approved by Creative + Commons as essentially the equivalent of this Public License. + + d. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + e. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + f. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + g. License Elements means the license attributes listed in the name + of a Creative Commons Public License. The License Elements of this + Public License are Attribution and ShareAlike. + + h. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + i. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + j. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + k. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + l. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + m. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. Additional offer from the Licensor -- Adapted Material. + Every recipient of Adapted Material from You + automatically receives an offer from the Licensor to + exercise the Licensed Rights in the Adapted Material + under the conditions of the Adapter's License You apply. + + c. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + b. ShareAlike. + + In addition to the conditions in Section 3(a), if You Share + Adapted Material You produce, the following conditions also apply. + + 1. The Adapter's License You apply must be a Creative Commons + license with the same License Elements, this version or + later, or a BY-SA Compatible License. + + 2. You must include the text of, or the URI or hyperlink to, the + Adapter's License You apply. You may satisfy this condition + in any reasonable manner based on the medium, means, and + context in which You Share Adapted Material. + + 3. You may not offer or impose any additional or different terms + or conditions on, or apply any Effective Technological + Measures to, Adapted Material that restrict exercise of the + rights granted under the Adapter's License You apply. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material, + including for purposes of Section 3(b); and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. - Creative Commons may be contacted at https://creativecommons.org/. diff --git a/docs/deployment.md b/docs/deployment.md index ce3175b..4b9a68c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -51,9 +51,9 @@ docker build -t noitu:latest . docker run -p 8080:8080 noitu:latest ``` -The build downloads the 4.8 MB upstream wordlist in a builder stage and -derives the ~1.7 MB database the game uses. Only the derived file is copied into -the final image, so the upstream wordlist never ships. The result is a +The build downloads the ~62 MB upstream Wiktionary export in a builder stage and +derives the ~2 MB database the game uses. Only the derived file is copied into +the final image, so the upstream export never ships. The result is a distroless image of about 25 MB running as a non-root user. Passing `--build-arg FIXTURE_DICT=1` builds the same image against the @@ -62,7 +62,7 @@ exists so the image can be tested without the download; do not ship it. ### What travels with the data -The derived wordlist is CC BY-SA 3.0 while the code is Apache-2.0, so the image +The derived wordlist is CC BY-SA 4.0 while the code is Apache-2.0, so the image carries `data/LICENSE`, `data/ATTRIBUTION.md` and `NOTICE` alongside it. CI asserts all three are present, and that the upstream file is not. Removing them would put the image out of compliance. @@ -140,12 +140,12 @@ persistence, by design, in this version. ## Updating the dictionary -The wordlist is a build artifact, not runtime state. A new upstream release -means a new image: - -1. Update `DICT_URL` and `DICT_SHA256` in the `Dockerfile`, and the matching - values in the `Makefile`. -2. Record what changed in `data/ATTRIBUTION.md`. -3. Rebuild and redeploy. +The wordlist is a build artifact, not runtime state, and the upstream export is +fetched fresh rather than pinned: rebuilding the image picks up whatever +kaikki.org currently serves, and the database's `meta` table records the +SHA-256 of the file it was built from. To update the dictionary, rebuild and +redeploy. To change the source itself, update `DICT_URL` in the `Dockerfile`, +the `Makefile` and the builder's constant (a test asserts the three agree), +then record what changed in `data/ATTRIBUTION.md`. Nothing migrates, because nothing persists. diff --git a/server/cmd/build-dictionary/filter.go b/server/cmd/build-dictionary/filter.go index a680a11..849ce29 100644 --- a/server/cmd/build-dictionary/filter.go +++ b/server/cmd/build-dictionary/filter.go @@ -21,6 +21,9 @@ const ( rejectDigit rejectReason = "contains a digit" rejectPunct rejectReason = "contains punctuation" rejectNonVietnam rejectReason = "no Vietnamese letters" + // rejectNotVietnamese is an entry the upstream tags as another language + // altogether, decided on its lang_code before the word is looked at. + rejectNotVietnamese rejectReason = "not a Vietnamese-language entry" ) // accept normalizes a raw source entry and decides whether it belongs in the diff --git a/server/cmd/build-dictionary/kaikki_list.go b/server/cmd/build-dictionary/kaikki_list.go new file mode 100644 index 0000000..4203f48 --- /dev/null +++ b/server/cmd/build-dictionary/kaikki_list.go @@ -0,0 +1,169 @@ +package main + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// The upstream is kaikki.org's wiktextract export of Wiktionary tiếng Việt: +// one JSON object per entry, refreshed from the monthly Wikimedia dump about +// once a week. The file is fetched fresh for every build and is not pinned — +// there is no archived snapshot to pin to — so the builder records the SHA-256 +// of the bytes it actually read, and that hash is what identifies a build. +// +// The URL stays percent-encoded: the path has a space in it, and both make +// and sh would otherwise split it. +const kaikkiSourceURL = "https://kaikki.org/viwiktionary/Ti%E1%BA%BFng%20Vi%E1%BB%87t/kaikki.org-dictionary-Ti%E1%BA%BFngVi%E1%BB%87t.jsonl" + +// kaikkiRow is the part of a wiktextract entry the game cares about. Every +// other field — senses, translations, categories — is skipped by the decoder. +type kaikkiRow struct { + Word string `json:"word"` + Pos string `json:"pos"` + LangCode string `json:"lang_code"` +} + +// kaikkiProvenance identifies the bytes a build was made from. +type kaikkiProvenance struct { + sha256 string + rows int + fetchedAt time.Time +} + +// readKaikkiList streams the export, keeps Vietnamese-language entries and +// hands their word forms to accept(). Part of speech is tallied for the build +// log but never filters: the owner's decision that capitalization removes no +// word applies equally to the "name" tag. +// +// Lines are read with bufio.Reader rather than bufio.Scanner because a row +// carries every sense and translation of its entry and can run to hundreds of +// kilobytes; a scanner's fixed cap would be a guess that eventually fails. +func readKaikkiList(path string, maxSyllables int) (map[string]entry, map[rejectReason]int, map[string]int, kaikkiProvenance, error) { + var prov kaikkiProvenance + + f, err := os.Open(path) + if err != nil { + return nil, nil, nil, prov, fmt.Errorf("read kaikki export: %w", err) + } + defer f.Close() + info, err := f.Stat() + if err != nil { + // A provenance row must be right or absent, never a plausible zero. + return nil, nil, nil, prov, fmt.Errorf("stat kaikki export: %w", err) + } + prov.fetchedAt = info.ModTime().UTC() + + hash := sha256.New() + reader := bufio.NewReaderSize(io.TeeReader(f, hash), 1<<20) + + words := make(map[string]entry) + rejects := make(map[rejectReason]int) + pos := make(map[string]int) + + lineNo := 0 + for { + line, err := reader.ReadBytes('\n') + if len(line) > 0 { + lineNo++ + if trimmed := bytes.TrimSpace(line); len(trimmed) > 0 { + // A bare literal such as null would decode into an empty row + // and be miscounted as a foreign-language entry; only objects + // are entries. + if trimmed[0] != '{' { + return nil, nil, nil, prov, fmt.Errorf("%s:%d: malformed line: not a JSON object", path, lineNo) + } + var row kaikkiRow + if err := json.Unmarshal(trimmed, &row); err != nil { + return nil, nil, nil, prov, fmt.Errorf("%s:%d: malformed line: %w", path, lineNo, err) + } + prov.rows++ + if row.LangCode != "vi" { + rejects[rejectNotVietnamese]++ + } else { + pos[row.Pos]++ + if word, syllables, reason, ok := accept(row.Word, maxSyllables); !ok { + rejects[reason]++ + } else { + words[word] = entry{ + word: word, + first: syllables[0], + last: syllables[len(syllables)-1], + syllables: len(syllables), + } + } + } + } + } + if err != nil { + if errors.Is(err, io.EOF) { + break + } + // The failure is on the line being read: the one just counted if + // a partial line came back with the error, otherwise the next. + failed := lineNo + 1 + if len(line) > 0 { + failed = lineNo + } + return nil, nil, nil, prov, fmt.Errorf("%s:%d: %w", path, failed, err) + } + } + prov.sha256 = hex.EncodeToString(hash.Sum(nil)) + + return words, rejects, pos, prov, nil +} + +// formatPosTally renders the part-of-speech counts on one log line, largest +// first, so the build log says what kind of entries the export held. +func formatPosTally(pos map[string]int) string { + type kv struct { + name string + count int + } + tally := make([]kv, 0, len(pos)) + for name, count := range pos { + if name == "" { + name = "(none)" + } + tally = append(tally, kv{name, count}) + } + sort.Slice(tally, func(i, j int) bool { + if tally[i].count != tally[j].count { + return tally[i].count > tally[j].count + } + return tally[i].name < tally[j].name + }) + parts := make([]string, len(tally)) + for i, t := range tally { + parts[i] = fmt.Sprintf("%s %d", t.name, t.count) + } + return strings.Join(parts, ", ") +} + +// kaikkiSourceSpec describes a kaikki build for the meta table. With no commit +// or checksum pinned upstream, the hash and row count of the bytes read are the +// provenance. +func kaikkiSourceSpec(path string, prov kaikkiProvenance, maxSyllables int) sourceSpec { + return sourceSpec{ + table: "kaikki:" + filepath.Base(path), + url: kaikkiSourceURL, + license: "CC BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)", + attribution: "See data/ATTRIBUTION.md for required attribution and the list of modifications.", + maxSyllables: maxSyllables, + extra: [][2]string{ + {"source_sha256", prov.sha256}, + {"source_rows", fmt.Sprint(prov.rows)}, + {"source_fetched_at", prov.fetchedAt.Format(time.RFC3339)}, + }, + } +} diff --git a/server/cmd/build-dictionary/kaikki_list_test.go b/server/cmd/build-dictionary/kaikki_list_test.go new file mode 100644 index 0000000..f408711 --- /dev/null +++ b/server/cmd/build-dictionary/kaikki_list_test.go @@ -0,0 +1,243 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" +) + +// fixtureKaikki writes a miniature stand-in for the kaikki export: the same +// JSONL shape, a handful of rows. +func fixtureKaikki(t *testing.T, lines ...string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "kaikki.jsonl") + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func defaultKaikkiLines() []string { + return []string{ + `{"word": "Hà Nội", "pos": "name", "lang_code": "vi", "senses": [{"glosses": ["thủ đô"]}]}`, // capitalized, name POS — kept, lowercased + `{"word": "học sinh", "pos": "noun", "lang_code": "vi"}`, + `{"word": "học sinh", "pos": "verb", "lang_code": "vi"}`, // same word, second POS — kept once + `{"word": "student", "pos": "noun", "lang_code": "en"}`, // not Vietnamese-language — rejected and counted + `{"word": "pháp", "pos": "noun", "lang_code": "vi"}`, // single syllable — rejected downstream + ``, + } +} + +func TestKaikkiListKeepsVietnameseEntries(t *testing.T) { + words, rejects, pos, prov, err := readKaikkiList(fixtureKaikki(t, defaultKaikkiLines()...), 0) + if err != nil { + t.Fatal(err) + } + + var got []string + for w := range words { + got = append(got, w) + } + assertSameStrings(t, got, []string{"hà nội", "học sinh"}) + + if n := rejects[rejectNotVietnamese]; n != 1 { + t.Errorf("non-Vietnamese rejects = %d, want 1", n) + } + if n := rejects[rejectTooShort]; n != 1 { + t.Errorf("too-short rejects = %d, want 1 (pháp)", n) + } + if pos["noun"] != 2 || pos["verb"] != 1 || pos["name"] != 1 { + t.Errorf("pos tally = %v, want noun 2, verb 1, name 1 (en row excluded)", pos) + } + if prov.rows != 5 { + t.Errorf("rows = %d, want 5", prov.rows) + } +} + +func TestKaikkiListHashesTheBytesItRead(t *testing.T) { + path := fixtureKaikki(t, defaultKaikkiLines()...) + _, _, _, prov, err := readKaikkiList(path, 0) + if err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(raw) + if want := hex.EncodeToString(sum[:]); prov.sha256 != want { + t.Errorf("sha256 = %s, want %s", prov.sha256, want) + } + if prov.fetchedAt.IsZero() { + t.Error("fetchedAt is zero, want the file's modification time") + } +} + +// fixtureKaikkiRaw writes exact bytes, for the shapes fixtureKaikki's trailing +// newline would hide. +func fixtureKaikkiRaw(t *testing.T, raw string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "kaikki.jsonl") + if err := os.WriteFile(path, []byte(raw), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func TestKaikkiListHandlesDownloadShapes(t *testing.T) { + cases := []struct { + name string + raw string + wantWords int + wantRows int + wantErr string + }{ + {"final line without newline", + `{"word": "học sinh", "pos": "noun", "lang_code": "vi"}` + "\n" + `{"word": "bánh mì", "pos": "noun", "lang_code": "vi"}`, + 2, 2, ""}, + {"HTTP error page instead of JSONL", "503\n", 0, 0, ":1: malformed"}, + {"cut mid-line", `{"word": "học sinh", "pos": "noun", "lang_code": "vi"}` + "\n" + `{"word": "bánh`, 0, 0, ":2: malformed"}, + {"bare JSON literal", "null\n", 0, 0, ":1: malformed"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + words, _, _, prov, err := readKaikkiList(fixtureKaikkiRaw(t, tc.raw), 0) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("err = %v, want one containing %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if len(words) != tc.wantWords || prov.rows != tc.wantRows { + t.Errorf("words=%d rows=%d, want %d/%d", len(words), prov.rows, tc.wantWords, tc.wantRows) + } + }) + } +} + +func TestKaikkiListNamesMalformedLine(t *testing.T) { + path := fixtureKaikki(t, + `{"word": "học sinh", "pos": "noun", "lang_code": "vi"}`, + `{"word": "broken"`, + ) + _, _, _, _, err := readKaikkiList(path, 0) + if err == nil { + t.Fatal("malformed line was skipped, want error") + } + if !strings.Contains(err.Error(), ":2:") { + t.Errorf("error does not name line 2: %v", err) + } +} + +func TestKaikkiListReadsLongLines(t *testing.T) { + // A real row carries every sense and translation and can exceed any + // scanner buffer; the reader must not have a line cap. + padding := strings.Repeat("x", 2<<20) + path := fixtureKaikki(t, `{"word": "học sinh", "pos": "noun", "lang_code": "vi", "note": "`+padding+`"}`) + words, _, _, _, err := readKaikkiList(path, 0) + if err != nil { + t.Fatal(err) + } + if _, ok := words["học sinh"]; !ok { + t.Error("word on a 2 MB line was lost") + } +} + +func TestFormatPosTally(t *testing.T) { + got := formatPosTally(map[string]int{"verb": 2, "noun": 5, "": 1}) + if want := "noun 5, verb 2, (none) 1"; got != want { + t.Errorf("tally = %q, want %q", got, want) + } +} + +func TestKaikkiBuildRecordsProvenance(t *testing.T) { + out := filepath.Join(t.TempDir(), "noitu.db") + path := fixtureKaikki(t, defaultKaikkiLines()...) + if err := run(config{kaikki: path, out: out, minWords: 1}); err != nil { + t.Fatalf("run: %v", err) + } + db := openOut(t, out) + + raw, _ := os.ReadFile(path) + sum := sha256.Sum256(raw) + want := map[string]string{ + "source_url": kaikkiSourceURL, + "source_sha256": hex.EncodeToString(sum[:]), + "source_rows": "5", + "source_license": "CC BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)", + "word_count": "2", + } + for key, value := range want { + var got string + if err := db.QueryRow(`SELECT value FROM meta WHERE key = ?`, key).Scan(&got); err != nil { + t.Errorf("meta[%q] missing: %v", key, err) + continue + } + if got != value { + t.Errorf("meta[%q] = %q, want %q", key, got, value) + } + } + for _, gone := range []string{"source_commit", "sources_kept", "sources_excluded"} { + var got string + if err := db.QueryRow(`SELECT value FROM meta WHERE key = ?`, gone).Scan(&got); err == nil { + t.Errorf("meta[%q] = %q, want absent", gone, got) + } + } + var fetched string + if err := db.QueryRow(`SELECT value FROM meta WHERE key = 'source_fetched_at'`).Scan(&fetched); err != nil || fetched == "" { + t.Errorf("meta[source_fetched_at] missing or empty: %v", err) + } +} + +func TestRunInputSelection(t *testing.T) { + kaikki := fixtureKaikki(t, defaultKaikkiLines()...) + words := fixtureKaikki(t, "học sinh") + out := filepath.Join(t.TempDir(), "noitu.db") + + cases := []struct { + name string + cfg config + wantErr string + }{ + {"both inputs", config{kaikki: kaikki, words: words, out: out, minWords: 1}, "mutually exclusive"}, + {"neither input", config{out: out, minWords: 1}, "no input given"}, + {"missing kaikki file", config{kaikki: filepath.Join(t.TempDir(), "absent.jsonl"), out: out, minWords: 1}, "make fetch-dict"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := run(tc.cfg) + if err == nil { + t.Fatal("run succeeded, want error") + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("error %q does not mention %q", err, tc.wantErr) + } + }) + } +} + +// A fixture is hand-written data; its database must not claim the upstream's +// licence, because the server logs whatever the meta table says. +func TestWordListBuildRecordsNoUpstreamLicense(t *testing.T) { + out := filepath.Join(t.TempDir(), "noitu.db") + if err := run(config{words: fixtureKaikki(t, "học sinh", "bánh mì"), out: out, minWords: 1}); err != nil { + t.Fatalf("run: %v", err) + } + var license, url string + db := openOut(t, out) + if err := db.QueryRow(`SELECT value FROM meta WHERE key = 'source_license'`).Scan(&license); err != nil { + t.Fatal(err) + } + if err := db.QueryRow(`SELECT value FROM meta WHERE key = 'source_url'`).Scan(&url); err != nil { + t.Fatal(err) + } + if strings.Contains(license, "CC BY-SA") || url != "" { + t.Errorf("fixture build claims upstream provenance: license=%q url=%q", license, url) + } +} diff --git a/server/cmd/build-dictionary/main.go b/server/cmd/build-dictionary/main.go index fbd1b36..f136d26 100644 --- a/server/cmd/build-dictionary/main.go +++ b/server/cmd/build-dictionary/main.go @@ -1,18 +1,19 @@ -// Command build-dictionary derives the game's wordlist from the upstream -// undertheseanlp/dictionary merged wordlist. +// Command build-dictionary derives the game's wordlist from kaikki.org's +// wiktextract export of Wiktionary tiếng Việt. // -// The upstream is a 4.8 MB JSONL file: every word of three Vietnamese -// wordlists, tagged with which of them contain it. The game needs only word -// forms of at least two syllables from the wordlists whose license we accept, -// indexed by first and last syllable. This tool performs that reduction and -// records provenance in a meta table. +// The upstream is a ~62 MB JSONL file: one entry per line with its senses, +// translations and part of speech. The game needs only Vietnamese word forms +// of at least two syllables, indexed by first and last syllable. This tool +// performs that reduction and records provenance in a meta table — including +// the SHA-256 of the file it read, since the upstream is fetched fresh for +// every build rather than pinned. // -// The derived database is a modified version of CC BY-SA 3.0 licensed data. +// The derived database is a modified version of CC BY-SA 4.0 licensed data. // See data/ATTRIBUTION.md. // // Usage: // -// go run ./cmd/build-dictionary --merged ../data/undertheseanlp-words.jsonl --out ../data/noitu.db +// go run ./cmd/build-dictionary --kaikki ../data/kaikki-viwiktionary-vi.jsonl --out ../data/noitu.db package main import ( @@ -30,13 +31,13 @@ import ( _ "modernc.org/sqlite" ) -const builderVer = "2" +// builderVer changes whenever the meta table's contract does, so two databases +// with different provenance rows never claim the same builder. +const builderVer = "3" type config struct { - // merged is the corpus: the upstream JSONL wordlist, read for the - // wordlists named in sources. - merged string - sources string + // kaikki is the corpus: the upstream wiktextract JSONL export. + kaikki string // words is an alternative source: a plain list, one word per line, used to // build a small fixture database without the upstream download. words string @@ -49,12 +50,11 @@ func main() { log.SetFlags(0) var cfg config - flag.StringVar(&cfg.merged, "merged", "", "upstream merged JSONL wordlist to read") - flag.StringVar(&cfg.sources, "sources", "wiktionary", "comma-separated upstream wordlists a word may come from (hongocduc, tudientv, wiktionary)") - flag.StringVar(&cfg.words, "words", "", "read a plain word list instead of the upstream wordlist (one word per line, # comments)") + flag.StringVar(&cfg.kaikki, "kaikki", "", "upstream kaikki.org wiktextract JSONL export to read") + flag.StringVar(&cfg.words, "words", "", "read a plain word list instead of the upstream export (one word per line, # comments)") flag.StringVar(&cfg.out, "out", "../data/noitu.db", "derived database to write") flag.IntVar(&cfg.maxSyllables, "max-syllables", 0, "reject words longer than this (0 = no limit)") - flag.IntVar(&cfg.minWords, "min-words", 20000, "fail if fewer words survive filtering") + flag.IntVar(&cfg.minWords, "min-words", 30000, "fail if fewer words survive filtering") flag.Parse() if err := run(cfg); err != nil { @@ -66,36 +66,33 @@ func run(cfg config) error { // Exactly one input. Picking silently between two would let a stray flag // ship a corpus nobody meant to build. switch { - case cfg.merged == "" && cfg.words == "": - return errors.New("no input given: pass --merged (the corpus) or --words (a plain list)") - case cfg.merged != "" && cfg.words != "": - return errors.New("--merged and --words are mutually exclusive") - case cfg.merged != "": - return runFromMergedList(cfg) + case cfg.kaikki == "" && cfg.words == "": + return errors.New("no input given: pass --kaikki (the corpus) or --words (a plain list)") + case cfg.kaikki != "" && cfg.words != "": + return errors.New("--kaikki and --words are mutually exclusive") + case cfg.kaikki != "": + return runFromKaikkiList(cfg) default: return runFromWordList(cfg) } } -// runFromMergedList derives the database from the upstream merged wordlist, -// keeping only words present in the wordlists named by --sources. -func runFromMergedList(cfg config) error { - allowed, err := parseSources(cfg.sources) - if err != nil { - return err - } - if _, err := os.Stat(cfg.merged); err != nil { - return fmt.Errorf("merged wordlist not found at %s — run 'make fetch-dict' first: %w", cfg.merged, err) +// runFromKaikkiList derives the database from the kaikki.org export, keeping +// Vietnamese-language entries and recording the hash of the bytes it read. +func runFromKaikkiList(cfg config) error { + if _, err := os.Stat(cfg.kaikki); err != nil { + return fmt.Errorf("kaikki export not found at %s — run 'make fetch-dict' first: %w", cfg.kaikki, err) } - words, rejects, err := readMergedList(cfg.merged, allowed, cfg.maxSyllables) + words, rejects, pos, prov, err := readKaikkiList(cfg.kaikki, cfg.maxSyllables) if err != nil { return err } logRejects(rejects) - log.Printf("accepted %d distinct words from %s (sources: %s)", len(words), cfg.merged, cfg.sources) + log.Printf("parts of speech: %s", formatPosTally(pos)) + log.Printf("accepted %d distinct words from %s (%d rows, sha256 %s)", len(words), cfg.kaikki, prov.rows, prov.sha256) - return finish(cfg, words, mergedProvenance(cfg.merged, allowed, cfg.maxSyllables)) + return finish(cfg, words, kaikkiSourceSpec(cfg.kaikki, prov, cfg.maxSyllables)) } // finish is the tail every input mode shares: the size floor, alias @@ -223,7 +220,7 @@ type entry struct { // sourceSpec is what the meta table records about where the words came from. type sourceSpec struct { - // table names the input: "merged:" for the corpus, "wordlist:" + // table names the input: "kaikki:" for the corpus, "wordlist:" // for a fixture, so the output says which build produced it. table string maxSyllables int diff --git a/server/cmd/build-dictionary/main_test.go b/server/cmd/build-dictionary/main_test.go index aa4fe43..fabc78e 100644 --- a/server/cmd/build-dictionary/main_test.go +++ b/server/cmd/build-dictionary/main_test.go @@ -11,31 +11,31 @@ import ( _ "modernc.org/sqlite" ) -// fixtureSource writes a miniature stand-in for the upstream wordlist: the -// same JSONL shape, a handful of rows instead of 79k. Each row is a word and -// the upstream wordlists that carry it. Tests never touch the real download. +// fixtureSource writes a miniature stand-in for the kaikki export: the same +// JSONL shape, a handful of rows instead of 44k. Each row is a word and the +// language its Wiktionary entry is for. Tests never touch the real download. func fixtureSource(t *testing.T, rows [][2]string) string { t.Helper() lines := make([]string, 0, len(rows)) for _, r := range rows { - lines = append(lines, `{"text": "`+r[0]+`", "source": ["`+r[1]+`"]}`) + lines = append(lines, `{"word": "`+r[0]+`", "pos": "noun", "lang_code": "`+r[1]+`"}`) } - return fixtureMerged(t, lines...) + return fixtureKaikki(t, lines...) } func defaultRows() [][2]string { return [][2]string{ - {"pháp luật", "wiktionary"}, - {"pháp luật", "wiktionary"}, // listed twice — must dedupe to one word - {"luật lệ", "wiktionary"}, - {"ngôn ngữ", "wiktionary"}, - {"ngữ pháp", "wiktionary"}, - {"hòa bình", "wiktionary"}, - {"vô tuyến điện", "wiktionary"}, // three syllables - {"pháp", "wiktionary"}, // single syllable — rejected - {"covid 19", "wiktionary"}, // digit — rejected - {"hello world", "hongocduc"}, // excluded source — never selected + {"pháp luật", "vi"}, + {"pháp luật", "vi"}, // listed twice — must dedupe to one word + {"luật lệ", "vi"}, + {"ngôn ngữ", "vi"}, + {"ngữ pháp", "vi"}, + {"hòa bình", "vi"}, + {"vô tuyến điện", "vi"}, // three syllables + {"pháp", "vi"}, // single syllable — rejected + {"covid 19", "vi"}, // digit — rejected + {"hello world", "en"}, // another language's entry — never selected } } @@ -44,8 +44,7 @@ func buildFixture(t *testing.T, rows [][2]string, maxSyllables int) string { out := filepath.Join(t.TempDir(), "noitu.db") cfg := config{ - merged: fixtureSource(t, rows), - sources: "wiktionary", + kaikki: fixtureSource(t, rows), out: out, maxSyllables: maxSyllables, minWords: 1, @@ -165,8 +164,7 @@ func TestBuildRecordsProvenance(t *testing.T) { // of silently shipping a near-empty dictionary. func TestBuildFailsBelowMinWords(t *testing.T) { cfg := config{ - merged: fixtureSource(t, defaultRows()), - sources: "wiktionary", + kaikki: fixtureSource(t, defaultRows()), out: filepath.Join(t.TempDir(), "noitu.db"), minWords: 1000, } @@ -187,8 +185,7 @@ func TestFailedBuildPreservesPreviousOutput(t *testing.T) { // Same output path, but a floor no fixture can clear. cfg := config{ - merged: fixtureSource(t, defaultRows()), - sources: "wiktionary", + kaikki: fixtureSource(t, defaultRows()), out: out, minWords: 1000, } diff --git a/server/cmd/build-dictionary/merged_list.go b/server/cmd/build-dictionary/merged_list.go deleted file mode 100644 index 5197fc6..0000000 --- a/server/cmd/build-dictionary/merged_list.go +++ /dev/null @@ -1,148 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "encoding/json" - "fmt" - "os" - "path/filepath" - "slices" - "strings" -) - -// The upstream is undertheseanlp/dictionary: one JSONL file listing every word -// of three merged Vietnamese wordlists together with which of them contain it. -// -// Pinned by commit rather than branch so the URL and the checksum in the -// Makefile can never disagree. -const ( - mergedSourceCommit = "2c078cfc373b06e2980d324ce1d7bd13740c3319" - mergedSourceURL = "https://raw.githubusercontent.com/undertheseanlp/dictionary/" + mergedSourceCommit + "/dictionary/words.txt" -) - -// knownSources are the three wordlists merged upstream. Only names in this -// list may be passed to --sources; a typo must fail loudly rather than quietly -// select nothing. -var knownSources = []string{"hongocduc", "tudientv", "wiktionary"} - -// mergedRow is one line of the upstream file. -type mergedRow struct { - Text string `json:"text"` - Source []string `json:"source"` -} - -// parseSources turns the --sources flag into the set of upstream wordlists a -// word may come from. -func parseSources(list string) (map[string]bool, error) { - allowed := make(map[string]bool) - for _, name := range strings.Split(list, ",") { - name = strings.TrimSpace(name) - if name == "" { - continue - } - if !slices.Contains(knownSources, name) { - return nil, fmt.Errorf("unknown source %q in --sources; known: %s", name, strings.Join(knownSources, ", ")) - } - allowed[name] = true - } - if len(allowed) == 0 { - return nil, fmt.Errorf("--sources selects nothing; known: %s", strings.Join(knownSources, ", ")) - } - return allowed, nil -} - -// readMergedList reads the upstream file and returns the words that belong in -// the game dictionary. -// -// A row is considered only if one of its sources is allowed; rows from other -// wordlists are never read. Capitalization is not a filter: "Hà Nội" and -// "hà nội" are the same word to the game and both land as the lowercase form. -// Rows then pass through the same accept() every other input mode uses. -func readMergedList(path string, allowed map[string]bool, maxSyllables int) (map[string]entry, map[rejectReason]int, error) { - f, err := os.Open(path) - if err != nil { - return nil, nil, fmt.Errorf("read merged list: %w", err) - } - defer f.Close() - - words := make(map[string]entry) - rejects := make(map[rejectReason]int) - - scanner := bufio.NewScanner(f) - // The longest upstream line is well under a kilobyte, but a scanner that - // hits its limit stops with an error rather than truncating, and a generous - // buffer keeps that error from ever being the reason a build fails. - scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) - - lineNo := 0 - for scanner.Scan() { - lineNo++ - line := bytes.TrimSpace(scanner.Bytes()) - if len(line) == 0 { - continue - } - - var row mergedRow - if err := json.Unmarshal(line, &row); err != nil { - return nil, nil, fmt.Errorf("%s:%d: malformed line: %w", path, lineNo, err) - } - if !anyAllowed(row.Source, allowed) { - continue - } - - word, syllables, reason, ok := accept(row.Text, maxSyllables) - if !ok { - rejects[reason]++ - continue - } - words[word] = entry{ - word: word, - first: syllables[0], - last: syllables[len(syllables)-1], - syllables: len(syllables), - } - } - if err := scanner.Err(); err != nil { - // lineNo counts lines already returned; the failure is on the next one. - return nil, nil, fmt.Errorf("%s:%d: %w", path, lineNo+1, err) - } - - return words, rejects, nil -} - -func anyAllowed(sources []string, allowed map[string]bool) bool { - for _, s := range sources { - if allowed[s] { - return true - } - } - return false -} - -// mergedProvenance describes a merged-list build for the meta table: which -// upstream wordlists were kept and which were excluded. knownSources is -// already sorted, so kept and excluded come out sorted too. -func mergedProvenance(path string, allowed map[string]bool, maxSyllables int) sourceSpec { - var kept, excluded []string - for _, name := range knownSources { - if allowed[name] { - kept = append(kept, name) - } else { - excluded = append(excluded, name) - } - } - - return sourceSpec{ - table: "merged:" + filepath.Base(path), - url: mergedSourceURL, - license: "CC BY-SA 3.0 (https://creativecommons.org/licenses/by-sa/3.0/)", - attribution: "See data/ATTRIBUTION.md for required attribution and the list of modifications.", - maxSyllables: maxSyllables, - extra: [][2]string{ - {"source_commit", mergedSourceCommit}, - {"sources_kept", strings.Join(kept, ",")}, - {"sources_excluded", strings.Join(excluded, ",")}, - }, - } -} diff --git a/server/cmd/build-dictionary/merged_list_test.go b/server/cmd/build-dictionary/merged_list_test.go deleted file mode 100644 index 3e2c41f..0000000 --- a/server/cmd/build-dictionary/merged_list_test.go +++ /dev/null @@ -1,184 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "strings" - "testing" -) - -// fixtureMerged writes a miniature stand-in for the upstream JSONL: the same -// shape, a handful of rows. -func fixtureMerged(t *testing.T, lines ...string) string { - t.Helper() - path := filepath.Join(t.TempDir(), "words.txt") - if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { - t.Fatal(err) - } - return path -} - -func defaultMergedLines() []string { - return []string{ - `{"text": "Hà Nội", "source": ["wiktionary"]}`, // capitalized upstream — kept, lowercased - `{"text": "cánh diều", "source": ["hongocduc"]}`, // excluded source — never read - `{"text": "tủ lạnh", "source": ["tudientv"]}`, // excluded source — never read - `{"text": "học sinh", "source": ["hongocduc", "tudientv", "wiktionary"]}`, - `{"text": "Học sinh", "source": ["wiktionary"]}`, // same word twice — kept once - `{"text": "pháp", "source": ["wiktionary"]}`, // single syllable — rejected downstream - ``, - } -} - -func wiktionaryOnly(t *testing.T) map[string]bool { - t.Helper() - allowed, err := parseSources("wiktionary") - if err != nil { - t.Fatal(err) - } - return allowed -} - -func TestMergedListKeepsAllowedSourcesOnly(t *testing.T) { - words, rejects, err := readMergedList(fixtureMerged(t, defaultMergedLines()...), wiktionaryOnly(t), 0) - if err != nil { - t.Fatal(err) - } - - var got []string - for w := range words { - got = append(got, w) - } - assertSameStrings(t, got, []string{"hà nội", "học sinh"}) - - if n := rejects[rejectTooShort]; n != 1 { - t.Errorf("too-short rejects = %d, want 1 (pháp)", n) - } -} - -func TestMergedListWidensWithMoreSources(t *testing.T) { - allowed, err := parseSources("hongocduc,wiktionary") - if err != nil { - t.Fatal(err) - } - words, _, err := readMergedList(fixtureMerged(t, defaultMergedLines()...), allowed, 0) - if err != nil { - t.Fatal(err) - } - for _, want := range []string{"hà nội", "học sinh", "cánh diều"} { - if _, ok := words[want]; !ok { - t.Errorf("%q missing", want) - } - } - if _, ok := words["tủ lạnh"]; ok { - t.Error("tudientv-only word leaked into the output") - } -} - -func TestMergedListNamesMalformedLine(t *testing.T) { - path := fixtureMerged(t, - `{"text": "học sinh", "source": ["wiktionary"]}`, - `{"text": "broken"`, - ) - _, _, err := readMergedList(path, wiktionaryOnly(t), 0) - if err == nil { - t.Fatal("malformed line was skipped, want error") - } - if !strings.Contains(err.Error(), ":2:") { - t.Errorf("error does not name line 2: %v", err) - } -} - -func TestParseSourcesRejectsUnknownNames(t *testing.T) { - for _, bad := range []string{"", "wiktionary,soha", "Wiktionary"} { - if _, err := parseSources(bad); err == nil { - t.Errorf("parseSources(%q) succeeded, want error", bad) - } - } - allowed, err := parseSources(" hongocduc , wiktionary ") - if err != nil { - t.Fatal(err) - } - if !allowed["hongocduc"] || !allowed["wiktionary"] || allowed["tudientv"] { - t.Errorf("allowed = %v, want hongocduc and wiktionary only", allowed) - } -} - -func TestMergedBuildRecordsProvenance(t *testing.T) { - out := filepath.Join(t.TempDir(), "noitu.db") - cfg := config{ - merged: fixtureMerged(t, defaultMergedLines()...), - sources: "wiktionary", - out: out, - minWords: 1, - } - if err := run(cfg); err != nil { - t.Fatalf("run: %v", err) - } - db := openOut(t, out) - - want := map[string]string{ - "source_url": mergedSourceURL, - "source_commit": mergedSourceCommit, - "sources_kept": "wiktionary", - "sources_excluded": "hongocduc,tudientv", - "word_count": "2", - } - for key, value := range want { - var got string - if err := db.QueryRow(`SELECT value FROM meta WHERE key = ?`, key).Scan(&got); err != nil { - t.Errorf("meta[%q] missing: %v", key, err) - continue - } - if got != value { - t.Errorf("meta[%q] = %q, want %q", key, got, value) - } - } -} - -func TestRunInputSelection(t *testing.T) { - merged := fixtureMerged(t, defaultMergedLines()...) - words := fixtureMerged(t, "học sinh") - out := filepath.Join(t.TempDir(), "noitu.db") - - cases := []struct { - name string - cfg config - wantErr string - }{ - {"both inputs", config{merged: merged, words: words, sources: "wiktionary", out: out, minWords: 1}, "mutually exclusive"}, - {"neither input", config{sources: "wiktionary", out: out, minWords: 1}, "no input given"}, - {"missing merged file", config{merged: filepath.Join(t.TempDir(), "absent.jsonl"), sources: "wiktionary", out: out, minWords: 1}, "make fetch-dict"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - err := run(tc.cfg) - if err == nil { - t.Fatal("run succeeded, want error") - } - if !strings.Contains(err.Error(), tc.wantErr) { - t.Errorf("error %q does not mention %q", err, tc.wantErr) - } - }) - } -} - -// A fixture is hand-written data; its database must not claim the upstream's -// licence, because the server logs whatever the meta table says. -func TestWordListBuildRecordsNoUpstreamLicense(t *testing.T) { - out := filepath.Join(t.TempDir(), "noitu.db") - if err := run(config{words: fixtureMerged(t, "học sinh", "bánh mì"), out: out, minWords: 1}); err != nil { - t.Fatalf("run: %v", err) - } - var license, url string - db := openOut(t, out) - if err := db.QueryRow(`SELECT value FROM meta WHERE key = 'source_license'`).Scan(&license); err != nil { - t.Fatal(err) - } - if err := db.QueryRow(`SELECT value FROM meta WHERE key = 'source_url'`).Scan(&url); err != nil { - t.Fatal(err) - } - if strings.Contains(license, "CC BY-SA") || url != "" { - t.Errorf("fixture build claims upstream provenance: license=%q url=%q", license, url) - } -} diff --git a/server/cmd/noitu-server/main.go b/server/cmd/noitu-server/main.go index bc94738..7ee4420 100644 --- a/server/cmd/noitu-server/main.go +++ b/server/cmd/noitu-server/main.go @@ -56,7 +56,7 @@ func run() error { } // The store deliberately does not log — a library writing to the global - // logger fights the server's own handler — so the CC BY-SA 3.0 attribution + // logger fights the server's own handler — so the CC BY-SA 4.0 attribution // that ships with the data surfaces here or nowhere. slog.Info("dictionary loaded", "path", cfg.dbPath, diff --git a/server/internal/dictionary/store.go b/server/internal/dictionary/store.go index 81836c1..f2729a4 100644 --- a/server/internal/dictionary/store.go +++ b/server/internal/dictionary/store.go @@ -126,7 +126,7 @@ func dsn(path string) string { } func (s *Store) loadMeta(db *sql.DB) (declaredWords int, err error) { - // The data is CC BY-SA 3.0 and its provenance travels with it. + // The data is CC BY-SA 4.0 and its provenance travels with it. if err := db.QueryRow(`SELECT value FROM meta WHERE key = 'source_license'`).Scan(&s.license); err != nil { return 0, fmt.Errorf("read dictionary metadata (is this a noitu.db?): %w", err) } diff --git a/server/internal/dictionary/store_test.go b/server/internal/dictionary/store_test.go index 41492a6..51fb6e8 100644 --- a/server/internal/dictionary/store_test.go +++ b/server/internal/dictionary/store_test.go @@ -40,7 +40,7 @@ func fixtureAt(tb testing.TB, dir string) string { defer db.Close() data := fixtureSchema + ` -INSERT INTO meta VALUES ('source_license','CC BY-SA 3.0'),('word_count','7'); +INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','7'); INSERT INTO words VALUES ('pháp luật','pháp','luật',2), ('pháp lý','pháp','lý',2), @@ -106,7 +106,7 @@ func TestOpenWrongSchema(t *testing.T) { // every room creation. func TestOpenEmptyDictionary(t *testing.T) { path := writeDB(t, fixtureSchema+` -INSERT INTO meta VALUES ('source_license','CC BY-SA 3.0'),('word_count','26845');`) +INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','99999');`) _, err := Open(path) if err == nil { @@ -121,7 +121,7 @@ INSERT INTO meta VALUES ('source_license','CC BY-SA 3.0'),('word_count','26845') // syllable has continuations that cannot be supplied. func TestOpenInconsistentOutDegree(t *testing.T) { path := writeDB(t, fixtureSchema+` -INSERT INTO meta VALUES ('source_license','CC BY-SA 3.0'),('word_count','1'); +INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','1'); INSERT INTO words VALUES ('pháp luật','pháp','luật',2); INSERT INTO syllables VALUES ('pháp',7),('luật',0);`) @@ -132,7 +132,7 @@ INSERT INTO syllables VALUES ('pháp',7),('luật',0);`) func TestOpenOrphanAlias(t *testing.T) { path := writeDB(t, fixtureSchema+` -INSERT INTO meta VALUES ('source_license','CC BY-SA 3.0'),('word_count','1'); +INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','1'); INSERT INTO words VALUES ('pháp luật','pháp','luật',2); INSERT INTO syllables VALUES ('pháp',1),('luật',0); INSERT INTO aliases VALUES ('phap luat','không tồn tại');`) @@ -427,8 +427,8 @@ func TestCountsAndLicense(t *testing.T) { if got := s.AliasCount(); got != 2 { t.Errorf("AliasCount = %d, want 2", got) } - if got := s.License(); got != "CC BY-SA 3.0" { - t.Errorf("License = %q, want %q", got, "CC BY-SA 3.0") + if got := s.License(); got != "CC BY-SA 4.0" { + t.Errorf("License = %q, want %q", got, "CC BY-SA 4.0") } } diff --git a/web/e2e/bot-game.spec.js b/web/e2e/bot-game.spec.js index 79fa3ef..de0afdd 100644 --- a/web/e2e/bot-game.spec.js +++ b/web/e2e/bot-game.spec.js @@ -228,7 +228,7 @@ test.describe('playing the bot', () => { const link = page.getByRole('link', { name: 'Wiktionary tiếng Việt' }); await expect(link).toHaveAttribute('href', 'https://vi.wiktionary.org/'); - await expect(page.getByRole('link', { name: /CC BY-SA 3\.0/ })).toBeVisible(); + await expect(page.getByRole('link', { name: /CC BY-SA 4\.0/ })).toBeVisible(); }); test('a deep link is served by the binary, not a 404', async ({ page }) => { diff --git a/web/src/lib/components/AttributionFooter.svelte b/web/src/lib/components/AttributionFooter.svelte index 24526f9..7ccac03 100644 --- a/web/src/lib/components/AttributionFooter.svelte +++ b/web/src/lib/components/AttributionFooter.svelte @@ -3,7 +3,7 @@ @@ -14,7 +14,7 @@ {t.attributionSource} , {t.attributionMiddle} diff --git a/web/src/lib/i18n/vi.js b/web/src/lib/i18n/vi.js index fe3289a..c78a1b8 100644 --- a/web/src/lib/i18n/vi.js +++ b/web/src/lib/i18n/vi.js @@ -119,7 +119,7 @@ export const t = { attributionIntro: 'Từ điển dựa trên', attributionSource: 'Wiktionary tiếng Việt', - attributionLicense: 'giấy phép CC BY-SA 3.0', + attributionLicense: 'giấy phép CC BY-SA 4.0', attributionMiddle: 'phát hành theo' }; diff --git a/web/tests/dictionary-source.test.js b/web/tests/dictionary-source.test.js index 2c424a5..0a1d756 100644 --- a/web/tests/dictionary-source.test.js +++ b/web/tests/dictionary-source.test.js @@ -1,4 +1,4 @@ -// The upstream dictionary is pinned in two places: the Makefile, which builds +// The upstream dictionary URL lives in two places: the Makefile, which builds // it for a developer, and the Dockerfile, which builds it for the image. They // have to agree, or the container ships a wordlist nobody tested against. // @@ -24,35 +24,29 @@ function pin(source, pattern, what) { return match?.[1].trim(); } -describe('the pinned upstream dictionary', () => { +describe('the upstream dictionary export', () => { const makeUrl = pin(makefile, /DICT_URL\s*:?=\s*(\S+)/, 'DICT_URL in the Makefile'); - const makeSha = pin(makefile, /DICT_SHA256\s*:?=\s*(\S+)/, 'DICT_SHA256 in the Makefile'); const dockerUrl = pin(dockerfile, /ARG DICT_URL=(\S+)/, 'DICT_URL in the Dockerfile'); - const dockerSha = pin(dockerfile, /ARG DICT_SHA256=(\S+)/, 'DICT_SHA256 in the Dockerfile'); - it('is the same release in both build files', () => { + it('is the same file in both build files', () => { expect(dockerUrl).toBe(makeUrl); }); - it('is pinned to the same checksum in both build files', () => { - expect(dockerSha).toBe(makeSha); + it('is the Vietnamese-language file of the Vietnamese Wiktionary edition', () => { + // The path carries a space, so it must stay percent-encoded or make and + // sh will split it; and it must be the vi edition, not the English one. + expect(makeUrl).toMatch(/^https:\/\/kaikki\.org\/viwiktionary\/Ti%E1%BA%BFng%20Vi%E1%BB%87t\/[^\s/]+\.jsonl$/); }); - it('is a checksum, not a placeholder', () => { - // A guard that only compared the two would pass happily if both were - // blanked, which is the one way this pin can silently stop pinning. - expect(makeSha).toMatch(/^[0-9a-f]{64}$/); - }); - - it('is the commit the builder stamps into the database', () => { - // The builder records the upstream commit in the meta table from its - // own constant. A pin bump that misses it would ship an attribution - // record naming bytes nobody downloaded, so the three copies must agree. + it('is the URL the builder stamps into the database', () => { + // The builder records the source URL in the meta table from its own + // constant. The three copies must agree or the attribution record names + // a file nobody downloaded. const builder = readFileSync( - fileURLToPath(new URL('../../server/cmd/build-dictionary/merged_list.go', import.meta.url)), + fileURLToPath(new URL('../../server/cmd/build-dictionary/kaikki_list.go', import.meta.url)), 'utf8' ); - const commit = pin(builder, /mergedSourceCommit\s*=\s*"([0-9a-f]{40})"/, 'mergedSourceCommit in the builder'); - expect(makeUrl).toContain(`/${commit}/`); + const builderUrl = pin(builder, /kaikkiSourceURL\s*=\s*"([^"]+)"/, 'kaikkiSourceURL in the builder'); + expect(builderUrl).toBe(makeUrl); }); });