diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e72cb1b..f1d7f26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ # Tests and packaging. # -# Nothing here downloads the 179 MB upstream dictionary except the release -# image build. Everything else plays against the small database derived from +# Nothing here downloads the upstream wordlist except the release image +# build. Everything else plays against the small database derived from # the checked-in word sample, which goes through the same builder the real one # does. # @@ -123,7 +123,7 @@ jobs: docker build --build-arg FIXTURE_DICT=1 -t noitu:ci . fi - # CC BY-SA 4.0 applies to the derived wordlist wherever it is + # CC BY-SA 3.0 applies to the derived wordlist wherever it is # distributed, and an image is distribution. This is the assertion that # the obligation actually shipped. - name: The licence travels with the data @@ -137,9 +137,9 @@ jobs: grep -qx "$required" files.txt || { echo "missing from the image: $required"; exit 1; } done - # The 179 MB source must never reach the final image. - if grep -q 'dictionary\.db$' files.txt; then - echo "the upstream dictionary leaked into the image" + # The upstream wordlist must never reach the final image. + if grep -q 'words\.jsonl$' files.txt; then + echo "the upstream wordlist leaked into the image" exit 1 fi diff --git a/.github/workflows/proto.yml b/.github/workflows/proto.yml index b15308b..d95be8d 100644 --- a/.github/workflows/proto.yml +++ b/.github/workflows/proto.yml @@ -2,7 +2,7 @@ # broken compatibility, and the committed generated code matches it. # # The suites that decode the cross-language fixtures run in ci.yml, where the -# rest of the tests are. Nothing here downloads the 179 MB upstream dictionary. +# rest of the tests are. Nothing here downloads the upstream wordlist. name: proto on: diff --git a/.gitignore b/.gitignore index 65630fa..5c51091 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Dictionary data — never committed. -# data/dictionary.db is the 179 MB upstream asset; data/noitu.db is derived from it. -# Both are build artifacts produced by `make fetch-dict` and `make dict`. +# data/undertheseanlp-words.jsonl is the upstream wordlist; data/noitu.db is +# derived from it. Both are build artifacts produced by `make fetch-dict` and +# `make dict`. +data/undertheseanlp-words.jsonl data/*.db data/*.db-journal data/*.db-wal diff --git a/Dockerfile b/Dockerfile index 7224f80..d9edbc2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # One image: the binary, the built frontend, and the derived dictionary. # -# The 179 MB upstream release is downloaded in a builder stage and never -# reaches the final image — only the ~3 MB database derived from it does. That -# derived database is CC BY-SA 4.0 while the code is Apache-2.0, so it is +# The upstream wordlist is downloaded in a builder stage and never reaches the +# final image — only the ~1.7 MB database derived from it does. That +# derived database is CC BY-SA 3.0 while the code is Apache-2.0, so it is # copied in as its own layer alongside its licence and attribution rather than # being embedded in the binary. @@ -30,15 +30,15 @@ RUN CGO_ENABLED=0 go build -trimpath -o /out/build-dictionary ./cmd/build-dictio # --- the dictionary --------------------------------------------------------- FROM alpine:3.22 AS dict -# Pinned by tag and checked by digest: the derived wordlist is the one thing in -# this image that cannot be rebuilt from the repository alone. The Makefile -# pins the same release for local builds, and a test asserts the two agree. -ARG DICT_URL=https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db -ARG DICT_SHA256=9259403f0675b2991a1bd0ef6d0dbc5933afdb135632af095a60662f09bbf1d3 +# Pinned by commit and checked by digest: the derived wordlist is the one thing +# in this image that cannot be rebuilt from the repository alone. The Makefile +# pins the same file for local builds, and a test asserts the two agree. +ARG DICT_URL=https://raw.githubusercontent.com/undertheseanlp/dictionary/2c078cfc373b06e2980d324ce1d7bd13740c3319/dictionary/words.txt +ARG DICT_SHA256=4c3e0e6117e4bdfa97731e135c3d4a05881889909267394a8de8d88ef79f13f0 # Set to 1 to build from the checked-in word sample instead of downloading the -# upstream release. That produces a playable but tiny dictionary, and exists so -# the image itself can be smoke-tested without a 179 MB download. +# upstream wordlist. That produces a playable but tiny dictionary, and exists so +# the image itself can be smoke-tested without network access. ARG FIXTURE_DICT=0 RUN apk add --no-cache curl @@ -51,9 +51,9 @@ RUN set -eu; \ if [ "$FIXTURE_DICT" = "1" ]; then \ build-dictionary --words ./fixture-words.txt --out /out/noitu.db --min-words 150; \ else \ - curl -fsSL -o dictionary.db "$DICT_URL"; \ - echo "$DICT_SHA256 dictionary.db" | sha256sum -c -; \ - build-dictionary --in ./dictionary.db --out /out/noitu.db; \ + curl -fsSL -o undertheseanlp-words.jsonl "$DICT_URL"; \ + echo "$DICT_SHA256 undertheseanlp-words.jsonl" | sha256sum -c -; \ + build-dictionary --merged ./undertheseanlp-words.jsonl --out /out/noitu.db; \ fi # --- the image -------------------------------------------------------------- @@ -64,7 +64,7 @@ COPY --from=build /out/noitu-server /app/noitu-server COPY --from=web /src/web/build /app/web # The share-alike half of the image. data/LICENSE and data/ATTRIBUTION.md ship -# with the derived wordlist because CC BY-SA 4.0 applies to it wherever it is +# with the derived wordlist because CC BY-SA 3.0 applies to it wherever it is # distributed, and a container image is distribution. COPY --from=dict /out/noitu.db /app/data/noitu.db COPY data/LICENSE /app/data/LICENSE diff --git a/Makefile b/Makefile index f2d0786..34f7b29 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,11 @@ # Every target has a raw equivalent documented in README.md, so contributors # without `make` (notably on Windows) are never blocked. -DICT_URL := https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db -DICT_SRC := data/dictionary.db -# Pinned to the v2.0.0 release asset. A mismatch means the upstream artifact -# changed under the same tag, or the download was truncated. -DICT_SHA256 := 9259403f0675b2991a1bd0ef6d0dbc5933afdb135632af095a60662f09bbf1d3 +DICT_URL := https://raw.githubusercontent.com/undertheseanlp/dictionary/2c078cfc373b06e2980d324ce1d7bd13740c3319/dictionary/words.txt +DICT_SRC := data/undertheseanlp-words.jsonl +# Pinned to a commit, not a branch, so the URL is immutable and cannot drift +# from the checksum. A mismatch means the download was truncated or tampered. +DICT_SHA256 := 4c3e0e6117e4bdfa97731e135c3d4a05881889909267394a8de8d88ef79f13f0 DICT_OUT := data/noitu.db FIXTURE_WORDS := testdata/fixture-words.txt FIXTURE_DB := data/fixture.db @@ -16,7 +16,7 @@ SERVER_BIN := noitu-server .PHONY: help fetch-dict verify-dict dict fixture-dict proto proto-check server web web-dev test test-go test-web test-e2e run clean help: - @echo "fetch-dict download + checksum the upstream dictionary (~179 MB) into data/" + @echo "fetch-dict download + checksum the upstream wordlist (~4.8 MB) into data/" @echo "verify-dict re-check the downloaded dictionary against its pinned SHA-256" @echo "dict derive $(DICT_OUT) from $(DICT_SRC)" @echo "fixture-dict build the small test dictionary — no download needed" @@ -30,10 +30,11 @@ help: @echo "run build and run the server locally" @echo "clean remove build artifacts (keeps downloaded dictionary)" -# One-time download. Resumable (-C -) so an interrupted 179 MB fetch can continue. +# One-time download. -f so an HTTP error page fails here, not as a confusing +# checksum mismatch. fetch-dict: @mkdir -p data - curl -L -C - -o $(DICT_SRC) $(DICT_URL) + curl -fL -o $(DICT_SRC) $(DICT_URL) @echo "$(DICT_SHA256) $(DICT_SRC)" | sha256sum -c - @echo "downloaded and verified $(DICT_SRC)" @@ -46,11 +47,11 @@ $(DICT_SRC): @exit 1 dict: $(DICT_SRC) - cd server && go run ./cmd/build-dictionary --in ../$(DICT_SRC) --out ../$(DICT_OUT) + cd server && go run ./cmd/build-dictionary --merged ../$(DICT_SRC) --out ../$(DICT_OUT) # The dictionary tests, end-to-end runs and CI all play against. Built from a # checked-in word list through the same pipeline as the real one, so nothing -# has to download 179 MB to get a working database. +# has to download anything to get a working database. $(FIXTURE_DB): $(FIXTURE_WORDS) cd server && go run ./cmd/build-dictionary --words ../$(FIXTURE_WORDS) --out ../$(FIXTURE_DB) --min-words 150 diff --git a/NOTICE b/NOTICE index 3674af5..9330b1b 100644 --- a/NOTICE +++ b/NOTICE @@ -13,19 +13,22 @@ Version 2.0. See the LICENSE file at the repository root. This covers everything under server/, tools/, web/, and proto/. ------------------------------------------------------------------------------ -2. DICTIONARY DATA — Creative Commons Attribution-ShareAlike 4.0 International +2. DICTIONARY DATA — Creative Commons Attribution-ShareAlike 3.0 Unported ------------------------------------------------------------------------------ The Vietnamese dictionary database is NOT covered by the Apache License. It is -derived from a third-party dataset and remains under CC BY-SA 4.0: +derived from Wiktionary text and remains under the licence that text carried, +CC BY-SA 3.0: - Affected artifacts: data/noitu.db (and data/dictionary.db, its source) + Affected artifacts: data/noitu.db (and data/undertheseanlp-words.jsonl, its source) License text: data/LICENSE Attribution and list of changes: data/ATTRIBUTION.md - Upstream source: https://github.com/minhqnd/dictionary (release v2.0.0) + Original work: Wiktionary tiếng Việt (https://vi.wiktionary.org/), by its contributors + Upstream source: https://github.com/undertheseanlp/dictionary (commit 2c078cf, + file dictionary/words.txt, rows tagged "wiktionary" only) -CC BY-SA 4.0 is a share-alike license. Any distribution of the derived database +CC BY-SA 3.0 is a share-alike license. Any distribution of the derived database -- including inside a container image or other packaged build -- must carry the same license, this attribution, and the record of modifications. @@ -33,5 +36,5 @@ The two regimes are kept on separate artifacts deliberately: the database is loaded at runtime from a file and is never embedded, compiled, or linked into the Go binary. -Neither database file is committed to version control. Both are build artifacts -produced by `make fetch-dict` and `make dict`. +Neither the source wordlist nor the derived database is committed to version +control. Both are build artifacts produced by `make fetch-dict` and `make dict`. diff --git a/README.md b/README.md index 1e0cce2..9f18067 100644 --- a/README.md +++ b/README.md @@ -159,14 +159,16 @@ is needed only to change the WebSocket schema — the generated code is committe building and running the project does not require it. ```sh -make fetch-dict # one-time: downloads the ~179 MB upstream dictionary into data/ +make fetch-dict # one-time: downloads the ~4.8 MB upstream wordlist into data/ make dict # derives data/noitu.db (the game's wordlist) from it make test # run all tests make run # build and start the server ``` -`make fetch-dict` is a one-time cost per machine. Neither database file is committed; -both are build artifacts. See [`data/ATTRIBUTION.md`](./data/ATTRIBUTION.md). +`make fetch-dict` is a one-time cost per machine. Neither the wordlist nor the derived +database is committed; both are build artifacts. See +[`data/ATTRIBUTION.md`](./data/ATTRIBUTION.md). If you have a `data/dictionary.db` from +before the corpus switch, it is no longer read and can be deleted. ## Running the server @@ -218,8 +220,8 @@ dev-only URL to get wrong. | Target | Does | |---|---| -| `fetch-dict` | Download the upstream `dictionary.db` (~179 MB) into `data/` | -| `dict` | Derive `data/noitu.db` from the upstream database | +| `fetch-dict` | Download the upstream wordlist (~4.8 MB) into `data/` and verify its checksum | +| `dict` | Derive `data/noitu.db` from the upstream wordlist | | `server` | Build the Go server binary | | `web` | Build the SvelteKit frontend to static assets | | `web-dev` | Run the frontend dev server, proxying `/ws` to a local server | @@ -236,11 +238,12 @@ dev-only URL to get wrong. `make` is not installed everywhere (notably Windows). Every target is a thin wrapper: ```sh -# fetch-dict -curl -L -C - -o data/dictionary.db https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db +# fetch-dict (URL and checksum are pinned in the Makefile) +curl -fL -o data/undertheseanlp-words.jsonl https://raw.githubusercontent.com/undertheseanlp/dictionary/2c078cfc373b06e2980d324ce1d7bd13740c3319/dictionary/words.txt +echo "4c3e0e6117e4bdfa97731e135c3d4a05881889909267394a8de8d88ef79f13f0 data/undertheseanlp-words.jsonl" | sha256sum -c - # dict -cd server && go run ./cmd/build-dictionary --in ../data/dictionary.db --out ../data/noitu.db +cd server && go run ./cmd/build-dictionary --merged ../data/undertheseanlp-words.jsonl --out ../data/noitu.db # test cd server && go vet ./... && go test ./... -race @@ -275,7 +278,7 @@ buf generate && buf lint The end-to-end suite plays against a small dictionary derived from [`testdata/fixture-words.txt`](./testdata/fixture-words.txt) through the same -builder the real one uses, so CI never downloads the 179 MB upstream release. +builder the real one uses, so CI never downloads the upstream wordlist. ## Deployment @@ -296,11 +299,12 @@ See [`NOTICE`](./NOTICE) for the full statement. | Artifact | License | |---|---| | All source code (`server/`, `web/`, `proto/`) | [Apache-2.0](./LICENSE) | -| Dictionary data (`data/noitu.db`) | [CC BY-SA 4.0](./data/LICENSE) | +| Dictionary data (`data/noitu.db`) | [CC BY-SA 3.0](./data/LICENSE) | -The dictionary is derived from [minhqnd/dictionary](https://github.com/minhqnd/dictionary) -(data licensed CC BY-SA 4.0), which itself aggregates Wiktionary and other Vietnamese -dictionary sources. CC BY-SA is a **share-alike** license: any redistribution of the derived +The dictionary is derived from the [Wiktionary tiếng Việt](https://vi.wiktionary.org/) +entries (CC BY-SA, by their contributors) as scraped and redistributed by +[undertheseanlp/dictionary](https://github.com/undertheseanlp/dictionary); the other two +wordlists in that file are not used. CC BY-SA is a **share-alike** license: any redistribution of the derived database — including inside a container image — must carry the same license, the attribution, and the record of modifications recorded in [`data/ATTRIBUTION.md`](./data/ATTRIBUTION.md). diff --git a/data/ATTRIBUTION.md b/data/ATTRIBUTION.md index 0d87958..2e35011 100644 --- a/data/ATTRIBUTION.md +++ b/data/ATTRIBUTION.md @@ -1,40 +1,57 @@ # Dictionary Data Attribution The Vietnamese dictionary data used by this game is **not** original work of this project. -It is derived from a third-party dataset licensed under **CC BY-SA 4.0**, and this file +It is derived from Wiktionary text licensed under **CC BY-SA 3.0 Unported**, and this file records the attribution and the modifications required by that license. ## Source | Field | Value | |---|---| -| Project | [minhqnd/dictionary](https://github.com/minhqnd/dictionary) | -| Asset | [`dictionary.db`, release v2.0.0](https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db) (~179 MB) | -| Author | [minhqnd](https://github.com/minhqnd) | -| Data license | [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/) — full text in [`LICENSE`](./LICENSE) | -| Code license (not used here) | MIT | +| Original work | Entries of [Wiktionary tiếng Việt](https://vi.wiktionary.org/), written by its contributors | +| Original license | [CC BY-SA 3.0 Unported](https://creativecommons.org/licenses/by-sa/3.0/) at the time of the 2018 scrape (Wikimedia projects moved to 4.0 on 2023-06-29); Wiktionary text is dual-licensed CC BY-SA / GFDL | +| This database's license | **CC BY-SA 3.0 Unported**, the same licence the source text carried — full text in [`LICENSE`](./LICENSE) | +| Redistributed by | [undertheseanlp/dictionary](https://github.com/undertheseanlp/dictionary), a 2018-12-10 scrape of vi.wiktionary.org merged with two other wordlists | +| Asset | [`dictionary/words.txt` at commit `2c078cf`](https://raw.githubusercontent.com/undertheseanlp/dictionary/2c078cfc373b06e2980d324ce1d7bd13740c3319/dictionary/words.txt) — 4,813,111 bytes, 79,226 JSONL rows | +| SHA-256 | `4c3e0e6117e4bdfa97731e135c3d4a05881889909267394a8de8d88ef79f13f0` | +| Rows used | the 32,484 tagged `wiktionary` | +| Repository code license (not used here) | GPL-3.0 | -That project is itself an aggregation. Its own upstream sources include -[Wiktionary](https://vi.wiktionary.org/) (CC BY-SA) and -[vntk/dictionary](https://github.com/vntk/dictionary), among other Vietnamese dictionary -projects. Those upstream attributions carry through this file. +The asset is pinned by commit, so the URL is immutable and the checksum above always +describes the same bytes. The attribution chain has two links — Wiktionary's contributors, +then undertheseanlp as the intermediary that scraped and redistributed — and both are named +here because CC BY-SA attribution belongs to the authors, not only to the last host. + +### Branches deliberately not used + +The merged file tags every word with which of three wordlists contain it. Only +`wiktionary` rows are read; the other two are never consulted — not for words, not for +anything. + +- **`hongocduc`** — Hồ Ngọc Đức's Free Vietnamese Dictionary Project wordlist, distributed + under the GNU GPL according to the branch README (a mirror; the canonical site no longer + resolves). Using it would make this database GPL-derived and force a relicense to GPLv3. + Excluded so the data stays CC BY-SA. +- **`tudientv`** — its branch README declares copyright *"Chưa rõ"* (unknown) and names the + Soha/Vietlex dictionary (Hoàng Phê, editor) as its primary source: an unlicensed + derivative of a commercial work. Excluded, and not to be reopened. ## Modifications made by this project -`server/cmd/build-dictionary` transforms the upstream `dictionary.db` into `data/noitu.db`. -The derived database is a **modified version** of the source data. Changes: +`server/cmd/build-dictionary` transforms the upstream wordlist into `data/noitu.db`. The +derived database is a **modified version** of the source data. Changes: -1. **Language filter** — kept only entries with `lang_code = 'vi'`; all other languages - (of 1,500+ language pairs in the source) were dropped. +1. **Source selection** — kept only rows whose source list contains `wiktionary`; rows from + the two other wordlists were not read. 2. **Length filter** — kept only words of **2 or more space-separated syllables**, as required by the nối từ game rules. Single-syllable entries were dropped. 3. **Content filter** — dropped entries containing digits or punctuation, entries using letters Vietnamese does not have (f, j, w, z), and entries whose syllables do not fit - Vietnamese phonotactics (a closed inventory of onsets, nuclei and codas). This removes - loanwords and foreign phrases such as "credit card" and "world cup" that the - multilingual source tags as Vietnamese. Diacritic-free Vietnamese words ("con cua") - are kept. -4. **Normalization** — all words Unicode NFC-normalized, lowercased, and whitespace-collapsed. + Vietnamese phonotactics (a closed inventory of onsets, nuclei and codas). Diacritic-free + Vietnamese words ("con cua") are kept. +4. **Normalization** — all words Unicode NFC-normalized, lowercased, and + whitespace-collapsed. Capitalized headwords (`Hà Nội`) become lowercase entries; nothing + is removed on the basis of capitalization. 5. **Spelling aliases** — added an `aliases` table mapping alternative Vietnamese spellings to canonical entries. Two kinds: competing tone placement in open oa/oe/uy syllables (`hoà` → `hòa`, `thuý` → `thúy`), and i/y alternation in Sino-Vietnamese syllables @@ -42,18 +59,19 @@ The derived database is a **modified version** of the source data. Changes: by this project and are not present upstream. 6. **Added columns and tables** — `first` and `last` syllable columns, a `syllables` count, an index on `first`, a `syllables` out-degree table, and a `meta` table recording - provenance. All added for game lookups. -7. **Deduplication** — the source lists a word once per sense; entries were deduplicated by - normalized form. A generated spelling variant that is itself a real word, or that more - than one word would claim, is discarded rather than recorded as an alias. -8. **Dropped fields** — all definitions, translations, pronunciations, examples, part-of-speech - tags, and relations from the source were discarded. The derived database contains - **only word forms**, not meanings. + provenance (source URL, commit, sources kept and excluded). All added for game lookups. +7. **Deduplication** — entries were deduplicated by normalized form, so a word listed both + capitalized and lowercase upstream appears once. A generated spelling variant that is + itself a real word, or that more than one word would claim, is discarded rather than + recorded as an alias. +8. **Dropped fields** — the per-word source tags were discarded after selection. The + derived database contains **only word forms**, not meanings; the upstream wordlist + carries none either. ## Share-alike obligation -CC BY-SA 4.0 is a **share-alike** license. The derived database `data/noitu.db`, and any -distribution of it, remains licensed under **CC BY-SA 4.0** — including when it is shipped +CC BY-SA 3.0 is a **share-alike** license. The derived database `data/noitu.db`, and any +distribution of it, remains licensed under **CC BY-SA 3.0** — including when it is shipped inside a container image or any other packaged build of this project. This obligation applies to the **data only**. The source code of this project is licensed @@ -64,7 +82,7 @@ keeping the two licensing regimes on separate artifacts. ## How to reproduce the derived data ```sh -make fetch-dict # downloads the ~179 MB upstream dictionary.db into data/ +make fetch-dict # downloads the ~4.8 MB upstream wordlist into data/ and verifies its checksum make dict # derives data/noitu.db from it ``` diff --git a/data/LICENSE b/data/LICENSE index 2d58298..604209a 100644 --- a/data/LICENSE +++ b/data/LICENSE @@ -1,428 +1,359 @@ -Attribution-ShareAlike 4.0 International +Creative Commons Legal Code -======================================================================= +Attribution-ShareAlike 3.0 Unported -Creative Commons Corporation ("Creative Commons") is not a law firm and -does not provide legal services or legal advice. Distribution of -Creative Commons public licenses does not create a lawyer-client or -other relationship. Creative Commons makes its licenses and related -information available on an "as-is" basis. Creative Commons gives no -warranties regarding its licenses, any material licensed under their -terms and conditions, or any related information. Creative Commons -disclaims all liability for damages resulting from their use to the -fullest extent possible. + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR + DAMAGES RESULTING FROM ITS USE. -Using Creative Commons Public Licenses - -Creative Commons public licenses provide a standard set of terms and -conditions that creators and other rights holders may use to share -original works of authorship and other material subject to copyright -and certain other rights specified in the public license below. The -following considerations are for informational purposes only, are not -exhaustive, and do not form part of our licenses. - - Considerations for licensors: Our public licenses are - intended for use by those authorized to give the public - permission to use material in ways otherwise restricted by - copyright and certain other rights. Our licenses are - irrevocable. Licensors should read and understand the terms - and conditions of the license they choose before applying it. - Licensors should also secure all rights necessary before - applying our licenses so that the public can reuse the - material as expected. Licensors should clearly mark any - material not subject to the license. This includes other CC- - licensed material, or material used under an exception or - limitation to copyright. More considerations for licensors: - wiki.creativecommons.org/Considerations_for_licensors - - Considerations for the public: By using one of our public - licenses, a licensor grants the public permission to use the - licensed material under specified terms and conditions. If - the licensor's permission is not necessary for any reason--for - example, because of any applicable exception or limitation to - copyright--then that use is not regulated by the license. Our - licenses grant only permissions under copyright and certain - other rights that a licensor has authority to grant. Use of - the licensed material may still be restricted for other - reasons, including because others have copyright or other - rights in the material. A licensor may make special requests, - such as asking that all changes be marked or described. - Although not required by our licenses, you are encouraged to - respect those requests where reasonable. More considerations - for the public: - wiki.creativecommons.org/Considerations_for_licensees - -======================================================================= - -Creative Commons Attribution-ShareAlike 4.0 International Public License -By exercising the Licensed Rights (defined below), You accept and agree -to be bound by the terms and conditions of this Creative Commons -Attribution-ShareAlike 4.0 International Public License ("Public -License"). To the extent this Public License may be interpreted as a -contract, You are granted the Licensed Rights in consideration of Your -acceptance of these terms and conditions, and the Licensor grants You -such rights in consideration of benefits the Licensor receives from -making the Licensed Material available under these terms and -conditions. - - -Section 1 -- Definitions. - - a. Adapted Material means material subject to Copyright and Similar - Rights that is derived from or based upon the Licensed Material - and in which the Licensed Material is translated, altered, - arranged, transformed, or otherwise modified in a manner requiring - permission under the Copyright and Similar Rights held by the - Licensor. For purposes of this Public License, where the Licensed - Material is a musical work, performance, or sound recording, - Adapted Material is always produced where the Licensed Material is - synched in timed relation with a moving image. - - b. Adapter's License means the license You apply to Your Copyright - and Similar Rights in Your contributions to Adapted Material in - accordance with the terms and conditions of this Public License. - - c. BY-SA Compatible License means a license listed at - creativecommons.org/compatiblelicenses, approved by Creative - Commons as essentially the equivalent of this Public License. - - d. Copyright and Similar Rights means copyright and/or similar rights - closely related to copyright including, without limitation, - performance, broadcast, sound recording, and Sui Generis Database - Rights, without regard to how the rights are labeled or - categorized. For purposes of this Public License, the rights - specified in Section 2(b)(1)-(2) are not Copyright and Similar - Rights. - - e. Effective Technological Measures means those measures that, in the - absence of proper authority, may not be circumvented under laws - fulfilling obligations under Article 11 of the WIPO Copyright - Treaty adopted on December 20, 1996, and/or similar international - agreements. - - f. Exceptions and Limitations means fair use, fair dealing, and/or - any other exception or limitation to Copyright and Similar Rights - that applies to Your use of the Licensed Material. - - g. License Elements means the license attributes listed in the name - of a Creative Commons Public License. The License Elements of this - Public License are Attribution and ShareAlike. - - h. Licensed Material means the artistic or literary work, database, - or other material to which the Licensor applied this Public - License. - - i. Licensed Rights means the rights granted to You subject to the - terms and conditions of this Public License, which are limited to - all Copyright and Similar Rights that apply to Your use of the - Licensed Material and that the Licensor has authority to license. - - j. Licensor means the individual(s) or entity(ies) granting rights - under this Public License. - - k. Share means to provide material to the public by any means or - process that requires permission under the Licensed Rights, such - as reproduction, public display, public performance, distribution, - dissemination, communication, or importation, and to make material - available to the public including in ways that members of the - public may access the material from a place and at a time - individually chosen by them. - - l. Sui Generis Database Rights means rights other than copyright - resulting from Directive 96/9/EC of the European Parliament and of - the Council of 11 March 1996 on the legal protection of databases, - as amended and/or succeeded, as well as other essentially - equivalent rights anywhere in the world. - - m. You means the individual or entity exercising the Licensed Rights - under this Public License. Your has a corresponding meaning. - - -Section 2 -- Scope. - - a. License grant. - - 1. Subject to the terms and conditions of this Public License, - the Licensor hereby grants You a worldwide, royalty-free, - non-sublicensable, non-exclusive, irrevocable license to - exercise the Licensed Rights in the Licensed Material to: - - a. reproduce and Share the Licensed Material, in whole or - in part; and - - b. produce, reproduce, and Share Adapted Material. - - 2. Exceptions and Limitations. For the avoidance of doubt, where - Exceptions and Limitations apply to Your use, this Public - License does not apply, and You do not need to comply with - its terms and conditions. - - 3. Term. The term of this Public License is specified in Section - 6(a). - - 4. Media and formats; technical modifications allowed. The - Licensor authorizes You to exercise the Licensed Rights in - all media and formats whether now known or hereafter created, - and to make technical modifications necessary to do so. The - Licensor waives and/or agrees not to assert any right or - authority to forbid You from making technical modifications - necessary to exercise the Licensed Rights, including - technical modifications necessary to circumvent Effective - Technological Measures. For purposes of this Public License, - simply making modifications authorized by this Section 2(a) - (4) never produces Adapted Material. - - 5. Downstream recipients. - - a. Offer from the Licensor -- Licensed Material. Every - recipient of the Licensed Material automatically - receives an offer from the Licensor to exercise the - Licensed Rights under the terms and conditions of this - Public License. - - b. Additional offer from the Licensor -- Adapted Material. - Every recipient of Adapted Material from You - automatically receives an offer from the Licensor to - exercise the Licensed Rights in the Adapted Material - under the conditions of the Adapter's License You apply. - - c. No downstream restrictions. You may not offer or impose - any additional or different terms or conditions on, or - apply any Effective Technological Measures to, the - Licensed Material if doing so restricts exercise of the - Licensed Rights by any recipient of the Licensed - Material. - - 6. No endorsement. Nothing in this Public License constitutes or - may be construed as permission to assert or imply that You - are, or that Your use of the Licensed Material is, connected - with, or sponsored, endorsed, or granted official status by, - the Licensor or others designated to receive attribution as - provided in Section 3(a)(1)(A)(i). - - b. Other rights. - - 1. Moral rights, such as the right of integrity, are not - licensed under this Public License, nor are publicity, - privacy, and/or other similar personality rights; however, to - the extent possible, the Licensor waives and/or agrees not to - assert any such rights held by the Licensor to the limited - extent necessary to allow You to exercise the Licensed - Rights, but not otherwise. - - 2. Patent and trademark rights are not licensed under this - Public License. - - 3. To the extent possible, the Licensor waives any right to - collect royalties from You for the exercise of the Licensed - Rights, whether directly or through a collecting society - under any voluntary or waivable statutory or compulsory - licensing scheme. In all other cases the Licensor expressly - reserves any right to collect such royalties. - - -Section 3 -- License Conditions. - -Your exercise of the Licensed Rights is expressly made subject to the -following conditions. - - a. Attribution. - - 1. If You Share the Licensed Material (including in modified - form), You must: - - a. retain the following if it is supplied by the Licensor - with the Licensed Material: - - i. identification of the creator(s) of the Licensed - Material and any others designated to receive - attribution, in any reasonable manner requested by - the Licensor (including by pseudonym if - designated); - - ii. a copyright notice; - - iii. a notice that refers to this Public License; - - iv. a notice that refers to the disclaimer of - warranties; - - v. a URI or hyperlink to the Licensed Material to the - extent reasonably practicable; - - b. indicate if You modified the Licensed Material and - retain an indication of any previous modifications; and - - c. indicate the Licensed Material is licensed under this - Public License, and include the text of, or the URI or - hyperlink to, this Public License. - - 2. You may satisfy the conditions in Section 3(a)(1) in any - reasonable manner based on the medium, means, and context in - which You Share the Licensed Material. For example, it may be - reasonable to satisfy the conditions by providing a URI or - hyperlink to a resource that includes the required - information. - - 3. If requested by the Licensor, You must remove any of the - information required by Section 3(a)(1)(A) to the extent - reasonably practicable. - - b. ShareAlike. - - In addition to the conditions in Section 3(a), if You Share - Adapted Material You produce, the following conditions also apply. - - 1. The Adapter's License You apply must be a Creative Commons - license with the same License Elements, this version or - later, or a BY-SA Compatible License. - - 2. You must include the text of, or the URI or hyperlink to, the - Adapter's License You apply. You may satisfy this condition - in any reasonable manner based on the medium, means, and - context in which You Share Adapted Material. - - 3. You may not offer or impose any additional or different terms - or conditions on, or apply any Effective Technological - Measures to, Adapted Material that restrict exercise of the - rights granted under the Adapter's License You apply. - - -Section 4 -- Sui Generis Database Rights. - -Where the Licensed Rights include Sui Generis Database Rights that -apply to Your use of the Licensed Material: - - a. for the avoidance of doubt, Section 2(a)(1) grants You the right - to extract, reuse, reproduce, and Share all or a substantial - portion of the contents of the database; - - b. if You include all or a substantial portion of the database - contents in a database in which You have Sui Generis Database - Rights, then the database in which You have Sui Generis Database - Rights (but not its individual contents) is Adapted Material, - including for purposes of Section 3(b); and - - c. You must comply with the conditions in Section 3(a) if You Share - all or a substantial portion of the contents of the database. - -For the avoidance of doubt, this Section 4 supplements and does not -replace Your obligations under this Public License where the Licensed -Rights include other Copyright and Similar Rights. - - -Section 5 -- Disclaimer of Warranties and Limitation of Liability. - - a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE - EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS - AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF - ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, - IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, - WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR - PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, - ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT - KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT - ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. - - b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE - TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, - NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, - INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, - COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR - USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN - ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR - DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR - IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. - - c. The disclaimer of warranties and limitation of liability provided - above shall be interpreted in a manner that, to the extent - possible, most closely approximates an absolute disclaimer and - waiver of all liability. - - -Section 6 -- Term and Termination. - - a. This Public License applies for the term of the Copyright and - Similar Rights licensed here. However, if You fail to comply with - this Public License, then Your rights under this Public License - terminate automatically. - - b. Where Your right to use the Licensed Material has terminated under - Section 6(a), it reinstates: - - 1. automatically as of the date the violation is cured, provided - it is cured within 30 days of Your discovery of the - violation; or - - 2. upon express reinstatement by the Licensor. - - For the avoidance of doubt, this Section 6(b) does not affect any - right the Licensor may have to seek remedies for Your violations - of this Public License. - - c. For the avoidance of doubt, the Licensor may also offer the - Licensed Material under separate terms or conditions or stop - distributing the Licensed Material at any time; however, doing so - will not terminate this Public License. - - d. Sections 1, 5, 6, 7, and 8 survive termination of this Public - License. - - -Section 7 -- Other Terms and Conditions. - - a. The Licensor shall not be bound by any additional or different - terms or conditions communicated by You unless expressly agreed. - - b. Any arrangements, understandings, or agreements regarding the - Licensed Material not stated herein are separate from and - independent of the terms and conditions of this Public License. - - -Section 8 -- Interpretation. - - a. For the avoidance of doubt, this Public License does not, and - shall not be interpreted to, reduce, limit, restrict, or impose - conditions on any use of the Licensed Material that could lawfully - be made without permission under this Public License. - - b. To the extent possible, if any provision of this Public License is - deemed unenforceable, it shall be automatically reformed to the - minimum extent necessary to make it enforceable. If the provision - cannot be reformed, it shall be severed from this Public License - without affecting the enforceability of the remaining terms and - conditions. - - c. No term or condition of this Public License will be waived and no - failure to comply consented to unless expressly agreed to by the - Licensor. - - d. Nothing in this Public License constitutes or may be interpreted - as a limitation upon, or waiver of, any privileges and immunities - that apply to the Licensor or You, including from the legal - processes of any jurisdiction or authority. - - -======================================================================= - -Creative Commons is not a party to its public -licenses. Notwithstanding, Creative Commons may elect to apply one of -its public licenses to material it publishes and in those instances -will be considered the “Licensor.” The text of the Creative Commons -public licenses is dedicated to the public domain under the CC0 Public -Domain Dedication. Except for the limited purpose of indicating that -material is shared under a Creative Commons public license or as -otherwise permitted by the Creative Commons policies published at -creativecommons.org/policies, Creative Commons does not authorize the -use of the trademark "Creative Commons" or any other trademark or logo -of Creative Commons without its prior written consent including, -without limitation, in connection with any unauthorized modifications -to any of its public licenses or any other arrangements, -understandings, or agreements concerning use of licensed material. For -the avoidance of doubt, this paragraph does not form part of the -public licenses. - -Creative Commons may be contacted at creativecommons.org. - +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE +COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY +COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS +AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE +TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY +BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS +CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND +CONDITIONS. + +1. Definitions + + a. "Adaptation" means a work based upon the Work, or upon the Work and + other pre-existing works, such as a translation, adaptation, + derivative work, arrangement of music or other alterations of a + literary or artistic work, or phonogram or performance and includes + cinematographic adaptations or any other form in which the Work may be + recast, transformed, or adapted including in any form recognizably + derived from the original, except that a work that constitutes a + Collection will not be considered an Adaptation for the purpose of + this License. For the avoidance of doubt, where the Work is a musical + work, performance or phonogram, the synchronization of the Work in + timed-relation with a moving image ("synching") will be considered an + Adaptation for the purpose of this License. + b. "Collection" means a collection of literary or artistic works, such as + encyclopedias and anthologies, or performances, phonograms or + broadcasts, or other works or subject matter other than works listed + in Section 1(f) below, which, by reason of the selection and + arrangement of their contents, constitute intellectual creations, in + which the Work is included in its entirety in unmodified form along + with one or more other contributions, each constituting separate and + independent works in themselves, which together are assembled into a + collective whole. A work that constitutes a Collection will not be + considered an Adaptation (as defined below) for the purposes of this + License. + c. "Creative Commons Compatible License" means a license that is listed + at https://creativecommons.org/compatiblelicenses that has been + approved by Creative Commons as being essentially equivalent to this + License, including, at a minimum, because that license: (i) contains + terms that have the same purpose, meaning and effect as the License + Elements of this License; and, (ii) explicitly permits the relicensing + of adaptations of works made available under that license under this + License or a Creative Commons jurisdiction license with the same + License Elements as this License. + d. "Distribute" means to make available to the public the original and + copies of the Work or Adaptation, as appropriate, through sale or + other transfer of ownership. + e. "License Elements" means the following high-level license attributes + as selected by Licensor and indicated in the title of this License: + Attribution, ShareAlike. + f. "Licensor" means the individual, individuals, entity or entities that + offer(s) the Work under the terms of this License. + g. "Original Author" means, in the case of a literary or artistic work, + the individual, individuals, entity or entities who created the Work + or if no individual or entity can be identified, the publisher; and in + addition (i) in the case of a performance the actors, singers, + musicians, dancers, and other persons who act, sing, deliver, declaim, + play in, interpret or otherwise perform literary or artistic works or + expressions of folklore; (ii) in the case of a phonogram the producer + being the person or legal entity who first fixes the sounds of a + performance or other sounds; and, (iii) in the case of broadcasts, the + organization that transmits the broadcast. + h. "Work" means the literary and/or artistic work offered under the terms + of this License including without limitation any production in the + literary, scientific and artistic domain, whatever may be the mode or + form of its expression including digital form, such as a book, + pamphlet and other writing; a lecture, address, sermon or other work + of the same nature; a dramatic or dramatico-musical work; a + choreographic work or entertainment in dumb show; a musical + composition with or without words; a cinematographic work to which are + assimilated works expressed by a process analogous to cinematography; + a work of drawing, painting, architecture, sculpture, engraving or + lithography; a photographic work to which are assimilated works + expressed by a process analogous to photography; a work of applied + art; an illustration, map, plan, sketch or three-dimensional work + relative to geography, topography, architecture or science; a + performance; a broadcast; a phonogram; a compilation of data to the + extent it is protected as a copyrightable work; or a work performed by + a variety or circus performer to the extent it is not otherwise + considered a literary or artistic work. + i. "You" means an individual or entity exercising rights under this + License who has not previously violated the terms of this License with + respect to the Work, or who has received express permission from the + Licensor to exercise rights under this License despite a previous + violation. + j. "Publicly Perform" means to perform public recitations of the Work and + to communicate to the public those public recitations, by any means or + process, including by wire or wireless means or public digital + performances; to make available to the public Works in such a way that + members of the public may access these Works from a place and at a + place individually chosen by them; to perform the Work to the public + by any means or process and the communication to the public of the + performances of the Work, including by public digital performance; to + broadcast and rebroadcast the Work by any means including signs, + sounds or images. + k. "Reproduce" means to make copies of the Work by any means including + without limitation by sound or visual recordings and the right of + fixation and reproducing fixations of the Work, including storage of a + protected performance or phonogram in digital form or other electronic + medium. + +2. Fair Dealing Rights. Nothing in this License is intended to reduce, +limit, or restrict any uses free from copyright or rights arising from +limitations or exceptions that are provided for in connection with the +copyright protection under copyright law or other applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, +Licensor hereby grants You a worldwide, royalty-free, non-exclusive, +perpetual (for the duration of the applicable copyright) license to +exercise the rights in the Work as stated below: + + a. to Reproduce the Work, to incorporate the Work into one or more + Collections, and to Reproduce the Work as incorporated in the + Collections; + b. to create and Reproduce Adaptations provided that any such Adaptation, + including any translation in any medium, takes reasonable steps to + clearly label, demarcate or otherwise identify that changes were made + to the original Work. For example, a translation could be marked "The + original work was translated from English to Spanish," or a + modification could indicate "The original work has been modified."; + c. to Distribute and Publicly Perform the Work including as incorporated + in Collections; and, + d. to Distribute and Publicly Perform Adaptations. + e. For the avoidance of doubt: + + i. Non-waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme cannot be waived, the Licensor + reserves the exclusive right to collect such royalties for any + exercise by You of the rights granted under this License; + ii. Waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme can be waived, the Licensor waives the + exclusive right to collect such royalties for any exercise by You + of the rights granted under this License; and, + iii. Voluntary License Schemes. The Licensor waives the right to + collect royalties, whether individually or, in the event that the + Licensor is a member of a collecting society that administers + voluntary licensing schemes, via that society, from any exercise + by You of the rights granted under this License. + +The above rights may be exercised in all media and formats whether now +known or hereafter devised. The above rights include the right to make +such modifications as are technically necessary to exercise the rights in +other media and formats. Subject to Section 8(f), all rights not expressly +granted by Licensor are hereby reserved. + +4. Restrictions. The license granted in Section 3 above is expressly made +subject to and limited by the following restrictions: + + a. You may Distribute or Publicly Perform the Work only under the terms + of this License. You must include a copy of, or the Uniform Resource + Identifier (URI) for, this License with every copy of the Work You + Distribute or Publicly Perform. You may not offer or impose any terms + on the Work that restrict the terms of this License or the ability of + the recipient of the Work to exercise the rights granted to that + recipient under the terms of the License. You may not sublicense the + Work. You must keep intact all notices that refer to this License and + to the disclaimer of warranties with every copy of the Work You + Distribute or Publicly Perform. When You Distribute or Publicly + Perform the Work, You may not impose any effective technological + measures on the Work that restrict the ability of a recipient of the + Work from You to exercise the rights granted to that recipient under + the terms of the License. This Section 4(a) applies to the Work as + incorporated in a Collection, but this does not require the Collection + apart from the Work itself to be made subject to the terms of this + License. If You create a Collection, upon notice from any Licensor You + must, to the extent practicable, remove from the Collection any credit + as required by Section 4(c), as requested. If You create an + Adaptation, upon notice from any Licensor You must, to the extent + practicable, remove from the Adaptation any credit as required by + Section 4(c), as requested. + b. You may Distribute or Publicly Perform an Adaptation only under the + terms of: (i) this License; (ii) a later version of this License with + the same License Elements as this License; (iii) a Creative Commons + jurisdiction license (either this or a later license version) that + contains the same License Elements as this License (e.g., + Attribution-ShareAlike 3.0 US)); (iv) a Creative Commons Compatible + License. If you license the Adaptation under one of the licenses + mentioned in (iv), you must comply with the terms of that license. If + you license the Adaptation under the terms of any of the licenses + mentioned in (i), (ii) or (iii) (the "Applicable License"), you must + comply with the terms of the Applicable License generally and the + following provisions: (I) You must include a copy of, or the URI for, + the Applicable License with every copy of each Adaptation You + Distribute or Publicly Perform; (II) You may not offer or impose any + terms on the Adaptation that restrict the terms of the Applicable + License or the ability of the recipient of the Adaptation to exercise + the rights granted to that recipient under the terms of the Applicable + License; (III) You must keep intact all notices that refer to the + Applicable License and to the disclaimer of warranties with every copy + of the Work as included in the Adaptation You Distribute or Publicly + Perform; (IV) when You Distribute or Publicly Perform the Adaptation, + You may not impose any effective technological measures on the + Adaptation that restrict the ability of a recipient of the Adaptation + from You to exercise the rights granted to that recipient under the + terms of the Applicable License. This Section 4(b) applies to the + Adaptation as incorporated in a Collection, but this does not require + the Collection apart from the Adaptation itself to be made subject to + the terms of the Applicable License. + c. If You Distribute, or Publicly Perform the Work or any Adaptations or + Collections, You must, unless a request has been made pursuant to + Section 4(a), keep intact all copyright notices for the Work and + provide, reasonable to the medium or means You are utilizing: (i) the + name of the Original Author (or pseudonym, if applicable) if supplied, + and/or if the Original Author and/or Licensor designate another party + or parties (e.g., a sponsor institute, publishing entity, journal) for + attribution ("Attribution Parties") in Licensor's copyright notice, + terms of service or by other reasonable means, the name of such party + or parties; (ii) the title of the Work if supplied; (iii) to the + extent reasonably practicable, the URI, if any, that Licensor + specifies to be associated with the Work, unless such URI does not + refer to the copyright notice or licensing information for the Work; + and (iv) , consistent with Ssection 3(b), in the case of an + Adaptation, a credit identifying the use of the Work in the Adaptation + (e.g., "French translation of the Work by Original Author," or + "Screenplay based on original Work by Original Author"). The credit + required by this Section 4(c) may be implemented in any reasonable + manner; provided, however, that in the case of a Adaptation or + Collection, at a minimum such credit will appear, if a credit for all + contributing authors of the Adaptation or Collection appears, then as + part of these credits and in a manner at least as prominent as the + credits for the other contributing authors. For the avoidance of + doubt, You may only use the credit required by this Section for the + purpose of attribution in the manner set out above and, by exercising + Your rights under this License, You may not implicitly or explicitly + assert or imply any connection with, sponsorship or endorsement by the + Original Author, Licensor and/or Attribution Parties, as appropriate, + of You or Your use of the Work, without the separate, express prior + written permission of the Original Author, Licensor and/or Attribution + Parties. + d. Except as otherwise agreed in writing by the Licensor or as may be + otherwise permitted by applicable law, if You Reproduce, Distribute or + Publicly Perform the Work either by itself or as part of any + Adaptations or Collections, You must not distort, mutilate, modify or + take other derogatory action in relation to the Work which would be + prejudicial to the Original Author's honor or reputation. Licensor + agrees that in those jurisdictions (e.g. Japan), in which any exercise + of the right granted in Section 3(b) of this License (the right to + make Adaptations) would be deemed to be a distortion, mutilation, + modification or other derogatory action prejudicial to the Original + Author's honor and reputation, the Licensor will waive or not assert, + as appropriate, this Section, to the fullest extent permitted by the + applicable national law, to enable You to reasonably exercise Your + right under Section 3(b) of this License (right to make Adaptations) + but not otherwise. + +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR +OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY +KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, +INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, +FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF +LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, +WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION +OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE +LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR +ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES +ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. Termination + + a. This License and the rights granted hereunder will terminate + automatically upon any breach by You of the terms of this License. + Individuals or entities who have received Adaptations or Collections + from You under this License, however, will not have their licenses + terminated provided such individuals or entities remain in full + compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will + survive any termination of this License. + b. Subject to the above terms and conditions, the license granted here is + perpetual (for the duration of the applicable copyright in the Work). + Notwithstanding the above, Licensor reserves the right to release the + Work under different license terms or to stop distributing the Work at + any time; provided, however that any such election will not serve to + withdraw this License (or any other license that has been, or is + required to be, granted under the terms of this License), and this + License will continue in full force and effect unless terminated as + stated above. + +8. Miscellaneous + + a. Each time You Distribute or Publicly Perform the Work or a Collection, + the Licensor offers to the recipient a license to the Work on the same + terms and conditions as the license granted to You under this License. + b. Each time You Distribute or Publicly Perform an Adaptation, Licensor + offers to the recipient a license to the original Work on the same + terms and conditions as the license granted to You under this License. + c. If any provision of this License is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this License, and without further action + by the parties to this agreement, such provision shall be reformed to + the minimum extent necessary to make such provision valid and + enforceable. + d. No term or provision of this License shall be deemed waived and no + breach consented to unless such waiver or consent shall be in writing + and signed by the party to be charged with such waiver or consent. + e. This License constitutes the entire agreement between the parties with + respect to the Work licensed here. There are no understandings, + agreements or representations with respect to the Work not specified + here. Licensor shall not be bound by any additional provisions that + may appear in any communication from You. This License may not be + modified without the mutual written agreement of the Licensor and You. + f. The rights granted under, and the subject matter referenced, in this + License were drafted utilizing the terminology of the Berne Convention + for the Protection of Literary and Artistic Works (as amended on + September 28, 1979), the Rome Convention of 1961, the WIPO Copyright + Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 + and the Universal Copyright Convention (as revised on July 24, 1971). + These rights and subject matter take effect in the relevant + jurisdiction in which the License terms are sought to be enforced + according to the corresponding provisions of the implementation of + those treaty provisions in the applicable national law. If the + standard suite of rights granted under applicable copyright law + includes additional rights not granted under this License, such + additional rights are deemed to be included in the License; this + License is not intended to restrict the license of any rights under + applicable law. + + +Creative Commons Notice + + Creative Commons is not a party to this License, and makes no warranty + whatsoever in connection with the Work. Creative Commons will not be + liable to You or any party on any legal theory for any damages + whatsoever, including without limitation any general, special, + incidental or consequential damages arising in connection to this + license. Notwithstanding the foregoing two (2) sentences, if Creative + Commons has expressly identified itself as the Licensor hereunder, it + shall have all rights and obligations of Licensor. + + Except for the limited purpose of indicating to the public that the + Work is licensed under the CCPL, Creative Commons does not authorize + the use by either party of the trademark "Creative Commons" or any + related trademark or logo of Creative Commons without the prior + written consent of Creative Commons. Any permitted use will be in + compliance with Creative Commons' then-current trademark usage + guidelines, as may be published on its website or otherwise made + available upon request from time to time. For the avoidance of doubt, + this trademark restriction does not form part of the License. + + Creative Commons may be contacted at https://creativecommons.org/. diff --git a/docs/deployment.md b/docs/deployment.md index 90e2cb3..ce3175b 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -51,9 +51,9 @@ docker build -t noitu:latest . docker run -p 8080:8080 noitu:latest ``` -The build downloads the 179 MB upstream dictionary in a builder stage and -derives the ~3 MB database the game uses. Only the derived file is copied into -the final image, so the upstream release never ships. The result is a +The build downloads the 4.8 MB upstream wordlist in a builder stage and +derives the ~1.7 MB database the game uses. Only the derived file is copied into +the final image, so the upstream wordlist never ships. The result is a distroless image of about 25 MB running as a non-root user. Passing `--build-arg FIXTURE_DICT=1` builds the same image against the @@ -62,7 +62,7 @@ exists so the image can be tested without the download; do not ship it. ### What travels with the data -The derived wordlist is CC BY-SA 4.0 while the code is Apache-2.0, so the image +The derived wordlist is CC BY-SA 3.0 while the code is Apache-2.0, so the image carries `data/LICENSE`, `data/ATTRIBUTION.md` and `NOTICE` alongside it. CI asserts all three are present, and that the upstream file is not. Removing them would put the image out of compliance. diff --git a/server/cmd/build-dictionary/main.go b/server/cmd/build-dictionary/main.go index dfba588..fbd1b36 100644 --- a/server/cmd/build-dictionary/main.go +++ b/server/cmd/build-dictionary/main.go @@ -1,17 +1,18 @@ // Command build-dictionary derives the game's wordlist from the upstream -// minhqnd/dictionary SQLite database. +// undertheseanlp/dictionary merged wordlist. // -// The upstream database is ~179 MB and covers 1,500+ language pairs with full -// definitions. The game needs none of that: only Vietnamese word forms of at -// least two syllables, indexed by first and last syllable. This tool performs -// that reduction and records provenance in a meta table. +// The upstream is a 4.8 MB JSONL file: every word of three Vietnamese +// wordlists, tagged with which of them contain it. The game needs only word +// forms of at least two syllables from the wordlists whose license we accept, +// indexed by first and last syllable. This tool performs that reduction and +// records provenance in a meta table. // -// The derived database is a modified version of CC BY-SA 4.0 licensed data. +// The derived database is a modified version of CC BY-SA 3.0 licensed data. // See data/ATTRIBUTION.md. // // Usage: // -// go run ./cmd/build-dictionary --in ../data/dictionary.db --out ../data/noitu.db +// go run ./cmd/build-dictionary --merged ../data/undertheseanlp-words.jsonl --out ../data/noitu.db package main import ( @@ -29,39 +30,31 @@ import ( _ "modernc.org/sqlite" ) -const ( - sourceURL = "https://github.com/minhqnd/dictionary/releases/download/v2.0.0/dictionary.db" - sourceLicense = "CC BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/)" - builderVer = "1" -) +const builderVer = "2" type config struct { - in string + // merged is the corpus: the upstream JSONL wordlist, read for the + // wordlists named in sources. + merged string + sources string // words is an alternative source: a plain list, one word per line, used to // build a small fixture database without the upstream download. words string out string maxSyllables int minWords int - table string - wordCol string - langCol string - lang string } func main() { log.SetFlags(0) var cfg config - flag.StringVar(&cfg.in, "in", "../data/dictionary.db", "upstream dictionary.db to read") - flag.StringVar(&cfg.words, "words", "", "read a plain word list instead of a source database (one word per line, # comments)") + flag.StringVar(&cfg.merged, "merged", "", "upstream merged JSONL wordlist to read") + flag.StringVar(&cfg.sources, "sources", "wiktionary", "comma-separated upstream wordlists a word may come from (hongocduc, tudientv, wiktionary)") + flag.StringVar(&cfg.words, "words", "", "read a plain word list instead of the upstream wordlist (one word per line, # comments)") flag.StringVar(&cfg.out, "out", "../data/noitu.db", "derived database to write") flag.IntVar(&cfg.maxSyllables, "max-syllables", 0, "reject words longer than this (0 = no limit)") - flag.IntVar(&cfg.minWords, "min-words", 40000, "fail if fewer words survive filtering") - flag.StringVar(&cfg.table, "table", "", "source table (default: auto-detect)") - flag.StringVar(&cfg.wordCol, "word-col", "", "source word column (default: auto-detect)") - flag.StringVar(&cfg.langCol, "lang-col", "", "source language column (default: auto-detect)") - flag.StringVar(&cfg.lang, "lang", "vi", "language code to keep") + flag.IntVar(&cfg.minWords, "min-words", 20000, "fail if fewer words survive filtering") flag.Parse() if err := run(cfg); err != nil { @@ -70,41 +63,55 @@ func main() { } func run(cfg config) error { - if cfg.words != "" { + // Exactly one input. Picking silently between two would let a stray flag + // ship a corpus nobody meant to build. + switch { + case cfg.merged == "" && cfg.words == "": + return errors.New("no input given: pass --merged (the corpus) or --words (a plain list)") + case cfg.merged != "" && cfg.words != "": + return errors.New("--merged and --words are mutually exclusive") + case cfg.merged != "": + return runFromMergedList(cfg) + default: return runFromWordList(cfg) } - if _, err := os.Stat(cfg.in); err != nil { - return fmt.Errorf("source database not found at %s — run 'make fetch-dict' first: %w", cfg.in, err) - } +} - src, err := sql.Open("sqlite", "file:"+cfg.in+"?mode=ro") - if err != nil { - return fmt.Errorf("open source: %w", err) - } - defer src.Close() - - source, err := resolveSource(src, cfg) +// runFromMergedList derives the database from the upstream merged wordlist, +// keeping only words present in the wordlists named by --sources. +func runFromMergedList(cfg config) error { + allowed, err := parseSources(cfg.sources) if err != nil { return err } - log.Printf("source: %s.%s filtered by %s = %q", source.table, source.wordCol, source.langCol, cfg.lang) + if _, err := os.Stat(cfg.merged); err != nil { + return fmt.Errorf("merged wordlist not found at %s — run 'make fetch-dict' first: %w", cfg.merged, err) + } - words, rejects, err := extract(src, source, cfg) + words, rejects, err := readMergedList(cfg.merged, allowed, cfg.maxSyllables) if err != nil { return err } logRejects(rejects) - log.Printf("accepted %d distinct words", len(words)) + log.Printf("accepted %d distinct words from %s (sources: %s)", len(words), cfg.merged, cfg.sources) + return finish(cfg, words, mergedProvenance(cfg.merged, allowed, cfg.maxSyllables)) +} + +// finish is the tail every input mode shares: the size floor, alias +// generation, the atomic write and the re-read verification. Keeping it in one +// place is what stops a fixture from drifting into a different shape from the +// database production loads. +func finish(cfg config, words map[string]entry, src sourceSpec) error { if len(words) < cfg.minWords { return fmt.Errorf("only %d words survived filtering, expected at least %d — "+ - "the source schema or content may have changed", len(words), cfg.minWords) + "the source content may have changed", len(words), cfg.minWords) } aliases, collisions := buildAliases(words) log.Printf("generated %d spelling aliases (%d skipped as ambiguous or already real words)", len(aliases), collisions) - if err := write(cfg.out, words, aliases, source); err != nil { + if err := write(cfg.out, words, aliases, src); err != nil { return err } if err := verify(cfg.out, cfg.minWords); err != nil { @@ -119,7 +126,7 @@ func run(cfg config) error { // upstream release. // // It exists so tests and CI have a real dictionary to play against without the -// 179 MB download. The filtering, alias generation, writing and verification +// upstream download. The filtering, alias generation, writing and verification // below are the same functions the real build uses — only the source of the // raw strings differs — so a fixture cannot drift into being shaped // differently from what production loads. @@ -153,26 +160,15 @@ func runFromWordList(cfg config) error { logRejects(rejects) log.Printf("accepted %d distinct words from %s", len(words), cfg.words) - if len(words) < cfg.minWords { - return fmt.Errorf("only %d words survived filtering, expected at least %d", len(words), cfg.minWords) - } - - aliases, collisions := buildAliases(words) - log.Printf("generated %d spelling aliases (%d skipped as ambiguous or already real words)", len(aliases), collisions) - // The source spec is what lands in the meta table. Naming the list rather // than a table makes it obvious in the output which build produced a given - // database. - src := sourceSpec{table: "wordlist:" + filepath.Base(cfg.words)} - if err := write(cfg.out, words, aliases, src); err != nil { - return err - } - if err := verify(cfg.out, cfg.minWords); err != nil { - return fmt.Errorf("output failed verification: %w", err) - } - - log.Printf("wrote %s", cfg.out) - return nil + // database — and a hand-written list carries no upstream licence, so the + // fixture must not claim one. + return finish(cfg, words, sourceSpec{ + table: "wordlist:" + filepath.Base(cfg.words), + license: "none: hand-written fixture wordlist, no upstream data", + attribution: "Fixture written by this project; no third-party attribution applies.", + }) } // verify re-opens the finished database and re-checks the invariants the game @@ -225,208 +221,21 @@ type entry struct { syllables int } +// sourceSpec is what the meta table records about where the words came from. type sourceSpec struct { + // table names the input: "merged:" for the corpus, "wordlist:" + // for a fixture, so the output says which build produced it. table string - wordCol string - langCol string maxSyllables int -} - -// quoteIdent renders a SQLite identifier. Go's %q escapes an embedded quote as -// \" but SQLite requires it doubled, so fmt.Sprintf("%q") is not correct here. -func quoteIdent(name string) string { - return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` -} - -// validateSource confirms operator-supplied identifiers actually exist. -// -// SQLite accepts a double-quoted string that matches no column as a string -// literal rather than erroring, so a typo in --word-col silently yields one -// row per distinct value of that literal. Checking up front turns a confusing -// near-empty build into a clear message. -func validateSource(db *sql.DB, spec sourceSpec) error { - tables, err := listTables(db) - if err != nil { - return err - } - if !contains(tables, spec.table) { - return fmt.Errorf("table %q not found; available: %s", spec.table, strings.Join(tables, ", ")) - } - - cols, err := listColumns(db, spec.table) - if err != nil { - return fmt.Errorf("read columns of %q: %w", spec.table, err) - } - for _, c := range []struct{ role, name string }{{"--word-col", spec.wordCol}, {"--lang-col", spec.langCol}} { - if !contains(cols, c.name) { - return fmt.Errorf("%s %q not found in table %q; available: %s", - c.role, c.name, spec.table, strings.Join(cols, ", ")) - } - } - - return nil -} - -func contains(haystack []string, needle string) bool { - for _, h := range haystack { - if h == needle { - return true - } - } - return false -} - -// resolveSource finds the table holding word forms. The upstream schema is not -// contractual — it is someone else's release artifact — so rather than hardcode -// it, look for a table carrying both a word-like and a language-like column. -// Explicit flags override detection entirely. -func resolveSource(db *sql.DB, cfg config) (sourceSpec, error) { - given := 0 - for _, f := range []string{cfg.table, cfg.wordCol, cfg.langCol} { - if f != "" { - given++ - } - } - switch { - case given == 3: - spec := sourceSpec{table: cfg.table, wordCol: cfg.wordCol, langCol: cfg.langCol, maxSyllables: cfg.maxSyllables} - if err := validateSource(db, spec); err != nil { - return sourceSpec{}, err - } - return spec, nil - case given > 0: - return sourceSpec{}, fmt.Errorf( - "--table, --word-col and --lang-col must be given together (got %d of 3); "+ - "omit all three to auto-detect", given) - } - - tables, err := listTables(db) - if err != nil { - return sourceSpec{}, err - } - - wordNames := []string{"word", "term", "headword", "text", "lemma", "entry"} - langNames := []string{"lang_code", "language", "lang", "lang_name"} - - for _, t := range tables { - cols, err := listColumns(db, t) - if err != nil { - log.Printf("warning: could not read columns of %q: %v", t, err) - continue - } - wordCol := pickColumn(cols, wordNames) - langCol := pickColumn(cols, langNames) - if wordCol != "" && langCol != "" { - return sourceSpec{table: t, wordCol: wordCol, langCol: langCol, maxSyllables: cfg.maxSyllables}, nil - } - } - - // Detection failed. Dump the schema so the fix is a single flag away rather - // than a debugging session. - var b strings.Builder - b.WriteString("could not auto-detect the source table.\n") - b.WriteString("Pass --table, --word-col and --lang-col explicitly. Schema found:\n") - for _, t := range tables { - cols, err := listColumns(db, t) - if err != nil { - continue - } - fmt.Fprintf(&b, " %s(%s)\n", t, strings.Join(cols, ", ")) - } - return sourceSpec{}, errors.New(b.String()) -} - -func listTables(db *sql.DB) ([]string, error) { - rows, err := db.Query(`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`) - if err != nil { - return nil, fmt.Errorf("list tables: %w", err) - } - defer rows.Close() - - var out []string - for rows.Next() { - var name string - if err := rows.Scan(&name); err != nil { - return nil, err - } - out = append(out, name) - } - return out, rows.Err() -} - -func listColumns(db *sql.DB, table string) ([]string, error) { - rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", quoteIdent(table))) - if err != nil { - return nil, err - } - defer rows.Close() - - var out []string - for rows.Next() { - var ( - cid int - name, ctype string - notNull, pk int - dflt sql.NullString - ) - if err := rows.Scan(&cid, &name, &ctype, ¬Null, &dflt, &pk); err != nil { - return nil, err - } - out = append(out, name) - } - return out, rows.Err() -} - -func pickColumn(cols []string, candidates []string) string { - for _, want := range candidates { - for _, c := range cols { - if strings.EqualFold(c, want) { - return c - } - } - } - return "" -} - -// extract streams every candidate word past the filters. Results are deduped by -// normalized form, since the source lists a word once per sense. -func extract(db *sql.DB, src sourceSpec, cfg config) (map[string]entry, map[rejectReason]int, error) { - query := fmt.Sprintf("SELECT DISTINCT %s FROM %s WHERE %s = ?", - quoteIdent(src.wordCol), quoteIdent(src.table), quoteIdent(src.langCol)) - rows, err := db.Query(query, cfg.lang) - if err != nil { - return nil, nil, fmt.Errorf("query source (%s): %w", query, err) - } - defer rows.Close() - - words := make(map[string]entry) - rejects := make(map[rejectReason]int) - - for rows.Next() { - var raw sql.NullString - if err := rows.Scan(&raw); err != nil { - return nil, nil, err - } - if !raw.Valid { - rejects[rejectEmpty]++ - continue - } - - word, syllables, reason, ok := accept(raw.String, cfg.maxSyllables) - if !ok { - rejects[reason]++ - continue - } - - words[word] = entry{ - word: word, - first: syllables[0], - last: syllables[len(syllables)-1], - syllables: len(syllables), - } - } - - return words, rejects, rows.Err() + // url is the upstream artifact; empty for fixture builds. + url string + // license and attribution describe the data's licence obligations. The + // server logs the licence at startup, so a build must state its own rather + // than inherit a constant it may not deserve. + license string + attribution string + // extra holds provenance rows specific to one input mode. + extra [][2]string } func logRejects(rejects map[rejectReason]int) { @@ -593,18 +402,17 @@ CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); } defer insertMeta.Close() meta := [][2]string{ - {"source_url", sourceURL}, - {"source_license", sourceLicense}, - {"attribution", "See data/ATTRIBUTION.md for required attribution and the list of modifications."}, + {"source_url", src.url}, + {"source_license", src.license}, + {"attribution", src.attribution}, {"built_at", time.Now().UTC().Format(time.RFC3339)}, {"builder_version", builderVer}, {"word_count", fmt.Sprint(len(words))}, {"alias_count", fmt.Sprint(len(aliases))}, {"source_table", src.table}, - {"source_word_column", src.wordCol}, - {"source_lang_column", src.langCol}, {"max_syllables", fmt.Sprint(src.maxSyllables)}, } + meta = append(meta, src.extra...) for _, kv := range meta { if _, err := insertMeta.Exec(kv[0], kv[1]); err != nil { return err diff --git a/server/cmd/build-dictionary/main_test.go b/server/cmd/build-dictionary/main_test.go index da5732a..aa4fe43 100644 --- a/server/cmd/build-dictionary/main_test.go +++ b/server/cmd/build-dictionary/main_test.go @@ -6,64 +6,49 @@ import ( "errors" "os" "path/filepath" - "strings" "testing" _ "modernc.org/sqlite" ) -// fixtureSource writes a miniature stand-in for the upstream dictionary: the -// same shape (a word column plus a language column, one row per sense) but a -// handful of rows instead of 357k. Tests never touch the real 179 MB download. +// fixtureSource writes a miniature stand-in for the upstream wordlist: the +// same JSONL shape, a handful of rows instead of 79k. Each row is a word and +// the upstream wordlists that carry it. Tests never touch the real download. func fixtureSource(t *testing.T, rows [][2]string) string { t.Helper() - path := filepath.Join(t.TempDir(), "source.db") - db, err := sql.Open("sqlite", "file:"+path) - if err != nil { - t.Fatal(err) - } - defer db.Close() - - if _, err := db.Exec(`CREATE TABLE entries (word TEXT, lang_code TEXT, definition TEXT)`); err != nil { - t.Fatal(err) - } + lines := make([]string, 0, len(rows)) for _, r := range rows { - if _, err := db.Exec(`INSERT INTO entries (word, lang_code, definition) VALUES (?, ?, ?)`, - r[0], r[1], "unused"); err != nil { - t.Fatal(err) - } + lines = append(lines, `{"text": "`+r[0]+`", "source": ["`+r[1]+`"]}`) } - return path + return fixtureMerged(t, lines...) } func defaultRows() [][2]string { return [][2]string{ - {"pháp luật", "vi"}, - {"pháp luật", "vi"}, // duplicate sense — must dedupe to one word - {"luật lệ", "vi"}, - {"ngôn ngữ", "vi"}, - {"ngữ pháp", "vi"}, - {"hòa bình", "vi"}, - {"vô tuyến điện", "vi"}, // three syllables - {"pháp", "vi"}, // single syllable — rejected - {"covid 19", "vi"}, // digit — rejected - {"hello world", "en"}, // wrong language — never selected + {"pháp luật", "wiktionary"}, + {"pháp luật", "wiktionary"}, // listed twice — must dedupe to one word + {"luật lệ", "wiktionary"}, + {"ngôn ngữ", "wiktionary"}, + {"ngữ pháp", "wiktionary"}, + {"hòa bình", "wiktionary"}, + {"vô tuyến điện", "wiktionary"}, // three syllables + {"pháp", "wiktionary"}, // single syllable — rejected + {"covid 19", "wiktionary"}, // digit — rejected + {"hello world", "hongocduc"}, // excluded source — never selected } } func buildFixture(t *testing.T, rows [][2]string, maxSyllables int) string { t.Helper() - in := fixtureSource(t, rows) out := filepath.Join(t.TempDir(), "noitu.db") - cfg := config{ - in: in, + merged: fixtureSource(t, rows), + sources: "wiktionary", out: out, maxSyllables: maxSyllables, minWords: 1, - lang: "vi", } if err := run(cfg); err != nil { t.Fatalf("run: %v", err) @@ -179,12 +164,11 @@ func TestBuildRecordsProvenance(t *testing.T) { // The floor exists so a schema change upstream fails the build loudly instead // of silently shipping a near-empty dictionary. func TestBuildFailsBelowMinWords(t *testing.T) { - in := fixtureSource(t, defaultRows()) cfg := config{ - in: in, + merged: fixtureSource(t, defaultRows()), + sources: "wiktionary", out: filepath.Join(t.TempDir(), "noitu.db"), minWords: 1000, - lang: "vi", } if err := run(cfg); err == nil { t.Fatal("run succeeded with an unreachable min-words floor, want error") @@ -203,10 +187,10 @@ func TestFailedBuildPreservesPreviousOutput(t *testing.T) { // Same output path, but a floor no fixture can clear. cfg := config{ - in: fixtureSource(t, defaultRows()), + merged: fixtureSource(t, defaultRows()), + sources: "wiktionary", out: out, minWords: 1000, - lang: "vi", } if err := run(cfg); err == nil { t.Fatal("run succeeded with an unreachable floor, want error") @@ -223,81 +207,3 @@ func TestFailedBuildPreservesPreviousOutput(t *testing.T) { t.Error("temp database left behind after a failed build") } } - -func TestExplicitSourceFlagsMustBeComplete(t *testing.T) { - in := fixtureSource(t, defaultRows()) - out := filepath.Join(t.TempDir(), "noitu.db") - - // Two of three: ambiguous, so fail loudly rather than silently auto-detect. - err := run(config{in: in, out: out, minWords: 1, lang: "vi", table: "entries", wordCol: "word"}) - if err == nil { - t.Fatal("run succeeded with a partial flag override, want error") - } -} - -// SQLite reads a double-quoted name that matches no column as a string literal, -// so a typo would otherwise build a dictionary out of that literal. -func TestUnknownColumnIsRejected(t *testing.T) { - in := fixtureSource(t, defaultRows()) - out := filepath.Join(t.TempDir(), "noitu.db") - - err := run(config{ - in: in, out: out, minWords: 1, lang: "vi", - table: "entries", wordCol: "does_not_exist", langCol: "lang_code", - }) - if err == nil { - t.Fatal("run succeeded with an unknown word column, want error") - } - if !strings.Contains(err.Error(), "does_not_exist") { - t.Errorf("error %q does not name the bad column", err) - } -} - -func TestBuildRecordsSourceProvenance(t *testing.T) { - db := openOut(t, buildFixture(t, defaultRows(), 0)) - - for key, want := range map[string]string{ - "source_table": "entries", - "source_word_column": "word", - "source_lang_column": "lang_code", - } { - var got string - if err := db.QueryRow(`SELECT value FROM meta WHERE key = ?`, key).Scan(&got); err != nil { - t.Errorf("meta[%q] missing: %v", key, err) - continue - } - if got != want { - t.Errorf("meta[%q] = %q, want %q", key, got, want) - } - } -} - -func TestBuildAutoDetectsSchema(t *testing.T) { - // Same data under different column names: detection should still find it. - path := filepath.Join(t.TempDir(), "odd.db") - db, err := sql.Open("sqlite", "file:"+path) - if err != nil { - t.Fatal(err) - } - if _, err := db.Exec(`CREATE TABLE lexicon (headword TEXT, language TEXT)`); err != nil { - t.Fatal(err) - } - if _, err := db.Exec(`INSERT INTO lexicon VALUES ('pháp luật', 'vi'), ('luật lệ', 'vi')`); err != nil { - t.Fatal(err) - } - db.Close() - - out := filepath.Join(t.TempDir(), "noitu.db") - if err := run(config{in: path, out: out, minWords: 1, lang: "vi"}); err != nil { - t.Fatalf("run with alternate schema: %v", err) - } - - got := openOut(t, out) - var count int - if err := got.QueryRow(`SELECT COUNT(*) FROM words`).Scan(&count); err != nil { - t.Fatal(err) - } - if count != 2 { - t.Errorf("word count = %d, want 2", count) - } -} diff --git a/server/cmd/build-dictionary/merged_list.go b/server/cmd/build-dictionary/merged_list.go new file mode 100644 index 0000000..5197fc6 --- /dev/null +++ b/server/cmd/build-dictionary/merged_list.go @@ -0,0 +1,148 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" +) + +// The upstream is undertheseanlp/dictionary: one JSONL file listing every word +// of three merged Vietnamese wordlists together with which of them contain it. +// +// Pinned by commit rather than branch so the URL and the checksum in the +// Makefile can never disagree. +const ( + mergedSourceCommit = "2c078cfc373b06e2980d324ce1d7bd13740c3319" + mergedSourceURL = "https://raw.githubusercontent.com/undertheseanlp/dictionary/" + mergedSourceCommit + "/dictionary/words.txt" +) + +// knownSources are the three wordlists merged upstream. Only names in this +// list may be passed to --sources; a typo must fail loudly rather than quietly +// select nothing. +var knownSources = []string{"hongocduc", "tudientv", "wiktionary"} + +// mergedRow is one line of the upstream file. +type mergedRow struct { + Text string `json:"text"` + Source []string `json:"source"` +} + +// parseSources turns the --sources flag into the set of upstream wordlists a +// word may come from. +func parseSources(list string) (map[string]bool, error) { + allowed := make(map[string]bool) + for _, name := range strings.Split(list, ",") { + name = strings.TrimSpace(name) + if name == "" { + continue + } + if !slices.Contains(knownSources, name) { + return nil, fmt.Errorf("unknown source %q in --sources; known: %s", name, strings.Join(knownSources, ", ")) + } + allowed[name] = true + } + if len(allowed) == 0 { + return nil, fmt.Errorf("--sources selects nothing; known: %s", strings.Join(knownSources, ", ")) + } + return allowed, nil +} + +// readMergedList reads the upstream file and returns the words that belong in +// the game dictionary. +// +// A row is considered only if one of its sources is allowed; rows from other +// wordlists are never read. Capitalization is not a filter: "Hà Nội" and +// "hà nội" are the same word to the game and both land as the lowercase form. +// Rows then pass through the same accept() every other input mode uses. +func readMergedList(path string, allowed map[string]bool, maxSyllables int) (map[string]entry, map[rejectReason]int, error) { + f, err := os.Open(path) + if err != nil { + return nil, nil, fmt.Errorf("read merged list: %w", err) + } + defer f.Close() + + words := make(map[string]entry) + rejects := make(map[rejectReason]int) + + scanner := bufio.NewScanner(f) + // The longest upstream line is well under a kilobyte, but a scanner that + // hits its limit stops with an error rather than truncating, and a generous + // buffer keeps that error from ever being the reason a build fails. + scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) + + lineNo := 0 + for scanner.Scan() { + lineNo++ + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 { + continue + } + + var row mergedRow + if err := json.Unmarshal(line, &row); err != nil { + return nil, nil, fmt.Errorf("%s:%d: malformed line: %w", path, lineNo, err) + } + if !anyAllowed(row.Source, allowed) { + continue + } + + word, syllables, reason, ok := accept(row.Text, maxSyllables) + if !ok { + rejects[reason]++ + continue + } + words[word] = entry{ + word: word, + first: syllables[0], + last: syllables[len(syllables)-1], + syllables: len(syllables), + } + } + if err := scanner.Err(); err != nil { + // lineNo counts lines already returned; the failure is on the next one. + return nil, nil, fmt.Errorf("%s:%d: %w", path, lineNo+1, err) + } + + return words, rejects, nil +} + +func anyAllowed(sources []string, allowed map[string]bool) bool { + for _, s := range sources { + if allowed[s] { + return true + } + } + return false +} + +// mergedProvenance describes a merged-list build for the meta table: which +// upstream wordlists were kept and which were excluded. knownSources is +// already sorted, so kept and excluded come out sorted too. +func mergedProvenance(path string, allowed map[string]bool, maxSyllables int) sourceSpec { + var kept, excluded []string + for _, name := range knownSources { + if allowed[name] { + kept = append(kept, name) + } else { + excluded = append(excluded, name) + } + } + + return sourceSpec{ + table: "merged:" + filepath.Base(path), + url: mergedSourceURL, + license: "CC BY-SA 3.0 (https://creativecommons.org/licenses/by-sa/3.0/)", + attribution: "See data/ATTRIBUTION.md for required attribution and the list of modifications.", + maxSyllables: maxSyllables, + extra: [][2]string{ + {"source_commit", mergedSourceCommit}, + {"sources_kept", strings.Join(kept, ",")}, + {"sources_excluded", strings.Join(excluded, ",")}, + }, + } +} diff --git a/server/cmd/build-dictionary/merged_list_test.go b/server/cmd/build-dictionary/merged_list_test.go new file mode 100644 index 0000000..3e2c41f --- /dev/null +++ b/server/cmd/build-dictionary/merged_list_test.go @@ -0,0 +1,184 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// fixtureMerged writes a miniature stand-in for the upstream JSONL: the same +// shape, a handful of rows. +func fixtureMerged(t *testing.T, lines ...string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "words.txt") + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func defaultMergedLines() []string { + return []string{ + `{"text": "Hà Nội", "source": ["wiktionary"]}`, // capitalized upstream — kept, lowercased + `{"text": "cánh diều", "source": ["hongocduc"]}`, // excluded source — never read + `{"text": "tủ lạnh", "source": ["tudientv"]}`, // excluded source — never read + `{"text": "học sinh", "source": ["hongocduc", "tudientv", "wiktionary"]}`, + `{"text": "Học sinh", "source": ["wiktionary"]}`, // same word twice — kept once + `{"text": "pháp", "source": ["wiktionary"]}`, // single syllable — rejected downstream + ``, + } +} + +func wiktionaryOnly(t *testing.T) map[string]bool { + t.Helper() + allowed, err := parseSources("wiktionary") + if err != nil { + t.Fatal(err) + } + return allowed +} + +func TestMergedListKeepsAllowedSourcesOnly(t *testing.T) { + words, rejects, err := readMergedList(fixtureMerged(t, defaultMergedLines()...), wiktionaryOnly(t), 0) + if err != nil { + t.Fatal(err) + } + + var got []string + for w := range words { + got = append(got, w) + } + assertSameStrings(t, got, []string{"hà nội", "học sinh"}) + + if n := rejects[rejectTooShort]; n != 1 { + t.Errorf("too-short rejects = %d, want 1 (pháp)", n) + } +} + +func TestMergedListWidensWithMoreSources(t *testing.T) { + allowed, err := parseSources("hongocduc,wiktionary") + if err != nil { + t.Fatal(err) + } + words, _, err := readMergedList(fixtureMerged(t, defaultMergedLines()...), allowed, 0) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"hà nội", "học sinh", "cánh diều"} { + if _, ok := words[want]; !ok { + t.Errorf("%q missing", want) + } + } + if _, ok := words["tủ lạnh"]; ok { + t.Error("tudientv-only word leaked into the output") + } +} + +func TestMergedListNamesMalformedLine(t *testing.T) { + path := fixtureMerged(t, + `{"text": "học sinh", "source": ["wiktionary"]}`, + `{"text": "broken"`, + ) + _, _, err := readMergedList(path, wiktionaryOnly(t), 0) + if err == nil { + t.Fatal("malformed line was skipped, want error") + } + if !strings.Contains(err.Error(), ":2:") { + t.Errorf("error does not name line 2: %v", err) + } +} + +func TestParseSourcesRejectsUnknownNames(t *testing.T) { + for _, bad := range []string{"", "wiktionary,soha", "Wiktionary"} { + if _, err := parseSources(bad); err == nil { + t.Errorf("parseSources(%q) succeeded, want error", bad) + } + } + allowed, err := parseSources(" hongocduc , wiktionary ") + if err != nil { + t.Fatal(err) + } + if !allowed["hongocduc"] || !allowed["wiktionary"] || allowed["tudientv"] { + t.Errorf("allowed = %v, want hongocduc and wiktionary only", allowed) + } +} + +func TestMergedBuildRecordsProvenance(t *testing.T) { + out := filepath.Join(t.TempDir(), "noitu.db") + cfg := config{ + merged: fixtureMerged(t, defaultMergedLines()...), + sources: "wiktionary", + out: out, + minWords: 1, + } + if err := run(cfg); err != nil { + t.Fatalf("run: %v", err) + } + db := openOut(t, out) + + want := map[string]string{ + "source_url": mergedSourceURL, + "source_commit": mergedSourceCommit, + "sources_kept": "wiktionary", + "sources_excluded": "hongocduc,tudientv", + "word_count": "2", + } + for key, value := range want { + var got string + if err := db.QueryRow(`SELECT value FROM meta WHERE key = ?`, key).Scan(&got); err != nil { + t.Errorf("meta[%q] missing: %v", key, err) + continue + } + if got != value { + t.Errorf("meta[%q] = %q, want %q", key, got, value) + } + } +} + +func TestRunInputSelection(t *testing.T) { + merged := fixtureMerged(t, defaultMergedLines()...) + words := fixtureMerged(t, "học sinh") + out := filepath.Join(t.TempDir(), "noitu.db") + + cases := []struct { + name string + cfg config + wantErr string + }{ + {"both inputs", config{merged: merged, words: words, sources: "wiktionary", out: out, minWords: 1}, "mutually exclusive"}, + {"neither input", config{sources: "wiktionary", out: out, minWords: 1}, "no input given"}, + {"missing merged file", config{merged: filepath.Join(t.TempDir(), "absent.jsonl"), sources: "wiktionary", out: out, minWords: 1}, "make fetch-dict"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := run(tc.cfg) + if err == nil { + t.Fatal("run succeeded, want error") + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("error %q does not mention %q", err, tc.wantErr) + } + }) + } +} + +// A fixture is hand-written data; its database must not claim the upstream's +// licence, because the server logs whatever the meta table says. +func TestWordListBuildRecordsNoUpstreamLicense(t *testing.T) { + out := filepath.Join(t.TempDir(), "noitu.db") + if err := run(config{words: fixtureMerged(t, "học sinh", "bánh mì"), out: out, minWords: 1}); err != nil { + t.Fatalf("run: %v", err) + } + var license, url string + db := openOut(t, out) + if err := db.QueryRow(`SELECT value FROM meta WHERE key = 'source_license'`).Scan(&license); err != nil { + t.Fatal(err) + } + if err := db.QueryRow(`SELECT value FROM meta WHERE key = 'source_url'`).Scan(&url); err != nil { + t.Fatal(err) + } + if strings.Contains(license, "CC BY-SA") || url != "" { + t.Errorf("fixture build claims upstream provenance: license=%q url=%q", license, url) + } +} diff --git a/server/cmd/noitu-server/main.go b/server/cmd/noitu-server/main.go index 7ee4420..bc94738 100644 --- a/server/cmd/noitu-server/main.go +++ b/server/cmd/noitu-server/main.go @@ -56,7 +56,7 @@ func run() error { } // The store deliberately does not log — a library writing to the global - // logger fights the server's own handler — so the CC BY-SA 4.0 attribution + // logger fights the server's own handler — so the CC BY-SA 3.0 attribution // that ships with the data surfaces here or nowhere. slog.Info("dictionary loaded", "path", cfg.dbPath, diff --git a/server/internal/dictionary/store.go b/server/internal/dictionary/store.go index f2729a4..81836c1 100644 --- a/server/internal/dictionary/store.go +++ b/server/internal/dictionary/store.go @@ -126,7 +126,7 @@ func dsn(path string) string { } func (s *Store) loadMeta(db *sql.DB) (declaredWords int, err error) { - // The data is CC BY-SA 4.0 and its provenance travels with it. + // The data is CC BY-SA 3.0 and its provenance travels with it. if err := db.QueryRow(`SELECT value FROM meta WHERE key = 'source_license'`).Scan(&s.license); err != nil { return 0, fmt.Errorf("read dictionary metadata (is this a noitu.db?): %w", err) } diff --git a/server/internal/dictionary/store_test.go b/server/internal/dictionary/store_test.go index 86889ee..41492a6 100644 --- a/server/internal/dictionary/store_test.go +++ b/server/internal/dictionary/store_test.go @@ -24,8 +24,8 @@ CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); ` // fixtureAt builds a miniature dictionary with the same schema the builder -// emits. Tests never depend on the real 48k-word database, so they stay fast -// and run in CI without the 179 MB upstream download. +// emits. Tests never depend on the real database, so they stay fast +// and run in CI without the upstream download. // // Takes testing.TB so benchmarks get working cleanup: a zero-value testing.T // never runs its Cleanup funcs, which leaks a temp directory per benchmark. @@ -40,7 +40,7 @@ func fixtureAt(tb testing.TB, dir string) string { defer db.Close() data := fixtureSchema + ` -INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','7'); +INSERT INTO meta VALUES ('source_license','CC BY-SA 3.0'),('word_count','7'); INSERT INTO words VALUES ('pháp luật','pháp','luật',2), ('pháp lý','pháp','lý',2), @@ -106,7 +106,7 @@ func TestOpenWrongSchema(t *testing.T) { // every room creation. func TestOpenEmptyDictionary(t *testing.T) { path := writeDB(t, fixtureSchema+` -INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','48216');`) +INSERT INTO meta VALUES ('source_license','CC BY-SA 3.0'),('word_count','26845');`) _, err := Open(path) if err == nil { @@ -121,7 +121,7 @@ INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','48216') // syllable has continuations that cannot be supplied. func TestOpenInconsistentOutDegree(t *testing.T) { path := writeDB(t, fixtureSchema+` -INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','1'); +INSERT INTO meta VALUES ('source_license','CC BY-SA 3.0'),('word_count','1'); INSERT INTO words VALUES ('pháp luật','pháp','luật',2); INSERT INTO syllables VALUES ('pháp',7),('luật',0);`) @@ -132,7 +132,7 @@ INSERT INTO syllables VALUES ('pháp',7),('luật',0);`) func TestOpenOrphanAlias(t *testing.T) { path := writeDB(t, fixtureSchema+` -INSERT INTO meta VALUES ('source_license','CC BY-SA 4.0'),('word_count','1'); +INSERT INTO meta VALUES ('source_license','CC BY-SA 3.0'),('word_count','1'); INSERT INTO words VALUES ('pháp luật','pháp','luật',2); INSERT INTO syllables VALUES ('pháp',1),('luật',0); INSERT INTO aliases VALUES ('phap luat','không tồn tại');`) @@ -427,8 +427,8 @@ func TestCountsAndLicense(t *testing.T) { if got := s.AliasCount(); got != 2 { t.Errorf("AliasCount = %d, want 2", got) } - if got := s.License(); got != "CC BY-SA 4.0" { - t.Errorf("License = %q, want %q", got, "CC BY-SA 4.0") + if got := s.License(); got != "CC BY-SA 3.0" { + t.Errorf("License = %q, want %q", got, "CC BY-SA 3.0") } } diff --git a/server/internal/game/engine_test.go b/server/internal/game/engine_test.go index 725cf57..286b2e1 100644 --- a/server/internal/game/engine_test.go +++ b/server/internal/game/engine_test.go @@ -10,7 +10,7 @@ import ( // fakeDict is a hand-built word graph. Tests state the exact edges they need, // so a rule can be exercised on a board small enough to reason about — no -// SQLite, no 48k-word corpus. +// SQLite, no real corpus. type fakeDict struct { words map[string][2]string // canonical -> {first, last} aliases map[string]string diff --git a/server/internal/wsapi/wsapi_test.go b/server/internal/wsapi/wsapi_test.go index e526420..aa50fa4 100644 --- a/server/internal/wsapi/wsapi_test.go +++ b/server/internal/wsapi/wsapi_test.go @@ -25,7 +25,7 @@ import ( // // A dozen words with edges a reader can follow beats a real dictionary here: // when a test says the bot must lose, the reason is visible in the graph -// rather than buried in 48,000 entries. +// rather than buried in tens of thousands of entries. type testDict struct { // words maps a canonical word to its first and last syllable. words map[string][2]string diff --git a/testdata/fixture-words.txt b/testdata/fixture-words.txt index 5388060..5720926 100644 --- a/testdata/fixture-words.txt +++ b/testdata/fixture-words.txt @@ -1,7 +1,7 @@ # Fixture wordlist for tests, end-to-end runs and CI. # # Hand-written common Vietnamese compounds, not an extract of the upstream -# dictionary: CI must never download the 179 MB release, and a fixture derived +# dictionary: CI must never download the upstream wordlist, and a fixture derived # from it would carry that release's licence into every test artifact. # # `make fixture-dict` turns this into a real derived database through the same diff --git a/web/e2e/bot-game.spec.js b/web/e2e/bot-game.spec.js index 729c453..79fa3ef 100644 --- a/web/e2e/bot-game.spec.js +++ b/web/e2e/bot-game.spec.js @@ -226,9 +226,9 @@ test.describe('playing the bot', () => { test('the attribution footer credits the dictionary source', async ({ page }) => { await page.goto('/'); - const link = page.getByRole('link', { name: 'minhqnd/dictionary' }); - await expect(link).toHaveAttribute('href', 'https://github.com/minhqnd/dictionary'); - await expect(page.getByRole('link', { name: /CC BY-SA 4\.0/ })).toBeVisible(); + const link = page.getByRole('link', { name: 'Wiktionary tiếng Việt' }); + await expect(link).toHaveAttribute('href', 'https://vi.wiktionary.org/'); + await expect(page.getByRole('link', { name: /CC BY-SA 3\.0/ })).toBeVisible(); }); test('a deep link is served by the binary, not a 404', async ({ page }) => { diff --git a/web/playwright.config.js b/web/playwright.config.js index 4032c09..b55f0ba 100644 --- a/web/playwright.config.js +++ b/web/playwright.config.js @@ -30,7 +30,7 @@ export default defineConfig({ projects: [{ name: 'chromium' }], // The real binary, serving the real built frontend, against the fixture - // dictionary. Nothing here downloads the 179 MB upstream release. + // dictionary. Nothing here downloads the upstream wordlist. webServer: { command: 'go run ./cmd/noitu-server', cwd: '../server', diff --git a/web/src/lib/components/AttributionFooter.svelte b/web/src/lib/components/AttributionFooter.svelte index bb5926e..24526f9 100644 --- a/web/src/lib/components/AttributionFooter.svelte +++ b/web/src/lib/components/AttributionFooter.svelte @@ -3,18 +3,18 @@