mirror of
https://github.com/tiennm99/noitu.git
synced 2026-10-13 10:15:55 +00:00
tiennm99
0a5c5b06fe
Replace the pinned 2018 undertheseanlp wordlist with kaikki.org's current wiktextract export of the Vietnamese Wiktionary, read with --kaikki. The same authors and website, eight years fresher: 34,813 words instead of 26,845, with every graph metric up and bot game length unchanged. The export is fetched fresh for every build and is not pinned, by the owner's decision: kaikki keeps no dated snapshots, so a checksum would break weekly. The builder therefore hashes the file as it streams it and records source_sha256, source_rows and source_fetched_at in meta; the fetch downloads to a .part name and renames on success; a truncated or non-JSON body fails the build, and the --min-words floor rises to 30,000. DICT_SHA256 and verify-dict are gone; the Makefile, Dockerfile and the builder's URL constant are held in agreement by a test. Current Wiktionary text is CC BY-SA 4.0, so the data licence returns to 4.0: data/LICENSE is restored, and NOTICE, ATTRIBUTION, README, the image docs and the in-game footer credit Wiktionary tiếng Việt's contributors and wiktextract/kaikki.org. builder_version becomes 3 for the changed meta contract.
53 lines
2.2 KiB
JavaScript
53 lines
2.2 KiB
JavaScript
// The upstream dictionary URL lives in two places: the Makefile, which builds
|
|
// it for a developer, and the Dockerfile, which builds it for the image. They
|
|
// have to agree, or the container ships a wordlist nobody tested against.
|
|
//
|
|
// This lives in the JavaScript suite for no better reason than that it is the
|
|
// suite that already reads other files in the repository. It is checking two
|
|
// build files, not the frontend.
|
|
|
|
import { readFileSync } from 'node:fs';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { describe, expect, it } from 'vitest';
|
|
|
|
const makefile = readFileSync(fileURLToPath(new URL('../../Makefile', import.meta.url)), 'utf8');
|
|
const dockerfile = readFileSync(fileURLToPath(new URL('../../Dockerfile', import.meta.url)), 'utf8');
|
|
|
|
/**
|
|
* @param {string} source
|
|
* @param {RegExp} pattern
|
|
* @param {string} what
|
|
*/
|
|
function pin(source, pattern, what) {
|
|
const match = source.match(pattern);
|
|
expect(match, `${what} not found`).not.toBeNull();
|
|
return match?.[1].trim();
|
|
}
|
|
|
|
describe('the upstream dictionary export', () => {
|
|
const makeUrl = pin(makefile, /DICT_URL\s*:?=\s*(\S+)/, 'DICT_URL in the Makefile');
|
|
const dockerUrl = pin(dockerfile, /ARG DICT_URL=(\S+)/, 'DICT_URL in the Dockerfile');
|
|
|
|
it('is the same file in both build files', () => {
|
|
expect(dockerUrl).toBe(makeUrl);
|
|
});
|
|
|
|
it('is the Vietnamese-language file of the Vietnamese Wiktionary edition', () => {
|
|
// The path carries a space, so it must stay percent-encoded or make and
|
|
// sh will split it; and it must be the vi edition, not the English one.
|
|
expect(makeUrl).toMatch(/^https:\/\/kaikki\.org\/viwiktionary\/Ti%E1%BA%BFng%20Vi%E1%BB%87t\/[^\s/]+\.jsonl$/);
|
|
});
|
|
|
|
it('is the URL the builder stamps into the database', () => {
|
|
// The builder records the source URL in the meta table from its own
|
|
// constant. The three copies must agree or the attribution record names
|
|
// a file nobody downloaded.
|
|
const builder = readFileSync(
|
|
fileURLToPath(new URL('../../server/cmd/build-dictionary/kaikki_list.go', import.meta.url)),
|
|
'utf8'
|
|
);
|
|
const builderUrl = pin(builder, /kaikkiSourceURL\s*=\s*"([^"]+)"/, 'kaikkiSourceURL in the builder');
|
|
expect(builderUrl).toBe(makeUrl);
|
|
});
|
|
});
|