refactor(scripts): share data paths and barrel emitter across generators

Extract common path resolution and event-emitter barrel into shared modules
to reduce duplication and enable consistent data pipeline configuration.
This commit is contained in:
tiennm99 committed 2026-08-30 20:52:28 +07:00
1 parent fd64e9239b
commit 3a965b7ede
6 files changed
+122 -67

No files matched your search

+1 -3
View File
@@ -31,9 +31,7 @@ import {
coverageVerdict,
loadBoundary,
} from './lib/assign-districts.mjs';
const PANO_DIR = 'src/data/panos';
const TREE_DIR = 'src/data/regions';
import { PANO_DIR, TREE_DIR } from './lib/paths.mjs';
const childrenOf = (code) => REGIONS[code]?.children ?? [];
const provinces = () => childrenOf('VN');
+5 -21
View File
@@ -18,6 +18,8 @@ import { PbfReader } from 'pbf';
import * as turf from '@turf/turf';
import { REGIONS } from '../src/data/regions/index.js';
import { assignPanos } from './lib/assign-districts.mjs';
import { BOUNDARY_DIR, PANO_DIR as OUT_DIR } from './lib/paths.mjs';
import { panoEntries, writeBarrelFile } from './lib/barrel.mjs';
// The image layer, the one carrying per-image points, only exists at z14.
const ZOOM = 14;
@@ -43,9 +45,6 @@ const GRID_DEG = 0.0003;
// full set reach the same places. It did cost density.
const MAX_PER_CITY = Infinity;
const BOUNDARY_DIR = 'src/data/boundaries';
const OUT_DIR = 'src/data/panos';
function loadToken() {
const env = Object.fromEntries(
readFileSync('.env', 'utf8')
@@ -316,27 +315,12 @@ function writeBarrel() {
);
}
const entries = files.map((name) => ({
code: name.replace(/\.json$/, '').toUpperCase(),
// Sanitised because region codes contain hyphens, which are legal in a
// filename but not in a bare identifier or an unquoted object key. Emitting
// them raw makes the generated barrel a syntax error.
ident: name.replace(/\.json$/, '').replace(/[^a-z0-9]/gi, '_'),
file: name,
}));
const body =
const header =
'// Generated by scripts/build-pano-index.mjs. Do not edit by hand.\n' +
'//\n' +
'// Lists only the province indexes that have actually been built, so a\n' +
'// partial build still compiles.\n\n' +
entries.map((e) => `import ${e.ident} from './${e.file}';`).join('\n') +
'\n\nexport const PANO_INDEXES = {\n' +
entries.map((e) => ` ${JSON.stringify(e.code)}: ${e.ident},`).join('\n') +
'\n};\n';
writeFileSync(`${OUT_DIR}/index.js`, body);
return entries.map((e) => e.code);
'// partial build still compiles.\n\n';
return writeBarrelFile(`${OUT_DIR}/index.js`, header, 'PANO_INDEXES', panoEntries(OUT_DIR));
}
let tilesRequested = 0;
+6 -41
View File
@@ -25,9 +25,11 @@
// build -- see the Coverage section of docs/project-overview.md for why absent
// coverage is not automatically a defect.
import { mkdirSync, readdirSync, readFileSync, writeFileSync, existsSync } from 'node:fs';
import { mkdirSync, readFileSync, writeFileSync, existsSync } from 'node:fs';
import { dirname } from 'node:path';
import * as turf from '@turf/turf';
import { BOUNDARY_DIR as OUT_DIR, TREE_DIR } from './lib/paths.mjs';
import { boundaryEntries, writeBarrelFile } from './lib/barrel.mjs';
const NOMINATIM = 'https://nominatim.openstreetmap.org/search';
// Nominatim asks for a real identifier and at most one request per second.
@@ -41,9 +43,6 @@ const REQUEST_GAP_MS = 1100;
const LEAF_TOLERANCE = 0.0001; // ~11m
const PROVINCE_TOLERANCE = 0.0005; // ~55m
const OUT_DIR = 'src/data/boundaries';
const TREE_DIR = 'src/data/regions';
// The region tree. One entry per node.
//
// level 'country' | 'province' | 'district'
@@ -174,19 +173,6 @@ function fileFor(code) {
return folder ? `${folder}/${code.toLowerCase()}.json` : null;
}
/**
* A JS identifier safe to use in a generated import.
*
* Region codes are hyphenated (TPHCM-Q7), and a hyphen is legal in a filename
* but not in a bare identifier or an unquoted object key. Without this the
* generated barrel is a syntax error and nothing in the app compiles.
* @param {string} file Filename, with or without extension.
* @returns {string} Identifier.
*/
function identFor(file) {
return file.replace(/\.json$/, '').replace(/[^a-z0-9]/gi, '_');
}
/** Direct children of a node, in declaration order. */
function childrenOf(code) {
return Object.keys(REGIONS).filter((key) => REGIONS[key].parent === code);
@@ -379,33 +365,12 @@ function buildProvince(code) {
* @returns {string[]} Region codes now in the barrel.
*/
function writeBoundaryBarrel() {
// Grouped by province, so walk one level of subdirectories.
const entries = readdirSync(OUT_DIR, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.flatMap((dir) =>
readdirSync(`${OUT_DIR}/${dir.name}`)
.filter((name) => name.endsWith('.json'))
.map((name) => `${dir.name}/${name}`)
)
.sort()
.map((file) => ({
code: file.split('/').pop().replace(/\.json$/, '').toUpperCase(),
ident: identFor(file.split('/').pop()),
file,
}));
const body =
const header =
'// Generated by scripts/build-region-boundaries.mjs. Do not edit by hand.\n' +
'//\n' +
'// Lists only the boundaries that have actually been built, so a partial\n' +
'// build still compiles.\n\n' +
entries.map((e) => `import ${e.ident} from './${e.file}';`).join('\n') +
'\n\nexport const REGION_BOUNDARIES = {\n' +
entries.map((e) => ` ${JSON.stringify(e.code)}: ${e.ident},`).join('\n') +
'\n};\n';
writeFileSync(`${OUT_DIR}/index.js`, body);
return entries.map((e) => e.code);
'// build still compiles.\n\n';
return writeBarrelFile(`${OUT_DIR}/index.js`, header, 'REGION_BOUNDARIES', boundaryEntries(OUT_DIR));
}
/**
+1 -2
View File
@@ -8,8 +8,7 @@
import { readFileSync } from 'node:fs';
import * as turf from '@turf/turf';
import { REGIONS } from '../../src/data/regions/index.js';
const BOUNDARY_DIR = 'src/data/boundaries';
import { BOUNDARY_DIR } from './paths.mjs';
// Distinct-place resolution. The index is thinned at ~33m, so a single street
// corridor yields dozens of entries and a raw count overstates how many
+96
View File
@@ -0,0 +1,96 @@
// Shared barrel emitter for the two generated import barrels:
// src/data/boundaries/index.js (scripts/build-region-boundaries.mjs) and
// src/data/panos/index.js (scripts/build-pano-index.mjs).
//
// Both rewrite the same shape from whatever is actually on disk -- a header
// comment, one sanitised import per entry, then an export map keyed by region
// code -- so a partial build still compiles. One copy, because the identifier
// sanitiser is the kind of detail that drifts silently when it lives in two
// places.
import { readdirSync, writeFileSync } from 'node:fs';
/**
* A JS identifier safe to use in a generated import.
*
* Region codes are hyphenated (TPHCM-Q7), and a hyphen is legal in a filename
* but not in a bare identifier or an unquoted object key. Without this the
* generated barrel is a syntax error and nothing in the app compiles.
* @param {string} file Filename, with or without extension.
* @returns {string} Identifier.
*/
export function identFor(file) {
return file.replace(/\.json$/, '').replace(/[^a-z0-9]/gi, '_');
}
/**
* Entries for the boundary barrel: every polygon actually on disk.
*
* Boundaries are grouped by province, so this walks one level of
* subdirectories rather than the flat listing the panorama barrel uses.
* @param {string} dir src/data/boundaries.
* @returns {Array<{code: string, ident: string, file: string}>} Entries.
*/
export function boundaryEntries(dir) {
return readdirSync(dir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.flatMap((entry) =>
readdirSync(`${dir}/${entry.name}`)
.filter((name) => name.endsWith('.json'))
.map((name) => `${entry.name}/${name}`)
)
.sort()
.map((file) => ({
code: file.split('/').pop().replace(/\.json$/, '').toUpperCase(),
ident: identFor(file.split('/').pop()),
file,
}));
}
/**
* Entries for the panorama index barrel: every province index actually on
* disk, one file per province directly under `dir`.
* @param {string} dir src/data/panos.
* @returns {Array<{code: string, ident: string, file: string}>} Entries.
*/
export function panoEntries(dir) {
return readdirSync(dir)
.filter((name) => name.endsWith('.json'))
.sort()
.map((name) => ({
code: name.replace(/\.json$/, '').toUpperCase(),
ident: identFor(name),
file: name,
}));
}
/**
* Render one generated barrel module's contents.
* @param {string} header Full header comment block, ending in a blank line.
* @param {string} exportName Name of the exported map, e.g. 'REGION_BOUNDARIES'.
* @param {Array<{code: string, ident: string, file: string}>} entries Entries
* to import and export, already in the order they should appear.
* @returns {string} File contents, ready to write.
*/
export function renderBarrel(header, exportName, entries) {
return (
header +
entries.map((e) => `import ${e.ident} from './${e.file}';`).join('\n') +
`\n\nexport const ${exportName} = {\n` +
entries.map((e) => ` ${JSON.stringify(e.code)}: ${e.ident},`).join('\n') +
'\n};\n'
);
}
/**
* Render and write one generated barrel module.
* @param {string} path Output file path.
* @param {string} header Full header comment block, ending in a blank line.
* @param {string} exportName Name of the exported map.
* @param {Array<{code: string, ident: string, file: string}>} entries Entries.
* @returns {string[]} Codes now in the barrel.
*/
export function writeBarrelFile(path, header, exportName, entries) {
writeFileSync(path, renderBarrel(header, exportName, entries));
return entries.map((e) => e.code);
}
+13
View File
@@ -0,0 +1,13 @@
// Absolute paths to the generated data directories, shared by every script
// that reads or writes them.
//
// Resolved from this file's own location rather than the process's cwd. npm
// always runs scripts from the project root anyway, but `node
// scripts/whatever.mjs` from an unexpected directory would otherwise silently
// read or write the wrong tree instead of failing loudly.
import { fileURLToPath } from 'node:url';
export const BOUNDARY_DIR = fileURLToPath(new URL('../../src/data/boundaries', import.meta.url));
export const PANO_DIR = fileURLToPath(new URL('../../src/data/panos', import.meta.url));
export const TREE_DIR = fileURLToPath(new URL('../../src/data/regions', import.meta.url));