feat(scripts): phase 05 — backfill + verify + wipe (local node, no admin routes)

Operator-run migration scripts for KV→Mongo and D1→trading_trades, plus a
parity verifier and a rollback wiper. Pure local Node — no Worker code,
no /__admin/* routes, no new Worker secrets. Complies with
docs/architecture.md §10.

Scripts
- backfill-kv-to-mongo.js: paginates CF KV REST API per module, fetches
  values, $setOnInsert upsert into per-module Mongo collection. Resumes
  from .backfill-cursor-<module>.json on restart. Throttles 50 ops/sec.
  expiresAt derived from KV metadata.expiration (debugger #10). --dry-run
  and --module flags for incremental work.
- backfill-d1-to-mongo.js: wrangler d1 execute --remote --json → parse →
  insertMany batches into trading_trades, preserving original integer id
  as legacy_id (code-reviewer #13). Pre-flight aborts if collection
  non-empty unless --force.
- verify-mongo-parity.js: count parity ±1%, SHA256 value compare,
  expiresAt ±5min bucket. Full-scan when <10K docs, sqrt-sample
  capped at 500 otherwise (code-reviewer #21). Trading: full-scan
  on legacy_id/ts/user_id/symbol/qty.
- wipe-mongo.js: rollback helper. deleteMany across all collections
  with readline confirm. --yes for CI.
- lib/migration-helpers.js: shared sleep, sha256, checkpoint I/O,
  cfKvList/cfKvGet, MongoClient singleton, sample strategy.

Surface updates
- .env.deploy.example: CF account/token/namespace placeholders.
- package.json: backfill:kv[:dry], backfill:d1[:dry], verify:mongo,
  wipe:mongo scripts.
- check-secret-leaks.js: SECRETS array gains CLOUDFLARE_API_TOKEN +
  CLOUDFLARE_ACCOUNT_ID for defense-in-depth.
- .gitignore: .backfill-cursor-*.json excluded.

Tests: 638 → 667 (+29 pure-logic tests for sha256, checkpoint round-trip,
count-diff, sample-size, fetch-mocked CF REST). Lint clean.

Operator-run sequence (after Phase 06 deploy):
  npm run backfill:kv:dry   # preview
  npm run backfill:kv
  npm run backfill:d1:dry
  npm run backfill:d1
  npm run verify:mongo      # exit 0 = parity ok

.env.deploy.example

@@ -20,3 +20,11 @@ MODULES=util,wordle,loldle,misc,trading,lolschedule,semantle,doantu,twentyq
 # as TELEGRAM_BOT_TOKEN / TELEGRAM_WEBHOOK_SECRET.
 # Format: mongodb+srv://miti99bot-worker:<pass>@<host>/miti99bot?retryWrites=true&w=majority
 MONGODB_URI=
+
+# Cloudflare API creds for backfill scripts (Phase 05). Read-only scope is enough.
+# Create at: dash.cloudflare.com → My Profile → API Tokens → Create Token.
+# Permissions needed: Account → Workers KV Storage → Read; Account → D1 → Read.
+CLOUDFLARE_ACCOUNT_ID=
+CLOUDFLARE_API_TOKEN=
+# KV namespace ID (production). Same value as wrangler.toml [[kv_namespaces]] id.
+KV_NAMESPACE_ID=

.gitignore

@@ -76,6 +76,9 @@ web_modules/
 !.dev.vars.example
 .wrangler/
 
+# Phase 05 backfill cursor checkpoints — persisted locally, must not be committed.
+.backfill-cursor-*.json
+
 # parcel-bundler cache (https://parceljs.org/)
 .cache
 .parcel-cache

package.json

@@ -18,6 +18,12 @@
     "db:migrate": "node scripts/migrate.js",
     "register": "node --env-file-if-exists=.env.deploy scripts/register.js",
     "register:dry": "node --env-file-if-exists=.env.deploy scripts/register.js --dry-run",
+    "backfill:kv": "node --env-file-if-exists=.env.deploy scripts/backfill-kv-to-mongo.js",
+    "backfill:kv:dry": "node --env-file-if-exists=.env.deploy scripts/backfill-kv-to-mongo.js --dry-run",
+    "backfill:d1": "node --env-file-if-exists=.env.deploy scripts/backfill-d1-to-mongo.js",
+    "backfill:d1:dry": "node --env-file-if-exists=.env.deploy scripts/backfill-d1-to-mongo.js --dry-run",
+    "verify:mongo": "node --env-file-if-exists=.env.deploy scripts/verify-mongo-parity.js",
+    "wipe:mongo": "node --env-file-if-exists=.env.deploy scripts/wipe-mongo.js",
     "lint": "biome check . && eslint src && node scripts/check-secret-leaks.js",
     "format": "biome format --write .",
     "test": "vitest run"

scripts/backfill-d1-to-mongo.js

@@ -0,0 +1,149 @@
+#!/usr/bin/env node
+/**
+ * @file backfill-d1-to-mongo — copy trading_trades from D1 into MongoDB Atlas.
+ *
+ * Uses `npx wrangler d1 execute --remote --json` (mirrors scripts/migrate.js pattern)
+ * to read all rows, then insertMany in batches of 100 into the `trading_trades`
+ * Mongo collection.
+ *
+ * Flags:
+ *   --dry-run  Count rows + log first 5 without writing.
+ *   --force    Bypass pre-flight abort when trading_trades collection is non-empty.
+ *
+ * Required env (loaded via --env-file-if-exists=.env.deploy):
+ *   MONGODB_URI
+ *
+ * Usage:
+ *   node --env-file-if-exists=.env.deploy scripts/backfill-d1-to-mongo.js
+ *   node --env-file-if-exists=.env.deploy scripts/backfill-d1-to-mongo.js --dry-run
+ *   node --env-file-if-exists=.env.deploy scripts/backfill-d1-to-mongo.js --force
+ */
+
+import { execSync } from "node:child_process";
+import { ObjectId } from "mongodb";
+import { closeMongoClient, getMongoClient, sleep } from "./lib/migration-helpers.js";
+
+const DB_NAME = "miti99bot-db";
+const COLLECTION = "trading_trades";
+const BATCH_SIZE = 100;
+const BATCH_SLEEP_MS = 100; // brief pause between batches
+
+const dryRun = process.argv.includes("--dry-run");
+const force = process.argv.includes("--force");
+
+// ─── Preflight ────────────────────────────────────────────────────────────────
+
+function validateEnv() {
+  const missingVars = [["MONGODB_URI", "Atlas connection string"]].filter(([k]) => !process.env[k]);
+  if (missingVars.length) {
+    for (const [k, desc] of missingVars) console.error(`[backfill-d1] Missing: ${k} (${desc})`);
+    console.error("  Copy .env.deploy.example to .env.deploy and fill in values.");
+    process.exit(1);
+  }
+}
+
+// ─── D1 query ─────────────────────────────────────────────────────────────────
+
+/**
+ * Execute a SQL query against the remote D1 database via wrangler.
+ * Mirrors the wranglerExecute pattern in scripts/migrate.js.
+ *
+ * @param {string} sql
+ * @returns {any[]} rows from D1
+ */
+function queryD1(sql) {
+  const cmd = `npx wrangler d1 execute ${DB_NAME} --remote --command "${sql.replace(/"/g, '\\"')}" --json`;
+  let stdout;
+  try {
+    stdout = execSync(cmd, { stdio: ["ignore", "pipe", "pipe"] }).toString();
+  } catch (err) {
+    const stderr = err.stderr?.toString() ?? "";
+    throw new Error(`wrangler d1 execute failed:\n${stderr || err.stdout?.toString()}`);
+  }
+  // wrangler --json wraps results: [{results: [...], success: bool}]
+  const parsed = JSON.parse(stdout);
+  const results = Array.isArray(parsed) ? (parsed[0]?.results ?? []) : [];
+  return results;
+}
+
+// ─── Row mapping ──────────────────────────────────────────────────────────────
+
+/**
+ * Map a D1 row to a Mongo document.
+ * Preserves legacy_id so round-trip verification can match records.
+ *
+ * @param {{ id: number, user_id: string, symbol: string, side: string,
+ *           qty: number, price_vnd: number, ts: number }} row
+ * @returns {object}
+ */
+function rowToDoc(row) {
+  return {
+    _id: new ObjectId(),
+    legacy_id: row.id,
+    user_id: row.user_id,
+    symbol: row.symbol,
+    side: row.side,
+    qty: row.qty,
+    price_vnd: row.price_vnd,
+    ts: row.ts,
+  };
+}
+
+// ─── Main ─────────────────────────────────────────────────────────────────────
+
+async function main() {
+  validateEnv();
+
+  if (dryRun) console.log("[backfill-d1] DRY RUN — no writes will be made");
+
+  // Fetch all rows from D1.
+  console.log("[backfill-d1] Querying D1 trading_trades...");
+  const rows = queryD1(
+    "SELECT id, user_id, symbol, side, qty, price_vnd, ts FROM trading_trades ORDER BY id",
+  );
+  console.log(`[backfill-d1] Found ${rows.length} rows in D1`);
+
+  if (dryRun) {
+    console.log("[backfill-d1] Sample (first 5 rows):");
+    for (const row of rows.slice(0, 5)) {
+      // Log structural info only — no PII values printed.
+      console.log(`  id=${row.id} symbol=${row.symbol} side=${row.side} ts=${row.ts}`);
+    }
+    console.log("[backfill-d1] DRY RUN complete. No writes made.");
+    return;
+  }
+
+  const client = await getMongoClient(process.env.MONGODB_URI);
+  const db = client.db();
+  const coll = db.collection(COLLECTION);
+
+  // Pre-flight: abort if collection already has data (prevents double-insert).
+  const existingCount = await coll.countDocuments();
+  if (existingCount > 0 && !force) {
+    console.error(`[backfill-d1] ABORT: ${COLLECTION} already has ${existingCount} document(s).`);
+    console.error("  Run with --force to bypass this check (e.g. after wipe-mongo).");
+    await closeMongoClient();
+    process.exit(1);
+  }
+  if (existingCount > 0 && force) {
+    console.log(`[backfill-d1] --force: proceeding despite ${existingCount} existing doc(s).`);
+  }
+
+  // Insert in batches of BATCH_SIZE.
+  let inserted = 0;
+  for (let i = 0; i < rows.length; i += BATCH_SIZE) {
+    const batch = rows.slice(i, i + BATCH_SIZE).map(rowToDoc);
+    await coll.insertMany(batch);
+    inserted += batch.length;
+    console.log(`[backfill-d1] Inserted ${inserted}/${rows.length}`);
+    if (i + BATCH_SIZE < rows.length) await sleep(BATCH_SLEEP_MS);
+  }
+
+  await closeMongoClient();
+  console.log(`[backfill-d1] Done — ${inserted} documents inserted into ${COLLECTION}.`);
+}
+
+main().catch((err) => {
+  console.error("[backfill-d1] Fatal:", err.message ?? err);
+  process.exit(1);
+});

scripts/backfill-kv-to-mongo.js

@@ -0,0 +1,203 @@
+#!/usr/bin/env node
+/**
+ * @file backfill-kv-to-mongo — copy historical KV data into MongoDB Atlas.
+ *
+ * Uses the CF KV REST API to enumerate keys per module, then upserts each
+ * key into the corresponding Mongo collection using $setOnInsert (skip-if-exists)
+ * so live dual-write data written after Phase 04 is never overwritten.
+ *
+ * Flags:
+ *   --dry-run        List + count without writing to Mongo.
+ *   --module <name>  Backfill only a single module (default: all).
+ *
+ * Required env (loaded via --env-file-if-exists=.env.deploy):
+ *   MONGODB_URI, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_API_TOKEN,
+ *   KV_NAMESPACE_ID, MODULES (comma-separated)
+ *
+ * Usage:
+ *   node --env-file-if-exists=.env.deploy scripts/backfill-kv-to-mongo.js
+ *   node --env-file-if-exists=.env.deploy scripts/backfill-kv-to-mongo.js --dry-run
+ *   node --env-file-if-exists=.env.deploy scripts/backfill-kv-to-mongo.js --module wordle
+ */
+
+import {
+  cfKvGet,
+  cfKvList,
+  clearCheckpoint,
+  closeMongoClient,
+  getMongoClient,
+  loadCheckpoint,
+  saveCheckpoint,
+  sleep,
+} from "./lib/migration-helpers.js";
+
+// ─── Config ───────────────────────────────────────────────────────────────────
+
+const {
+  MONGODB_URI,
+  CLOUDFLARE_ACCOUNT_ID,
+  CLOUDFLARE_API_TOKEN,
+  KV_NAMESPACE_ID,
+  MODULES: MODULES_ENV,
+} = process.env;
+
+const dryRun = process.argv.includes("--dry-run");
+const moduleFlag = (() => {
+  const idx = process.argv.indexOf("--module");
+  return idx !== -1 ? process.argv[idx + 1] : null;
+})();
+
+/** Normalize module name to MongoDB collection name (mirrors mongo-kv-store.js). */
+const toCollName = (mod) => mod.replace(/-/g, "_");
+
+// ─── Preflight ────────────────────────────────────────────────────────────────
+
+function validateEnv() {
+  const missing = [
+    "MONGODB_URI",
+    "CLOUDFLARE_ACCOUNT_ID",
+    "CLOUDFLARE_API_TOKEN",
+    "KV_NAMESPACE_ID",
+    "MODULES",
+  ].filter((k) => !process.env[k]);
+  if (missing.length) {
+    console.error(`[backfill-kv] Missing required env vars: ${missing.join(", ")}`);
+    console.error("  Copy .env.deploy.example → .env.deploy and fill in values.");
+    process.exit(1);
+  }
+}
+
+// ─── Per-module backfill ──────────────────────────────────────────────────────
+
+/**
+ * Backfill one module from KV into Mongo.
+ *
+ * @param {import("mongodb").Db} db
+ * @param {string} mod — module name (e.g. "wordle", "loldle-emoji")
+ * @returns {Promise<{copied: number, skipped: number, failed: number}>}
+ */
+async function backfillModule(db, mod) {
+  const coll = db.collection(toCollName(mod));
+  const prefix = `${mod}:`;
+  const cp = loadCheckpoint(mod);
+  let cursor = cp?.cursor ?? null;
+  let page = 0;
+  let copied = 0;
+  let skipped = 0;
+  let failed = 0;
+
+  if (cp)
+    console.log(
+      `[${mod}] resuming from checkpoint (cursor=${cursor ? cursor.slice(0, 12) : "start"})`,
+    );
+
+  let hasMore = true;
+  while (hasMore) {
+    page++;
+    const {
+      keys,
+      cursor: nextCursor,
+      list_complete,
+    } = await cfKvList(
+      CLOUDFLARE_ACCOUNT_ID,
+      KV_NAMESPACE_ID,
+      CLOUDFLARE_API_TOKEN,
+      prefix,
+      cursor,
+    );
+
+    for (const keyObj of keys) {
+      const key = keyObj.name;
+      // expiresAt: KV metadata.expiration is unix-seconds → convert to JS Date.
+      const expSecs = keyObj.metadata?.expiration;
+      const expiresAt = expSecs ? new Date(expSecs * 1000) : undefined;
+
+      if (dryRun) {
+        copied++; // count as "would copy" in dry-run
+        continue;
+      }
+
+      try {
+        const value = await cfKvGet(
+          CLOUDFLARE_ACCOUNT_ID,
+          KV_NAMESPACE_ID,
+          CLOUDFLARE_API_TOKEN,
+          key,
+        );
+
+        const doc = { value };
+        if (expiresAt) doc.expiresAt = expiresAt;
+
+        // $setOnInsert: skip-if-exists — preserves any newer Mongo state from dual-write.
+        const result = await coll.updateOne({ _id: key }, { $setOnInsert: doc }, { upsert: true });
+        if (result.upsertedCount > 0) {
+          copied++;
+        } else {
+          skipped++;
+        }
+        await sleep(20); // throttle: ~50 ops/sec to stay within Atlas M0 headroom
+      } catch (err) {
+        failed++;
+        console.error(`[${mod}] ERROR key="${key.slice(0, 40)}…": ${err.message}`);
+      }
+    }
+
+    // Checkpoint after each page so a crash doesn't lose progress.
+    saveCheckpoint(mod, { cursor: nextCursor, lastKey: keys.at(-1)?.name ?? null });
+
+    const verb = dryRun ? "(dry-run)" : `${copied} copied, ${skipped} skipped, ${failed} failed`;
+    console.log(`[${mod}] page ${page}: ${keys.length} keys ${verb}`);
+
+    cursor = nextCursor;
+    hasMore = !list_complete && !!cursor;
+
+    // Brief pause between pages to respect CF REST rate limits (1200 req/5 min).
+    if (hasMore) await sleep(250);
+  }
+
+  if (!dryRun) clearCheckpoint(mod);
+  return { copied, skipped, failed };
+}
+
+// ─── Main ─────────────────────────────────────────────────────────────────────
+
+async function main() {
+  validateEnv();
+
+  const allModules = MODULES_ENV.split(",")
+    .map((m) => m.trim())
+    .filter(Boolean);
+  const modules = moduleFlag ? [moduleFlag] : allModules;
+
+  if (moduleFlag && !allModules.includes(moduleFlag)) {
+    console.error(
+      `[backfill-kv] Unknown module "${moduleFlag}". Available: ${allModules.join(", ")}`,
+    );
+    process.exit(1);
+  }
+
+  if (dryRun) console.log("[backfill-kv] DRY RUN — no writes will be made");
+  console.log(`[backfill-kv] Modules: ${modules.join(", ")}`);
+
+  const client = await getMongoClient(MONGODB_URI);
+  const db = client.db();
+
+  let totalFailed = 0;
+  for (const mod of modules) {
+    const { copied, skipped, failed } = await backfillModule(db, mod);
+    console.log(`[${mod}] DONE — ${copied} copied, ${skipped} skipped, ${failed} failed`);
+    totalFailed += failed;
+  }
+
+  await closeMongoClient();
+  if (totalFailed > 0) {
+    console.error(`[backfill-kv] Completed with ${totalFailed} failed key(s). Check logs above.`);
+    process.exit(1);
+  }
+  console.log("[backfill-kv] All modules complete.");
+}
+
+main().catch((err) => {
+  console.error("[backfill-kv] Fatal:", err.message ?? err);
+  process.exit(1);
+});

scripts/check-secret-leaks.js

@@ -31,7 +31,15 @@ const PROJECT_ROOT = resolve(import.meta.dirname, "..");
 const SCAN_DIRS = ["src", "scripts"];
 const SCAN_EXTS = new Set([".js", ".mjs", ".ts"]);
 
-const SECRETS = ["MONGODB_URI", "TELEGRAM_BOT_TOKEN", "TELEGRAM_WEBHOOK_SECRET", "ADMIN_TOKEN"];
+const SECRETS = [
+  "MONGODB_URI",
+  "TELEGRAM_BOT_TOKEN",
+  "TELEGRAM_WEBHOOK_SECRET",
+  "ADMIN_TOKEN",
+  // Phase 05: CF API creds used by backfill scripts — defense-in-depth.
+  "CLOUDFLARE_API_TOKEN",
+  "CLOUDFLARE_ACCOUNT_ID",
+];
 
 // A line is suspicious if it both names a secret AND looks like it's emitting
 // the value (console.*, JSON.stringify(env...), throw with interpolation).

scripts/lib/migration-helpers.js

@@ -0,0 +1,194 @@
+#!/usr/bin/env node
+/**
+ * @file migration-helpers — shared utilities for Phase 05 backfill scripts.
+ *
+ * Exports:
+ *  - sleep, sha256
+ *  - loadCheckpoint, saveCheckpoint, clearCheckpoint
+ *  - cfKvList, cfKvGet
+ *  - getMongoClient, closeMongoClient
+ *  - countDiffRatio, sampleStrategy
+ */
+
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { MongoClient } from "mongodb";
+
+const CF_API_BASE = "https://api.cloudflare.com/client/v4";
+
+// ─── Primitives ───────────────────────────────────────────────────────────────
+
+/** @param {number} ms @returns {Promise<void>} */
+export const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
+
+/** @param {string} text @returns {string} hex SHA-256 */
+export const sha256 = (text) => createHash("sha256").update(text, "utf8").digest("hex");
+
+// ─── Checkpoint ───────────────────────────────────────────────────────────────
+
+const cpPath = (mod) => resolve(process.cwd(), `.backfill-cursor-${mod}.json`);
+
+/**
+ * Load saved cursor state for a module. Returns null if none.
+ *
+ * @param {string} moduleName
+ * @returns {{ cursor: string|null, lastKey: string|null }|null}
+ */
+export function loadCheckpoint(moduleName) {
+  const p = cpPath(moduleName);
+  if (!existsSync(p)) return null;
+  try {
+    return JSON.parse(readFileSync(p, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Persist cursor state so a crash can resume from this page.
+ *
+ * @param {string} moduleName
+ * @param {{ cursor: string|null, lastKey: string|null }} state
+ */
+export function saveCheckpoint(moduleName, state) {
+  writeFileSync(cpPath(moduleName), JSON.stringify(state), "utf8");
+}
+
+/**
+ * Remove checkpoint file on successful module completion.
+ *
+ * @param {string} moduleName
+ */
+export function clearCheckpoint(moduleName) {
+  const p = cpPath(moduleName);
+  if (existsSync(p)) {
+    try {
+      unlinkSync(p);
+    } catch {
+      // Non-fatal — may already be gone.
+    }
+  }
+}
+
+// ─── CF KV REST ───────────────────────────────────────────────────────────────
+
+/**
+ * List one page of KV keys for a given prefix.
+ *
+ * @param {string} accountId
+ * @param {string} nsId
+ * @param {string} token
+ * @param {string} prefix
+ * @param {string|null} cursor
+ * @returns {Promise<{
+ *   keys: Array<{name: string, metadata?: {expiration?: number}}>,
+ *   cursor: string|null,
+ *   list_complete: boolean
+ * }>}
+ */
+export async function cfKvList(accountId, nsId, token, prefix, cursor) {
+  const url = new URL(`${CF_API_BASE}/accounts/${accountId}/storage/kv/namespaces/${nsId}/keys`);
+  url.searchParams.set("prefix", prefix);
+  url.searchParams.set("limit", "1000");
+  if (cursor) url.searchParams.set("cursor", cursor);
+
+  const res = await fetch(url.toString(), {
+    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
+  });
+  if (!res.ok) {
+    const body = await res.text().catch(() => "");
+    throw new Error(`CF KV list ${res.status}: ${body}`);
+  }
+  const json = await res.json();
+  if (!json.success) throw new Error(`CF KV list error: ${JSON.stringify(json.errors ?? json)}`);
+  return {
+    keys: json.result ?? [],
+    cursor: json.result_info?.cursor ?? null,
+    list_complete: !json.result_info?.cursor,
+  };
+}
+
+/**
+ * Fetch the string value for a single KV key.
+ *
+ * @param {string} accountId
+ * @param {string} nsId
+ * @param {string} token
+ * @param {string} key
+ * @returns {Promise<string>}
+ */
+export async function cfKvGet(accountId, nsId, token, key) {
+  const url = `${CF_API_BASE}/accounts/${accountId}/storage/kv/namespaces/${nsId}/values/${encodeURIComponent(key)}`;
+  const res = await fetch(url, { headers: { Authorization: `Bearer ${token}` } });
+  if (!res.ok) {
+    const body = await res.text().catch(() => "");
+    throw new Error(`CF KV get "${key}" ${res.status}: ${body}`);
+  }
+  return res.text();
+}
+
+// ─── MongoDB singleton ────────────────────────────────────────────────────────
+
+/** @type {MongoClient|null} */
+let _client = null;
+
+/**
+ * Return a shared MongoClient, connecting on first call.
+ * Registers process exit / signal handlers to close cleanly.
+ *
+ * @param {string} uri
+ * @returns {Promise<MongoClient>}
+ */
+export async function getMongoClient(uri) {
+  if (!uri) throw new Error("MongoDB URI is required — set it in .env.deploy and re-run");
+  if (_client) return _client;
+  const client = new MongoClient(uri);
+  await client.connect();
+  _client = client;
+  const close = () => {
+    _client?.close().catch(() => {});
+    _client = null;
+  };
+  process.once("exit", close);
+  process.once("SIGINT", () => {
+    close();
+    process.exit(0);
+  });
+  process.once("SIGTERM", () => {
+    close();
+    process.exit(0);
+  });
+  return _client;
+}
+
+/** Close the shared client (test teardown / forced close). */
+export async function closeMongoClient() {
+  if (_client) {
+    await _client.close();
+    _client = null;
+  }
+}
+
+// ─── Parity math ─────────────────────────────────────────────────────────────
+
+/**
+ * Relative difference between two counts — 0 means identical.
+ *
+ * @param {number} a
+ * @param {number} b
+ * @returns {number} value in [0, 1]
+ */
+export const countDiffRatio = (a, b) => Math.abs(a - b) / Math.max(a, b, 1);
+
+/**
+ * Determine verify strategy: full-scan when total < 10 000,
+ * otherwise random-sample N = min(500, ceil(sqrt(total))).
+ *
+ * @param {number} total
+ * @returns {{ fullScan: boolean, sampleSize: number }}
+ */
+export function sampleStrategy(total) {
+  if (total < 10000) return { fullScan: true, sampleSize: total };
+  return { fullScan: false, sampleSize: Math.min(500, Math.ceil(Math.sqrt(total))) };
+}

scripts/verify-mongo-parity.js

@@ -0,0 +1,215 @@
+#!/usr/bin/env node
+/**
+ * @file verify-mongo-parity — cross-check KV / D1 data against MongoDB Atlas.
+ *
+ * Per KV module: count via CF REST vs Mongo countDocuments (±1% tolerance),
+ * then value-compare via SHA256. Full-scan when total < 10 000; random-sample
+ * N = min(500, ceil(sqrt(N))) otherwise. expiresAt checked within ±5 min.
+ * For trading_trades: full-scan on legacy_id, ts, user_id, symbol, qty.
+ *
+ * Flags: --dry-run (counts only, no value compare)
+ * Required env: MONGODB_URI, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_API_TOKEN,
+ *               KV_NAMESPACE_ID, MODULES
+ * Exit: 0 all-pass | 1 any failure
+ */
+
+import { execSync } from "node:child_process";
+import {
+  cfKvGet,
+  cfKvList,
+  closeMongoClient,
+  countDiffRatio,
+  getMongoClient,
+  sampleStrategy,
+  sha256,
+  sleep,
+} from "./lib/migration-helpers.js";
+
+const DB_NAME = "miti99bot-db";
+const EXPIRES_TOL_MS = 5 * 60 * 1000; // ±5 minutes
+const dryRun = process.argv.includes("--dry-run");
+const {
+  MONGODB_URI,
+  CLOUDFLARE_ACCOUNT_ID,
+  CLOUDFLARE_API_TOKEN,
+  KV_NAMESPACE_ID,
+  MODULES: MODULES_ENV,
+} = process.env;
+const toCollName = (mod) => mod.replace(/-/g, "_");
+const redactKey = (k) => {
+  const c = k.indexOf(":");
+  return `${c >= 0 ? k.slice(0, c + 1) : ""}sha256:${sha256(k).slice(0, 8)}…`;
+};
+
+function validateEnv() {
+  const missing = [
+    "MONGODB_URI",
+    "CLOUDFLARE_ACCOUNT_ID",
+    "CLOUDFLARE_API_TOKEN",
+    "KV_NAMESPACE_ID",
+    "MODULES",
+  ].filter((k) => !process.env[k]);
+  if (missing.length) {
+    console.error(`[verify] Missing env vars: ${missing.join(", ")}`);
+    process.exit(1);
+  }
+}
+
+function queryD1(sql) {
+  const cmd = `npx wrangler d1 execute ${DB_NAME} --remote --command "${sql.replace(/"/g, '\\"')}" --json`;
+  const parsed = JSON.parse(execSync(cmd, { stdio: ["ignore", "pipe", "pipe"] }).toString());
+  return Array.isArray(parsed) ? (parsed[0]?.results ?? []) : [];
+}
+
+function shuffle(arr) {
+  const a = [...arr];
+  for (let i = a.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [a[i], a[j]] = [a[j], a[i]];
+  }
+  return a;
+}
+
+/** Enumerate ALL key objects for a module prefix via paginated CF REST. */
+async function allKvKeys(mod) {
+  const keys = [];
+  let cursor = null;
+  do {
+    const page = await cfKvList(
+      CLOUDFLARE_ACCOUNT_ID,
+      KV_NAMESPACE_ID,
+      CLOUDFLARE_API_TOKEN,
+      `${mod}:`,
+      cursor,
+    );
+    keys.push(...page.keys);
+    cursor = page.list_complete ? null : page.cursor;
+    if (cursor) await sleep(250);
+  } while (cursor);
+  return keys;
+}
+
+/**
+ * Compare one KV key against its Mongo doc.
+ * Returns null on match, mismatch description string on failure.
+ *
+ * @param {import("mongodb").Collection} coll
+ * @param {{ name: string, metadata?: { expiration?: number } }} keyObj
+ * @returns {Promise<string|null>}
+ */
+async function compareKey(coll, keyObj) {
+  const value = await cfKvGet(
+    CLOUDFLARE_ACCOUNT_ID,
+    KV_NAMESPACE_ID,
+    CLOUDFLARE_API_TOKEN,
+    keyObj.name,
+  );
+  await sleep(5);
+  const doc = await coll.findOne({ _id: keyObj.name });
+  if (!doc) return "missing in Mongo";
+  if (sha256(value) !== sha256(doc.value ?? "")) return "value hash mismatch";
+  const kvExpMs = keyObj.metadata?.expiration ? keyObj.metadata.expiration * 1000 : null;
+  const mgExpMs = doc.expiresAt ? new Date(doc.expiresAt).getTime() : null;
+  if ((kvExpMs === null) !== (mgExpMs === null)) return "expiresAt presence mismatch";
+  if (kvExpMs !== null && Math.abs(kvExpMs - mgExpMs) > EXPIRES_TOL_MS)
+    return `expiresAt diff ${Math.abs(kvExpMs - mgExpMs)}ms`;
+  return null;
+}
+
+async function verifyModule(db, mod) {
+  const coll = db.collection(toCollName(mod));
+  const kvKeys = await allKvKeys(mod);
+  const nKv = kvKeys.length;
+  const nMongo = await coll.countDocuments();
+  const ratio = countDiffRatio(nKv, nMongo);
+  const countOk = ratio < 0.01;
+  console.log(
+    `[${mod}] KV=${nKv} Mongo=${nMongo} diff=${(ratio * 100).toFixed(2)}% ${countOk ? "OK" : "FAIL"}`,
+  );
+  if (!countOk)
+    return { pass: false, mismatches: [`count diff ${(ratio * 100).toFixed(2)}% > 1%`] };
+  if (dryRun) return { pass: true, mismatches: [] };
+
+  const { fullScan, sampleSize } = sampleStrategy(nKv);
+  const sample = fullScan ? kvKeys : shuffle(kvKeys).slice(0, sampleSize);
+  console.log(`[${mod}] ${fullScan ? "full-scan" : `sample ${sampleSize}/${nKv}`}`);
+
+  const mismatches = [];
+  for (const keyObj of sample) {
+    const m = await compareKey(coll, keyObj);
+    if (m) mismatches.push(`${redactKey(keyObj.name)}: ${m}`);
+  }
+  console.log(
+    `[${mod}] value compare: ${mismatches.length === 0 ? "PASS" : `${mismatches.length} mismatch(es)`}`,
+  );
+  return { pass: mismatches.length === 0, mismatches };
+}
+
+async function verifyTrading(db) {
+  const coll = db.collection("trading_trades");
+  const rows = queryD1(
+    "SELECT id, user_id, symbol, side, qty, price_vnd, ts FROM trading_trades ORDER BY id",
+  );
+  const nMongo = await coll.countDocuments();
+  const ratio = countDiffRatio(rows.length, nMongo);
+  const countOk = ratio < 0.01;
+  console.log(
+    `[trading] D1=${rows.length} Mongo=${nMongo} diff=${(ratio * 100).toFixed(2)}% ${countOk ? "OK" : "FAIL"}`,
+  );
+  if (!countOk) return { pass: false, mismatches: [`count diff ${(ratio * 100).toFixed(2)}%`] };
+  if (dryRun) return { pass: true, mismatches: [] };
+
+  const mismatches = [];
+  for (const row of rows) {
+    const doc = await coll.findOne({ legacy_id: row.id });
+    if (!doc) {
+      mismatches.push(`legacy_id=${row.id} missing`);
+      continue;
+    }
+    for (const f of ["user_id", "symbol", "qty", "ts"]) {
+      if (String(doc[f]) !== String(row[f])) mismatches.push(`legacy_id=${row.id} ${f} mismatch`);
+    }
+  }
+  console.log(
+    `[trading] full-scan: ${mismatches.length === 0 ? "PASS" : `${mismatches.length} mismatch(es)`}`,
+  );
+  return { pass: mismatches.length === 0, mismatches };
+}
+
+async function main() {
+  validateEnv();
+  if (dryRun) console.log("[verify] DRY RUN — counts only");
+  const modules = MODULES_ENV.split(",")
+    .map((m) => m.trim())
+    .filter(Boolean);
+  const db = (await getMongoClient(MONGODB_URI)).db();
+
+  const report = [];
+  let anyFail = false;
+  for (const mod of modules) {
+    const r = await verifyModule(db, mod);
+    report.push({ mod, ...r });
+    if (!r.pass) anyFail = true;
+  }
+  const tr = await verifyTrading(db);
+  report.push({ mod: "trading", ...tr });
+  if (!tr.pass) anyFail = true;
+
+  await closeMongoClient();
+  console.log("\n── Parity Report ─────────────────────────────");
+  for (const r of report) {
+    console.log(`  ${r.pass ? "PASS" : "FAIL"}  ${r.mod}`);
+    for (const m of r.mismatches) console.log(`       mismatch: ${m}`);
+  }
+  console.log("──────────────────────────────────────────────");
+  if (anyFail) {
+    console.error("[verify] FAILED.");
+    process.exit(1);
+  }
+  console.log("[verify] All modules PASS.");
+}
+
+main().catch((err) => {
+  console.error("[verify] Fatal:", err.message ?? err);
+  process.exit(1);
+});

scripts/wipe-mongo.js

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+/**
+ * @file wipe-mongo — rollback helper: delete all documents from every backfill
+ * collection in MongoDB Atlas.
+ *
+ * WARNING: THIS IS IRREVERSIBLE. Run only when you want to start the backfill
+ * from scratch (e.g. after discovering a systematic mapping bug).
+ *
+ * Flags:
+ *   --yes  Skip the interactive confirmation prompt (for CI / scripted rollback).
+ *          Even with --yes, prints a loud warning so logs capture intent.
+ *
+ * Required env: MONGODB_URI, MODULES (comma-separated KV module names)
+ *
+ * Usage:
+ *   node --env-file-if-exists=.env.deploy scripts/wipe-mongo.js
+ *   node --env-file-if-exists=.env.deploy scripts/wipe-mongo.js --yes
+ */
+
+import { createInterface } from "node:readline";
+import { closeMongoClient, getMongoClient } from "./lib/migration-helpers.js";
+
+const { MONGODB_URI, MODULES: MODULES_ENV } = process.env;
+const skipPrompt = process.argv.includes("--yes");
+
+function validateEnv() {
+  const needed = { MONGODB_URI, MODULES: MODULES_ENV };
+  const missing = Object.entries(needed)
+    .filter(([, v]) => !v)
+    .map(([k]) => k);
+  if (missing.length) {
+    console.error(`[wipe-mongo] Missing required env vars: ${missing.join(", ")}`);
+    console.error("  Copy .env.deploy.example to .env.deploy and fill in values.");
+    process.exit(1);
+  }
+}
+
+/** @param {string} question @returns {Promise<string>} */
+function prompt(question) {
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+  return new Promise((resolve) => {
+    rl.question(question, (answer) => {
+      rl.close();
+      resolve(answer);
+    });
+  });
+}
+
+async function main() {
+  validateEnv();
+
+  // Build the list of collections: one per KV module + trading_trades.
+  const kvModules = MODULES_ENV.split(",")
+    .map((m) => m.trim())
+    .filter(Boolean);
+  const collections = [
+    ...kvModules.map((m) => m.replace(/-/g, "_")), // mirrors mongo-kv-store.js normalization
+    "trading_trades",
+  ];
+
+  console.error("╔══════════════════════════════════════════════════════╗");
+  console.error("║  WARNING: wipe-mongo will DELETE ALL DOCUMENTS from  ║");
+  console.error("║  the Atlas database and CANNOT BE UNDONE.            ║");
+  console.error("╚══════════════════════════════════════════════════════╝");
+  console.log(`Collections to wipe (${collections.length}): ${collections.join(", ")}`);
+
+  if (!skipPrompt) {
+    const answer = await prompt("\nType CONFIRM to wipe Atlas database miti99bot: ");
+    if (answer.trim() !== "CONFIRM") {
+      console.log("Aborted — nothing was deleted.");
+      process.exit(0);
+    }
+  } else {
+    console.error("[wipe-mongo] --yes passed: skipping interactive prompt. Proceeding with wipe.");
+  }
+
+  const client = await getMongoClient(MONGODB_URI);
+  const db = client.db();
+
+  let totalDeleted = 0;
+  for (const name of collections) {
+    const result = await db.collection(name).deleteMany({});
+    console.log(`[wipe-mongo] ${name}: deleted ${result.deletedCount} document(s)`);
+    totalDeleted += result.deletedCount;
+  }
+
+  await closeMongoClient();
+  console.log(`[wipe-mongo] Done — ${totalDeleted} total document(s) removed.`);
+}
+
+main().catch((err) => {
+  console.error("[wipe-mongo] Fatal:", err.message ?? err);
+  process.exit(1);
+});

tests/scripts/migration-helpers.test.js

@@ -0,0 +1,265 @@
+/**
+ * @file migration-helpers.test.js — pure-logic unit tests for migration-helpers.
+ *
+ * Tests: sha256, checkpoint round-trip, countDiffRatio, sampleStrategy,
+ * cfKvList / cfKvGet response parsing (via fetch mock).
+ *
+ * No real Atlas connection, no real CF REST calls.
+ */
+
+import { existsSync, unlinkSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  cfKvGet,
+  cfKvList,
+  clearCheckpoint,
+  closeMongoClient,
+  countDiffRatio,
+  loadCheckpoint,
+  sampleStrategy,
+  saveCheckpoint,
+  sha256,
+  sleep,
+} from "../../scripts/lib/migration-helpers.js";
+
+// ─── sleep ────────────────────────────────────────────────────────────────────
+
+describe("sleep", () => {
+  it("resolves after at least ms milliseconds", async () => {
+    const start = Date.now();
+    await sleep(20);
+    expect(Date.now() - start).toBeGreaterThanOrEqual(15); // allow 5ms jitter
+  });
+});
+
+// ─── sha256 ───────────────────────────────────────────────────────────────────
+
+describe("sha256", () => {
+  it("produces a 64-char hex string", () => {
+    expect(sha256("hello")).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  it("is deterministic for the same input", () => {
+    expect(sha256("test")).toBe(sha256("test"));
+  });
+
+  it("differs for different inputs", () => {
+    expect(sha256("a")).not.toBe(sha256("b"));
+  });
+
+  it("known vector: empty string", () => {
+    // SHA256("") = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+    expect(sha256("")).toBe("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+  });
+});
+
+// ─── Checkpoint ───────────────────────────────────────────────────────────────
+
+describe("checkpoint round-trip", () => {
+  const TEST_MODULE = "__test_module_unit__";
+  const cpFile = resolve(process.cwd(), `.backfill-cursor-${TEST_MODULE}.json`);
+
+  afterEach(() => {
+    if (existsSync(cpFile)) unlinkSync(cpFile);
+  });
+
+  it("loadCheckpoint returns null when no file exists", () => {
+    expect(loadCheckpoint(TEST_MODULE)).toBeNull();
+  });
+
+  it("saveCheckpoint + loadCheckpoint round-trip", () => {
+    const state = { cursor: "abc123", lastKey: "wordle:games:42" };
+    saveCheckpoint(TEST_MODULE, state);
+    expect(loadCheckpoint(TEST_MODULE)).toEqual(state);
+  });
+
+  it("clearCheckpoint removes the file", () => {
+    saveCheckpoint(TEST_MODULE, { cursor: "x", lastKey: "k" });
+    expect(existsSync(cpFile)).toBe(true);
+    clearCheckpoint(TEST_MODULE);
+    expect(existsSync(cpFile)).toBe(false);
+  });
+
+  it("clearCheckpoint is a no-op when file absent", () => {
+    expect(() => clearCheckpoint(TEST_MODULE)).not.toThrow();
+  });
+
+  it("loadCheckpoint returns null on corrupt JSON", () => {
+    writeFileSync(cpFile, "{bad json", "utf8");
+    expect(loadCheckpoint(TEST_MODULE)).toBeNull();
+  });
+});
+
+// ─── countDiffRatio ───────────────────────────────────────────────────────────
+
+describe("countDiffRatio", () => {
+  it("returns 0 for equal counts", () => {
+    expect(countDiffRatio(100, 100)).toBe(0);
+  });
+
+  it("returns 1 for (0, N)", () => {
+    expect(countDiffRatio(0, 100)).toBe(1);
+  });
+
+  it("returns 1 for (N, 0)", () => {
+    expect(countDiffRatio(50, 0)).toBe(1);
+  });
+
+  it("handles (0, 0) without division-by-zero — max(a,b,1)=1", () => {
+    expect(countDiffRatio(0, 0)).toBe(0);
+  });
+
+  it("1% diff on 1000", () => {
+    expect(countDiffRatio(1000, 990)).toBeCloseTo(0.01);
+  });
+
+  it("below 1% threshold for 5 diff on 1000", () => {
+    expect(countDiffRatio(1000, 995)).toBeLessThan(0.01);
+  });
+});
+
+// ─── sampleStrategy ───────────────────────────────────────────────────────────
+
+describe("sampleStrategy", () => {
+  it("full-scan for 0 total", () => {
+    const { fullScan, sampleSize } = sampleStrategy(0);
+    expect(fullScan).toBe(true);
+    expect(sampleSize).toBe(0);
+  });
+
+  it("full-scan for 9999 total", () => {
+    const { fullScan, sampleSize } = sampleStrategy(9999);
+    expect(fullScan).toBe(true);
+    expect(sampleSize).toBe(9999);
+  });
+
+  it("full-scan boundary: 10000 triggers sampling", () => {
+    const { fullScan } = sampleStrategy(10000);
+    expect(fullScan).toBe(false);
+  });
+
+  it("sample size = ceil(sqrt(total)) for mid range", () => {
+    // sqrt(40000) = 200 → sampleSize = 200
+    const { sampleSize } = sampleStrategy(40000);
+    expect(sampleSize).toBe(200);
+  });
+
+  it("sample size capped at 500", () => {
+    // sqrt(1_000_000) = 1000, capped to 500
+    const { sampleSize } = sampleStrategy(1_000_000);
+    expect(sampleSize).toBe(500);
+  });
+
+  it("ceil applied: sqrt(10001) ≈ 100.005 → 101", () => {
+    const { sampleSize } = sampleStrategy(10201); // sqrt = 101 exactly
+    expect(sampleSize).toBe(101);
+  });
+});
+
+// ─── cfKvList ─────────────────────────────────────────────────────────────────
+
+describe("cfKvList (fetch mock)", () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  function mockFetch(body, status = 200) {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn().mockResolvedValue({
+        ok: status >= 200 && status < 300,
+        status,
+        statusText: status === 200 ? "OK" : "Error",
+        text: () => Promise.resolve(JSON.stringify(body)),
+        json: () => Promise.resolve(body),
+      }),
+    );
+  }
+
+  it("parses a successful list response", async () => {
+    mockFetch({
+      success: true,
+      result: [
+        { name: "wordle:game:1", metadata: { expiration: 1700000000 } },
+        { name: "wordle:game:2" },
+      ],
+      result_info: { count: 2 },
+    });
+    const result = await cfKvList("acct", "ns", "tok", "wordle:", null);
+    expect(result.keys).toHaveLength(2);
+    expect(result.keys[0].name).toBe("wordle:game:1");
+    expect(result.keys[0].metadata?.expiration).toBe(1700000000);
+    expect(result.list_complete).toBe(true);
+    expect(result.cursor).toBeNull();
+  });
+
+  it("includes cursor when result_info.cursor present", async () => {
+    mockFetch({
+      success: true,
+      result: Array.from({ length: 1000 }, (_, i) => ({ name: `k:${i}` })),
+      result_info: { count: 1000, cursor: "next-page-cursor" },
+    });
+    const result = await cfKvList("acct", "ns", "tok", "k:", null);
+    expect(result.cursor).toBe("next-page-cursor");
+    expect(result.list_complete).toBe(false);
+  });
+
+  it("throws on HTTP error", async () => {
+    mockFetch({ errors: ["unauthorized"] }, 401);
+    await expect(cfKvList("acct", "ns", "tok", "x:", null)).rejects.toThrow("401");
+  });
+
+  it("throws when success=false", async () => {
+    mockFetch({ success: false, errors: [{ message: "namespace not found" }] });
+    await expect(cfKvList("acct", "ns", "tok", "x:", null)).rejects.toThrow(/error/i);
+  });
+});
+
+// ─── cfKvGet ──────────────────────────────────────────────────────────────────
+
+describe("cfKvGet (fetch mock)", () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  it("returns value text on success", async () => {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn().mockResolvedValue({
+        ok: true,
+        status: 200,
+        text: () => Promise.resolve('{"score":42}'),
+      }),
+    );
+    const val = await cfKvGet("acct", "ns", "tok", "wordle:game:1");
+    expect(val).toBe('{"score":42}');
+  });
+
+  it("URL-encodes the key in the request URL", async () => {
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: true,
+      text: () => Promise.resolve("v"),
+    });
+    vi.stubGlobal("fetch", fetchMock);
+    await cfKvGet("acct", "ns", "tok", "some key/with spaces");
+    const calledUrl = fetchMock.mock.calls[0][0];
+    expect(calledUrl).toContain("some%20key%2Fwith%20spaces");
+  });
+
+  it("throws on HTTP 404", async () => {
+    vi.stubGlobal(
+      "fetch",
+      vi.fn().mockResolvedValue({
+        ok: false,
+        status: 404,
+        statusText: "Not Found",
+        text: () => Promise.resolve("not found"),
+      }),
+    );
+    await expect(cfKvGet("acct", "ns", "tok", "missing:key")).rejects.toThrow("404");
+  });
+});
+
+// ─── getMongoClient teardown ──────────────────────────────────────────────────
+
+// Ensure any singleton client opened during tests is closed (prevents open handles).
+afterEach(async () => {
+  await closeMongoClient();
+});