From f3a235de000b1b75acdb757e107884e2045e562e Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Tue, 5 May 2026 21:15:18 +0700 Subject: [PATCH] =?UTF-8?q?feat:=20add=20Atlas=20=E2=86=92=20KV=20one-shot?= =?UTF-8?q?=20migration=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator runs `npm run migrate` (reads admin + group docs from Atlas) followed by `npm run migrate:bulk` (uploads via wrangler kv bulk put). Cache collections are skipped by default since they auto-rebuild from upstream APIs; --include-cache flag migrates them with TTL preserved. - mongodb is added as a devDependency only — never enters the Worker bundle, the Worker still talks to KV exclusively. - scripts/.atlas-export.json is gitignored (contains exported state). - README documents the one-time runbook. --- .gitignore | 3 + README.md | 16 +++++ package.json | 3 + scripts/migrate-atlas-to-kv.js | 118 +++++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+) create mode 100644 scripts/migrate-atlas-to-kv.js diff --git a/.gitignore b/.gitignore index f191b65..b1b15f7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ pnpm-debug.log* .tmp-deploy/ .wrangler/ +# Atlas → KV migration intermediate (contains exported bot state) +scripts/.atlas-export.json + # Editor / IDE # .idea/ # .vscode/ diff --git a/README.md b/README.md index d57fac7..b4a978c 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,22 @@ Or via Docker Compose: docker compose up --build ``` +## Migrating from MongoDB Atlas (one-time) + +If you're moving an existing bot off Atlas to Cloudflare KV: + +```sh +# 1. Make sure .env has MONGODB_URI pointing at the Atlas cluster (read-only is fine). +npm install # pulls the mongodb devDep needed by the migration script +npm run migrate # writes scripts/.atlas-export.json (admin + groups) +# Optionally also migrate cached app entries with their TTL preserved: +# npm run migrate -- --include-cache +npm run migrate:bulk # uploads the JSON to the production KV namespace +rm scripts/.atlas-export.json # contains your bot state — delete after success +``` + +Cache collections (`apple_app`, `google_app`) are skipped by default since they auto-rebuild from upstream APIs within `APP_CACHE_SECONDS`. Re-running the migration is idempotent. + ## Project Layout ``` diff --git a/package.json b/package.json index 602faa1..c2f047b 100644 --- a/package.json +++ b/package.json @@ -13,9 +13,12 @@ "deploy": "wrangler deploy && npm run register", "register": "node --env-file=.env.deploy scripts/register-webhook.js", "register:dry": "node --env-file=.env.deploy scripts/register-webhook.js --dry-run", + "migrate": "node --env-file=.env scripts/migrate-atlas-to-kv.js", + "migrate:bulk": "wrangler kv bulk put --binding STORE_KV --remote scripts/.atlas-export.json", "lint": "node scripts/check-secret-leaks.js" }, "devDependencies": { + "mongodb": "^6.10.0", "wrangler": "^3.90.0" }, "license": "Apache-2.0" diff --git a/scripts/migrate-atlas-to-kv.js b/scripts/migrate-atlas-to-kv.js new file mode 100644 index 0000000..3dc6e7d --- /dev/null +++ b/scripts/migrate-atlas-to-kv.js @@ -0,0 +1,118 @@ +// One-shot Atlas → Cloudflare KV exporter. +// Reads `common` (admin singleton) and `group` collections from MongoDB and +// writes a bulk-put JSON file consumable by: +// +// wrangler kv bulk put --binding STORE_KV --remote scripts/.atlas-export.json +// +// Cache collections (apple_app, google_app) are skipped by default — they +// auto-rebuild from upstream within `APP_CACHE_SECONDS`. Pass --include-cache +// to migrate them with their TTL recomputed (already-expired entries skipped). +// +// Run: npm run migrate # then: npm run migrate:bulk +// +// Requires .env with MONGODB_URI; loaded by package.json's `node --env-file=.env`. + +import { writeFile } from 'node:fs/promises'; +import { MongoClient } from 'mongodb'; + +const OUT_PATH = 'scripts/.atlas-export.json'; +const KV_MIN_TTL_SECONDS = 60; +const APP_CACHE_SECONDS = Number(process.env.APP_CACHE_SECONDS ?? 600); + +function exitWith(message) { + console.error(`migrate-atlas-to-kv: ${message}`); + process.exit(1); +} + +function log(line) { + console.log(`migrate-atlas-to-kv: ${line}`); +} + +// Compute remaining TTL in seconds for a cached app entry, given its +// stored `millis` (= cache write time). Returns null if already expired. +function remainingTtl(millis, nowMs) { + const expiresAt = millis + APP_CACHE_SECONDS * 1000; + const remainingSec = Math.floor((expiresAt - nowMs) / 1000); + if (remainingSec <= 0) return null; + return Math.max(KV_MIN_TTL_SECONDS, remainingSec); +} + +async function main() { + const uri = process.env.MONGODB_URI; + if (!uri) exitWith('MONGODB_URI not set; check .env'); + + const includeCache = process.argv.includes('--include-cache'); + + const client = new MongoClient(uri, { + serverSelectionTimeoutMS: 5000, + socketTimeoutMS: 10000, + appName: 'migrate-atlas-to-kv', + }); + await client.connect(); + const db = client.db(); + + const entries = []; + const counts = { admin: 0, group: 0, apple: 0, appleSkipped: 0, google: 0, googleSkipped: 0 }; + + const adminDoc = await db.collection('common').findOne({ _id: 'admin' }); + if (adminDoc) { + entries.push({ key: 'admin', value: JSON.stringify(adminDoc) }); + counts.admin = 1; + } else { + log('warning: no admin doc found in common collection'); + } + + const groupDocs = await db.collection('group').find({}).toArray(); + for (const doc of groupDocs) { + entries.push({ key: `group:${doc._id}`, value: JSON.stringify(doc) }); + } + counts.group = groupDocs.length; + + if (includeCache) { + const now = Date.now(); + const appleDocs = await db.collection('apple_app').find({}).toArray(); + for (const doc of appleDocs) { + const ttl = remainingTtl(doc.millis ?? 0, now); + if (ttl == null) { + counts.appleSkipped++; + continue; + } + entries.push({ key: `apple:${doc._id}`, value: JSON.stringify(doc), expiration_ttl: ttl }); + counts.apple++; + } + + const googleDocs = await db.collection('google_app').find({}).toArray(); + for (const doc of googleDocs) { + const ttl = remainingTtl(doc.millis ?? 0, now); + if (ttl == null) { + counts.googleSkipped++; + continue; + } + entries.push({ key: `google:${doc._id}`, value: JSON.stringify(doc), expiration_ttl: ttl }); + counts.google++; + } + } + + await client.close(); + + if (entries.length > 10000) { + exitWith(`bulk put limit is 10000; got ${entries.length}. Chunk the export manually.`); + } + + await writeFile(OUT_PATH, JSON.stringify(entries, null, 2)); + + log(`wrote ${entries.length} entries to ${OUT_PATH}`); + log(` admin: ${counts.admin}`); + log(` groups: ${counts.group}`); + if (includeCache) { + log(` apple: ${counts.apple} (skipped ${counts.appleSkipped} expired)`); + log(` google: ${counts.google} (skipped ${counts.googleSkipped} expired)`); + } else { + log(' caches: skipped (use --include-cache to migrate them)'); + } + log(''); + log('next: npm run migrate:bulk'); + log('then: rm scripts/.atlas-export.json (contains your data)'); +} + +main().catch((err) => exitWith(err.stack ?? err.message ?? String(err)));