feat: add Atlas → KV one-shot migration script

Operator runs `npm run migrate` (reads admin + group docs from Atlas)
followed by `npm run migrate:bulk` (uploads via wrangler kv bulk put).
Cache collections are skipped by default since they auto-rebuild from
upstream APIs; --include-cache flag migrates them with TTL preserved.

- mongodb is added as a devDependency only — never enters the Worker
  bundle, the Worker still talks to KV exclusively.
- scripts/.atlas-export.json is gitignored (contains exported state).
- README documents the one-time runbook.
This commit is contained in:
tiennm99 committed 2026-05-05 21:15:18 +07:00
1 parent 971c9c311f
commit d33bfeae3d
4 files changed
+140

No files matched your search

+3
View File
@@ -14,6 +14,9 @@ pnpm-debug.log*
.tmp-deploy/
.wrangler/
# Atlas → KV migration intermediate (contains exported bot state)
scripts/.atlas-export.json
# Editor / IDE
# .idea/
# .vscode/
+16
View File
@@ -55,6 +55,22 @@ Or via Docker Compose:
docker compose up --build
```
## Migrating from MongoDB Atlas (one-time)
If you're moving an existing bot off Atlas to Cloudflare KV:
```sh
# 1. Make sure .env has MONGODB_URI pointing at the Atlas cluster (read-only is fine).
npm install # pulls the mongodb devDep needed by the migration script
npm run migrate # writes scripts/.atlas-export.json (admin + groups)
# Optionally also migrate cached app entries with their TTL preserved:
# npm run migrate -- --include-cache
npm run migrate:bulk # uploads the JSON to the production KV namespace
rm scripts/.atlas-export.json # contains your bot state — delete after success
```
Cache collections (`apple_app`, `google_app`) are skipped by default since they auto-rebuild from upstream APIs within `APP_CACHE_SECONDS`. Re-running the migration is idempotent.
## Project Layout
```
+3
View File
@@ -13,9 +13,12 @@
"deploy": "wrangler deploy && npm run register",
"register": "node --env-file=.env.deploy scripts/register-webhook.js",
"register:dry": "node --env-file=.env.deploy scripts/register-webhook.js --dry-run",
"migrate": "node --env-file=.env scripts/migrate-atlas-to-kv.js",
"migrate:bulk": "wrangler kv bulk put --binding STORE_KV --remote scripts/.atlas-export.json",
"lint": "node scripts/check-secret-leaks.js"
},
"devDependencies": {
"mongodb": "^6.10.0",
"wrangler": "^3.90.0"
},
"license": "Apache-2.0"
+118
View File
@@ -0,0 +1,118 @@
// One-shot Atlas → Cloudflare KV exporter.
// Reads `common` (admin singleton) and `group` collections from MongoDB and
// writes a bulk-put JSON file consumable by:
//
// wrangler kv bulk put --binding STORE_KV --remote scripts/.atlas-export.json
//
// Cache collections (apple_app, google_app) are skipped by default — they
// auto-rebuild from upstream within `APP_CACHE_SECONDS`. Pass --include-cache
// to migrate them with their TTL recomputed (already-expired entries skipped).
//
// Run: npm run migrate # then: npm run migrate:bulk
//
// Requires .env with MONGODB_URI; loaded by package.json's `node --env-file=.env`.
import { writeFile } from 'node:fs/promises';
import { MongoClient } from 'mongodb';
const OUT_PATH = 'scripts/.atlas-export.json';
const KV_MIN_TTL_SECONDS = 60;
const APP_CACHE_SECONDS = Number(process.env.APP_CACHE_SECONDS ?? 600);
function exitWith(message) {
console.error(`migrate-atlas-to-kv: ${message}`);
process.exit(1);
}
function log(line) {
console.log(`migrate-atlas-to-kv: ${line}`);
}
// Compute remaining TTL in seconds for a cached app entry, given its
// stored `millis` (= cache write time). Returns null if already expired.
function remainingTtl(millis, nowMs) {
const expiresAt = millis + APP_CACHE_SECONDS * 1000;
const remainingSec = Math.floor((expiresAt - nowMs) / 1000);
if (remainingSec <= 0) return null;
return Math.max(KV_MIN_TTL_SECONDS, remainingSec);
}
async function main() {
const uri = process.env.MONGODB_URI;
if (!uri) exitWith('MONGODB_URI not set; check .env');
const includeCache = process.argv.includes('--include-cache');
const client = new MongoClient(uri, {
serverSelectionTimeoutMS: 5000,
socketTimeoutMS: 10000,
appName: 'migrate-atlas-to-kv',
});
await client.connect();
const db = client.db();
const entries = [];
const counts = { admin: 0, group: 0, apple: 0, appleSkipped: 0, google: 0, googleSkipped: 0 };
const adminDoc = await db.collection('common').findOne({ _id: 'admin' });
if (adminDoc) {
entries.push({ key: 'admin', value: JSON.stringify(adminDoc) });
counts.admin = 1;
} else {
log('warning: no admin doc found in common collection');
}
const groupDocs = await db.collection('group').find({}).toArray();
for (const doc of groupDocs) {
entries.push({ key: `group:${doc._id}`, value: JSON.stringify(doc) });
}
counts.group = groupDocs.length;
if (includeCache) {
const now = Date.now();
const appleDocs = await db.collection('apple_app').find({}).toArray();
for (const doc of appleDocs) {
const ttl = remainingTtl(doc.millis ?? 0, now);
if (ttl == null) {
counts.appleSkipped++;
continue;
}
entries.push({ key: `apple:${doc._id}`, value: JSON.stringify(doc), expiration_ttl: ttl });
counts.apple++;
}
const googleDocs = await db.collection('google_app').find({}).toArray();
for (const doc of googleDocs) {
const ttl = remainingTtl(doc.millis ?? 0, now);
if (ttl == null) {
counts.googleSkipped++;
continue;
}
entries.push({ key: `google:${doc._id}`, value: JSON.stringify(doc), expiration_ttl: ttl });
counts.google++;
}
}
await client.close();
if (entries.length > 10000) {
exitWith(`bulk put limit is 10000; got ${entries.length}. Chunk the export manually.`);
}
await writeFile(OUT_PATH, JSON.stringify(entries, null, 2));
log(`wrote ${entries.length} entries to ${OUT_PATH}`);
log(` admin: ${counts.admin}`);
log(` groups: ${counts.group}`);
if (includeCache) {
log(` apple: ${counts.apple} (skipped ${counts.appleSkipped} expired)`);
log(` google: ${counts.google} (skipped ${counts.googleSkipped} expired)`);
} else {
log(' caches: skipped (use --include-cache to migrate them)');
}
log('');
log('next: npm run migrate:bulk');
log('then: rm scripts/.atlas-export.json (contains your data)');
}
main().catch((err) => exitWith(err.stack ?? err.message ?? String(err)));