mirror of
https://github.com/tiennm99/store-scraper-bot.git
synced 2026-10-13 22:16:29 +00:00
feat: add Atlas → KV one-shot migration script
Operator runs `npm run migrate` (reads admin + group docs from Atlas) followed by `npm run migrate:bulk` (uploads via wrangler kv bulk put). Cache collections are skipped by default since they auto-rebuild from upstream APIs; --include-cache flag migrates them with TTL preserved. - mongodb is added as a devDependency only — never enters the Worker bundle, the Worker still talks to KV exclusively. - scripts/.atlas-export.json is gitignored (contains exported state). - README documents the one-time runbook.
This commit is contained in:
1 parent
971c9c311f
commit
d33bfeae3d
4 files changed
+140
No files matched your search
@@ -14,6 +14,9 @@ pnpm-debug.log*
|
||||
.tmp-deploy/
|
||||
.wrangler/
|
||||
|
||||
# Atlas → KV migration intermediate (contains exported bot state)
|
||||
scripts/.atlas-export.json
|
||||
|
||||
# Editor / IDE
|
||||
# .idea/
|
||||
# .vscode/
|
||||
|
||||
@@ -55,6 +55,22 @@ Or via Docker Compose:
|
||||
docker compose up --build
|
||||
```
|
||||
|
||||
## Migrating from MongoDB Atlas (one-time)
|
||||
|
||||
If you're moving an existing bot off Atlas to Cloudflare KV:
|
||||
|
||||
```sh
|
||||
# 1. Make sure .env has MONGODB_URI pointing at the Atlas cluster (read-only is fine).
|
||||
npm install # pulls the mongodb devDep needed by the migration script
|
||||
npm run migrate # writes scripts/.atlas-export.json (admin + groups)
|
||||
# Optionally also migrate cached app entries with their TTL preserved:
|
||||
# npm run migrate -- --include-cache
|
||||
npm run migrate:bulk # uploads the JSON to the production KV namespace
|
||||
rm scripts/.atlas-export.json # contains your bot state — delete after success
|
||||
```
|
||||
|
||||
Cache collections (`apple_app`, `google_app`) are skipped by default since they auto-rebuild from upstream APIs within `APP_CACHE_SECONDS`. Re-running the migration is idempotent.
|
||||
|
||||
## Project Layout
|
||||
|
||||
```
|
||||
|
||||
@@ -13,9 +13,12 @@
|
||||
"deploy": "wrangler deploy && npm run register",
|
||||
"register": "node --env-file=.env.deploy scripts/register-webhook.js",
|
||||
"register:dry": "node --env-file=.env.deploy scripts/register-webhook.js --dry-run",
|
||||
"migrate": "node --env-file=.env scripts/migrate-atlas-to-kv.js",
|
||||
"migrate:bulk": "wrangler kv bulk put --binding STORE_KV --remote scripts/.atlas-export.json",
|
||||
"lint": "node scripts/check-secret-leaks.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"mongodb": "^6.10.0",
|
||||
"wrangler": "^3.90.0"
|
||||
},
|
||||
"license": "Apache-2.0"
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
// One-shot Atlas → Cloudflare KV exporter.
|
||||
// Reads `common` (admin singleton) and `group` collections from MongoDB and
|
||||
// writes a bulk-put JSON file consumable by:
|
||||
//
|
||||
// wrangler kv bulk put --binding STORE_KV --remote scripts/.atlas-export.json
|
||||
//
|
||||
// Cache collections (apple_app, google_app) are skipped by default — they
|
||||
// auto-rebuild from upstream within `APP_CACHE_SECONDS`. Pass --include-cache
|
||||
// to migrate them with their TTL recomputed (already-expired entries skipped).
|
||||
//
|
||||
// Run: npm run migrate # then: npm run migrate:bulk
|
||||
//
|
||||
// Requires .env with MONGODB_URI; loaded by package.json's `node --env-file=.env`.
|
||||
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import { MongoClient } from 'mongodb';
|
||||
|
||||
const OUT_PATH = 'scripts/.atlas-export.json';
|
||||
const KV_MIN_TTL_SECONDS = 60;
|
||||
const APP_CACHE_SECONDS = Number(process.env.APP_CACHE_SECONDS ?? 600);
|
||||
|
||||
function exitWith(message) {
|
||||
console.error(`migrate-atlas-to-kv: ${message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function log(line) {
|
||||
console.log(`migrate-atlas-to-kv: ${line}`);
|
||||
}
|
||||
|
||||
// Compute remaining TTL in seconds for a cached app entry, given its
|
||||
// stored `millis` (= cache write time). Returns null if already expired.
|
||||
function remainingTtl(millis, nowMs) {
|
||||
const expiresAt = millis + APP_CACHE_SECONDS * 1000;
|
||||
const remainingSec = Math.floor((expiresAt - nowMs) / 1000);
|
||||
if (remainingSec <= 0) return null;
|
||||
return Math.max(KV_MIN_TTL_SECONDS, remainingSec);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const uri = process.env.MONGODB_URI;
|
||||
if (!uri) exitWith('MONGODB_URI not set; check .env');
|
||||
|
||||
const includeCache = process.argv.includes('--include-cache');
|
||||
|
||||
const client = new MongoClient(uri, {
|
||||
serverSelectionTimeoutMS: 5000,
|
||||
socketTimeoutMS: 10000,
|
||||
appName: 'migrate-atlas-to-kv',
|
||||
});
|
||||
await client.connect();
|
||||
const db = client.db();
|
||||
|
||||
const entries = [];
|
||||
const counts = { admin: 0, group: 0, apple: 0, appleSkipped: 0, google: 0, googleSkipped: 0 };
|
||||
|
||||
const adminDoc = await db.collection('common').findOne({ _id: 'admin' });
|
||||
if (adminDoc) {
|
||||
entries.push({ key: 'admin', value: JSON.stringify(adminDoc) });
|
||||
counts.admin = 1;
|
||||
} else {
|
||||
log('warning: no admin doc found in common collection');
|
||||
}
|
||||
|
||||
const groupDocs = await db.collection('group').find({}).toArray();
|
||||
for (const doc of groupDocs) {
|
||||
entries.push({ key: `group:${doc._id}`, value: JSON.stringify(doc) });
|
||||
}
|
||||
counts.group = groupDocs.length;
|
||||
|
||||
if (includeCache) {
|
||||
const now = Date.now();
|
||||
const appleDocs = await db.collection('apple_app').find({}).toArray();
|
||||
for (const doc of appleDocs) {
|
||||
const ttl = remainingTtl(doc.millis ?? 0, now);
|
||||
if (ttl == null) {
|
||||
counts.appleSkipped++;
|
||||
continue;
|
||||
}
|
||||
entries.push({ key: `apple:${doc._id}`, value: JSON.stringify(doc), expiration_ttl: ttl });
|
||||
counts.apple++;
|
||||
}
|
||||
|
||||
const googleDocs = await db.collection('google_app').find({}).toArray();
|
||||
for (const doc of googleDocs) {
|
||||
const ttl = remainingTtl(doc.millis ?? 0, now);
|
||||
if (ttl == null) {
|
||||
counts.googleSkipped++;
|
||||
continue;
|
||||
}
|
||||
entries.push({ key: `google:${doc._id}`, value: JSON.stringify(doc), expiration_ttl: ttl });
|
||||
counts.google++;
|
||||
}
|
||||
}
|
||||
|
||||
await client.close();
|
||||
|
||||
if (entries.length > 10000) {
|
||||
exitWith(`bulk put limit is 10000; got ${entries.length}. Chunk the export manually.`);
|
||||
}
|
||||
|
||||
await writeFile(OUT_PATH, JSON.stringify(entries, null, 2));
|
||||
|
||||
log(`wrote ${entries.length} entries to ${OUT_PATH}`);
|
||||
log(` admin: ${counts.admin}`);
|
||||
log(` groups: ${counts.group}`);
|
||||
if (includeCache) {
|
||||
log(` apple: ${counts.apple} (skipped ${counts.appleSkipped} expired)`);
|
||||
log(` google: ${counts.google} (skipped ${counts.googleSkipped} expired)`);
|
||||
} else {
|
||||
log(' caches: skipped (use --include-cache to migrate them)');
|
||||
}
|
||||
log('');
|
||||
log('next: npm run migrate:bulk');
|
||||
log('then: rm scripts/.atlas-export.json (contains your data)');
|
||||
}
|
||||
|
||||
main().catch((err) => exitWith(err.stack ?? err.message ?? String(err)));
|
||||
Reference in new issue
Block a user