mirror of
https://github.com/tiennm99/store-scraper-bot.git
synced 2026-10-13 22:16:29 +00:00
feat: migrate to Vercel + Upstash with KEY_PREFIX namespacing
Phases 1-5 of consolidate-vercel-upstash plan. Replaces Cloudflare Workers + KV with Vercel serverless functions + Upstash Redis. Inlines app-store-scraper / google-play-scraper npm libs (drops the store-scraper.vercel.app HTTP roundtrip). KEY_PREFIX (default 'store-scraper-bot:') namespaces all Redis keys so the Upstash DB can be safely shared with other Vercel projects. - vercel.json + .vercelignore + Vercel-aware package.json scripts - api/webhook.js + api/cron.js Vercel functions (with shared src/app-builder.js); cron auth fails closed when CRON_SECRET unset - src/repository/upstash.js replaces kv.js; all 4 repos take a handle bundling client + prefix - scripts/migrate-atlas-to-upstash.js writes legacy Java Atlas state directly to Upstash with --dry-run + --include-cache flags - .env.example refreshed for the new env surface Phases 6 (Vercel deploy + webhook cutover) and 7 (Docker + wrangler cleanup) remain operator-driven post-deploy.
This commit is contained in:
tiennm99 committed
1 parent
cf17071679
commit
73762fa0a0
33 files changed
+4011
-488
No files matched your search
@@ -1,118 +0,0 @@
|
||||
// One-shot Atlas → Cloudflare KV exporter.
|
||||
// Reads `common` (admin singleton) and `group` collections from MongoDB and
|
||||
// writes a bulk-put JSON file consumable by:
|
||||
//
|
||||
// wrangler kv bulk put --binding STORE_KV --remote scripts/.atlas-export.json
|
||||
//
|
||||
// Cache collections (apple_app, google_app) are skipped by default — they
|
||||
// auto-rebuild from upstream within `APP_CACHE_SECONDS`. Pass --include-cache
|
||||
// to migrate them with their TTL recomputed (already-expired entries skipped).
|
||||
//
|
||||
// Run: npm run migrate # then: npm run migrate:bulk
|
||||
//
|
||||
// Requires .env with MONGODB_URI; loaded by package.json's `node --env-file=.env`.
|
||||
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import { MongoClient } from 'mongodb';
|
||||
|
||||
const OUT_PATH = 'scripts/.atlas-export.json';
|
||||
const KV_MIN_TTL_SECONDS = 60;
|
||||
const APP_CACHE_SECONDS = Number(process.env.APP_CACHE_SECONDS ?? 600);
|
||||
|
||||
function exitWith(message) {
|
||||
console.error(`migrate-atlas-to-kv: ${message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function log(line) {
|
||||
console.log(`migrate-atlas-to-kv: ${line}`);
|
||||
}
|
||||
|
||||
// Compute remaining TTL in seconds for a cached app entry, given its
|
||||
// stored `millis` (= cache write time). Returns null if already expired.
|
||||
function remainingTtl(millis, nowMs) {
|
||||
const expiresAt = millis + APP_CACHE_SECONDS * 1000;
|
||||
const remainingSec = Math.floor((expiresAt - nowMs) / 1000);
|
||||
if (remainingSec <= 0) return null;
|
||||
return Math.max(KV_MIN_TTL_SECONDS, remainingSec);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const uri = process.env.MONGODB_URI;
|
||||
if (!uri) exitWith('MONGODB_URI not set; check .env');
|
||||
|
||||
const includeCache = process.argv.includes('--include-cache');
|
||||
|
||||
const client = new MongoClient(uri, {
|
||||
serverSelectionTimeoutMS: 5000,
|
||||
socketTimeoutMS: 10000,
|
||||
appName: 'migrate-atlas-to-kv',
|
||||
});
|
||||
await client.connect();
|
||||
const db = client.db();
|
||||
|
||||
const entries = [];
|
||||
const counts = { admin: 0, group: 0, apple: 0, appleSkipped: 0, google: 0, googleSkipped: 0 };
|
||||
|
||||
const adminDoc = await db.collection('common').findOne({ _id: 'admin' });
|
||||
if (adminDoc) {
|
||||
entries.push({ key: 'admin', value: JSON.stringify(adminDoc) });
|
||||
counts.admin = 1;
|
||||
} else {
|
||||
log('warning: no admin doc found in common collection');
|
||||
}
|
||||
|
||||
const groupDocs = await db.collection('group').find({}).toArray();
|
||||
for (const doc of groupDocs) {
|
||||
entries.push({ key: `group:${doc._id}`, value: JSON.stringify(doc) });
|
||||
}
|
||||
counts.group = groupDocs.length;
|
||||
|
||||
if (includeCache) {
|
||||
const now = Date.now();
|
||||
const appleDocs = await db.collection('apple_app').find({}).toArray();
|
||||
for (const doc of appleDocs) {
|
||||
const ttl = remainingTtl(doc.millis ?? 0, now);
|
||||
if (ttl == null) {
|
||||
counts.appleSkipped++;
|
||||
continue;
|
||||
}
|
||||
entries.push({ key: `apple:${doc._id}`, value: JSON.stringify(doc), expiration_ttl: ttl });
|
||||
counts.apple++;
|
||||
}
|
||||
|
||||
const googleDocs = await db.collection('google_app').find({}).toArray();
|
||||
for (const doc of googleDocs) {
|
||||
const ttl = remainingTtl(doc.millis ?? 0, now);
|
||||
if (ttl == null) {
|
||||
counts.googleSkipped++;
|
||||
continue;
|
||||
}
|
||||
entries.push({ key: `google:${doc._id}`, value: JSON.stringify(doc), expiration_ttl: ttl });
|
||||
counts.google++;
|
||||
}
|
||||
}
|
||||
|
||||
await client.close();
|
||||
|
||||
if (entries.length > 10000) {
|
||||
exitWith(`bulk put limit is 10000; got ${entries.length}. Chunk the export manually.`);
|
||||
}
|
||||
|
||||
await writeFile(OUT_PATH, JSON.stringify(entries, null, 2));
|
||||
|
||||
log(`wrote ${entries.length} entries to ${OUT_PATH}`);
|
||||
log(` admin: ${counts.admin}`);
|
||||
log(` groups: ${counts.group}`);
|
||||
if (includeCache) {
|
||||
log(` apple: ${counts.apple} (skipped ${counts.appleSkipped} expired)`);
|
||||
log(` google: ${counts.google} (skipped ${counts.googleSkipped} expired)`);
|
||||
} else {
|
||||
log(' caches: skipped (use --include-cache to migrate them)');
|
||||
}
|
||||
log('');
|
||||
log('next: npm run migrate:bulk');
|
||||
log('then: rm scripts/.atlas-export.json (contains your data)');
|
||||
}
|
||||
|
||||
main().catch((err) => exitWith(err.stack ?? err.message ?? String(err)));
|
||||
@@ -0,0 +1,156 @@
|
||||
// One-shot legacy-DB migrator: MongoDB Atlas (java-store-scraper-bot) → Upstash Redis.
|
||||
// Direct write — no on-disk JSON intermediate.
|
||||
//
|
||||
// common._id="admin" → SET <prefix>admin <json>
|
||||
// group.find({}) → SET <prefix>group:<_id> <json> (per group)
|
||||
// apple_app.find({}) → SET <prefix>apple:<_id> <json> EX <ttl> (only with --include-cache)
|
||||
// google_app.find({}) → SET <prefix>google:<_id> <json> EX <ttl> (only with --include-cache)
|
||||
//
|
||||
// KEY_PREFIX defaults to 'store-scraper-bot:' — must match what the bot
|
||||
// runtime reads or migrated data is invisible after cutover.
|
||||
//
|
||||
// Run: npm run migrate
|
||||
// Dry: npm run migrate:dry
|
||||
// Cache: npm run migrate -- --include-cache
|
||||
//
|
||||
// Reads .env.deploy for: MONGODB_URI, UPSTASH_REDIS_REST_URL,
|
||||
// UPSTASH_REDIS_REST_TOKEN, KEY_PREFIX (optional), APP_CACHE_SECONDS (optional).
|
||||
|
||||
import { MongoClient } from 'mongodb';
|
||||
import { Redis } from '@upstash/redis';
|
||||
|
||||
const MIN_TTL_SECONDS = 60;
|
||||
const DEFAULT_KEY_PREFIX = 'store-scraper-bot:';
|
||||
const APP_CACHE_SECONDS = Number(process.env.APP_CACHE_SECONDS ?? 600);
|
||||
|
||||
function exitWith(message) {
|
||||
console.error(`migrate-atlas-to-upstash: ${message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function log(line) {
|
||||
console.log(`migrate-atlas-to-upstash: ${line}`);
|
||||
}
|
||||
|
||||
// Compute remaining TTL in seconds for a cached app entry, given its
|
||||
// stored `millis` (= cache write time). Returns null if already expired.
|
||||
function remainingTtl(millis, nowMs) {
|
||||
const expiresAt = millis + APP_CACHE_SECONDS * 1000;
|
||||
const remainingSec = Math.floor((expiresAt - nowMs) / 1000);
|
||||
if (remainingSec <= 0) return null;
|
||||
return Math.max(MIN_TTL_SECONDS, remainingSec);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const mongoUri = process.env.MONGODB_URI;
|
||||
if (!mongoUri) exitWith('MONGODB_URI not set; check .env.deploy');
|
||||
|
||||
const upstashUrl = process.env.UPSTASH_REDIS_REST_URL;
|
||||
const upstashToken = process.env.UPSTASH_REDIS_REST_TOKEN;
|
||||
const dryRun = process.argv.includes('--dry-run');
|
||||
if (!dryRun) {
|
||||
if (!upstashUrl) exitWith('UPSTASH_REDIS_REST_URL not set; check .env.deploy');
|
||||
if (!upstashToken) exitWith('UPSTASH_REDIS_REST_TOKEN not set; check .env.deploy');
|
||||
}
|
||||
|
||||
const includeCache = process.argv.includes('--include-cache');
|
||||
const prefix = process.env.KEY_PREFIX ?? DEFAULT_KEY_PREFIX;
|
||||
|
||||
log(`mode: ${dryRun ? 'DRY RUN (no writes)' : 'LIVE (writes to Upstash)'}`);
|
||||
log(`KEY_PREFIX: ${prefix} (must match bot runtime KEY_PREFIX)`);
|
||||
log(`include-cache: ${includeCache}`);
|
||||
|
||||
const redis = dryRun
|
||||
? null
|
||||
: new Redis({ url: upstashUrl, token: upstashToken });
|
||||
|
||||
// Long timeout because Atlas free tier auto-pauses idle clusters; first hit
|
||||
// can take 10–30 s to wake up. Migration is one-shot, not perf-critical.
|
||||
const mongo = new MongoClient(mongoUri, {
|
||||
serverSelectionTimeoutMS: 30000,
|
||||
socketTimeoutMS: 30000,
|
||||
appName: 'migrate-atlas-to-upstash',
|
||||
});
|
||||
await mongo.connect();
|
||||
const db = mongo.db();
|
||||
|
||||
const counts = {
|
||||
admin: 0,
|
||||
group: 0,
|
||||
apple: 0,
|
||||
appleSkipped: 0,
|
||||
google: 0,
|
||||
googleSkipped: 0,
|
||||
};
|
||||
|
||||
async function writeKey(logicalKey, value, ttlSeconds = null) {
|
||||
const physical = `${prefix}${logicalKey}`;
|
||||
if (dryRun) {
|
||||
log(` DRY SET ${physical}${ttlSeconds != null ? ` EX ${ttlSeconds}` : ''}`);
|
||||
return;
|
||||
}
|
||||
await redis.set(
|
||||
physical,
|
||||
JSON.stringify(value),
|
||||
ttlSeconds != null ? { ex: ttlSeconds } : undefined,
|
||||
);
|
||||
}
|
||||
|
||||
// 1. admin singleton (common._id = "admin")
|
||||
const adminDoc = await db.collection('common').findOne({ _id: 'admin' });
|
||||
if (adminDoc) {
|
||||
await writeKey('admin', adminDoc);
|
||||
counts.admin = 1;
|
||||
} else {
|
||||
log('warning: no admin doc found in common collection');
|
||||
}
|
||||
|
||||
// 2. groups
|
||||
const groupDocs = await db.collection('group').find({}).toArray();
|
||||
for (const doc of groupDocs) {
|
||||
await writeKey(`group:${doc._id}`, doc);
|
||||
}
|
||||
counts.group = groupDocs.length;
|
||||
|
||||
// 3. caches (opt-in)
|
||||
if (includeCache) {
|
||||
const now = Date.now();
|
||||
const appleDocs = await db.collection('apple_app').find({}).toArray();
|
||||
for (const doc of appleDocs) {
|
||||
const ttl = remainingTtl(doc.millis ?? 0, now);
|
||||
if (ttl == null) {
|
||||
counts.appleSkipped++;
|
||||
continue;
|
||||
}
|
||||
await writeKey(`apple:${doc._id}`, doc, ttl);
|
||||
counts.apple++;
|
||||
}
|
||||
|
||||
const googleDocs = await db.collection('google_app').find({}).toArray();
|
||||
for (const doc of googleDocs) {
|
||||
const ttl = remainingTtl(doc.millis ?? 0, now);
|
||||
if (ttl == null) {
|
||||
counts.googleSkipped++;
|
||||
continue;
|
||||
}
|
||||
await writeKey(`google:${doc._id}`, doc, ttl);
|
||||
counts.google++;
|
||||
}
|
||||
}
|
||||
|
||||
await mongo.close();
|
||||
|
||||
log('---');
|
||||
log(`admin: ${counts.admin}`);
|
||||
log(`groups: ${counts.group}`);
|
||||
if (includeCache) {
|
||||
log(`apple: ${counts.apple} (skipped ${counts.appleSkipped} expired)`);
|
||||
log(`google: ${counts.google} (skipped ${counts.googleSkipped} expired)`);
|
||||
} else {
|
||||
log('caches: skipped (use --include-cache to migrate them)');
|
||||
}
|
||||
log('');
|
||||
log(dryRun ? 'dry run complete — no Upstash writes performed' : 'migration complete');
|
||||
}
|
||||
|
||||
main().catch((err) => exitWith(err.stack ?? err.message ?? String(err)));
|
||||
Reference in new issue
Block a user