tiennm99/store-scraper-bot
1
0 Fork 0
mirror of https://github.com/tiennm99/store-scraper-bot.git synced 2026-10-11 03:13:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: remove cloudflare + docker + legacy migration scripts

Browse Source 
Phase 7 cleanup of the Vercel + Upstash consolidation plan:

- delete wrangler.toml, Dockerfile, docker-compose{,.dev}.yml,
  scripts/migrate-atlas-to-upstash.js (one-shot migration done)
- drop wrangler + mongodb devDeps and migrate* npm scripts;
  regenerate package-lock.json (-70 packages)
- prune CF/Wrangler/Atlas-export entries from .gitignore + .vercelignore
- drop MONGODB_URI from .env.deploy.example
- rewrite README for Vercel + Upstash architecture
- refresh stale Cloudflare comments in src/{logger,models,repository}
This commit is contained in:
tiennm99 committed 2026-05-09 21:49:48 +07:00
1 parent 1799ebf29f
commit f1710a5698
19 files changed
+55 -2126
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/migrate-atlas-to-upstash.js
-158
View File
@@ -1,158 +0,0 @@
// One-shot legacy-DB migrator: MongoDB Atlas (java-store-scraper-bot) → Upstash Redis.
// Direct write — no on-disk JSON intermediate.
//
// common._id="admin" → SET <prefix>admin <json>
// group.find({}) → SET <prefix>group:<_id> <json> (per group)
// apple_app.find({}) → SET <prefix>apple:<_id> <json> EX <ttl> (only with --include-cache)
// google_app.find({}) → SET <prefix>google:<_id> <json> EX <ttl> (only with --include-cache)
//
// KEY_PREFIX defaults to 'store-scraper-bot:' — must match what the bot
// runtime reads or migrated data is invisible after cutover.
//
// Run: npm run migrate
// Dry: npm run migrate:dry
// Cache: npm run migrate -- --include-cache
//
// Reads .env.deploy for: MONGODB_URI, UPSTASH_REDIS_REST_URL,
// UPSTASH_REDIS_REST_TOKEN, KEY_PREFIX (optional), APP_CACHE_SECONDS (optional).
import { MongoClient } from 'mongodb';
import { Redis } from '@upstash/redis';
const MIN_TTL_SECONDS = 60;
const DEFAULT_KEY_PREFIX = 'store-scraper-bot:';
const APP_CACHE_SECONDS = Number(process.env.APP_CACHE_SECONDS ?? 600);
function exitWith(message) {
console.error(`migrate-atlas-to-upstash: ${message}`);
process.exit(1);
}
function log(line) {
console.log(`migrate-atlas-to-upstash: ${line}`);
}
// Compute remaining TTL in seconds for a cached app entry, given its
// stored `millis` (= cache write time). Returns null if already expired.
function remainingTtl(millis, nowMs) {
const expiresAt = millis + APP_CACHE_SECONDS * 1000;
const remainingSec = Math.floor((expiresAt - nowMs) / 1000);
if (remainingSec <= 0) return null;
return Math.max(MIN_TTL_SECONDS, remainingSec);
}
async function main() {
const mongoUri = process.env.MONGODB_URI;
if (!mongoUri) exitWith('MONGODB_URI not set; check .env.deploy');
// Accept either Upstash naming convention (vanilla signup) or Vercel
// Marketplace integration names — same as the runtime adapter.
const upstashUrl = process.env.UPSTASH_REDIS_REST_URL ?? process.env.KV_REST_API_URL;
const upstashToken = process.env.UPSTASH_REDIS_REST_TOKEN ?? process.env.KV_REST_API_TOKEN;
const dryRun = process.argv.includes('--dry-run');
if (!dryRun) {
if (!upstashUrl) exitWith('UPSTASH_REDIS_REST_URL (or KV_REST_API_URL) not set; check .env.deploy');
if (!upstashToken) exitWith('UPSTASH_REDIS_REST_TOKEN (or KV_REST_API_TOKEN) not set; check .env.deploy');
}
const includeCache = process.argv.includes('--include-cache');
const prefix = process.env.KEY_PREFIX ?? DEFAULT_KEY_PREFIX;
log(`mode: ${dryRun ? 'DRY RUN (no writes)' : 'LIVE (writes to Upstash)'}`);
log(`KEY_PREFIX: ${prefix} (must match bot runtime KEY_PREFIX)`);
log(`include-cache: ${includeCache}`);
const redis = dryRun
? null
: new Redis({ url: upstashUrl, token: upstashToken });
// Long timeout because Atlas free tier auto-pauses idle clusters; first hit
// can take 10–30 s to wake up. Migration is one-shot, not perf-critical.
const mongo = new MongoClient(mongoUri, {
serverSelectionTimeoutMS: 30000,
socketTimeoutMS: 30000,
appName: 'migrate-atlas-to-upstash',
});
await mongo.connect();
const db = mongo.db();
const counts = {
admin: 0,
group: 0,
apple: 0,
appleSkipped: 0,
google: 0,
googleSkipped: 0,
};
async function writeKey(logicalKey, value, ttlSeconds = null) {
const physical = `${prefix}${logicalKey}`;
if (dryRun) {
log(` DRY SET ${physical}${ttlSeconds != null ? ` EX ${ttlSeconds}` : ''}`);
return;
}
await redis.set(
physical,
JSON.stringify(value),
ttlSeconds != null ? { ex: ttlSeconds } : undefined,
);
}
// 1. admin singleton (common._id = "admin")
const adminDoc = await db.collection('common').findOne({ _id: 'admin' });
if (adminDoc) {
await writeKey('admin', adminDoc);
counts.admin = 1;
} else {
log('warning: no admin doc found in common collection');
}
// 2. groups
const groupDocs = await db.collection('group').find({}).toArray();
for (const doc of groupDocs) {
await writeKey(`group:${doc._id}`, doc);
}
counts.group = groupDocs.length;
// 3. caches (opt-in)
if (includeCache) {
const now = Date.now();
const appleDocs = await db.collection('apple_app').find({}).toArray();
for (const doc of appleDocs) {
const ttl = remainingTtl(doc.millis ?? 0, now);
if (ttl == null) {
counts.appleSkipped++;
continue;
}
await writeKey(`apple:${doc._id}`, doc, ttl);
counts.apple++;
}
const googleDocs = await db.collection('google_app').find({}).toArray();
for (const doc of googleDocs) {
const ttl = remainingTtl(doc.millis ?? 0, now);
if (ttl == null) {
counts.googleSkipped++;
continue;
}
await writeKey(`google:${doc._id}`, doc, ttl);
counts.google++;
}
}
await mongo.close();
log('---');
log(`admin: ${counts.admin}`);
log(`groups: ${counts.group}`);
if (includeCache) {
log(`apple: ${counts.apple} (skipped ${counts.appleSkipped} expired)`);
log(`google: ${counts.google} (skipped ${counts.googleSkipped} expired)`);
} else {
log('caches: skipped (use --include-cache to migrate them)');
}
log('');
log(dryRun ? 'dry run complete — no Upstash writes performed' : 'migration complete');
}
main().catch((err) => exitWith(err.stack ?? err.message ?? String(err)));