mirror of
https://github.com/tiennm99/store-scraper-bot.git
synced 2026-10-11 03:13:50 +00:00
feat: replace MongoDB driver with Cloudflare KV storage
Drops the mongodb dependency entirely; all four logical collections (admin singleton, group, apple_app, google_app) now live in a single KV namespace bound as STORE_KV with prefixed keys. Cache TTL is delegated to KV via expirationTtl (clamped to the 60s minimum). Document shape, field names, and Java parity at the doc level are preserved. - Adds src/repository/kv.js helper (getJson/putJson/del with TTL clamp) - Rewrites all four *-repository.js modules on top of KV - Removes src/repository/mongodb.js and the MONGODB_URI env requirement - Adds an early STORE_KV-binding guard in src/index.js - Bumps to 0.3.0
This commit is contained in:
1 parent
00f49531e0
commit
9898f408a2
12 files changed
+81
-108
No files matched your search
+2
-5
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "js-store-scraper-bot",
|
||||
"version": "0.2.0",
|
||||
"description": "JavaScript port of store-scraper-bot — Telegram bot tracking Apple App Store + Google Play app updates. Deploys to Cloudflare Workers.",
|
||||
"version": "0.3.0",
|
||||
"description": "JavaScript port of store-scraper-bot — Telegram bot tracking Apple App Store + Google Play app updates. Deploys to Cloudflare Workers, backed by Cloudflare KV.",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"engines": {
|
||||
@@ -15,9 +15,6 @@
|
||||
"register:dry": "node --env-file=.env.deploy scripts/register-webhook.js --dry-run",
|
||||
"lint": "node scripts/check-secret-leaks.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"mongodb": "^6.10.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"wrangler": "^3.90.0"
|
||||
},
|
||||
|
||||
@@ -16,7 +16,6 @@ export function loadConfig(env) {
|
||||
'TELEGRAM_BOT_TOKEN',
|
||||
'TELEGRAM_BOT_USERNAME',
|
||||
'TELEGRAM_WEBHOOK_SECRET',
|
||||
'MONGODB_URI',
|
||||
'ADMIN_IDS',
|
||||
];
|
||||
for (const k of required) {
|
||||
|
||||
+4
-3
@@ -6,9 +6,10 @@ import { createBot } from './bot/bot.js';
|
||||
import { dispatch } from './bot/dispatch.js';
|
||||
import { runDailyCheck } from './scheduler/scheduler.js';
|
||||
|
||||
// Builds the per-invocation context. Cheap; relies on memoized MongoClient
|
||||
// inside the store factory chain.
|
||||
// Builds the per-invocation context. Cheap — KV binding is exposed by the
|
||||
// runtime; no connection setup needed.
|
||||
function build(env) {
|
||||
if (!env.STORE_KV) throw new Error('STORE_KV binding missing');
|
||||
const config = loadConfig(env);
|
||||
const store = createStore(env, config.appCacheSeconds);
|
||||
const appleScraper = createAppleScraper(config, store);
|
||||
@@ -19,7 +20,7 @@ function build(env) {
|
||||
|
||||
export default {
|
||||
// Telegram webhook entry. Validates the `secret_token` header, acks fast,
|
||||
// then dispatches in `ctx.waitUntil` so Telegram doesn't retry on slow Mongo.
|
||||
// then dispatches in `ctx.waitUntil` so Telegram doesn't retry on slow downstream calls.
|
||||
async fetch(request, env, ctx) {
|
||||
if (request.method !== 'POST') {
|
||||
return new Response('Not found', { status: 404 });
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
// AppleApp cache entry — Java parity (_id=appId, class="AppleApp").
|
||||
// TTL is enforced by Cloudflare KV via expirationTtl, so no isExpired helper.
|
||||
export function newAppleApp(appId, response, millis) {
|
||||
return { _id: appId, class: 'AppleApp', app: response, millis };
|
||||
}
|
||||
|
||||
export function isAppleAppExpired(entry, nowMillis, cacheMillis) {
|
||||
return nowMillis - entry.millis > cacheMillis;
|
||||
}
|
||||
@@ -1,8 +1,5 @@
|
||||
// GoogleApp cache entry — Java parity (_id=appId, class="GoogleApp").
|
||||
// TTL is enforced by Cloudflare KV via expirationTtl, so no isExpired helper.
|
||||
export function newGoogleApp(appId, response, millis) {
|
||||
return { _id: appId, class: 'GoogleApp', app: response, millis };
|
||||
}
|
||||
|
||||
export function isGoogleAppExpired(entry, nowMillis, cacheMillis) {
|
||||
return nowMillis - entry.millis > cacheMillis;
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { getCollection } from './mongodb.js';
|
||||
import { getJson, putJson } from './kv.js';
|
||||
import {
|
||||
ADMIN_ID,
|
||||
adminAddGroup,
|
||||
@@ -7,28 +7,22 @@ import {
|
||||
newAdmin,
|
||||
} from '../models/admin.js';
|
||||
|
||||
// Stored in "common" collection at _id="admin" (Java parity).
|
||||
// KV-backed admin singleton — Java parity at the document level
|
||||
// (key 'admin' holds the same shape Mongo stored at _id="admin").
|
||||
export function createAdminRepository(env) {
|
||||
function collection() {
|
||||
return getCollection('common', env);
|
||||
}
|
||||
|
||||
async function init() {
|
||||
const c = await collection();
|
||||
const count = await c.countDocuments({ _id: ADMIN_ID });
|
||||
if (count > 0) return;
|
||||
const existing = await getJson(env, ADMIN_ID);
|
||||
if (existing) return;
|
||||
await save(newAdmin());
|
||||
}
|
||||
|
||||
async function getAdmin() {
|
||||
const c = await collection();
|
||||
const doc = await c.findOne({ _id: ADMIN_ID });
|
||||
const doc = await getJson(env, ADMIN_ID);
|
||||
return doc ?? newAdmin();
|
||||
}
|
||||
|
||||
async function save(admin) {
|
||||
const c = await collection();
|
||||
await c.replaceOne({ _id: ADMIN_ID }, admin, { upsert: true });
|
||||
await putJson(env, ADMIN_ID, admin);
|
||||
}
|
||||
|
||||
async function addGroup(groupId) {
|
||||
|
||||
@@ -1,26 +1,23 @@
|
||||
import { getCollection } from './mongodb.js';
|
||||
import { isAppleAppExpired } from '../models/apple-app.js';
|
||||
import { getJson, putJson } from './kv.js';
|
||||
|
||||
// KV-backed Apple app cache. Key shape: `apple:{appId}`.
|
||||
// KV's expirationTtl replaces Java/Mongo's manual `(now - millis) > cacheMillis`
|
||||
// check — expired keys are deleted, so a get() returning null is the cache miss.
|
||||
export function createAppleAppRepository(env, appCacheSeconds) {
|
||||
function collection() {
|
||||
return getCollection('apple_app', env);
|
||||
function key(appId) {
|
||||
return `apple:${appId}`;
|
||||
}
|
||||
|
||||
async function get(appId) {
|
||||
const c = await collection();
|
||||
return c.findOne({ _id: appId });
|
||||
return getJson(env, key(appId));
|
||||
}
|
||||
|
||||
async function save(entry) {
|
||||
const c = await collection();
|
||||
await c.replaceOne({ _id: entry._id }, entry, { upsert: true });
|
||||
await putJson(env, key(entry._id), entry, { expirationTtl: appCacheSeconds });
|
||||
}
|
||||
|
||||
async function getCached(appId) {
|
||||
const entry = await get(appId);
|
||||
if (!entry) return null;
|
||||
if (isAppleAppExpired(entry, Date.now(), appCacheSeconds * 1000)) return null;
|
||||
return entry;
|
||||
return get(appId);
|
||||
}
|
||||
|
||||
return { get, save, getCached };
|
||||
|
||||
@@ -1,26 +1,23 @@
|
||||
import { getCollection } from './mongodb.js';
|
||||
import { isGoogleAppExpired } from '../models/google-app.js';
|
||||
import { getJson, putJson } from './kv.js';
|
||||
|
||||
// KV-backed Google app cache. Key shape: `google:{appId}`.
|
||||
// KV's expirationTtl replaces Java/Mongo's manual `(now - millis) > cacheMillis`
|
||||
// check — expired keys are deleted, so a get() returning null is the cache miss.
|
||||
export function createGoogleAppRepository(env, appCacheSeconds) {
|
||||
function collection() {
|
||||
return getCollection('google_app', env);
|
||||
function key(appId) {
|
||||
return `google:${appId}`;
|
||||
}
|
||||
|
||||
async function get(appId) {
|
||||
const c = await collection();
|
||||
return c.findOne({ _id: appId });
|
||||
return getJson(env, key(appId));
|
||||
}
|
||||
|
||||
async function save(entry) {
|
||||
const c = await collection();
|
||||
await c.replaceOne({ _id: entry._id }, entry, { upsert: true });
|
||||
await putJson(env, key(entry._id), entry, { expirationTtl: appCacheSeconds });
|
||||
}
|
||||
|
||||
async function getCached(appId) {
|
||||
const entry = await get(appId);
|
||||
if (!entry) return null;
|
||||
if (isGoogleAppExpired(entry, Date.now(), appCacheSeconds * 1000)) return null;
|
||||
return entry;
|
||||
return get(appId);
|
||||
}
|
||||
|
||||
return { get, save, getCached };
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { getCollection } from './mongodb.js';
|
||||
import { del, getJson, putJson } from './kv.js';
|
||||
import {
|
||||
groupAddAppleApp,
|
||||
groupAddGoogleApp,
|
||||
@@ -8,26 +8,24 @@ import {
|
||||
newGroup,
|
||||
} from '../models/group.js';
|
||||
|
||||
// KV-backed per-group state. Key shape: `group:{chatId}`.
|
||||
export function createGroupRepository(env) {
|
||||
function collection() {
|
||||
return getCollection('group', env);
|
||||
function key(groupId) {
|
||||
return `group:${groupIdToKey(groupId)}`;
|
||||
}
|
||||
|
||||
async function exists(groupId) {
|
||||
const c = await collection();
|
||||
const count = await c.countDocuments({ _id: groupIdToKey(groupId) });
|
||||
return count > 0;
|
||||
const doc = await getJson(env, key(groupId));
|
||||
return doc !== null;
|
||||
}
|
||||
|
||||
async function getGroup(groupId) {
|
||||
const c = await collection();
|
||||
const doc = await c.findOne({ _id: groupIdToKey(groupId) });
|
||||
const doc = await getJson(env, key(groupId));
|
||||
return doc ?? newGroup(groupId);
|
||||
}
|
||||
|
||||
async function saveGroup(group) {
|
||||
const c = await collection();
|
||||
await c.replaceOne({ _id: group._id }, group, { upsert: true });
|
||||
await putJson(env, key(group._id), group);
|
||||
}
|
||||
|
||||
async function initGroup(groupId) {
|
||||
@@ -36,8 +34,7 @@ export function createGroupRepository(env) {
|
||||
}
|
||||
|
||||
async function deleteGroup(groupId) {
|
||||
const c = await collection();
|
||||
await c.deleteOne({ _id: groupIdToKey(groupId) });
|
||||
await del(env, key(groupId));
|
||||
}
|
||||
|
||||
async function mutateAndSave(groupId, mutator) {
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
// Thin wrapper around the Cloudflare KV binding `env.STORE_KV`.
|
||||
// All four logical collections live in one namespace, separated by key prefix:
|
||||
// admin singleton
|
||||
// group:{chatId} per-group state
|
||||
// apple:{appId} cached Apple response (with KV TTL)
|
||||
// google:{appId} cached Google response (with KV TTL)
|
||||
|
||||
// KV's minimum expirationTtl is 60s. Java/Mongo had no such floor; clamp here
|
||||
// so a low APP_CACHE_SECONDS override doesn't make put() reject.
|
||||
const KV_MIN_TTL_SECONDS = 60;
|
||||
|
||||
export class KvUnavailable extends Error {
|
||||
constructor() {
|
||||
super('STORE_KV binding is missing — check wrangler.toml [[kv_namespaces]]');
|
||||
this.name = 'KvUnavailable';
|
||||
}
|
||||
}
|
||||
|
||||
function binding(env) {
|
||||
if (!env || !env.STORE_KV) throw new KvUnavailable();
|
||||
return env.STORE_KV;
|
||||
}
|
||||
|
||||
export async function getJson(env, key) {
|
||||
return binding(env).get(key, 'json');
|
||||
}
|
||||
|
||||
export async function putJson(env, key, value, opts = {}) {
|
||||
const putOpts = { ...opts };
|
||||
if (putOpts.expirationTtl != null) {
|
||||
putOpts.expirationTtl = Math.max(KV_MIN_TTL_SECONDS, putOpts.expirationTtl);
|
||||
}
|
||||
await binding(env).put(key, JSON.stringify(value), putOpts);
|
||||
}
|
||||
|
||||
export async function del(env, key) {
|
||||
await binding(env).delete(key);
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
import { MongoClient } from 'mongodb';
|
||||
|
||||
// Thrown when the driver fails to reach Atlas (e.g. paused cluster, network).
|
||||
// Command handlers catch this and reply with "Internal server error".
|
||||
export class MongoUnavailable extends Error {
|
||||
constructor(cause) {
|
||||
super(`MongoDB unavailable: ${cause.message}`);
|
||||
this.name = 'MongoUnavailable';
|
||||
this.cause = cause;
|
||||
}
|
||||
}
|
||||
|
||||
// Memoized per warm Worker isolate. Module-scope is per-isolate in Workers,
|
||||
// so this caches one Promise<{client, db}> for the isolate's lifetime.
|
||||
let memoized = null;
|
||||
|
||||
export async function getMongo(env) {
|
||||
if (memoized) return memoized;
|
||||
memoized = (async () => {
|
||||
try {
|
||||
const client = new MongoClient(env.MONGODB_URI, {
|
||||
serverSelectionTimeoutMS: 5000,
|
||||
socketTimeoutMS: 10000,
|
||||
appName: 'js-store-scraper-bot',
|
||||
});
|
||||
await client.connect();
|
||||
// db() with no arg uses the database from the URI path.
|
||||
const db = client.db();
|
||||
return { client, db };
|
||||
} catch (err) {
|
||||
memoized = null; // allow retry on next request
|
||||
throw new MongoUnavailable(err);
|
||||
}
|
||||
})();
|
||||
return memoized;
|
||||
}
|
||||
|
||||
export async function getCollection(name, env) {
|
||||
const { db } = await getMongo(env);
|
||||
return db.collection(name);
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import { createAppleAppRepository } from './apple-app-repository.js';
|
||||
import { createGoogleAppRepository } from './google-app-repository.js';
|
||||
|
||||
// Single binding point for all repositories. Threads `env` once so command
|
||||
// handlers don't need to know about the Worker `env` argument.
|
||||
// handlers don't need to know about the Worker `env` argument or the KV binding.
|
||||
export function createStore(env, appCacheSeconds) {
|
||||
return {
|
||||
admin: createAdminRepository(env),
|
||||
|
||||
Reference in new issue
Block a user