mirror of
https://github.com/tiennm99/store-scraper-bot.git
synced 2026-10-14 02:16:19 +00:00
feat: Cloudflare Workers code port (deploy pending)
Refactors source to be Worker-shaped. No live deploy yet — sister deploy plan
runs Atlas provisioning + smoke later.
- wrangler.toml with nodejs_compat_v2, daily UTC 0 cron (= 7am Asia/Ho_Chi_Minh)
- package.json: drop node-telegram-bot-api, node-cron, dotenv, pino,
pino-pretty; add wrangler devDep; bump to 0.2.0
- src/bot/telegram-api.js: raw fetch wrapper for Telegram Bot API
- src/bot/dispatch.js: per-message dispatcher extracted from polling loop
- src/repository/mongodb.js: memoized MongoClient per warm isolate, typed
MongoUnavailable error, fast-fail timeouts
- src/repository/store.js: factory binding env once
- All 4 repositories converted to factory shape
- src/api/{apple,google}-scraper.js: take store instead of importing repos
- src/index.js: Worker entry exporting { fetch, scheduled }; webhook validates
X-Telegram-Bot-Api-Secret-Token; ack-then-waitUntil pattern
- src/scheduler/scheduler.js: trimmed; runDailyCheck only (no node-cron)
- src/config.js, src/logger.js: env-driven, console.log JSON output
- scripts/register-webhook.js: setWebhook + setMyCommands; --dry-run supported
- scripts/check-secret-leaks.js: lint blocks console.log(env.<SECRET>)
- plans/260426-2015-cloudflare-worker-code-port: this code port plan
- plans/260426-2327-cloudflare-deploy-and-smoke: sister deploy plan
Validated via node --check on all 32 source files; lint clean. Real deploy
gates (bundle size, cold-start CPU) run in deploy plan.
This commit is contained in:
1 parent
3c97553c69
commit
797f040574
44 files changed
+2121
-492
No files matched your search
@@ -1,4 +1,3 @@
|
||||
import { getCachedAppleApp, saveAppleApp } from '../repository/apple-app-repository.js';
|
||||
import { newAppleApp } from '../models/apple-app.js';
|
||||
|
||||
// Mirrors Java AppStoreScraper (api/apple/AppStoreScraper.java).
|
||||
@@ -12,8 +11,9 @@ export function buildAppleRequestByBundleId(appId, country) {
|
||||
return { appId, country, ratings: true };
|
||||
}
|
||||
|
||||
export function createAppleScraper(config) {
|
||||
const { logger, appCacheSeconds } = config;
|
||||
export function createAppleScraper(config, store) {
|
||||
const { logger } = config;
|
||||
const repo = store.appleApp;
|
||||
|
||||
async function rawApp(req) {
|
||||
const res = await fetch(`${BASE_URL}/app`, {
|
||||
@@ -33,14 +33,14 @@ export function createAppleScraper(config) {
|
||||
async function cache(resp) {
|
||||
if (!resp || !resp.appId) return;
|
||||
try {
|
||||
await saveAppleApp(newAppleApp(resp.appId, resp, Date.now()));
|
||||
await repo.save(newAppleApp(resp.appId, resp, Date.now()));
|
||||
} catch (err) {
|
||||
logger.warn({ appId: resp.appId, err: err.message }, 'failed to cache apple app');
|
||||
}
|
||||
}
|
||||
|
||||
async function getApp(appId, country) {
|
||||
const cached = await getCachedAppleApp(appId, appCacheSeconds);
|
||||
const cached = await repo.getCached(appId);
|
||||
if (cached) return cached.app;
|
||||
const resp = await app(buildAppleRequestByBundleId(appId, country));
|
||||
await cache(resp);
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { getCachedGoogleApp, saveGoogleApp } from '../repository/google-app-repository.js';
|
||||
import { newGoogleApp } from '../models/google-app.js';
|
||||
|
||||
// Mirrors Java GooglePlayScraper (api/google/GooglePlayScraper.java).
|
||||
@@ -8,8 +7,9 @@ export function buildGoogleRequest(appId, country) {
|
||||
return { appId, country: country || 'vn' };
|
||||
}
|
||||
|
||||
export function createGoogleScraper(config) {
|
||||
const { logger, appCacheSeconds } = config;
|
||||
export function createGoogleScraper(config, store) {
|
||||
const { logger } = config;
|
||||
const repo = store.googleApp;
|
||||
|
||||
async function rawApp(req) {
|
||||
const res = await fetch(`${BASE_URL}/app`, {
|
||||
@@ -31,14 +31,14 @@ export function createGoogleScraper(config) {
|
||||
const id = resp.appId || fallbackId;
|
||||
if (!id) return;
|
||||
try {
|
||||
await saveGoogleApp(newGoogleApp(id, resp, Date.now()));
|
||||
await repo.save(newGoogleApp(id, resp, Date.now()));
|
||||
} catch (err) {
|
||||
logger.warn({ appId: id, err: err.message }, 'failed to cache google app');
|
||||
}
|
||||
}
|
||||
|
||||
async function getApp(appId, country) {
|
||||
const cached = await getCachedGoogleApp(appId, appCacheSeconds);
|
||||
const cached = await repo.getCached(appId);
|
||||
if (cached) return cached.app;
|
||||
const resp = await app(buildGoogleRequest(appId, country));
|
||||
await cache(resp, appId);
|
||||
|
||||
+17
-63
@@ -1,4 +1,4 @@
|
||||
import TelegramBot from 'node-telegram-bot-api';
|
||||
import { createTelegramApi } from './telegram-api.js';
|
||||
import { createInfoCommand } from './commands/info.js';
|
||||
import { createAddGroupCommand } from './commands/add-group.js';
|
||||
import { createDeleteGroupCommand } from './commands/delete-group.js';
|
||||
@@ -16,14 +16,14 @@ import { createRawGoogleAppCommand } from './commands/raw-google-app.js';
|
||||
// HTML parse mode for all messages (Java parity).
|
||||
const PARSE_MODE = 'HTML';
|
||||
|
||||
export function createBot(config, appleScraper, googleScraper) {
|
||||
const tg = new TelegramBot(config.telegramBotToken, { polling: true });
|
||||
export function createBot(config, store, appleScraper, googleScraper) {
|
||||
const api = createTelegramApi(config.telegramBotToken);
|
||||
const logger = config.logger;
|
||||
|
||||
const sender = {
|
||||
async sendMessage(chatId, html) {
|
||||
try {
|
||||
await tg.sendMessage(chatId, html, {
|
||||
await api.sendMessage(chatId, html, {
|
||||
parse_mode: PARSE_MODE,
|
||||
disable_web_page_preview: true,
|
||||
});
|
||||
@@ -33,7 +33,7 @@ export function createBot(config, appleScraper, googleScraper) {
|
||||
},
|
||||
async sendMessageSilent(chatId, html) {
|
||||
try {
|
||||
await tg.sendMessage(chatId, html, {
|
||||
await api.sendMessage(chatId, html, {
|
||||
parse_mode: PARSE_MODE,
|
||||
disable_web_page_preview: true,
|
||||
disable_notification: true,
|
||||
@@ -44,12 +44,7 @@ export function createBot(config, appleScraper, googleScraper) {
|
||||
},
|
||||
async sendDocument(chatId, filename, body) {
|
||||
try {
|
||||
await tg.sendDocument(
|
||||
chatId,
|
||||
Buffer.from(body, 'utf8'),
|
||||
{},
|
||||
{ filename, contentType: 'application/json' },
|
||||
);
|
||||
await api.sendDocument(chatId, filename, body);
|
||||
} catch (err) {
|
||||
logger.warn({ chatId, err: err.message }, 'send document failed');
|
||||
}
|
||||
@@ -59,60 +54,19 @@ export function createBot(config, appleScraper, googleScraper) {
|
||||
// Java command identifiers — keep names matching exactly.
|
||||
const commands = {
|
||||
info: createInfoCommand(),
|
||||
addgroup: createAddGroupCommand(config),
|
||||
delgroup: createDeleteGroupCommand(config),
|
||||
listgroup: createListGroupCommand(config),
|
||||
addapple: createAddAppleAppCommand(appleScraper),
|
||||
delapple: createDeleteAppleAppCommand(),
|
||||
addgoogle: createAddGoogleAppCommand(googleScraper),
|
||||
delgoogle: createDeleteGoogleAppCommand(),
|
||||
listapp: createListAppCommand(),
|
||||
checkapp: createCheckAppCommand(config, appleScraper, googleScraper),
|
||||
checkappscore: createCheckAppScoresCommand(appleScraper, googleScraper),
|
||||
addgroup: createAddGroupCommand(config, store),
|
||||
delgroup: createDeleteGroupCommand(config, store),
|
||||
listgroup: createListGroupCommand(config, store),
|
||||
addapple: createAddAppleAppCommand(store, appleScraper),
|
||||
delapple: createDeleteAppleAppCommand(store),
|
||||
addgoogle: createAddGoogleAppCommand(store, googleScraper),
|
||||
delgoogle: createDeleteGoogleAppCommand(store),
|
||||
listapp: createListAppCommand(store),
|
||||
checkapp: createCheckAppCommand(config, store, appleScraper, googleScraper),
|
||||
checkappscore: createCheckAppScoresCommand(store, appleScraper, googleScraper),
|
||||
rawappleapp: createRawAppleAppCommand(appleScraper),
|
||||
rawgoogleapp: createRawGoogleAppCommand(googleScraper),
|
||||
};
|
||||
|
||||
tg.on('message', async (msg) => {
|
||||
const name = parseCommandName(msg.text, config.telegramBotUsername);
|
||||
if (!name) return;
|
||||
const handler = commands[name];
|
||||
if (!handler) {
|
||||
logger.debug({ command: name }, 'Unknown command');
|
||||
return;
|
||||
}
|
||||
logger.info(
|
||||
{ command: name, userId: msg.from?.id, chatId: msg.chat.id },
|
||||
'Executing command',
|
||||
);
|
||||
try {
|
||||
await handler(msg, sender);
|
||||
} catch (err) {
|
||||
logger.error({ err: err.message, command: name }, 'panic in command');
|
||||
await sender.sendMessage(msg.chat.id, 'Internal server error');
|
||||
}
|
||||
});
|
||||
|
||||
tg.on('polling_error', (err) => {
|
||||
logger.warn({ err: err.message }, 'polling error');
|
||||
});
|
||||
|
||||
tg.getMe()
|
||||
.then((me) => logger.info({ username: me.username }, 'Authorized on account'))
|
||||
.catch((err) => logger.error({ err: err.message }, 'getMe failed'));
|
||||
|
||||
return { sender, telegram: tg };
|
||||
}
|
||||
|
||||
// Extracts "info" from "/info", "/info arg", "/info@bot", "/info@bot arg".
|
||||
function parseCommandName(text, botUsername) {
|
||||
if (!text || text[0] !== '/') return null;
|
||||
const space = text.indexOf(' ');
|
||||
const head = space < 0 ? text.slice(1) : text.slice(1, space);
|
||||
const at = head.indexOf('@');
|
||||
if (at < 0) return head;
|
||||
const cmd = head.slice(0, at);
|
||||
const target = head.slice(at + 1);
|
||||
if (botUsername && target && target.toLowerCase() !== botUsername.toLowerCase()) return null;
|
||||
return cmd;
|
||||
return { sender, commands, api };
|
||||
}
|
||||
@@ -1,11 +1,10 @@
|
||||
import { buildAppleRequestByBundleId, buildAppleRequestByTrackId } from '../../api/apple-scraper.js';
|
||||
import * as groupRepo from '../../repository/group-repository.js';
|
||||
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
|
||||
|
||||
// /addapple <id|appId> [country=vn] — Java AddAppleAppCommand.
|
||||
export function createAddAppleAppCommand(appleScraper) {
|
||||
export function createAddAppleAppCommand(store, appleScraper) {
|
||||
return async (msg, sender) => {
|
||||
if (!(await authorizeGroup(msg.chat.id, sender))) return;
|
||||
if (!(await authorizeGroup(msg.chat.id, store, sender))) return;
|
||||
const args = splitArgs(getCommandArguments(msg.text));
|
||||
if (args.length < 1 || args.length > 2) {
|
||||
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
|
||||
@@ -13,7 +12,6 @@ export function createAddAppleAppCommand(appleScraper) {
|
||||
}
|
||||
const country = args.length === 2 ? args[1] : 'vn';
|
||||
|
||||
// Java: try parsing arg[0] as Long (trackId); else treat as bundleId.
|
||||
const trackId = Number.parseInt(args[0], 10);
|
||||
const req =
|
||||
Number.isFinite(trackId) && String(trackId) === args[0]
|
||||
@@ -32,7 +30,7 @@ export function createAddAppleAppCommand(appleScraper) {
|
||||
}
|
||||
|
||||
try {
|
||||
const added = await groupRepo.addAppleApp(msg.chat.id, resp.appId, country);
|
||||
const added = await store.group.addAppleApp(msg.chat.id, resp.appId, country);
|
||||
if (!added) {
|
||||
await sender.sendMessage(msg.chat.id, `Apple app <code>${resp.appId}</code> is already added`);
|
||||
return;
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
import { buildGoogleRequest } from '../../api/google-scraper.js';
|
||||
import * as groupRepo from '../../repository/group-repository.js';
|
||||
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
|
||||
|
||||
// /addgoogle <appId> [country=vn] — Java AddGoogleAppCommand.
|
||||
export function createAddGoogleAppCommand(googleScraper) {
|
||||
export function createAddGoogleAppCommand(store, googleScraper) {
|
||||
return async (msg, sender) => {
|
||||
if (!(await authorizeGroup(msg.chat.id, sender))) return;
|
||||
if (!(await authorizeGroup(msg.chat.id, store, sender))) return;
|
||||
const args = splitArgs(getCommandArguments(msg.text));
|
||||
if (args.length < 1 || args.length > 2) {
|
||||
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
|
||||
@@ -26,7 +25,7 @@ export function createAddGoogleAppCommand(googleScraper) {
|
||||
}
|
||||
|
||||
try {
|
||||
const added = await groupRepo.addGoogleApp(msg.chat.id, appId, country);
|
||||
const added = await store.group.addGoogleApp(msg.chat.id, appId, country);
|
||||
if (!added) {
|
||||
await sender.sendMessage(msg.chat.id, `Google app <code>${appId}</code> is already added`);
|
||||
return;
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
import * as adminRepo from '../../repository/admin-repository.js';
|
||||
import * as groupRepo from '../../repository/group-repository.js';
|
||||
import { getCommandArguments, requireAdminUser, splitArgs } from './command-utils.js';
|
||||
|
||||
// /addgroup [groupId] — Java AddGroupCommand. Admin-only.
|
||||
export function createAddGroupCommand(config) {
|
||||
export function createAddGroupCommand(config, store) {
|
||||
return async (msg, sender) => {
|
||||
if (!(await requireAdminUser(msg.from.id, msg.chat.id, config, sender))) return;
|
||||
const args = splitArgs(getCommandArguments(msg.text));
|
||||
@@ -21,12 +19,12 @@ export function createAddGroupCommand(config) {
|
||||
groupId = parsed;
|
||||
}
|
||||
try {
|
||||
const added = await adminRepo.addGroup(groupId);
|
||||
const added = await store.admin.addGroup(groupId);
|
||||
if (!added) {
|
||||
await sender.sendMessage(msg.chat.id, 'Group is already added');
|
||||
return;
|
||||
}
|
||||
await groupRepo.initGroup(groupId);
|
||||
await store.group.initGroup(groupId);
|
||||
await sender.sendMessage(msg.chat.id, 'Group added successfully');
|
||||
} catch {
|
||||
await sender.sendMessage(msg.chat.id, 'Internal server error');
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
import * as groupRepo from '../../repository/group-repository.js';
|
||||
import { buildTable } from '../../util/table.js';
|
||||
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
|
||||
|
||||
// /checkappscore — Java CheckAppScoreCommand. Reports score + ratings.
|
||||
// Score rounded to 1 decimal (Java Precision.round(score, 1) parity).
|
||||
export function createCheckAppScoresCommand(appleScraper, googleScraper) {
|
||||
export function createCheckAppScoresCommand(store, appleScraper, googleScraper) {
|
||||
return async (msg, sender) => {
|
||||
if (!(await authorizeGroup(msg.chat.id, sender))) return;
|
||||
if (!(await authorizeGroup(msg.chat.id, store, sender))) return;
|
||||
if (splitArgs(getCommandArguments(msg.text)).length !== 0) {
|
||||
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const group = await groupRepo.getGroup(msg.chat.id);
|
||||
const group = await store.group.getGroup(msg.chat.id);
|
||||
const headers = ['AppId', 'Score', 'Ratings'];
|
||||
const appleRows = await scoreRowsFor(group.appleApps, appleScraper);
|
||||
const googleRows = await scoreRowsFor(group.googleApps, googleScraper);
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
import * as groupRepo from '../../repository/group-repository.js';
|
||||
import { buildTable } from '../../util/table.js';
|
||||
import { daysBetween, formatDateInTz } from '../../util/time.js';
|
||||
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
|
||||
|
||||
// /checkapp — Java CheckAppCommand. Reports update status per app, per store.
|
||||
export function createCheckAppCommand(config, appleScraper, googleScraper) {
|
||||
export function createCheckAppCommand(config, store, appleScraper, googleScraper) {
|
||||
return async (msg, sender) => {
|
||||
if (!(await authorizeGroup(msg.chat.id, sender))) return;
|
||||
if (!(await authorizeGroup(msg.chat.id, store, sender))) return;
|
||||
if (splitArgs(getCommandArguments(msg.text)).length !== 0) {
|
||||
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const group = await groupRepo.getGroup(msg.chat.id);
|
||||
const group = await store.group.getGroup(msg.chat.id);
|
||||
const nowMs = Date.now();
|
||||
const threshold = config.numDaysWarningNotUpdated;
|
||||
const headers = ['AppId', 'Updated', 'Days', 'OK'];
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import * as adminRepo from '../../repository/admin-repository.js';
|
||||
|
||||
export function splitArgs(text) {
|
||||
if (!text) return [];
|
||||
return text.trim().split(/\s+/).filter((s) => s.length > 0);
|
||||
@@ -14,9 +12,9 @@ export function getCommandArguments(text) {
|
||||
return trimmed.slice(space + 1).trim();
|
||||
}
|
||||
|
||||
export async function authorizeGroup(chatId, sender) {
|
||||
export async function authorizeGroup(chatId, store, sender) {
|
||||
try {
|
||||
const ok = await adminRepo.hasGroup(chatId);
|
||||
const ok = await store.admin.hasGroup(chatId);
|
||||
if (!ok) {
|
||||
await sender.sendMessage(chatId, 'Group is not allowed to use bot');
|
||||
return false;
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
import * as groupRepo from '../../repository/group-repository.js';
|
||||
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
|
||||
|
||||
// /delapple <appId> — Java DeleteAppleAppCommand.
|
||||
export function createDeleteAppleAppCommand() {
|
||||
export function createDeleteAppleAppCommand(store) {
|
||||
return async (msg, sender) => {
|
||||
if (!(await authorizeGroup(msg.chat.id, sender))) return;
|
||||
if (!(await authorizeGroup(msg.chat.id, store, sender))) return;
|
||||
const args = splitArgs(getCommandArguments(msg.text));
|
||||
if (args.length !== 1) {
|
||||
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const removed = await groupRepo.removeAppleApp(msg.chat.id, args[0]);
|
||||
const removed = await store.group.removeAppleApp(msg.chat.id, args[0]);
|
||||
if (!removed) {
|
||||
await sender.sendMessage(msg.chat.id, 'Apple app is not added');
|
||||
return;
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
import * as groupRepo from '../../repository/group-repository.js';
|
||||
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
|
||||
|
||||
// /delgoogle <appId> — Java DeleteGoogleAppCommand.
|
||||
export function createDeleteGoogleAppCommand() {
|
||||
export function createDeleteGoogleAppCommand(store) {
|
||||
return async (msg, sender) => {
|
||||
if (!(await authorizeGroup(msg.chat.id, sender))) return;
|
||||
if (!(await authorizeGroup(msg.chat.id, store, sender))) return;
|
||||
const args = splitArgs(getCommandArguments(msg.text));
|
||||
if (args.length !== 1) {
|
||||
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const removed = await groupRepo.removeGoogleApp(msg.chat.id, args[0]);
|
||||
const removed = await store.group.removeGoogleApp(msg.chat.id, args[0]);
|
||||
if (!removed) {
|
||||
await sender.sendMessage(msg.chat.id, 'Google app is not added');
|
||||
return;
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import * as adminRepo from '../../repository/admin-repository.js';
|
||||
import { getCommandArguments, requireAdminUser, splitArgs } from './command-utils.js';
|
||||
|
||||
// /delgroup [groupId] — Java DeleteGroupCommand. Admin-only.
|
||||
export function createDeleteGroupCommand(config) {
|
||||
export function createDeleteGroupCommand(config, store) {
|
||||
return async (msg, sender) => {
|
||||
if (!(await requireAdminUser(msg.from.id, msg.chat.id, config, sender))) return;
|
||||
const args = splitArgs(getCommandArguments(msg.text));
|
||||
@@ -20,7 +19,7 @@ export function createDeleteGroupCommand(config) {
|
||||
groupId = parsed;
|
||||
}
|
||||
try {
|
||||
const removed = await adminRepo.removeGroup(groupId);
|
||||
const removed = await store.admin.removeGroup(groupId);
|
||||
if (!removed) {
|
||||
await sender.sendMessage(msg.chat.id, 'Group is not added');
|
||||
return;
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
import * as groupRepo from '../../repository/group-repository.js';
|
||||
import { buildTable } from '../../util/table.js';
|
||||
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
|
||||
|
||||
// /listapp — Java ListAppCommand. Two tables (Apple / Google) of tracked apps.
|
||||
export function createListAppCommand() {
|
||||
export function createListAppCommand(store) {
|
||||
return async (msg, sender) => {
|
||||
if (!(await authorizeGroup(msg.chat.id, sender))) return;
|
||||
if (!(await authorizeGroup(msg.chat.id, store, sender))) return;
|
||||
if (splitArgs(getCommandArguments(msg.text)).length !== 0) {
|
||||
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const group = await groupRepo.getGroup(msg.chat.id);
|
||||
const group = await store.group.getGroup(msg.chat.id);
|
||||
const out =
|
||||
'<b>Apple Apps</b>\n' +
|
||||
formatAppTable(group.appleApps) +
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import * as adminRepo from '../../repository/admin-repository.js';
|
||||
import { getCommandArguments, requireAdminUser, splitArgs } from './command-utils.js';
|
||||
|
||||
// /listgroup — Java ListGroupCommand. Admin-only.
|
||||
export function createListGroupCommand(config) {
|
||||
export function createListGroupCommand(config, store) {
|
||||
return async (msg, sender) => {
|
||||
if (!(await requireAdminUser(msg.from.id, msg.chat.id, config, sender))) return;
|
||||
if (splitArgs(getCommandArguments(msg.text)).length !== 0) {
|
||||
@@ -10,7 +9,7 @@ export function createListGroupCommand(config) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const groups = await adminRepo.getAllGroups();
|
||||
const groups = await store.admin.getAllGroups();
|
||||
if (groups.length === 0) {
|
||||
await sender.sendMessage(msg.chat.id, 'No groups found');
|
||||
return;
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
// Per-message dispatcher. Routes Telegram update messages to the matching
|
||||
// command handler. Equivalent to the inner loop of the old polling bot.
|
||||
export async function dispatch(message, deps) {
|
||||
const { sender, commands, config, logger } = deps;
|
||||
if (!message?.text || message.text[0] !== '/') return;
|
||||
|
||||
const name = parseCommandName(message.text, config.telegramBotUsername);
|
||||
if (!name) return;
|
||||
|
||||
const handler = commands[name];
|
||||
if (!handler) {
|
||||
logger.debug({ command: name }, 'Unknown command');
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{ command: name, userId: message.from?.id, chatId: message.chat.id },
|
||||
'Executing command',
|
||||
);
|
||||
try {
|
||||
await handler(message, sender);
|
||||
} catch (err) {
|
||||
logger.error({ err: err.message, command: name }, 'command failed');
|
||||
try {
|
||||
await sender.sendMessage(message.chat.id, 'Internal server error');
|
||||
} catch {
|
||||
// best-effort; don't double-log
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extracts "info" from "/info", "/info arg", "/info@bot", "/info@bot arg".
|
||||
function parseCommandName(text, botUsername) {
|
||||
const space = text.indexOf(' ');
|
||||
const head = space < 0 ? text.slice(1) : text.slice(1, space);
|
||||
const at = head.indexOf('@');
|
||||
if (at < 0) return head;
|
||||
const cmd = head.slice(0, at);
|
||||
const target = head.slice(at + 1);
|
||||
if (botUsername && target && target.toLowerCase() !== botUsername.toLowerCase()) return null;
|
||||
return cmd;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
// Raw fetch wrapper for the Telegram Bot API. Replaces node-telegram-bot-api
|
||||
// (which uses Node-only request/streams and bloats the Worker bundle).
|
||||
|
||||
const TELEGRAM_BASE = 'https://api.telegram.org';
|
||||
|
||||
export class TelegramApiError extends Error {
|
||||
constructor(method, status, body) {
|
||||
super(`telegram ${method} failed: ${status} ${body}`);
|
||||
this.name = 'TelegramApiError';
|
||||
this.method = method;
|
||||
this.status = status;
|
||||
this.body = body;
|
||||
}
|
||||
}
|
||||
|
||||
export function createTelegramApi(token) {
|
||||
const base = `${TELEGRAM_BASE}/bot${token}`;
|
||||
|
||||
async function callJson(method, payload) {
|
||||
const res = await fetch(`${base}/${method}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const text = await res.text();
|
||||
if (!res.ok) throw new TelegramApiError(method, res.status, text);
|
||||
return JSON.parse(text);
|
||||
}
|
||||
|
||||
// multipart/form-data — for sendDocument. WHATWG FormData/Blob is native to
|
||||
// Workers; no `form-data` npm dep needed.
|
||||
async function callMultipart(method, fields, file) {
|
||||
const form = new FormData();
|
||||
for (const [k, v] of Object.entries(fields)) form.set(k, String(v));
|
||||
if (file) {
|
||||
form.set(
|
||||
file.field,
|
||||
new Blob([file.body], { type: file.contentType }),
|
||||
file.filename,
|
||||
);
|
||||
}
|
||||
const res = await fetch(`${base}/${method}`, { method: 'POST', body: form });
|
||||
const text = await res.text();
|
||||
if (!res.ok) throw new TelegramApiError(method, res.status, text);
|
||||
return JSON.parse(text);
|
||||
}
|
||||
|
||||
return {
|
||||
getMe: () => callJson('getMe', {}),
|
||||
sendMessage: (chatId, text, opts = {}) =>
|
||||
callJson('sendMessage', { chat_id: chatId, text, ...opts }),
|
||||
sendDocument: (chatId, filename, body, opts = {}) =>
|
||||
callMultipart(
|
||||
'sendDocument',
|
||||
{ chat_id: chatId, ...opts },
|
||||
{ field: 'document', filename, body, contentType: 'application/json' },
|
||||
),
|
||||
};
|
||||
}
|
||||
+24
-70
@@ -1,87 +1,41 @@
|
||||
import 'dotenv/config';
|
||||
import { createLogger } from './logger.js';
|
||||
|
||||
const DEFAULT_DATABASE_NAME = 'store-scraper-bot';
|
||||
|
||||
function getEnv(key, fallback = '') {
|
||||
const v = process.env[key];
|
||||
return v && v.length > 0 ? v : fallback;
|
||||
}
|
||||
|
||||
function getEnvInt(key, fallback) {
|
||||
const v = process.env[key];
|
||||
if (!v) return fallback;
|
||||
const n = Number.parseInt(v, 10);
|
||||
return Number.isFinite(n) ? n : fallback;
|
||||
}
|
||||
|
||||
function parseAdminIds(raw) {
|
||||
return raw
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => s.length > 0)
|
||||
.map((s) => {
|
||||
try {
|
||||
return BigInt(s);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((v) => v !== null);
|
||||
.map((s) => Number.parseInt(s, 10))
|
||||
.filter((n) => Number.isFinite(n));
|
||||
}
|
||||
|
||||
// Mirrors Java/Go databaseFromURI: extract db name from connection string.
|
||||
function databaseFromUri(uri) {
|
||||
const idx = uri.indexOf('://');
|
||||
let rest = idx >= 0 ? uri.slice(idx + 3) : uri;
|
||||
const slash = rest.indexOf('/');
|
||||
if (slash < 0) return DEFAULT_DATABASE_NAME;
|
||||
let tail = rest.slice(slash + 1);
|
||||
const q = tail.indexOf('?');
|
||||
if (q >= 0) tail = tail.slice(0, q);
|
||||
tail = tail.trim();
|
||||
return tail.length > 0 ? tail : DEFAULT_DATABASE_NAME;
|
||||
}
|
||||
// Builds config from a Workers `env` binding. Called once per fetch / scheduled
|
||||
// invocation; cheap.
|
||||
export function loadConfig(env) {
|
||||
const required = [
|
||||
'TELEGRAM_BOT_TOKEN',
|
||||
'TELEGRAM_BOT_USERNAME',
|
||||
'TELEGRAM_WEBHOOK_SECRET',
|
||||
'MONGODB_URI',
|
||||
'ADMIN_IDS',
|
||||
];
|
||||
for (const k of required) {
|
||||
if (!env[k]) throw new Error(`${k} is required`);
|
||||
}
|
||||
|
||||
export function loadConfig() {
|
||||
const telegramBotToken = getEnv('TELEGRAM_BOT_TOKEN');
|
||||
if (!telegramBotToken) throw new Error('TELEGRAM_BOT_TOKEN is required');
|
||||
const telegramBotUsername = getEnv('TELEGRAM_BOT_USERNAME');
|
||||
if (!telegramBotUsername) throw new Error('TELEGRAM_BOT_USERNAME is required');
|
||||
|
||||
// Java parity: prefer MONGODB_CONNECTION_STRING, fall back to MONGO_URI.
|
||||
const mongoUri = getEnv('MONGODB_CONNECTION_STRING', getEnv('MONGO_URI', 'mongodb://localhost:27017'));
|
||||
const mongoDatabase = getEnv('MONGO_DATABASE') || databaseFromUri(mongoUri);
|
||||
const mongoTimeoutMs = getEnvInt('MONGO_TIMEOUT_SECONDS', 10) * 1000;
|
||||
|
||||
const env = getEnv('ENV', 'DEVELOPMENT') === 'PRODUCTION' ? 'PRODUCTION' : 'DEVELOPMENT';
|
||||
|
||||
const adminIdsRaw = getEnv('ADMIN_IDS');
|
||||
if (!adminIdsRaw) throw new Error('ADMIN_IDS is required');
|
||||
const adminIds = parseAdminIds(adminIdsRaw);
|
||||
const adminIds = parseAdminIds(env.ADMIN_IDS);
|
||||
if (adminIds.length === 0) throw new Error('at least one admin ID is required');
|
||||
|
||||
const config = {
|
||||
telegramBotToken,
|
||||
telegramBotUsername,
|
||||
mongoUri,
|
||||
mongoDatabase,
|
||||
mongoTimeoutMs,
|
||||
env,
|
||||
return {
|
||||
telegramBotToken: env.TELEGRAM_BOT_TOKEN,
|
||||
telegramBotUsername: env.TELEGRAM_BOT_USERNAME,
|
||||
telegramWebhookSecret: env.TELEGRAM_WEBHOOK_SECRET,
|
||||
adminIds,
|
||||
creatorId: adminIds[0],
|
||||
sourceCommit: getEnv('SOURCE_COMMIT', 'unknown'),
|
||||
appCacheSeconds: getEnvInt('APP_CACHE_SECONDS', 600),
|
||||
numDaysWarningNotUpdated: getEnvInt('NUM_DAYS_WARNING_NOT_UPDATED', 30),
|
||||
scheduleCheckAppTime: getEnv('SCHEDULE_CHECK_APP_TIME', '0 7 * * *'),
|
||||
isAdmin: (userId) => adminIds.includes(Number(userId)),
|
||||
appCacheSeconds: Number(env.APP_CACHE_SECONDS ?? 600),
|
||||
numDaysWarningNotUpdated: Number(env.NUM_DAYS_WARNING_NOT_UPDATED ?? 30),
|
||||
timezone: 'Asia/Ho_Chi_Minh',
|
||||
logger: createLogger(env),
|
||||
logger: createLogger(),
|
||||
};
|
||||
|
||||
config.isAdmin = (userId) => {
|
||||
const id = typeof userId === 'bigint' ? userId : BigInt(userId);
|
||||
return config.adminIds.some((a) => a === id);
|
||||
};
|
||||
|
||||
return config;
|
||||
}
|
||||
+66
-36
@@ -1,43 +1,73 @@
|
||||
import { loadConfig } from './config.js';
|
||||
import { closeMongoDB, initMongoDB } from './repository/mongodb.js';
|
||||
import { initAdmin } from './repository/admin-repository.js';
|
||||
import { createStore } from './repository/store.js';
|
||||
import { createAppleScraper } from './api/apple-scraper.js';
|
||||
import { createGoogleScraper } from './api/google-scraper.js';
|
||||
import { createBot } from './bot/bot.js';
|
||||
import { createScheduler } from './scheduler/scheduler.js';
|
||||
import { dispatch } from './bot/dispatch.js';
|
||||
import { runDailyCheck } from './scheduler/scheduler.js';
|
||||
|
||||
async function main() {
|
||||
const config = loadConfig();
|
||||
const logger = config.logger;
|
||||
logger.info({ env: config.env, commit: config.sourceCommit }, 'Starting Store Scraper Bot');
|
||||
|
||||
await initMongoDB(config);
|
||||
await initAdmin(); // Java parity: ensure singleton "common/admin" doc exists.
|
||||
|
||||
const appleScraper = createAppleScraper(config);
|
||||
const googleScraper = createGoogleScraper(config);
|
||||
|
||||
const bot = createBot(config, appleScraper, googleScraper);
|
||||
const scheduler = createScheduler(config, bot.sender, appleScraper, googleScraper);
|
||||
scheduler.start();
|
||||
|
||||
logger.info('Starting Telegram bot polling');
|
||||
|
||||
const shutdown = async (signal) => {
|
||||
logger.info({ signal }, 'Received shutdown signal, stopping bot...');
|
||||
try {
|
||||
scheduler.stop();
|
||||
await bot.telegram.stopPolling();
|
||||
await closeMongoDB();
|
||||
} finally {
|
||||
process.exit(0);
|
||||
}
|
||||
};
|
||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||
// Builds the per-invocation context. Cheap; relies on memoized MongoClient
|
||||
// inside the store factory chain.
|
||||
function build(env) {
|
||||
const config = loadConfig(env);
|
||||
const store = createStore(env, config.appCacheSeconds);
|
||||
const appleScraper = createAppleScraper(config, store);
|
||||
const googleScraper = createGoogleScraper(config, store);
|
||||
const { sender, commands } = createBot(config, store, appleScraper, googleScraper);
|
||||
return { config, store, appleScraper, googleScraper, sender, commands };
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Fatal:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
export default {
|
||||
// Telegram webhook entry. Validates the `secret_token` header, acks fast,
|
||||
// then dispatches in `ctx.waitUntil` so Telegram doesn't retry on slow Mongo.
|
||||
async fetch(request, env, ctx) {
|
||||
if (request.method !== 'POST') {
|
||||
return new Response('Not found', { status: 404 });
|
||||
}
|
||||
|
||||
let app;
|
||||
try {
|
||||
app = build(env);
|
||||
} catch (err) {
|
||||
console.log(JSON.stringify({ level: 'error', msg: 'config error', err: err.message }));
|
||||
return new Response('Server misconfigured', { status: 500 });
|
||||
}
|
||||
|
||||
const secret = request.headers.get('X-Telegram-Bot-Api-Secret-Token');
|
||||
if (secret !== app.config.telegramWebhookSecret) {
|
||||
return new Response('Unauthorized', { status: 401 });
|
||||
}
|
||||
|
||||
let update;
|
||||
try {
|
||||
update = await request.json();
|
||||
} catch {
|
||||
return new Response('Bad request', { status: 400 });
|
||||
}
|
||||
if (!update?.message) return new Response('OK');
|
||||
|
||||
ctx.waitUntil(
|
||||
dispatch(update.message, {
|
||||
sender: app.sender,
|
||||
commands: app.commands,
|
||||
config: app.config,
|
||||
logger: app.config.logger,
|
||||
}),
|
||||
);
|
||||
return new Response('OK');
|
||||
},
|
||||
|
||||
// Daily cron handler. Schedule lives in wrangler.toml.
|
||||
async scheduled(event, env, ctx) {
|
||||
let app;
|
||||
try {
|
||||
app = build(env);
|
||||
} catch (err) {
|
||||
console.log(JSON.stringify({ level: 'error', msg: 'config error', err: err.message }));
|
||||
return;
|
||||
}
|
||||
ctx.waitUntil(
|
||||
runDailyCheck(app.config, app.store, app.sender, app.appleScraper, app.googleScraper),
|
||||
);
|
||||
},
|
||||
};
|
||||
+15
-15
@@ -1,16 +1,16 @@
|
||||
import { pino } from 'pino';
|
||||
|
||||
export function createLogger(env) {
|
||||
const isDev = env !== 'PRODUCTION';
|
||||
return pino(
|
||||
isDev
|
||||
? {
|
||||
level: 'debug',
|
||||
transport: {
|
||||
target: 'pino-pretty',
|
||||
options: { colorize: true, translateTime: 'SYS:HH:MM:ss', ignore: 'pid,hostname' },
|
||||
},
|
||||
}
|
||||
: { level: 'info' },
|
||||
);
|
||||
// Worker-friendly structured logger. Cloudflare Observability indexes JSON
|
||||
// console output, so we emit one JSON record per call.
|
||||
export function createLogger() {
|
||||
function log(level, payloadOrMsg, maybeMsg) {
|
||||
const isObj = payloadOrMsg !== null && typeof payloadOrMsg === 'object';
|
||||
const payload = isObj ? payloadOrMsg : {};
|
||||
const msg = isObj ? maybeMsg ?? '' : payloadOrMsg ?? '';
|
||||
console.log(JSON.stringify({ level, ts: new Date().toISOString(), msg, ...payload }));
|
||||
}
|
||||
return {
|
||||
debug: (p, m) => log('debug', p, m),
|
||||
info: (p, m) => log('info', p, m),
|
||||
warn: (p, m) => log('warn', p, m),
|
||||
error: (p, m) => log('error', p, m),
|
||||
};
|
||||
}
|
||||
@@ -1,47 +1,59 @@
|
||||
import { getCollection } from './mongodb.js';
|
||||
import { ADMIN_ID, adminAddGroup, adminHasGroup, adminRemoveGroup, newAdmin } from '../models/admin.js';
|
||||
import {
|
||||
ADMIN_ID,
|
||||
adminAddGroup,
|
||||
adminHasGroup,
|
||||
adminRemoveGroup,
|
||||
newAdmin,
|
||||
} from '../models/admin.js';
|
||||
|
||||
// Stored in "common" collection at _id="admin" (Java parity).
|
||||
function collection() {
|
||||
return getCollection('common');
|
||||
}
|
||||
export function createAdminRepository(env) {
|
||||
function collection() {
|
||||
return getCollection('common', env);
|
||||
}
|
||||
|
||||
export async function initAdmin() {
|
||||
const c = collection();
|
||||
const count = await c.countDocuments({ _id: ADMIN_ID });
|
||||
if (count > 0) return;
|
||||
await save(newAdmin());
|
||||
}
|
||||
async function init() {
|
||||
const c = await collection();
|
||||
const count = await c.countDocuments({ _id: ADMIN_ID });
|
||||
if (count > 0) return;
|
||||
await save(newAdmin());
|
||||
}
|
||||
|
||||
export async function getAdmin() {
|
||||
const doc = await collection().findOne({ _id: ADMIN_ID });
|
||||
return doc ?? newAdmin();
|
||||
}
|
||||
async function getAdmin() {
|
||||
const c = await collection();
|
||||
const doc = await c.findOne({ _id: ADMIN_ID });
|
||||
return doc ?? newAdmin();
|
||||
}
|
||||
|
||||
export async function save(admin) {
|
||||
await collection().replaceOne({ _id: ADMIN_ID }, admin, { upsert: true });
|
||||
}
|
||||
async function save(admin) {
|
||||
const c = await collection();
|
||||
await c.replaceOne({ _id: ADMIN_ID }, admin, { upsert: true });
|
||||
}
|
||||
|
||||
export async function addGroup(groupId) {
|
||||
const admin = await getAdmin();
|
||||
if (!adminAddGroup(admin, groupId)) return false;
|
||||
await save(admin);
|
||||
return true;
|
||||
}
|
||||
async function addGroup(groupId) {
|
||||
const admin = await getAdmin();
|
||||
if (!adminAddGroup(admin, groupId)) return false;
|
||||
await save(admin);
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function removeGroup(groupId) {
|
||||
const admin = await getAdmin();
|
||||
if (!adminRemoveGroup(admin, groupId)) return false;
|
||||
await save(admin);
|
||||
return true;
|
||||
}
|
||||
async function removeGroup(groupId) {
|
||||
const admin = await getAdmin();
|
||||
if (!adminRemoveGroup(admin, groupId)) return false;
|
||||
await save(admin);
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function hasGroup(groupId) {
|
||||
const admin = await getAdmin();
|
||||
return adminHasGroup(admin, groupId);
|
||||
}
|
||||
async function hasGroup(groupId) {
|
||||
const admin = await getAdmin();
|
||||
return adminHasGroup(admin, groupId);
|
||||
}
|
||||
|
||||
export async function getAllGroups() {
|
||||
const admin = await getAdmin();
|
||||
return admin.groups;
|
||||
async function getAllGroups() {
|
||||
const admin = await getAdmin();
|
||||
return admin.groups;
|
||||
}
|
||||
|
||||
return { init, getAdmin, save, addGroup, removeGroup, hasGroup, getAllGroups };
|
||||
}
|
||||
@@ -1,22 +1,27 @@
|
||||
import { getCollection } from './mongodb.js';
|
||||
import { isAppleAppExpired } from '../models/apple-app.js';
|
||||
|
||||
function collection() {
|
||||
return getCollection('apple_app');
|
||||
}
|
||||
export function createAppleAppRepository(env, appCacheSeconds) {
|
||||
function collection() {
|
||||
return getCollection('apple_app', env);
|
||||
}
|
||||
|
||||
export async function getAppleApp(appId) {
|
||||
return collection().findOne({ _id: appId });
|
||||
}
|
||||
async function get(appId) {
|
||||
const c = await collection();
|
||||
return c.findOne({ _id: appId });
|
||||
}
|
||||
|
||||
export async function saveAppleApp(entry) {
|
||||
await collection().replaceOne({ _id: entry._id }, entry, { upsert: true });
|
||||
}
|
||||
async function save(entry) {
|
||||
const c = await collection();
|
||||
await c.replaceOne({ _id: entry._id }, entry, { upsert: true });
|
||||
}
|
||||
|
||||
export async function getCachedAppleApp(appId, appCacheSeconds) {
|
||||
const entry = await getAppleApp(appId);
|
||||
if (!entry) return null;
|
||||
const cacheMillis = appCacheSeconds * 1000;
|
||||
if (isAppleAppExpired(entry, Date.now(), cacheMillis)) return null;
|
||||
return entry;
|
||||
async function getCached(appId) {
|
||||
const entry = await get(appId);
|
||||
if (!entry) return null;
|
||||
if (isAppleAppExpired(entry, Date.now(), appCacheSeconds * 1000)) return null;
|
||||
return entry;
|
||||
}
|
||||
|
||||
return { get, save, getCached };
|
||||
}
|
||||
@@ -1,22 +1,27 @@
|
||||
import { getCollection } from './mongodb.js';
|
||||
import { isGoogleAppExpired } from '../models/google-app.js';
|
||||
|
||||
function collection() {
|
||||
return getCollection('google_app');
|
||||
}
|
||||
export function createGoogleAppRepository(env, appCacheSeconds) {
|
||||
function collection() {
|
||||
return getCollection('google_app', env);
|
||||
}
|
||||
|
||||
export async function getGoogleApp(appId) {
|
||||
return collection().findOne({ _id: appId });
|
||||
}
|
||||
async function get(appId) {
|
||||
const c = await collection();
|
||||
return c.findOne({ _id: appId });
|
||||
}
|
||||
|
||||
export async function saveGoogleApp(entry) {
|
||||
await collection().replaceOne({ _id: entry._id }, entry, { upsert: true });
|
||||
}
|
||||
async function save(entry) {
|
||||
const c = await collection();
|
||||
await c.replaceOne({ _id: entry._id }, entry, { upsert: true });
|
||||
}
|
||||
|
||||
export async function getCachedGoogleApp(appId, appCacheSeconds) {
|
||||
const entry = await getGoogleApp(appId);
|
||||
if (!entry) return null;
|
||||
const cacheMillis = appCacheSeconds * 1000;
|
||||
if (isGoogleAppExpired(entry, Date.now(), cacheMillis)) return null;
|
||||
return entry;
|
||||
async function getCached(appId) {
|
||||
const entry = await get(appId);
|
||||
if (!entry) return null;
|
||||
if (isGoogleAppExpired(entry, Date.now(), appCacheSeconds * 1000)) return null;
|
||||
return entry;
|
||||
}
|
||||
|
||||
return { get, save, getCached };
|
||||
}
|
||||
@@ -8,52 +8,58 @@ import {
|
||||
newGroup,
|
||||
} from '../models/group.js';
|
||||
|
||||
function collection() {
|
||||
return getCollection('group');
|
||||
}
|
||||
export function createGroupRepository(env) {
|
||||
function collection() {
|
||||
return getCollection('group', env);
|
||||
}
|
||||
|
||||
export async function exists(groupId) {
|
||||
const count = await collection().countDocuments({ _id: groupIdToKey(groupId) });
|
||||
return count > 0;
|
||||
}
|
||||
async function exists(groupId) {
|
||||
const c = await collection();
|
||||
const count = await c.countDocuments({ _id: groupIdToKey(groupId) });
|
||||
return count > 0;
|
||||
}
|
||||
|
||||
export async function getGroup(groupId) {
|
||||
const doc = await collection().findOne({ _id: groupIdToKey(groupId) });
|
||||
return doc ?? newGroup(groupId);
|
||||
}
|
||||
async function getGroup(groupId) {
|
||||
const c = await collection();
|
||||
const doc = await c.findOne({ _id: groupIdToKey(groupId) });
|
||||
return doc ?? newGroup(groupId);
|
||||
}
|
||||
|
||||
export async function saveGroup(group) {
|
||||
await collection().replaceOne({ _id: group._id }, group, { upsert: true });
|
||||
}
|
||||
async function saveGroup(group) {
|
||||
const c = await collection();
|
||||
await c.replaceOne({ _id: group._id }, group, { upsert: true });
|
||||
}
|
||||
|
||||
export async function initGroup(groupId) {
|
||||
if (await exists(groupId)) return;
|
||||
await saveGroup(newGroup(groupId));
|
||||
}
|
||||
async function initGroup(groupId) {
|
||||
if (await exists(groupId)) return;
|
||||
await saveGroup(newGroup(groupId));
|
||||
}
|
||||
|
||||
export async function deleteGroup(groupId) {
|
||||
await collection().deleteOne({ _id: groupIdToKey(groupId) });
|
||||
}
|
||||
async function deleteGroup(groupId) {
|
||||
const c = await collection();
|
||||
await c.deleteOne({ _id: groupIdToKey(groupId) });
|
||||
}
|
||||
|
||||
async function mutateAndSave(groupId, mutator) {
|
||||
const group = await getGroup(groupId);
|
||||
if (!mutator(group)) return false;
|
||||
await saveGroup(group);
|
||||
return true;
|
||||
}
|
||||
async function mutateAndSave(groupId, mutator) {
|
||||
const group = await getGroup(groupId);
|
||||
if (!mutator(group)) return false;
|
||||
await saveGroup(group);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function addAppleApp(groupId, appId, country) {
|
||||
return mutateAndSave(groupId, (g) => groupAddAppleApp(g, appId, country));
|
||||
}
|
||||
|
||||
export function removeAppleApp(groupId, appId) {
|
||||
return mutateAndSave(groupId, (g) => groupRemoveAppleApp(g, appId));
|
||||
}
|
||||
|
||||
export function addGoogleApp(groupId, appId, country) {
|
||||
return mutateAndSave(groupId, (g) => groupAddGoogleApp(g, appId, country));
|
||||
}
|
||||
|
||||
export function removeGoogleApp(groupId, appId) {
|
||||
return mutateAndSave(groupId, (g) => groupRemoveGoogleApp(g, appId));
|
||||
return {
|
||||
exists,
|
||||
getGroup,
|
||||
saveGroup,
|
||||
initGroup,
|
||||
deleteGroup,
|
||||
addAppleApp: (groupId, appId, country) =>
|
||||
mutateAndSave(groupId, (g) => groupAddAppleApp(g, appId, country)),
|
||||
removeAppleApp: (groupId, appId) =>
|
||||
mutateAndSave(groupId, (g) => groupRemoveAppleApp(g, appId)),
|
||||
addGoogleApp: (groupId, appId, country) =>
|
||||
mutateAndSave(groupId, (g) => groupAddGoogleApp(g, appId, country)),
|
||||
removeGoogleApp: (groupId, appId) =>
|
||||
mutateAndSave(groupId, (g) => groupRemoveGoogleApp(g, appId)),
|
||||
};
|
||||
}
|
||||
+34
-23
@@ -1,30 +1,41 @@
|
||||
import { MongoClient } from 'mongodb';
|
||||
|
||||
let client;
|
||||
let database;
|
||||
|
||||
export async function initMongoDB(config) {
|
||||
client = new MongoClient(config.mongoUri, {
|
||||
serverSelectionTimeoutMS: config.mongoTimeoutMs,
|
||||
});
|
||||
await client.connect();
|
||||
await client.db(config.mongoDatabase).command({ ping: 1 });
|
||||
database = client.db(config.mongoDatabase);
|
||||
config.logger.info(
|
||||
{ database: config.mongoDatabase, uri: config.mongoUri },
|
||||
'Connected to MongoDB',
|
||||
);
|
||||
// Thrown when the driver fails to reach Atlas (e.g. paused cluster, network).
|
||||
// Command handlers catch this and reply with "Internal server error".
|
||||
export class MongoUnavailable extends Error {
|
||||
constructor(cause) {
|
||||
super(`MongoDB unavailable: ${cause.message}`);
|
||||
this.name = 'MongoUnavailable';
|
||||
this.cause = cause;
|
||||
}
|
||||
}
|
||||
|
||||
export async function closeMongoDB() {
|
||||
if (client) await client.close();
|
||||
// Memoized per warm Worker isolate. Module-scope is per-isolate in Workers,
|
||||
// so this caches one Promise<{client, db}> for the isolate's lifetime.
|
||||
let memoized = null;
|
||||
|
||||
export async function getMongo(env) {
|
||||
if (memoized) return memoized;
|
||||
memoized = (async () => {
|
||||
try {
|
||||
const client = new MongoClient(env.MONGODB_URI, {
|
||||
serverSelectionTimeoutMS: 5000,
|
||||
socketTimeoutMS: 10000,
|
||||
appName: 'js-store-scraper-bot',
|
||||
});
|
||||
await client.connect();
|
||||
// db() with no arg uses the database from the URI path.
|
||||
const db = client.db();
|
||||
return { client, db };
|
||||
} catch (err) {
|
||||
memoized = null; // allow retry on next request
|
||||
throw new MongoUnavailable(err);
|
||||
}
|
||||
})();
|
||||
return memoized;
|
||||
}
|
||||
|
||||
export function getDatabase() {
|
||||
if (!database) throw new Error('MongoDB not initialized');
|
||||
return database;
|
||||
}
|
||||
|
||||
export function getCollection(name) {
|
||||
return getDatabase().collection(name);
|
||||
export async function getCollection(name, env) {
|
||||
const { db } = await getMongo(env);
|
||||
return db.collection(name);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { createAdminRepository } from './admin-repository.js';
|
||||
import { createGroupRepository } from './group-repository.js';
|
||||
import { createAppleAppRepository } from './apple-app-repository.js';
|
||||
import { createGoogleAppRepository } from './google-app-repository.js';
|
||||
|
||||
// Single binding point for all repositories. Threads `env` once so command
|
||||
// handlers don't need to know about the Worker `env` argument.
|
||||
export function createStore(env, appCacheSeconds) {
|
||||
return {
|
||||
admin: createAdminRepository(env),
|
||||
group: createGroupRepository(env),
|
||||
appleApp: createAppleAppRepository(env, appCacheSeconds),
|
||||
googleApp: createGoogleAppRepository(env, appCacheSeconds),
|
||||
};
|
||||
}
|
||||
+93
-113
@@ -1,136 +1,116 @@
|
||||
import cron from 'node-cron';
|
||||
import * as adminRepo from '../repository/admin-repository.js';
|
||||
import * as groupRepo from '../repository/group-repository.js';
|
||||
import { buildTable, formatNumber, truncateString } from '../util/table.js';
|
||||
import { daysBetween, formatDateInTz, formatDateTimeInTz, weekdayInTz } from '../util/time.js';
|
||||
|
||||
export function createScheduler(config, sender, appleScraper, googleScraper) {
|
||||
// One-shot daily check, invoked from the Worker `scheduled` handler. The cron
|
||||
// schedule lives in wrangler.toml ("0 0 * * *" UTC = 7am Asia/Ho_Chi_Minh).
|
||||
export async function runDailyCheck(config, store, sender, appleScraper, googleScraper) {
|
||||
const logger = config.logger;
|
||||
let task;
|
||||
const now = new Date();
|
||||
const dow = weekdayInTz(now, config.timezone);
|
||||
const silent = dow === 0 || dow === 6;
|
||||
logger.info({ silent }, 'Running daily check job');
|
||||
|
||||
function start() {
|
||||
task = cron.schedule(config.scheduleCheckAppTime, runDailyCheck, {
|
||||
scheduled: true,
|
||||
timezone: config.timezone,
|
||||
});
|
||||
logger.info(
|
||||
{ schedule: config.scheduleCheckAppTime, timezone: config.timezone },
|
||||
'Scheduler started',
|
||||
);
|
||||
let groups;
|
||||
try {
|
||||
groups = await store.admin.getAllGroups();
|
||||
} catch (err) {
|
||||
logger.error({ err: err.message }, 'Failed to get groups');
|
||||
return;
|
||||
}
|
||||
|
||||
function stop() {
|
||||
if (task) task.stop();
|
||||
logger.info('Scheduler stopped');
|
||||
}
|
||||
|
||||
async function runDailyCheck() {
|
||||
const now = new Date();
|
||||
const dow = weekdayInTz(now, config.timezone);
|
||||
const silent = dow === 0 || dow === 6;
|
||||
logger.info({ silent }, 'Running daily check job');
|
||||
|
||||
let groups;
|
||||
for (const gid of groups) {
|
||||
try {
|
||||
groups = await adminRepo.getAllGroups();
|
||||
await checkGroup(gid, silent, now, config, store, sender, appleScraper, googleScraper);
|
||||
} catch (err) {
|
||||
logger.error({ err: err.message }, 'Failed to get groups');
|
||||
return;
|
||||
logger.error({ err: err.message, groupId: gid }, 'check group failed');
|
||||
}
|
||||
for (const gid of groups) {
|
||||
try {
|
||||
await checkGroup(gid, silent, now);
|
||||
} catch (err) {
|
||||
logger.error({ err: err.message, groupId: gid }, 'check group failed');
|
||||
}
|
||||
}
|
||||
logger.info({ groupsChecked: groups.length }, 'Daily check job completed');
|
||||
}
|
||||
logger.info({ groupsChecked: groups.length }, 'Daily check job completed');
|
||||
}
|
||||
|
||||
async function checkGroup(groupId, silent, now) {
|
||||
const group = await groupRepo.getGroup(groupId);
|
||||
if (group.appleApps.length === 0 && group.googleApps.length === 0) {
|
||||
logger.info({ groupId }, 'Group has no apps, skipping');
|
||||
return;
|
||||
}
|
||||
const threshold = config.numDaysWarningNotUpdated;
|
||||
const stale = [];
|
||||
async function checkGroup(groupId, silent, now, config, store, sender, appleScraper, googleScraper) {
|
||||
const logger = config.logger;
|
||||
const group = await store.group.getGroup(groupId);
|
||||
if (group.appleApps.length === 0 && group.googleApps.length === 0) {
|
||||
logger.info({ groupId }, 'Group has no apps, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
const threshold = config.numDaysWarningNotUpdated;
|
||||
const stale = [];
|
||||
|
||||
for (const info of group.appleApps) {
|
||||
try {
|
||||
const app = await appleScraper.getApp(info.appId, info.country);
|
||||
if (!app) continue;
|
||||
const updatedMs = Date.parse(app.updated);
|
||||
if (Number.isNaN(updatedMs)) continue;
|
||||
const days = daysBetween(updatedMs, now.getTime());
|
||||
if (days > threshold) {
|
||||
stale.push({
|
||||
appId: info.appId,
|
||||
title: app.title,
|
||||
days,
|
||||
updated: formatDateInTz(new Date(updatedMs), config.timezone),
|
||||
score: app.score,
|
||||
reviews: Number(app.reviews ?? 0),
|
||||
ratings: Number(app.ratings ?? 0),
|
||||
isApple: true,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error({ err: err.message, appId: info.appId }, 'Apple fetch failed');
|
||||
for (const info of group.appleApps) {
|
||||
try {
|
||||
const app = await appleScraper.getApp(info.appId, info.country);
|
||||
if (!app) continue;
|
||||
const updatedMs = Date.parse(app.updated);
|
||||
if (Number.isNaN(updatedMs)) continue;
|
||||
const days = daysBetween(updatedMs, now.getTime());
|
||||
if (days > threshold) {
|
||||
stale.push({
|
||||
appId: info.appId,
|
||||
title: app.title,
|
||||
days,
|
||||
updated: formatDateInTz(new Date(updatedMs), config.timezone),
|
||||
score: app.score,
|
||||
reviews: Number(app.reviews ?? 0),
|
||||
ratings: Number(app.ratings ?? 0),
|
||||
isApple: true,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error({ err: err.message, appId: info.appId }, 'Apple fetch failed');
|
||||
}
|
||||
}
|
||||
|
||||
for (const info of group.googleApps) {
|
||||
try {
|
||||
const app = await googleScraper.getApp(info.appId, info.country);
|
||||
if (!app) continue;
|
||||
const updatedMs = Number(app.updated);
|
||||
const days = daysBetween(updatedMs, now.getTime());
|
||||
if (days > threshold) {
|
||||
stale.push({
|
||||
appId: info.appId,
|
||||
title: app.title,
|
||||
days,
|
||||
updated: formatDateInTz(new Date(updatedMs), config.timezone),
|
||||
score: app.score,
|
||||
reviews: Number(app.reviews ?? 0),
|
||||
ratings: Number(app.ratings ?? 0),
|
||||
isApple: false,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error({ err: err.message, appId: info.appId }, 'Google fetch failed');
|
||||
for (const info of group.googleApps) {
|
||||
try {
|
||||
const app = await googleScraper.getApp(info.appId, info.country);
|
||||
if (!app) continue;
|
||||
const updatedMs = Number(app.updated);
|
||||
const days = daysBetween(updatedMs, now.getTime());
|
||||
if (days > threshold) {
|
||||
stale.push({
|
||||
appId: info.appId,
|
||||
title: app.title,
|
||||
days,
|
||||
updated: formatDateInTz(new Date(updatedMs), config.timezone),
|
||||
score: app.score,
|
||||
reviews: Number(app.reviews ?? 0),
|
||||
ratings: Number(app.ratings ?? 0),
|
||||
isApple: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (stale.length === 0) {
|
||||
logger.info({ groupId }, 'All apps up-to-date');
|
||||
return;
|
||||
} catch (err) {
|
||||
logger.error({ err: err.message, appId: info.appId }, 'Google fetch failed');
|
||||
}
|
||||
const message = buildReport(groupId, stale, now);
|
||||
if (silent) await sender.sendMessageSilent(groupId, message);
|
||||
else await sender.sendMessage(groupId, message);
|
||||
}
|
||||
|
||||
function buildReport(groupId, apps, now) {
|
||||
const headers = ['App', 'Store', 'Days', 'Updated', 'Score', 'Reviews', 'Ratings'];
|
||||
const rows = apps.map((a) => [
|
||||
truncateString(a.title || '', 30),
|
||||
a.isApple ? 'Apple' : 'Google',
|
||||
String(a.days),
|
||||
a.updated,
|
||||
Number(a.score ?? 0).toFixed(1),
|
||||
String(a.reviews),
|
||||
formatNumber(a.ratings),
|
||||
]);
|
||||
return (
|
||||
`<b>Daily App Check Report</b>\n` +
|
||||
`Date: ${formatDateTimeInTz(now, config.timezone)}\n` +
|
||||
`Group: <code>${groupId}</code>\n` +
|
||||
`Apps not updated in >${config.numDaysWarningNotUpdated} days: <b>${apps.length}</b>\n\n` +
|
||||
`<pre>${buildTable(headers, rows)}</pre>`
|
||||
);
|
||||
if (stale.length === 0) {
|
||||
logger.info({ groupId }, 'All apps up-to-date');
|
||||
return;
|
||||
}
|
||||
const message = buildReport(groupId, stale, now, config);
|
||||
if (silent) await sender.sendMessageSilent(groupId, message);
|
||||
else await sender.sendMessage(groupId, message);
|
||||
}
|
||||
|
||||
return { start, stop, runDailyCheck };
|
||||
function buildReport(groupId, apps, now, config) {
|
||||
const headers = ['App', 'Store', 'Days', 'Updated', 'Score', 'Reviews', 'Ratings'];
|
||||
const rows = apps.map((a) => [
|
||||
truncateString(a.title || '', 30),
|
||||
a.isApple ? 'Apple' : 'Google',
|
||||
String(a.days),
|
||||
a.updated,
|
||||
Number(a.score ?? 0).toFixed(1),
|
||||
String(a.reviews),
|
||||
formatNumber(a.ratings),
|
||||
]);
|
||||
return (
|
||||
`<b>Daily App Check Report</b>\n` +
|
||||
`Date: ${formatDateTimeInTz(now, config.timezone)}\n` +
|
||||
`Group: <code>${groupId}</code>\n` +
|
||||
`Apps not updated in >${config.numDaysWarningNotUpdated} days: <b>${apps.length}</b>\n\n` +
|
||||
`<pre>${buildTable(headers, rows)}</pre>`
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user