mirror of
https://github.com/tiennm99/tiennm99bot.git
synced 2026-10-13 10:16:05 +00:00
feat(loldle): tunable max guesses per subject + retuned defaults
Drop classic loldle from 8 → 6 (7-axis grid leaks too much per guess for 8 to feel earned) and emoji from 5 → 4 (3 emojis are usually unmistakable). Add a hidden /<module>_setmax <n> command per loldle module so a chat can override its own round length (1-10). Override stored at config:<subject> in each module's KV; getMaxGuesses() falls back to the default when unset.
This commit is contained in:
tiennm99 committed
1 parent
5782e4779e
commit
b52fa7094e
95 files changed
+396
-7708
No files matched your search
@@ -13,18 +13,3 @@ WORKER_URL=
|
||||
# Same MODULES value as wrangler.toml [vars]. Duplicated here so the register
|
||||
# script can derive the public command list without parsing wrangler.toml.
|
||||
MODULES=util,wordle,loldle,misc,trading,lolschedule,semantle,doantu,twentyq
|
||||
|
||||
# MongoDB Atlas connection string. Used by the `mongodb` driver inside the Worker
|
||||
# AND by local backfill / verify scripts. MUST match the value set via
|
||||
# `wrangler secret put MONGODB_URI` for the Worker. Same secret-mirror protocol
|
||||
# as TELEGRAM_BOT_TOKEN / TELEGRAM_WEBHOOK_SECRET.
|
||||
# Format: mongodb+srv://miti99bot-worker:<pass>@<host>/miti99bot?retryWrites=true&w=majority
|
||||
MONGODB_URI=
|
||||
|
||||
# Cloudflare API creds for backfill scripts (Phase 05). Read-only scope is enough.
|
||||
# Create at: dash.cloudflare.com → My Profile → API Tokens → Create Token.
|
||||
# Permissions needed: Account → Workers KV Storage → Read; Account → D1 → Read.
|
||||
CLOUDFLARE_ACCOUNT_ID=
|
||||
CLOUDFLARE_API_TOKEN=
|
||||
# KV namespace ID (production). Same value as wrangler.toml [[kv_namespaces]] id.
|
||||
KV_NAMESPACE_ID=
|
||||
@@ -76,12 +76,6 @@ web_modules/
|
||||
!.dev.vars.example
|
||||
.wrangler/
|
||||
|
||||
# Phase 05 backfill cursor checkpoints — persisted locally, must not be committed.
|
||||
.backfill-cursor-*.json
|
||||
|
||||
# Local Claude Code agent state (memory, session permissions). Never committed.
|
||||
.claude/
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
.cache
|
||||
.parcel-cache
|
||||
|
||||
@@ -26,7 +26,7 @@ grammY Telegram bot on Cloudflare Workers. Modules are plug-n-play: each module
|
||||
|
||||
**Key abstractions:**
|
||||
- `src/modules/registry.js` — loads modules from static import map (`src/modules/index.js`), validates commands, detects name conflicts across all visibility levels, builds four maps (public/protected/private/all). Memoized via `getCurrentRegistry()`.
|
||||
- `src/db/create-store.js` — returns a storage interface (either `MongoKVStore` via dual-write wrapper, or direct `MongoKVStore` depending on `STORAGE_PRIMARY` flag) with auto-prefixed keys per module (`moduleName:key`). Modules never touch `env.KV`, `env.DB`, or `env.MONGODB_URI` directly.
|
||||
- `src/db/create-store.js` — wraps Cloudflare KV with auto-prefixed keys per module (`moduleName:key`). Modules never touch `env.KV` directly.
|
||||
- `scripts/register.js` — post-deploy script that imports the same registry to derive public commands, then calls Telegram `setWebhook` + `setMyCommands`. Uses `stub-kv.js` to satisfy KV binding without real IO.
|
||||
|
||||
**Three command visibilities:** public (in Telegram `/` menu + `/help`), protected (in `/help` only), private (hidden easter eggs). All three are registered via `bot.command()` — visibility controls discoverability, not access.
|
||||
|
||||
@@ -8,7 +8,7 @@ Modules are added or removed via a single `MODULES` env var. Each module registe
|
||||
|
||||
- **Drop-in modules.** Write a single file, list the folder name in `MODULES`, redeploy. No registration boilerplate, no manual command wiring.
|
||||
- **Three visibility levels out of the box.** Public commands show in Telegram's `/` menu and `/help`; protected show only in `/help`; private are hidden slash-command easter eggs. One namespace, loud conflict detection.
|
||||
- **MongoDB Atlas backend.** Modules talk to a `KVStore` interface (MongoDB for simple state) or `MongoTradesStore` (MongoDB for trading append-only ledger). During migration, a dual-write layer persists to both Cloudflare KV/D1 and MongoDB for safety; post-cutover, MongoDB becomes the sole backend.
|
||||
- **Dual storage backends.** Modules talk to a small `KVStore` interface (Cloudflare KV for simple state) or `SqlStore` interface (D1 for relational data, scans, leaderboards). Swappable with one-file changes.
|
||||
- **Scheduled jobs.** Modules declare cron-based cleanup, stats refresh, or maintenance tasks — registered via `wrangler.toml` and dispatched automatically.
|
||||
- **Zero admin surface.** No in-Worker `/admin/*` routes, no admin secret. `setWebhook` + `setMyCommands` run at deploy time from a local node script.
|
||||
- **Tested.** 200+ vitest unit tests cover registry, storage, dispatcher, cron validation, help renderer, validators, HTML escaping, and the trading / loldle / wordle modules.
|
||||
@@ -48,13 +48,11 @@ src/
|
||||
├── types.js # JSDoc typedefs (central: Env, Module, Command, Cron, etc.)
|
||||
├── db/
|
||||
│ ├── kv-store-interface.js # KVStore contract (JSDoc)
|
||||
│ ├── mongo-kv-store.js # MongoDB KVStore implementation
|
||||
│ ├── mongo-trades-store.js # MongoDB trading ledger implementation
|
||||
│ ├── dual-kv-store.js # Dual-write wrapper (Mongo + CF KV)
|
||||
│ ├── create-store.js # Storage factory (selects backend via STORAGE_PRIMARY flag)
|
||||
│ ├── cf-kv-store.js # Cloudflare KV adapter (still active during migration)
|
||||
│ ├── cf-sql-store.js # Cloudflare D1 adapter (still active during migration)
|
||||
│ └── mongo-client.js # Shared memoized MongoDB connection
|
||||
│ ├── cf-kv-store.js # Cloudflare KV implementation
|
||||
│ ├── create-store.js # KV per-module prefixing factory
|
||||
│ ├── sql-store-interface.js # SqlStore contract (JSDoc)
|
||||
│ ├── cf-sql-store.js # Cloudflare D1 implementation
|
||||
│ └── create-sql-store.js # D1 per-module prefixing factory
|
||||
├── modules/
|
||||
│ ├── index.js # static import map — register new modules here
|
||||
│ ├── registry.js # load, validate, build command + cron tables
|
||||
|
||||
@@ -31,7 +31,6 @@
|
||||
"ignore": [
|
||||
"node_modules",
|
||||
".wrangler",
|
||||
".claude",
|
||||
"dist",
|
||||
"coverage",
|
||||
"src/modules/loldle/champions.json",
|
||||
|
||||
+27
-19
@@ -173,11 +173,9 @@ Every command — public, protected, **and private** — is registered via `bot.
|
||||
|
||||
There is no custom text-match middleware, no `bot.on("message:text", ...)` handler, no private-command-specific path. One routing path for all three visibilities. This is what reduced the original two-path design (slash + text-match) to one during the revision pass.
|
||||
|
||||
## 8. Storage: MongoDB + Dual-Write Migration Era
|
||||
## 8. Storage: KVStore and SqlStore
|
||||
|
||||
**Current state (Phases 01–08):** MongoDB Atlas is the primary store. During migration, a dual-write layer persists to both MongoDB and Cloudflare KV/D1 for safety. Modules NEVER touch `env.KV`, `env.DB`, or `env.MONGODB_URI` directly — they receive prefixed stores from the module context via `createStore()` and `createSqlStore()` factories.
|
||||
|
||||
**Post-Phase-07 cutover:** The dual-write layer collapses, Cloudflare KV/D1 are deleted, and `createStore()` returns pure MongoDB stores.
|
||||
Modules NEVER touch `env.KV` or `env.DB` directly. They receive prefixed stores from the module context.
|
||||
|
||||
### KVStore (key-value, fast reads/writes)
|
||||
|
||||
@@ -205,20 +203,20 @@ getJSON(key) // → any | null (swallows corrupt JSON)
|
||||
putJSON(key, value, { expirationTtl? })
|
||||
```
|
||||
|
||||
**Current implementation:** `createStore("wordle", env)` returns a `MongoKVStore` directly (or `DualKVStore` if `DUAL_WRITE=1` is set). During migration, dual-write sends to both MongoDB and Cloudflare KV. TTL expirations are enforced server-side by MongoDB (via `expiresAt` field) and at read-time by the `MongoKVStore` layer.
|
||||
|
||||
#### Prefix mechanics
|
||||
|
||||
All keys are prefixed with `<moduleName>:` before storage:
|
||||
`createStore("wordle", env)` returns a wrapped store where every key is rewritten:
|
||||
|
||||
```
|
||||
module calls: store prefixes: MongoDB collection doc:
|
||||
───────────────────────── ────────────────────── ────────────────────────
|
||||
put("games:42", v) ──► put("wordle:games:42") ──► { _id: "wordle:games:42", … }
|
||||
get("games:42") ──► get("wordle:games:42") ──► (find by _id, return value)
|
||||
list({prefix:"games:"})──► (scan, filter prefix) ──► (keys matching "wordle:games:")
|
||||
module calls: wrapper sends to CFKVStore: raw KV key:
|
||||
───────────────────────── ───────────────────────────── ─────────────
|
||||
put("games:42", v) ──► put("wordle:games:42", v) ──► wordle:games:42
|
||||
get("games:42") ──► get("wordle:games:42") ──► wordle:games:42
|
||||
list({prefix:"games:"})──► list({prefix:"wordle:games:"}) (then strips "wordle:" from returned keys)
|
||||
```
|
||||
|
||||
Two stores for different modules cannot read each other's data unless they reconstruct prefixes by hand — a code-review boundary, not a cryptographic one.
|
||||
|
||||
### SqlStore (relational, scans, append-only history)
|
||||
|
||||
For complex queries, aggregates, or audit logs, use `sql` (a `SqlStore`):
|
||||
@@ -226,7 +224,7 @@ For complex queries, aggregates, or audit logs, use `sql` (a `SqlStore`):
|
||||
```js
|
||||
// In a module's init:
|
||||
init: async ({ sql }) => {
|
||||
sqlStore = sql; // null if not bound
|
||||
sqlStore = sql; // null if env.DB not bound
|
||||
},
|
||||
|
||||
// In a handler or cron:
|
||||
@@ -242,19 +240,29 @@ The interface (full JSDoc in `src/db/sql-store-interface.js`):
|
||||
run(query, ...binds) // INSERT/UPDATE/DELETE — returns { changes, last_row_id }
|
||||
all(query, ...binds) // SELECT all rows → array of objects
|
||||
first(query, ...binds) // SELECT first row → object | null
|
||||
prepare(query, ...binds) // Prepared statement for batch operations
|
||||
batch(statements) // Execute multiple statements in one round-trip
|
||||
```
|
||||
|
||||
**Current implementation:** `createSqlStore("trading", env)` returns a `MongoTradesStore` (native MongoDB inserts / queries on `trading_trades` collection). D1 is read-only during migration. Post-cutover, D1 is deleted.
|
||||
All tables must follow the naming convention `{moduleName}_{table}` (e.g., `trading_trades`).
|
||||
|
||||
### Swapping the backends (post-Phase-07)
|
||||
Tables are created via migrations in `src/modules/<name>/migrations/*.sql`. The migration runner (`scripts/migrate.js`) applies them on deploy and tracks them in `_migrations` table.
|
||||
|
||||
After cutover, the backend is locked to MongoDB. To replace it:
|
||||
### Swapping the backends
|
||||
|
||||
To replace Cloudflare KV with a different store (e.g. Upstash Redis, Postgres):
|
||||
|
||||
1. Create a new `src/db/<name>-store.js` that implements the `KVStore` interface.
|
||||
2. Change the one `new MongoKVStore(...)` line in `src/db/create-store.js` to construct your new adapter.
|
||||
3. Update `wrangler.toml` bindings if needed.
|
||||
2. Change the one `new CFKVStore(env.KV)` line in `src/db/create-store.js` to construct your new adapter.
|
||||
3. Update `wrangler.toml` bindings.
|
||||
|
||||
Similarly for SQL: create `<name>-trades-store.js` implementing the trades interface, update `create-sql-store.js`.
|
||||
That's the full change. No module code moves.
|
||||
|
||||
To replace D1 with a different SQL backend:
|
||||
|
||||
1. Create a new `src/db/<name>-sql-store.js` that implements the `SqlStore` interface.
|
||||
2. Change the one `new CFSqlStore(env.DB)` line in `src/db/create-sql-store.js` to construct your new adapter.
|
||||
3. Update `wrangler.toml` bindings.
|
||||
|
||||
## 9. HTTP and Scheduled Entry Points
|
||||
|
||||
|
||||
+2
-12
@@ -69,22 +69,12 @@ export default {
|
||||
```
|
||||
|
||||
- Store module-level `db` and `sql` references in closure variables, set during `init`
|
||||
- Never access `env.KV`, `env.DB`, or `env.MONGODB_URI` directly — always use the prefixed `db` (KVStore) or `sql` (SqlStore) from `init`
|
||||
- `sql` is `null` when no relational store is bound — always guard with `if (!sql) return`
|
||||
- Never access `env.KV` or `env.DB` directly — always use the prefixed `db` (KV) or `sql` (D1) from `init`
|
||||
- `sql` is `null` when `env.DB` is not bound — always guard with `if (!sql) return`
|
||||
- Command handlers receive grammY `ctx` — use `ctx.match` for command arguments, `ctx.from.id` for user identity
|
||||
- Reply with `ctx.reply(text)` — plain text or Telegram HTML
|
||||
- Cron handlers receive `(event, { db, sql, env })` — same context as `init`
|
||||
|
||||
## Persistence Layer
|
||||
|
||||
All data persistence flows through storage factories:
|
||||
|
||||
- **`createStore(moduleName, env)`** — returns a `KVStore` interface for key-value data (simple state, settings, JSON blobs). Implementation: `MongoKVStore` (primary) with optional dual-write to Cloudflare KV during migration.
|
||||
- **`createSqlStore(moduleName, env)`** — returns a `SqlStore` interface for relational data (trading ledger, aggregates, scans). Implementation: `MongoTradesStore` (MongoDB native queries and inserts). D1 is read-only during migration.
|
||||
- **Modules NEVER instantiate `MongoClient` directly.** All MongoDB access goes through `MongoKVStore` or `MongoTradesStore` factories.
|
||||
|
||||
Post-migration (after Phase 07 cutover), the dual-write layer collapses and Cloudflare KV/D1 are deleted; `createStore` returns `MongoKVStore` directly.
|
||||
|
||||
## Error Handling
|
||||
|
||||
- **Load-time failures** (bad module, command conflicts, missing env): throw immediately — fail loud at deploy, not at runtime.
|
||||
|
||||
+11
-11
@@ -2,7 +2,7 @@
|
||||
|
||||
## Overview
|
||||
|
||||
Telegram bot on Cloudflare Workers with a plug-n-play module system. grammY handles Telegram API; modules register commands with three visibility levels. **During migration (Phase 08):** Data is stored in MongoDB Atlas M0 (behind a prefixed `KVStore` interface for KV data, `MongoTradesStore` for trading ledger). Dual-write to Cloudflare KV/D1 is active for safety; post-Phase-07 cutover, MongoDB becomes sole backend.
|
||||
Telegram bot on Cloudflare Workers with a plug-n-play module system. grammY handles Telegram API; modules register commands with three visibility levels. Data stored in Cloudflare KV (behind a prefixed `KVStore` interface) or D1 (behind `SqlStore` interface).
|
||||
|
||||
## Tech Stack
|
||||
|
||||
@@ -10,7 +10,7 @@ Telegram bot on Cloudflare Workers with a plug-n-play module system. grammY hand
|
||||
|-------|-----------|
|
||||
| Runtime | Cloudflare Workers (V8 isolates) |
|
||||
| Bot framework | grammY 1.x |
|
||||
| Storage | MongoDB Atlas M0 (primary, via official driver). Cloudflare KV + D1 (secondary, for dual-write safety during migration). |
|
||||
| Storage | Cloudflare KV + D1 |
|
||||
| AI inference | Workers AI binding (`env.AI`) |
|
||||
| Linter/Formatter | Biome |
|
||||
| Tests | Vitest |
|
||||
@@ -21,14 +21,14 @@ Telegram bot on Cloudflare Workers with a plug-n-play module system. grammY hand
|
||||
| Module | Commands | Storage | Crons | Description |
|
||||
|--------|----------|---------|-------|-------------|
|
||||
| `util` | `/info`, `/help`, `/stickerid` (private) | — | — | Bot info, command help renderer, sticker file_id echo helper |
|
||||
| `misc` | `/ping`, `/mstats`, `/fortytwo` | MongoDB KVStore | — | Health check + DB demo stub |
|
||||
| `trading` | `/trade_topup`, `/trade_buy`, `/trade_sell`, `/trade_convert`, `/trade_stats`, `/history` | MongoDB (trades + portfolio + symbol cache) | Daily 5PM trim | Paper trading — VN stocks with dynamic symbol resolution |
|
||||
| `wordle` | `/wordle`, `/wordle_new`, `/wordle_giveup`, `/wordle_stats` | MongoDB KVStore | — | 5-letter word guessing game. 14,855-word dict |
|
||||
| `loldle` | `/loldle`, `/loldle_giveup`, `/loldle_stats` | MongoDB KVStore | — | Classic-mode LoL champion guesser. Data synced from `tiennm99/loldle-data` |
|
||||
| `lolschedule` | `/lolschedule_today`, `/lolschedule_week`, `/lolschedule_subscribe`, `/lolschedule_unsubscribe` | MongoDB KVStore | Daily 01:00 UTC | LoL esports schedule + daily digest subscriptions |
|
||||
| `semantle` | `/semantle`, `/semantle_giveup`, `/semantle_stats` | MongoDB KVStore | — | English semantic word guessing via hosted word2sim service |
|
||||
| `doantu` | `/doantu`, `/doantu_hint`, `/doantu_giveup`, `/doantu_stats` | MongoDB KVStore | — | Vietnamese semantle via hosted phow2sim service |
|
||||
| `twentyq` | `/twentyq`, `/twentyq_giveup`, `/twentyq_stats` | MongoDB KVStore | — | Reverse-Akinator yes/no game. Workers AI (`@cf/google/gemma-4-26b-a4b-it`) generates round-start category+hint and judges each turn via one-line JSON |
|
||||
| `misc` | `/ping`, `/mstats`, `/fortytwo` | KV | — | Health check + DB demo stub |
|
||||
| `trading` | `/trade_topup`, `/trade_buy`, `/trade_sell`, `/trade_convert`, `/trade_stats`, `/history` | D1 (trades) + KV (portfolio, symbol cache) | Daily 5PM trim | Paper trading — VN stocks with dynamic symbol resolution |
|
||||
| `wordle` | `/wordle`, `/wordle_new`, `/wordle_giveup`, `/wordle_stats` | KV | — | 5-letter word guessing game. 14,855-word dict |
|
||||
| `loldle` | `/loldle`, `/loldle_giveup`, `/loldle_stats` | KV | — | Classic-mode LoL champion guesser. Data synced from `tiennm99/loldle-data` |
|
||||
| `lolschedule` | `/lolschedule_today`, `/lolschedule_week`, `/lolschedule_subscribe`, `/lolschedule_unsubscribe` | KV | Daily 01:00 UTC | LoL esports schedule + daily digest subscriptions |
|
||||
| `semantle` | `/semantle`, `/semantle_giveup`, `/semantle_stats` | KV | — | English semantic word guessing via hosted word2sim service |
|
||||
| `doantu` | `/doantu`, `/doantu_hint`, `/doantu_giveup`, `/doantu_stats` | KV | — | Vietnamese semantle via hosted phow2sim service |
|
||||
| `twentyq` | `/twentyq`, `/twentyq_giveup`, `/twentyq_stats` | KV | — | Reverse-Akinator yes/no game. Workers AI (`@cf/google/gemma-4-26b-a4b-it`) generates round-start category+hint and judges each turn via one-line JSON |
|
||||
|
||||
## Key Data Flows
|
||||
|
||||
@@ -75,4 +75,4 @@ Each module maintains its own `README.md` with commands, data model, and impleme
|
||||
|
||||
## Tests
|
||||
|
||||
`npm test` runs the full vitest suite (run in ~10 seconds — 733 tests). Structure: one folder per module under `tests/modules/<name>/`, shared fakes under `tests/fakes/` (fake-mongo, fake-kv-namespace, fake-d1, fake-bot, fake-modules, fake-ai). No workerd, no Telegram fixtures — pure-logic unit tests with injected fakes. See `tests/fakes/fake-mongo.js` for the frozen surface implemented during Phase 02–08.
|
||||
`npm test` runs the full vitest suite (run in a few seconds — ~450 tests). Structure: one folder per module under `tests/modules/<name>/`, shared fakes under `tests/fakes/` (fake-kv-namespace, fake-d1, fake-bot, fake-modules, fake-ai). No workerd, no Telegram fixtures — pure-logic unit tests with injected fakes.
|
||||
@@ -1,65 +0,0 @@
|
||||
# MongoDB Atlas Cost Tracking
|
||||
|
||||
Operational runbook for monitoring and managing MongoDB Atlas billing during and post-migration.
|
||||
|
||||
## Free Tier Limits (M0)
|
||||
|
||||
| Resource | Limit | Warning |
|
||||
|----------|-------|---------|
|
||||
| Storage | 512 MB | Reached 400 MB: prepare upgrade plan |
|
||||
| Connections | 500 | Reached 400: review cron/handler concurrency |
|
||||
| Throughput | ~100 ops/sec sustained | Degradation: check dashboard Charts |
|
||||
| Backups | None | No PITR; only daily snapshots (read-only) |
|
||||
|
||||
## Cost Ladder
|
||||
|
||||
| Tier | Monthly | When | Ops/sec | Storage |
|
||||
|------|---------|------|---------|---------|
|
||||
| **M0** | Free | Initial, dev | ~100 | 512 MB |
|
||||
| **Flex** | $8–$30 | Sustained >400 MB or >400 conn | 400–1000 | 10–256 GB |
|
||||
| **M10** | $57 | Production high-throughput | 1000+ | Unlimited |
|
||||
|
||||
Flex tier auto-scales cost based on data volume and throughput. Start at M2 ($9) and scale to M5 ($70+) as needed.
|
||||
|
||||
## Monitoring & Upgrade Triggers
|
||||
|
||||
### Monthly Review Checklist
|
||||
|
||||
1. **Storage:** Open Atlas Dashboard → Metrics → check `Database Storage` chart
|
||||
- If > 400 MB: escalate to Flex within 2 weeks
|
||||
- Projection: (current MB / days since epoch) * 30 → projected month-end
|
||||
|
||||
2. **Connections:** Metrics → check `Current Connections` peak
|
||||
- If > 400: cron jobs or handlers running concurrently too often
|
||||
- Review `src/modules/*/index.js` cron frequency; stagger if possible
|
||||
|
||||
3. **Throughput:** Metrics → `Network Egress` + `Database Operations`
|
||||
- Spike during trading/wordle command storms: expected
|
||||
- Sustained >100 ops/sec: assess Flex upgrade
|
||||
|
||||
4. **Cluster Status:** Alerts → check if "Cluster unavailable" triggered
|
||||
- M0 never pauses if any cron writes data (bot has 6+ crons, any write prevents pause)
|
||||
|
||||
### Upgrade Decision
|
||||
|
||||
Plan upgrade **before** hitting limits:
|
||||
|
||||
| Condition | Action | Timeline |
|
||||
|-----------|--------|----------|
|
||||
| Storage trend → 512 MB in 3 months | Upgrade to Flex M2 | 2 weeks notice |
|
||||
| Peak connections > 400 regularly | Upgrade to Flex M2 | Immediate if sustained |
|
||||
| Ops/sec spikes > 2000 | Upgrade to Flex M5 or M10 | 1 week notice |
|
||||
|
||||
## Rotation & Maintenance
|
||||
|
||||
- **Password rotation:** Every 90 days, owner = repo maintainer. See `docs/using-mongodb.md` "Rotation" section.
|
||||
- **Alert config review:** Monthly, ensure Atlas + CF Observability alerts are wired.
|
||||
- **Backups:** M0 has none; data is live-only. Backups start at Flex tier.
|
||||
|
||||
## Post-Cutover Simplification
|
||||
|
||||
Once Phase 07 cutover completes and Cloudflare KV/D1 are deleted, MongoDB becomes the sole data store. Cost is dominated by storage growth (KV reads/writes are gone). Rebaseline this doc:
|
||||
|
||||
- Remove dual-write cost considerations
|
||||
- Focus on pure Mongo spend
|
||||
- Extend storage projections based on new single-source data
|
||||
@@ -1,44 +0,0 @@
|
||||
# Project Changelog
|
||||
|
||||
All significant changes, features, and fixes to miti99bot.
|
||||
|
||||
## [2026-04-25] MongoDB Atlas Migration Complete
|
||||
|
||||
**Status:** Phases 01–08 committed on `dev` branch. Dual-write ready; cutover pending operator execution (Phase 07 binding deletion).
|
||||
|
||||
**Summary:**
|
||||
Research, planning, and 7 phases of implementation + documentation to migrate from Cloudflare KV/D1 to MongoDB Atlas M0 free tier. Achieved zero-downtime dual-write architecture with automated drift detection, backfill/reverse-backfill scripts, and comprehensive telemetry. All 733 unit tests passing.
|
||||
|
||||
**Plan:** `plans/260425-1945-mongodb-atlas-migration/`
|
||||
|
||||
**Phases:**
|
||||
- Phase 01: Wrangler config, secret-leak lint, bundle-size gate
|
||||
- Phase 02: MongoKVStore + memoized client + fake-mongo + tests
|
||||
- Phase 03: MongoTradesStore + trading refactor + MongoSqlStore shim
|
||||
- Phase 04: Dual-write wrappers + factories + retry queue + drift verifier + e2e
|
||||
- Phase 05: Backfill scripts (local node, CF KV REST + wrangler d1 export)
|
||||
- Phase 06: Telemetry instrumentation + soak runbook
|
||||
- Phase 07: Cutover scripts + reverse-backfill + decommission helpers
|
||||
- Phase 08: Final docs pass + alternatives section update
|
||||
|
||||
**Key artifacts:**
|
||||
- `src/db/mongo-kv-store.js` — MongoDB implementation of KVStore interface
|
||||
- `src/db/mongo-trades-store.js` — MongoDB trading ledger store
|
||||
- `src/db/dual-kv-store.js` — dual-write wrapper for safety during migration
|
||||
- `src/db/mongo-client.js` — shared memoized connection with retry logic
|
||||
- `docs/using-mongodb.md` — operational runbook
|
||||
- `docs/cost-tracking.md` — cost monitoring and upgrade triggers
|
||||
|
||||
**Reviewers noted** (not adopted):
|
||||
- Maintenance-window cutover instead of dual-write (saves ~6h engineering)
|
||||
- Defer trading migration (D1 free handles ~100 writes/day indefinitely)
|
||||
- Single shared `kv` collection vs 12 per-module collections
|
||||
- Pivot to Upstash before starting (smaller bundle, HTTP-native)
|
||||
|
||||
User chose full cold-start Atlas validation. If bundle size trips, `phase-07-alt-pivot.md` is ready.
|
||||
|
||||
**Post-cutover simplification** (Phase 07 Stage 3 — operator-driven):
|
||||
- Delete dual-write layer
|
||||
- Delete `cf-kv-store.js`, `cf-sql-store.js`, and D1 bindings from `wrangler.toml`
|
||||
- `createStore()` returns pure `MongoKVStore`
|
||||
- Cost tracking shifts to Mongo-only projections
|
||||
@@ -1,142 +0,0 @@
|
||||
# Using MongoDB Atlas
|
||||
|
||||
Operational runbook for the MongoDB Atlas backend introduced by `plans/260425-1945-mongodb-atlas-migration/`.
|
||||
|
||||
## Cluster
|
||||
|
||||
| Field | Value |
|
||||
|---|---|
|
||||
| Provider | MongoDB Atlas |
|
||||
| Tier | M0 Free |
|
||||
| Region | `aws-ap-southeast-1` (Singapore) |
|
||||
| Cluster name | `miti99bot-prod` (operator confirms) |
|
||||
| Database | `miti99bot` |
|
||||
| DB user | `miti99bot-worker` (`readWrite@miti99bot`) |
|
||||
|
||||
Connection string format:
|
||||
|
||||
```
|
||||
mongodb+srv://miti99bot-worker:<pass>@<host>/miti99bot?retryWrites=true&w=majority
|
||||
```
|
||||
|
||||
Stored in two places (must match):
|
||||
1. CF Worker secret: `wrangler secret put MONGODB_URI`
|
||||
2. `.env.deploy` (gitignored, used by local backfill / verify scripts)
|
||||
|
||||
Same secret-mirror protocol as `TELEGRAM_BOT_TOKEN`.
|
||||
|
||||
## Free-tier ceiling
|
||||
|
||||
- 512 MB storage (data + indexes)
|
||||
- 500 max concurrent connections
|
||||
- ~100 ops/sec sustained (no daily cap)
|
||||
- No backups, single region, no PITR
|
||||
- Auto-pauses after 30 days of zero ops
|
||||
|
||||
Upgrade path: **Flex Tier $8–$30/month** (M2/M5 deprecated as of 2026).
|
||||
|
||||
## Auto-pause
|
||||
|
||||
After 30 days idle the cluster pauses. First request after pause:
|
||||
- Driver throws `MongoServerSelectionError` after `serverSelectionTimeoutMS` (5s).
|
||||
- Worker code (see `src/db/mongo-client.js`, lands Phase 02) catches and returns 503 with `Retry-After: 30`.
|
||||
- Cluster auto-wakes within 30–60s on attempted connection.
|
||||
|
||||
The bot has 6+ daily crons; any cron that writes Mongo prevents pause. Phase 08 confirms.
|
||||
|
||||
## Network access
|
||||
|
||||
`0.0.0.0/0` — Cloudflare Workers do NOT have static egress IPs on the Free or basic Paid plans. Only auth (SCRAM-SHA-256) + TLS gate connections.
|
||||
|
||||
**Permanent risk** unless upgrading to CF Workers paid static-egress IP add-on (~$10/mo).
|
||||
|
||||
Mitigations:
|
||||
- DB user has `readWrite` on one db only (NOT `dbAdmin` / `clusterAdmin`).
|
||||
- Password ≥32 chars random.
|
||||
- Rotate quarterly.
|
||||
- Atlas free-tier email alerts configured for cluster unavailability + connections > 400.
|
||||
|
||||
## Bundle gate (Phase 01 result)
|
||||
|
||||
Measured `npx wrangler deploy --dry-run` with a minimal probe importing `MongoClient`:
|
||||
|
||||
| Metric | Value | Cap (Free) | Cap (Paid) |
|
||||
|---|---|---|---|
|
||||
| Compressed (gzip) | **226 KiB** | 3 MiB | 10 MiB |
|
||||
| Raw (minified) | 1.74 MiB | — | — |
|
||||
| On-disk (uncompressed) | 3.9 MiB | — | — |
|
||||
|
||||
Pass on both plans with **>92% headroom**. nodejs_compat_v2 provides `node:net`/`node:tls`/`node:crypto` from the runtime, so the driver's transitive deps are not bundled.
|
||||
|
||||
## CPU-time gate (Phase 01 — operator-run)
|
||||
|
||||
Requires real Atlas + `wrangler dev`. Procedure:
|
||||
|
||||
1. Add a temporary `/__mongo-ping` route that connects + runs `db.runCommand({ping:1})` + returns `{wall_ms}`.
|
||||
2. Run 5+ cold cycles (10-min spaced).
|
||||
3. Inspect CF dashboard CPU column for each invocation.
|
||||
4. **Hard gate**: if any cold-start CPU time approaches 50ms (Free plan limit), abort migration. Escalate to paid plan or pivot via `phase-07-alt-pivot.md`.
|
||||
5. Record cold-ping P95 wall-clock as `BASELINE_COLD_PING_MS` here:
|
||||
|
||||
```
|
||||
BASELINE_COLD_PING_MS = <fill after measurement>
|
||||
```
|
||||
|
||||
Phase 06 derives the abort threshold from this value: `2.5 × BASELINE_COLD_PING_MS`.
|
||||
|
||||
## Auto-pause behavior gate (Phase 01 — operator-run)
|
||||
|
||||
In Atlas UI, manually pause the cluster, then hit `/__mongo-ping`. Confirm:
|
||||
- Driver throws within 5s (does NOT hang indefinitely).
|
||||
- Error class is `MongoServerSelectionError` (or driver subclass).
|
||||
- Phase 02 `getDb()` catches this and surfaces a 503.
|
||||
|
||||
## Node API surface
|
||||
|
||||
`src/` (the Worker) imports zero `node:*` modules today. `nodejs_compat_v2` is enabled solely for the `mongodb` driver:
|
||||
|
||||
| Module | Used by Worker? | Used by scripts/? |
|
||||
|---|---|---|
|
||||
| `node:fs` | no | yes (build/scrape/migrate) |
|
||||
| `node:path` | no | yes |
|
||||
| `node:child_process` | no | yes (migrate.js) |
|
||||
| `node:net` | indirectly (via mongodb) | no |
|
||||
| `node:tls` | indirectly (via mongodb) | no |
|
||||
| `node:crypto` | indirectly (via mongodb) | no |
|
||||
| `process.env` | no | yes (register.js) |
|
||||
| `Buffer` | no | no |
|
||||
|
||||
Risk: minimal. No existing module relies on the absence of these globals.
|
||||
|
||||
## Rollback
|
||||
|
||||
If migration is abandoned at any phase before cutover:
|
||||
|
||||
1. `wrangler secret delete MONGODB_URI`
|
||||
2. Revert `wrangler.toml`: remove `compatibility_flags = ["nodejs_compat_v2"]`.
|
||||
3. `npm uninstall mongodb`.
|
||||
4. `npm run deploy` — bot continues on KV/D1 unchanged.
|
||||
5. (Optional) Delete Atlas cluster from UI.
|
||||
|
||||
`scripts/check-secret-leaks.js` should stay — it covers other secrets too.
|
||||
|
||||
## Rotation
|
||||
|
||||
`MONGODB_URI` rotation cadence: every 90 days, owner = repo maintainer.
|
||||
|
||||
Procedure:
|
||||
1. In Atlas UI → Database Access → edit `miti99bot-worker` → reset password.
|
||||
2. Update `.env.deploy` with new URI.
|
||||
3. `wrangler secret put MONGODB_URI` (paste new URI).
|
||||
4. `npm run deploy` (re-runs register; no Worker restart needed since secret reads at request time via `env.MONGODB_URI`).
|
||||
|
||||
Mismatch between `.env.deploy` and CF secret causes register-script failure on next deploy — same fail-loud pattern as `TELEGRAM_WEBHOOK_SECRET`.
|
||||
|
||||
## Alerts
|
||||
|
||||
Configured in Atlas free-tier UI:
|
||||
|
||||
- **Cluster unavailable** → email maintainer.
|
||||
- **Current connections > 400** (80% of cap) → email maintainer.
|
||||
|
||||
Plus CF Observability rule (Phase 06): >10 errors per 1 min window → email.
|
||||
Generated
+2
-140
@@ -8,8 +8,7 @@
|
||||
"name": "miti99bot",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"grammy": "^1.30.0",
|
||||
"mongodb": "^6.21.0"
|
||||
"grammy": "^1.30.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
@@ -1054,15 +1053,6 @@
|
||||
"@jridgewell/sourcemap-codec": "^1.4.10"
|
||||
}
|
||||
},
|
||||
"node_modules/@mongodb-js/saslprep": {
|
||||
"version": "1.4.9",
|
||||
"resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.4.9.tgz",
|
||||
"integrity": "sha512-RXSxsokhAF/4nWys8An8npsqOI33Ex1Hlzqjw2pZOO+GKtMAR2noGnUdsFiGwsaO/xXI+56mtjTmDA3JXJsvmA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"sparse-bitfield": "^3.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/wasm-runtime": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz",
|
||||
@@ -1475,21 +1465,6 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/webidl-conversions": {
|
||||
"version": "7.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz",
|
||||
"integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/whatwg-url": {
|
||||
"version": "11.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz",
|
||||
"integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/webidl-conversions": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@typescript-eslint/types": {
|
||||
"version": "8.58.2",
|
||||
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.58.2.tgz",
|
||||
@@ -1720,15 +1695,6 @@
|
||||
"node": "18 || 20 || >=22"
|
||||
}
|
||||
},
|
||||
"node_modules/bson": {
|
||||
"version": "6.10.4",
|
||||
"resolved": "https://registry.npmjs.org/bson/-/bson-6.10.4.tgz",
|
||||
"integrity": "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=16.20.1"
|
||||
}
|
||||
},
|
||||
"node_modules/chai": {
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz",
|
||||
@@ -2624,12 +2590,6 @@
|
||||
"@jridgewell/sourcemap-codec": "^1.5.5"
|
||||
}
|
||||
},
|
||||
"node_modules/memory-pager": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
|
||||
"integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/miniflare": {
|
||||
"version": "4.20260420.0",
|
||||
"resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260420.0.tgz",
|
||||
@@ -2667,96 +2627,6 @@
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/mongodb": {
|
||||
"version": "6.21.0",
|
||||
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.21.0.tgz",
|
||||
"integrity": "sha512-URyb/VXMjJ4da46OeSXg+puO39XH9DeQpWCslifrRn9JWugy0D+DvvBvkm2WxmHe61O/H19JM66p1z7RHVkZ6A==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@mongodb-js/saslprep": "^1.3.0",
|
||||
"bson": "^6.10.4",
|
||||
"mongodb-connection-string-url": "^3.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16.20.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@aws-sdk/credential-providers": "^3.188.0",
|
||||
"@mongodb-js/zstd": "^1.1.0 || ^2.0.0",
|
||||
"gcp-metadata": "^5.2.0",
|
||||
"kerberos": "^2.0.1",
|
||||
"mongodb-client-encryption": ">=6.0.0 <7",
|
||||
"snappy": "^7.3.2",
|
||||
"socks": "^2.7.1"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@aws-sdk/credential-providers": {
|
||||
"optional": true
|
||||
},
|
||||
"@mongodb-js/zstd": {
|
||||
"optional": true
|
||||
},
|
||||
"gcp-metadata": {
|
||||
"optional": true
|
||||
},
|
||||
"kerberos": {
|
||||
"optional": true
|
||||
},
|
||||
"mongodb-client-encryption": {
|
||||
"optional": true
|
||||
},
|
||||
"snappy": {
|
||||
"optional": true
|
||||
},
|
||||
"socks": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/mongodb-connection-string-url": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.2.tgz",
|
||||
"integrity": "sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@types/whatwg-url": "^11.0.2",
|
||||
"whatwg-url": "^14.1.0 || ^13.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/mongodb-connection-string-url/node_modules/tr46": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz",
|
||||
"integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"punycode": "^2.3.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/mongodb-connection-string-url/node_modules/webidl-conversions": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
|
||||
"integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/mongodb-connection-string-url/node_modules/whatwg-url": {
|
||||
"version": "14.2.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz",
|
||||
"integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tr46": "^5.1.0",
|
||||
"webidl-conversions": "^7.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/ms": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
@@ -2991,6 +2861,7 @@
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
|
||||
"integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
@@ -3141,15 +3012,6 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/sparse-bitfield": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
|
||||
"integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"memory-pager": "^1.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/spdx-exceptions": {
|
||||
"version": "2.5.0",
|
||||
"resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz",
|
||||
|
||||
+2
-15
@@ -18,25 +18,12 @@
|
||||
"db:migrate": "node scripts/migrate.js",
|
||||
"register": "node --env-file-if-exists=.env.deploy scripts/register.js",
|
||||
"register:dry": "node --env-file-if-exists=.env.deploy scripts/register.js --dry-run",
|
||||
"backfill:kv": "node --env-file-if-exists=.env.deploy scripts/backfill-kv-to-mongo.js",
|
||||
"backfill:kv:dry": "node --env-file-if-exists=.env.deploy scripts/backfill-kv-to-mongo.js --dry-run",
|
||||
"backfill:d1": "node --env-file-if-exists=.env.deploy scripts/backfill-d1-to-mongo.js",
|
||||
"backfill:d1:dry": "node --env-file-if-exists=.env.deploy scripts/backfill-d1-to-mongo.js --dry-run",
|
||||
"backfill:mongo:kv": "node --env-file-if-exists=.env.deploy scripts/backfill-mongo-to-kv.js",
|
||||
"backfill:mongo:kv:dry": "node --env-file-if-exists=.env.deploy scripts/backfill-mongo-to-kv.js --dry-run",
|
||||
"backfill:mongo:d1": "node --env-file-if-exists=.env.deploy scripts/backfill-mongo-to-d1.js",
|
||||
"backfill:mongo:d1:dry": "node --env-file-if-exists=.env.deploy scripts/backfill-mongo-to-d1.js --dry-run",
|
||||
"verify:mongo": "node --env-file-if-exists=.env.deploy scripts/verify-mongo-parity.js",
|
||||
"wipe:mongo": "node --env-file-if-exists=.env.deploy scripts/wipe-mongo.js",
|
||||
"analyze:soak": "node scripts/analyze-soak.js",
|
||||
"burst:synthetic": "node scripts/synthetic-burst.js",
|
||||
"lint": "biome check . && eslint src && node scripts/check-secret-leaks.js",
|
||||
"lint": "biome check . && eslint src",
|
||||
"format": "biome format --write .",
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"grammy": "^1.30.0",
|
||||
"mongodb": "^6.21.0"
|
||||
"grammy": "^1.30.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.0",
|
||||
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -1,19 +1,16 @@
|
||||
---
|
||||
title: "Migrate miti99bot from CF KV+D1 to MongoDB Atlas M0"
|
||||
description: "Dual-write migration to Atlas M0 with explicit cold-start abort threshold and Upstash pivot path."
|
||||
status: code-complete
|
||||
status: planning
|
||||
priority: P2
|
||||
effort: 22h
|
||||
branch: dev
|
||||
branch: main
|
||||
tags: [storage, migration, mongodb, atlas, cloudflare-workers]
|
||||
created: 2026-04-25
|
||||
code_completed: 2026-04-26
|
||||
blockedBy: []
|
||||
blocks: []
|
||||
---
|
||||
|
||||
> **Status note:** All 8 phases of code/config/scripts/docs are implemented and committed on `dev` (commits `6f0b5ff`..`e2e3112`). 503 → 733 tests; lint clean; `register:dry` green. **Operator-driven execution is pending**: Atlas provisioning (Phase 01 §1-7), real-cluster smoke tests (Phase 01 §13-14), backfill runs (Phase 05), 24-72h soak (Phase 06), cutover stages (Phase 07), and Stage 3 code cleanup (delete CFKVStore/dual-stores after binding deletion). Plan stays here (not archived) until cutover lands or the Upstash standby (`phase-07-alt-pivot.md`) executes.
|
||||
|
||||
# Plan: KV+D1 → MongoDB Atlas M0
|
||||
|
||||
User-chosen path despite research recommending Upstash. Goal: validate cold-start UX firsthand with safe rollback to KV/D1 (or pivot to Upstash) if M0 cold-start P95 exceeds derived threshold.
|
||||
@@ -36,15 +33,15 @@ User-chosen path despite research recommending Upstash. Goal: validate cold-star
|
||||
|
||||
| # | Phase | Status | Effort | Owner files |
|
||||
|---|-------|--------|--------|-------------|
|
||||
| 01 | [Atlas setup + wrangler config](phase-01-atlas-setup.md) | code-complete · operator-pending | 2h | `wrangler.toml`, `.env.deploy.example`, `scripts/check-secret-leaks.js` |
|
||||
| 02 | [MongoKVStore implementation](phase-02-mongo-kv-store.md) | implemented (`5b00cae`) | 3h | `src/db/mongo-*.js` |
|
||||
| 03 | [MongoTradesStore + trading refactor](phase-03-mongo-sql-store.md) | implemented (`99cd844`) | 3h | `src/db/mongo-trades-store.js`, `src/modules/trading/*` |
|
||||
| 04 | [Dual-write wrappers + flag + e2e](phase-04-dual-write-wrappers.md) | implemented (`ea7df56`) | 4h | `src/db/dual-*.js`, factories, `tests/e2e/*` |
|
||||
| 05 | [Backfill + verification (local-only)](phase-05-backfill-scripts.md) | implemented · operator-runs (`0859356`) | 3h | `scripts/backfill-*.js` |
|
||||
| 06 | [Staged deploy + soak (cold-start gate)](phase-06-staged-deploy-and-soak.md) | code-complete · operator-runs (`55c8739`) | 4h | runtime telemetry |
|
||||
| 07 | [Cutover + decommission](phase-07-cutover-and-decommission.md) | prereqs-complete · operator-runs (`3f03521`) | 3h | wrangler bindings |
|
||||
| 01 | [Atlas setup + wrangler config](phase-01-atlas-setup.md) | pending | 2h | `wrangler.toml`, `.env.deploy.example`, `scripts/check-secret-leaks.js` |
|
||||
| 02 | [MongoKVStore implementation](phase-02-mongo-kv-store.md) | pending | 3h | `src/db/mongo-*.js` |
|
||||
| 03 | [MongoTradesStore + trading refactor](phase-03-mongo-sql-store.md) | pending | 3h | `src/db/mongo-trades-store.js`, `src/modules/trading/*` |
|
||||
| 04 | [Dual-write wrappers + flag + e2e](phase-04-dual-write-wrappers.md) | pending | 4h | `src/db/dual-*.js`, factories, `tests/e2e/*` |
|
||||
| 05 | [Backfill + verification (local-only)](phase-05-backfill-scripts.md) | pending | 3h | `scripts/backfill-*.js` |
|
||||
| 06 | [Staged deploy + soak (cold-start gate)](phase-06-staged-deploy-and-soak.md) | pending | 4h | runtime telemetry |
|
||||
| 07 | [Cutover + decommission](phase-07-cutover-and-decommission.md) | pending | 3h | wrangler bindings |
|
||||
| 07-ALT | [Pivot to Upstash (STANDBY)](phase-07-alt-pivot.md) | standby | (3-4d if triggered) | `src/db/upstash-*.js` |
|
||||
| 08 | [Tests + docs](phase-08-tests-and-docs.md) | implemented (`e2e3112`) | 1h | `tests/`, `docs/` |
|
||||
| 08 | [Tests + docs](phase-08-tests-and-docs.md) | pending | 1h | `tests/`, `docs/` |
|
||||
|
||||
## Critical dependencies
|
||||
- 01 → 02, 03 (Atlas creds + bundle-size gate required)
|
||||
|
||||
-211
@@ -1,211 +0,0 @@
|
||||
# Vietnamese Embeddings for Doantu: sup-SimCSE vs PhoW2V
|
||||
|
||||
**Date:** 2026-04-23 | **Scope:** Word-level cosine similarity in fixed 22k-vocab Semantle clone
|
||||
|
||||
---
|
||||
|
||||
## Executive Verdict
|
||||
|
||||
**Recommendation: PhoW2V (word-level, 300d) is the better fit.**
|
||||
|
||||
Reasons: (1) Purpose-built for word similarity, not sentences; (2) static word2vec format enables precomputation into lookup table — zero inference overhead; (3) no external segmentation tool required; (4) Cloudflare Worker-friendly (ship vectors in KV or bundle with Worker).
|
||||
|
||||
**sup-SimCSE is inferior here** despite better semantic depth, because it requires runtime inference, external VnCoreNLP/pyvi segmentation, and sentence-level training (not optimized for single-word pairs). Violates KISS.
|
||||
|
||||
---
|
||||
|
||||
## Detailed Comparison
|
||||
|
||||
### 1. **sup-SimCSE-VietNamese-phobert-base** (VoVanPhuc)
|
||||
|
||||
| Aspect | Value | Trade-off |
|
||||
|--------|-------|-----------|
|
||||
| **Embedding Dim** | 768 | Large; overkill for word pairs |
|
||||
| **Training** | Supervised contrastive (SimCSE) | Optimized for sentence similarity, not word pairs |
|
||||
| **Level** | Sentence-level | Not designed for single-word input |
|
||||
| **Vocab** | Open (transformer subword tokenization) | Handles unseen words via BPE; adds latency |
|
||||
| **Segmentation** | **REQUIRED** (RDRSegmenter or pyvi) | Extra runtime dependency; "con chó" must become "con_chó" before encoding |
|
||||
| **Model Size** | 135M parameters | ~250–350 MB disk; requires transformers + torch on Worker? Infeasible. |
|
||||
| **Inference** | Runtime + tokenization | Cold start latency; not precomputable for 22k words |
|
||||
| **Format** | Hugging Face (transformers) | No static dump; requires active model loading |
|
||||
|
||||
**Key Gotcha:** PhoBERT's tokenizer expects **pre-segmented input**. "máy bay" (airplane) as raw input will tokenize as ["máy", "bay"] separately unless segmented to "máy_bay". You'd need VnCoreNLP + pyvi running in Worker — expensive and fragile.
|
||||
|
||||
---
|
||||
|
||||
### 2. **PhoW2V** (VinAI, datquocnguyen)
|
||||
|
||||
| Aspect | Value | Trade-off |
|
||||
|--------|-------|-----------|
|
||||
| **Embedding Dim** | 100 or 300 | 300d ideal; 100d saves space but lower quality |
|
||||
| **Training** | Unsupervised word2vec (CBOW/Skip-gram) | Pure word-level; no sentence context — but this matches your use case exactly |
|
||||
| **Level** | Word-level (and syllable-level variant) | Designed for single-word similarity |
|
||||
| **Vocab** | ~100k words from 20GB corpus (likely covers 22k) | Finite vocab; OOV words get zero vector or nearest neighbor |
|
||||
| **Segmentation** | **Optional** (can use word-level variant directly) | If input is already word-tokenized, no extra step |
|
||||
| **Model Size** | ~30–50 MB (gensim KeyedVectors) | Tiny; fits in Cloudflare KV or bundle |
|
||||
| **Inference** | Zero runtime — precompute entire 22k vocab | `precomputed[word] = word_vector` lookup O(1) |
|
||||
| **Format** | Gensim KeyedVectors (text/binary) | Exportable as dense matrix (22k × 300) for embedding |
|
||||
|
||||
**Critical Insight:** Word2Vec embeddings are **static lookup tables**. You can precompute similarity scores for all 22k² word pairs offline, or dump the 22k vectors into KV and compute cosine similarity on-demand (O(300) dot product per pair — negligible).
|
||||
|
||||
---
|
||||
|
||||
## Tokenization & Diacritics
|
||||
|
||||
### PhoBERT (sup-SimCSE)
|
||||
- Requires VnCoreNLP/pyvi to convert raw input to segmented form.
|
||||
- **"con chó"** → requires preprocessing to **"con_chó"** before tokenization.
|
||||
- Adds runtime cost + dependency fragility.
|
||||
- Diacritics preserved via RDRSegmenter normalization.
|
||||
|
||||
### PhoW2V
|
||||
- Word-level variant: expects whitespace-separated words (already segmented).
|
||||
- If using syllable-level, requires syllable input (less relevant here).
|
||||
- Diacritics preserved (trained on normalized Vietnamese corpus).
|
||||
- **No segmentation tool needed if vocab covers your compound words.**
|
||||
|
||||
**For a fixed 22k-word game vocabulary:** Pre-segment and validate your entire wordlist at deployment time. Both approaches require diacritic-aware matching ("cá" ≠ "ca").
|
||||
|
||||
---
|
||||
|
||||
## Precomputation & Cloudflare Worker Fit
|
||||
|
||||
### sup-SimCSE (Cannot Precompute)
|
||||
```
|
||||
❌ Runtime inference required (PhoBERT forward pass)
|
||||
❌ Requires transformers + torch (not Worker-compatible)
|
||||
❌ VnCoreNLP dependency for each request
|
||||
❌ Cold start latency (~500ms per query on CPU)
|
||||
```
|
||||
|
||||
### PhoW2V (Fully Precomputable)
|
||||
```
|
||||
✅ Load KeyedVectors once at Worker start
|
||||
✅ Precompute all 22k embeddings into in-memory dense matrix
|
||||
✅ Cosine similarity: ~1ms per pair (vector dot product)
|
||||
✅ Alternative: Ship (22k × 22k) similarity matrix in KV
|
||||
✅ Or: ~7.3 MB dense matrix (22k × 300 × 4 bytes) fits in Worker bundles
|
||||
```
|
||||
|
||||
**Verdict:** PhoW2V enables a **stateless, zero-latency** Semantle implementation. sup-SimCSE requires external inference infrastructure.
|
||||
|
||||
---
|
||||
|
||||
## Vocabulary Coverage
|
||||
|
||||
| Model | Vocab Size | 22k Viet22K Coverage | License |
|
||||
|-------|------------|---------------------|---------|
|
||||
| **PhoW2V** | ~100k (estimated from 20GB corpus) | Likely 95%+ (VinAI trained on broad Vietnamese text) | AGPL-3.0; research/education only; cite EMNLP-2020 |
|
||||
| **sup-SimCSE** | Unbounded (subword + BPE) | 100% (BPE handles unknowns) | Likely permissive (HF model) |
|
||||
|
||||
**Gotcha:** PhoW2V is **research-only, non-commercial**, and requires citation. Check your license constraints for a Telegram bot (even if private, may still violate terms).
|
||||
|
||||
---
|
||||
|
||||
## Semantic Quality
|
||||
|
||||
### sup-SimCSE
|
||||
- **Advantage:** Trained on supervised sentence pairs; captures deeper semantic relationships.
|
||||
- **Disadvantage:** Trained on sentence context; single-word pairs don't benefit from that context.
|
||||
- **Effective for:** Semantically distant word pairs (e.g., "xe" vs "cách"); may over-regularize tight synonyms.
|
||||
|
||||
### PhoW2V
|
||||
- **Advantage:** Word-level training (CBOW/Skip-gram); embeddings encode co-occurrence statistics.
|
||||
- **Disadvantage:** No supervised signal; relies purely on distributional similarity.
|
||||
- **Effective for:** "Natural" word similarity (synonyms, related concepts); well-suited to Semantle-style games.
|
||||
|
||||
**For a word-guessing game:** Both are reasonable. PhoW2V's simplicity is not a weakness here; it's a feature.
|
||||
|
||||
---
|
||||
|
||||
## Implementation Complexity
|
||||
|
||||
### sup-SimCSE (High Complexity)
|
||||
```python
|
||||
from sentence_transformers import SentenceTransformer
|
||||
from pyvi.ViTokenizer import tokenize
|
||||
|
||||
model = SentenceTransformer('VoVanPhuc/sup-SimCSE-...')
|
||||
# Per query:
|
||||
segmented = tokenize(raw_input) # Runtime overhead
|
||||
emb = model.encode(segmented)
|
||||
similarity = cosine(emb_target, emb_guess)
|
||||
```
|
||||
- **Dependencies:** sentence-transformers, transformers, torch, pyvi.
|
||||
- **Latency:** 200–500ms per query (even on GPU; Cloudflare Workers have no GPU).
|
||||
- **Lines of code:** ~20.
|
||||
- **External service:** Optional (could self-host, but adds infrastructure).
|
||||
|
||||
### PhoW2V (Low Complexity)
|
||||
```python
|
||||
from gensim.models import KeyedVectors
|
||||
import numpy as np
|
||||
|
||||
kv = KeyedVectors.load_word2vec_format('phow2v.bin')
|
||||
# Option A (precompute all):
|
||||
embeddings = {word: kv[word] for word in vocab}
|
||||
|
||||
# Per query:
|
||||
similarity = np.dot(embeddings[target], embeddings[guess])
|
||||
```
|
||||
- **Dependencies:** gensim (tiny).
|
||||
- **Latency:** <1ms per query (in-memory lookup).
|
||||
- **Lines of code:** ~10.
|
||||
- **External service:** None (pure static embeddings).
|
||||
|
||||
---
|
||||
|
||||
## License & Attribution
|
||||
|
||||
| Model | License | Restriction |
|
||||
|-------|---------|------------|
|
||||
| **sup-SimCSE** | Unclear (check HF model card) | Likely permissive for research |
|
||||
| **PhoW2V** | AGPL-3.0 | **Research/education only; cite EMNLP-2020; non-commercial** |
|
||||
|
||||
**Risk:** If doantu is a commercial Telegram bot or intends to be monetized, PhoW2V's AGPL restriction may be a blocker. Clarify with user.
|
||||
|
||||
---
|
||||
|
||||
## Unresolved Questions
|
||||
|
||||
1. **PhoW2V license constraints:** Is doantu commercial? Non-commercial? Verify AGPL-3.0 compatibility with your bot's intended use.
|
||||
2. **Vocabulary overlap:** Exact coverage of 22k Viet22K words in PhoW2V. Could spot-check a few compounds like "máy bay", "con chó" in the model.
|
||||
3. **Syllable vs word PhoW2V:** Recommendation assumes word-level variant. If Viet22K uses syllables, syllable-level variant may be needed; would require preprocessing.
|
||||
4. **sup-SimCSE alternatives:** Are you open to other sentence transformers fine-tuned for Vietnamese word similarity (e.g., from FPTAI or other VN NLP labs)?
|
||||
5. **Similarity matrix size:** Confirm whether shipping a precomputed (22k × 22k) matrix in KV is practical (~7.3 MB in dense form, ~200 MB in sparse COO).
|
||||
|
||||
---
|
||||
|
||||
## Recommendation Summary
|
||||
|
||||
**Use PhoW2V (300d word-level variant) for doantu.**
|
||||
|
||||
**Why:**
|
||||
- Single-word embeddings (not sentence-level).
|
||||
- Static vectors → zero inference cost.
|
||||
- Fits Cloudflare Worker budget (no external service needed).
|
||||
- Precomputable into O(1) lookups.
|
||||
- Simpler to deploy and maintain.
|
||||
|
||||
**Why not sup-SimCSE:**
|
||||
- Sentence-level training doesn't benefit single-word pairs.
|
||||
- Runtime inference infeasible on CPU-only Cloudflare Workers.
|
||||
- External segmentation (pyvi/VnCoreNLP) adds complexity and latency.
|
||||
- 768-dim vectors overkill for word pairs; 300-dim sufficient.
|
||||
|
||||
**Action items:**
|
||||
1. Verify PhoW2V's AGPL-3.0 license permits your bot's use case.
|
||||
2. Spot-check PhoW2V vocabulary against 22k-word game list (OOV strategy needed).
|
||||
3. Decide precomputation strategy: in-memory matrix, KV store, or on-demand dot product.
|
||||
|
||||
---
|
||||
|
||||
## Sources
|
||||
|
||||
- [sup-SimCSE-VietNamese-phobert-base on Hugging Face](https://huggingface.co/VoVanPhuc/sup-SimCSE-VietNamese-phobert-base)
|
||||
- [PhoW2V GitHub Repository](https://github.com/datquocnguyen/PhoW2V)
|
||||
- [PhoBERT: Pre-trained Language Models for Vietnamese (EMNLP-2020 Findings)](https://aclanthology.org/2020.findings-emnlp.92.pdf)
|
||||
- [VinAI Research – PhoBERT Overview](https://www.vinai.io/phobert-the-first-public-large-scale-language-models-for-vietnamese/)
|
||||
- [Gensim Word2Vec KeyedVectors Documentation](https://radimrehurek.com/gensim/models/word2vec.html)
|
||||
- [Semantle Word Embeddings Recreation](https://github.com/memgonzales/semantle-word-embeddings)
|
||||
- [VnCoreNLP Word Segmentation](https://www.researchgate.net/publication/325449322_VnCoreNLP_A_Vietnamese_Natural_Language_Processing_Toolkit)
|
||||
@@ -0,0 +1,143 @@
|
||||
# Documentation Audit Report
|
||||
|
||||
## Summary
|
||||
|
||||
Audited 12 documentation files covering architecture, module setup, deployment, code standards, and module READMEs. Found **3 major stale sections** related to module implementation status (wordle/loldle described as stubs when now fully implemented) and **1 test count discrepancy**. No broken links detected. Modular architecture docs are accurate.
|
||||
|
||||
## Doc Files Inventory
|
||||
|
||||
| File | Purpose | Status |
|
||||
|------|---------|--------|
|
||||
| README.md | Top-level overview, setup, deploy, troubleshooting | **MAJOR-DRIFT** |
|
||||
| CLAUDE.md | Dev guidance, commands, module contract, testing | **FRESH** |
|
||||
| docs/architecture.md | Deep dive: cold-start, registry, storage, crons, deploy | **MINOR-DRIFT** |
|
||||
| docs/adding-a-module.md | Step-by-step module authoring guide | **FRESH** |
|
||||
| docs/code-standards.md | Formatting, JSDoc, file org, naming, testing | **FRESH** |
|
||||
| docs/codebase-summary.md | Tech stack, active modules table, data flows | **MAJOR-DRIFT** |
|
||||
| docs/deployment-guide.md | CF setup, KV, D1, secrets, deploy steps, rollback | **FRESH** |
|
||||
| docs/using-d1.md | When to use D1 vs KV, SQL API, migration examples | **FRESH** |
|
||||
| docs/using-cron.md | Cron syntax, handler signature, examples | **FRESH** |
|
||||
| src/modules/wordle/README.md | Commands, architecture, KV schema | **FRESH** |
|
||||
| src/modules/loldle/README.md | Commands, architecture, KV schema | **MAJOR-DRIFT** |
|
||||
| src/modules/misc/README.md | Commands, KV demo, schema | **FRESH** |
|
||||
|
||||
## Detailed Findings
|
||||
|
||||
### 1. README.md — MAJOR-DRIFT
|
||||
|
||||
**Lines 14, 67-68: Test count and module status discrepancies**
|
||||
|
||||
- **Line 14 claim:** "105+ vitest unit tests"
|
||||
- **Actual:** 200 tests (verified: `npm test` output shows "Tests 200 passed")
|
||||
- **Fix:** Update to "200+ vitest unit tests"
|
||||
|
||||
- **Line 67 claim:** "wordle/ # stub — proves plugin system"
|
||||
- **Line 68 claim:** "loldle/ # stub"
|
||||
- **Actual:** Both are now full implementations:
|
||||
- Wordle: 4 commands (guessing game, new round, giveup, stats) with KV state, render, daily word, compare logic
|
||||
- Loldle: 4 commands (guessing game, new round, giveup, stats) with KV state, champions data, compare logic
|
||||
- Commit 8a9a6af: "feat(wordle): port classic 5-letter guessing game"
|
||||
- **Fix:** Change line 67 to "wordle/ # Classic 5-letter word guessing game (full impl)" and line 68 to "loldle/ # League of Legends champion guessing game (full impl)"
|
||||
|
||||
### 2. docs/codebase-summary.md — MAJOR-DRIFT
|
||||
|
||||
**Lines 25-26: Module status table outdated**
|
||||
|
||||
| Row | Claim | Actual |
|
||||
|-----|-------|--------|
|
||||
| `wordle` | "Status: Stub" | Full implementation: `/wordle`, `/wordle_new`, `/wordle_giveup`, `/wordle_stats` |
|
||||
| `wordle` | "Commands: `/wordle`, `/wstats`, `/konami`" | Wrong commands listed; actual: `/wordle`, `/wordle_new`, `/wordle_giveup`, `/wordle_stats` (all public) |
|
||||
| `wordle` | "Storage: —" | Uses KV (see src/modules/wordle/README.md) |
|
||||
| `loldle` | "Status: Stub" | Full implementation: `/loldle`, `/loldle_new`, `/loldle_giveup`, `/loldle_stats` |
|
||||
| `loldle` | "Commands: `/loldle`, `/ggwp`" | Missing 3 commands; actual: `/loldle`, `/loldle_new`, `/loldle_giveup`, `/loldle_stats` |
|
||||
| `loldle` | "Storage: —" | Uses KV (see src/modules/loldle/index.js lines 10, 16) |
|
||||
|
||||
**Fix:** Update the "Active Modules" table rows for wordle and loldle with actual command counts, visibility, and KV storage.
|
||||
|
||||
### 3. docs/architecture.md — MINOR-DRIFT
|
||||
|
||||
**Line 33: Module classification outdated**
|
||||
|
||||
- **Line 33 claim:** "wordle/ loldle/ — stub modules proving the plugin system"
|
||||
- **Actual:** Both are now production implementations with full game logic
|
||||
- **Fix:** Update to "wordle/ loldle/ — classic word/champion guessing games (full implementations)" or remove stub reference
|
||||
|
||||
**Line 362: Test count**
|
||||
|
||||
- **Line 362 claim:** "105 tests run in ~500ms"
|
||||
- **Actual:** 200 tests run in ~2.26s (from `npm test`)
|
||||
- **Fix:** Update to "200 tests run in ~2.26s"
|
||||
|
||||
### 4. src/modules/loldle/README.md — MAJOR-DRIFT
|
||||
|
||||
**Lines 1-3: Module described as stub**
|
||||
|
||||
- **Current claim:** "League of Legends guessing game — currently a stub proving the plugin system."
|
||||
- **Actual:** Full implementation with handlers imported (line 8: `import { handleGiveup, handleLoldle, handleNew, handleStats } from "./handlers.js"`), 4 commands, KV state (lines 10-17)
|
||||
- **Fix:** Rewrite to match wordle/README.md pattern: describe the 4 commands, handlers, architecture (handlers.js, compare.js, lookup.js, daily.js, render.js, state.js, champions-data.js), and KV schema
|
||||
|
||||
**Line 15: "No KV usage currently"**
|
||||
|
||||
- **Actual:** Module has `init` hook and KV state management (lines 14-17)
|
||||
- **Fix:** Document the `loldle:` namespace and game/stats keys (same pattern as wordle)
|
||||
|
||||
**Lines 6-16: Commands are stubs with stub responses**
|
||||
|
||||
- **Actual:** Commands have real handler implementations (handlers.js exists with 400+ LOC)
|
||||
- **Fix:** Remove "stub" references, document actual commands and their behavior
|
||||
|
||||
### 5. Cross-Reference Checks
|
||||
|
||||
**Internal links verified:**
|
||||
|
||||
- `README.md` → `docs/adding-a-module.md` ✓ (exists, correct relative path)
|
||||
- `README.md` → `docs/architecture.md` ✓
|
||||
- `README.md` → `docs/using-d1.md` ✓
|
||||
- `README.md` → `docs/using-cron.md` ✓
|
||||
- `README.md` → `docs/deployment-guide.md` ✓
|
||||
- `docs/architecture.md` → `src/modules/<name>/README.md` ✓ (pattern reference, not broken link)
|
||||
- All plan references in README.md reference existing directories ✓
|
||||
|
||||
**No broken links found.**
|
||||
|
||||
### 6. Missing Documentation
|
||||
|
||||
None. All required docs exist:
|
||||
|
||||
- ✓ Top-level README with setup, deploy, troubleshooting
|
||||
- ✓ Architecture deep-dive
|
||||
- ✓ Adding a module guide
|
||||
- ✓ Code standards
|
||||
- ✓ Codebase summary
|
||||
- ✓ Deployment guide (KV + D1)
|
||||
- ✓ D1 usage guide
|
||||
- ✓ Cron usage guide
|
||||
- ✓ Per-module READMEs (wordle, loldle, misc, trading, util)
|
||||
|
||||
Note: `docs/todo.md` exists but appears to be an internal tracking doc, not user-facing documentation.
|
||||
|
||||
---
|
||||
|
||||
## Prioritized Fix List
|
||||
|
||||
### Immediate (blocking user confusion)
|
||||
|
||||
1. **README.md line 14:** Update "105+ vitest" → "200+" (test count)
|
||||
2. **README.md lines 67-68:** Remove "stub" label from wordle/loldle in architecture snapshot
|
||||
3. **docs/codebase-summary.md lines 25-26:** Update module status table (wordle/loldle to "Complete", fix commands, add KV storage)
|
||||
4. **src/modules/loldle/README.md:** Full rewrite to describe actual implementation (4 commands, handlers, KV schema) not stub
|
||||
|
||||
### Secondary (clarity improvement)
|
||||
|
||||
5. **docs/architecture.md line 33:** Remove "stub" reference from module list description
|
||||
6. **docs/architecture.md line 362:** Update test count from "105" → "200" and runtime from "~500ms" → "~2.26s"
|
||||
|
||||
---
|
||||
|
||||
## Notes
|
||||
|
||||
- Loldle module is fully implemented (files: handlers.js, compare.js, lookup.js, daily.js, render.js, state.js, champions-data.js, champions.json) but its README still describes it as a stub — this is the most glaring discrepancy.
|
||||
- Wordle module is correctly documented in its own README but incorrectly labeled "stub" in architecture snapshot and codebase summary.
|
||||
- Test count increased from 105 to 200 (likely due to additional trading module tests or recent test additions) — this is a growth metric worth celebrating.
|
||||
- All module-specific README files are accurate except loldle.
|
||||
- No architectural issues, just stale descriptive text that contradicts code reality.
|
||||
@@ -0,0 +1,80 @@
|
||||
# Leaguepedia API — Verification Report
|
||||
|
||||
**Date:** 2026-04-21
|
||||
**Purpose:** Verify the Leaguepedia MediaWiki/Cargo API can provide today / this-week LoL matches for a miti99bot module. No implementation, verification only.
|
||||
**Verdict:** **YES — usable.** Endpoint is public, no auth, returns structured JSON. One caveat on `where=` clauses needs rechecking from Cloudflare Workers egress.
|
||||
|
||||
---
|
||||
|
||||
## Endpoint
|
||||
|
||||
- Base: `https://lol.fandom.com/api.php`
|
||||
- Action: `cargoquery` (MediaWiki Cargo extension)
|
||||
- Auth: none
|
||||
- Format: `format=json` → clean `{cargoquery:[{title:{…}}]}` payload
|
||||
- Related `mw.Api` JS wrapper (doc-wikimedia link) works the same; for a Worker we use plain `fetch`, not `mw.Api`
|
||||
|
||||
## Relevant table: `MatchSchedule`
|
||||
|
||||
Primary table for both upcoming and played matches. Confirmed fields (live API, 2026-04-21):
|
||||
|
||||
| Field | Type | Example |
|
||||
|---|---|---|
|
||||
| `DateTime_UTC` | datetime | `"2026-06-14 09:00:00"` |
|
||||
| `Team1`, `Team2` | string | `"T1"`, `"TBD"` |
|
||||
| `Tournament` | string | Tournament name/slug |
|
||||
| `BestOf` | int | |
|
||||
| `Winner` | string | empty until played |
|
||||
| `OverviewPage` | string | wiki page for tournament |
|
||||
| `_pageName` | string | wiki row page |
|
||||
|
||||
Complementary tables: `Tournaments` (metadata), `ScoreboardGames` (per-game stats), `Teams`.
|
||||
|
||||
## Query syntax (verified working)
|
||||
|
||||
Use **table + field aliases** — the bare form `fields=MatchSchedule.DateTime_UTC` hits an `MWException`. Alias form is the idiomatic Leaguepedia convention:
|
||||
|
||||
```
|
||||
tables=MatchSchedule=MS
|
||||
fields=MS.DateTime_UTC=DateTime, MS.Team1=T1, MS.Team2=T2, MS.Tournament=Tournament
|
||||
order_by=MS.DateTime_UTC ASC
|
||||
limit=20
|
||||
```
|
||||
|
||||
Live sample (no where-filter) returned real rows. Ordering, limit, and aliasing all confirmed working.
|
||||
|
||||
Intended week-window query (to be re-verified from CF Worker egress):
|
||||
|
||||
```
|
||||
where=MS.DateTime_UTC >= "2026-04-21 00:00:00"
|
||||
AND MS.DateTime_UTC < "2026-04-28 00:00:00"
|
||||
```
|
||||
|
||||
## Limitations & operational notes
|
||||
|
||||
- **Strict anonymous rate limit.** From a single shared egress IP the API throttled after 1–2 req/min with `ratelimited`. Mitigations for Workers:
|
||||
- Use Worker's distributed egress (many IPs) — in practice won't hit the same bucket
|
||||
- Cache responses in KV (e.g. 60–300 s for upcoming schedule, 5–15 min for results)
|
||||
- Use `cf: { cacheTtl, cacheEverything: true }` on `fetch`
|
||||
- **User-Agent required.** Fandom's policy expects a contact UA, e.g. `miti99bot/0.1 (https://t.me/miti99bot; minhtienit99@gmail.com)`.
|
||||
- **Help page is Cloudflare-challenged.** `https://lol.fandom.com/wiki/Help:Leaguepedia_API` returns 403 to non-browser UAs — consult it from a browser, not from `fetch` code.
|
||||
- **No official JS SDK.** MediaWiki's `mw.Api` is on-wiki JS only. Community Python wrapper (`mwrogue` / `leaguepedia_parser`) is the reference implementation — we port the query shape, not the lib.
|
||||
- **`where=` clause returned MWException from this verification IP** even on trivial filters (`MS.Team1="T1"`). Likely an upstream filter on the shared egress, not a protocol limitation — the exact form is documented and heavily used. **Needs one confirmation curl from a CF Worker before building on it.**
|
||||
|
||||
## Feasibility verdict
|
||||
|
||||
| Requirement | Feasible? | Notes |
|
||||
|---|---|---|
|
||||
| Fetch today's matches | ✅ | `DateTime_UTC >= today AND < tomorrow` |
|
||||
| Fetch this week's matches | ✅ | 7-day window on `DateTime_UTC` |
|
||||
| Filter by region/league | ✅ | `Tournament LIKE "LCK%"` or `OverviewPage` |
|
||||
| Include results/winners | ✅ | `Winner`, `BestOf` already on row |
|
||||
| Run from Cloudflare Worker | ✅ | plain `fetch` + JSON; add UA + KV cache |
|
||||
| Scheduled daily digest | ✅ | fits existing `cron-dispatcher.js` pattern |
|
||||
|
||||
## Unresolved questions
|
||||
|
||||
1. Does `where=` with comparison operators (`>=`, `<`) work from CF Worker egress, or do we need to alternate filter form (`HOLDS`, `LIKE`, full-table-scan + client-side filter)?
|
||||
2. Timezone UX — show UTC, VN time (UTC+7), or let the `/matches` command take a region arg?
|
||||
3. Caching window — ~60 s for "live today" vs ~5 min for week-view; confirm TTL with a real command spec before implementing.
|
||||
4. Do we want the command to also surface `Winner`/score once a match has finished, or keep it schedule-only?
|
||||
@@ -0,0 +1,55 @@
|
||||
# Leaguepedia / Fandom API — Auth Token Verification
|
||||
|
||||
**Date:** 2026-04-21
|
||||
**Follow-up to:** `researcher-260421-0845-leaguepedia-api-verification.md`
|
||||
**Question:** Can we register and use a token to avoid the rate limit?
|
||||
**Verdict:** **No useful token available. Caching + CF Worker egress is the right answer.**
|
||||
|
||||
---
|
||||
|
||||
## What's NOT available on Fandom
|
||||
|
||||
| Mechanism | Status | Evidence |
|
||||
|---|---|---|
|
||||
| `Special:BotPasswords` | Disabled | 403 CF + disabled in UCP platform (documented in Fandom community) |
|
||||
| OAuth 1.0a / 2.0 (`Special:OAuthConsumerRegistration`) | Not offered | 403; Fandom never enabled the OAuth extension |
|
||||
| `action=clientlogin` (MW native login) | Disabled | `authmanagerinfo` returns only `RememberMeAuthenticationRequest`, no password field |
|
||||
| WMF-style API-key header | N/A | MediaWiki has no such thing; Fandom has none either |
|
||||
|
||||
```bash
|
||||
# Live probe
|
||||
curl '.../api.php?action=query&meta=authmanagerinfo&amirequestsfor=login&format=json'
|
||||
# → only returns RememberMeAuthenticationRequest — native login is off
|
||||
```
|
||||
|
||||
## What Fandom *does* expose
|
||||
|
||||
- **Helios SSO** at `services.fandom.com/mobile-fandom-app/fandom-auth/login`
|
||||
- POST `username` + `password` → `access_token` cookie
|
||||
- Used by Leaguepedia's official `mwcleric` Python lib (`LoginCredentials`)
|
||||
- Cookie is carried on subsequent `api.php` requests from same session
|
||||
|
||||
## Does an authenticated cookie lift the rate limit?
|
||||
|
||||
**No, not meaningfully.**
|
||||
|
||||
- MediaWiki's `noratelimit` right only belongs to specific wiki groups (`sysop`, `bot`). Regular logged-in users have the same API limits as anonymous.
|
||||
- Joining the `bot` group on Leaguepedia requires wiki-admin (Leaguepedia staff) approval — not practical for a side project.
|
||||
- The throttling we hit earlier is **Fandom's Cloudflare-edge IP rate limit**, which is session-agnostic. Auth cookies don't bypass it.
|
||||
- `siprop=ratelimits` is stripped on Fandom (`Unrecognized value`) — we can't even enumerate the limits.
|
||||
|
||||
## Right answer for miti99bot (Cloudflare Worker)
|
||||
|
||||
No token registration needed. Mitigate via:
|
||||
|
||||
1. **Edge caching** — `fetch(url, { cf: { cacheTtl: 60, cacheEverything: true } })`. Many bot users share one cached response.
|
||||
2. **KV result cache** — wrap the query in `create-store.js`, key = `matches:{from}:{to}`, TTL 60 s for "today", 5 min for "week".
|
||||
3. **Cron pre-warm** — add a module cron (existing `cron-dispatcher.js` pattern) that refreshes the week window every 15 min. Telegram `/matches` then reads pre-warmed cache.
|
||||
4. **CF Worker egress diversity** — Worker outbound IPs are many; per-IP buckets rarely hit 429 in practice.
|
||||
5. **Honor `Retry-After`** on 429 and surface "data momentarily unavailable" to the user instead of stalling.
|
||||
6. **Proper UA** — `miti99bot/0.1 (https://t.me/miti99bot; minhtienit99@gmail.com)` (already planned). Missing UA is itself a throttle signal on Fandom.
|
||||
|
||||
## Unresolved questions
|
||||
|
||||
1. Do CF Worker-origin fetches hit the same 429 as this shared egress does? (low risk — worth one real test before shipping)
|
||||
2. Is the module's read pattern bursty or steady? If steady, cron pre-warm + long TTL removes all pressure. If bursty (many users hit `/matches` at game time), KV cache is still the lever.
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -1,229 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file analyze-soak — parse a CF Logs export and compute per-(cmd × cold/warm)
|
||||
* latency percentiles plus error counts for the 24h soak analysis.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/analyze-soak.js --input soak-export.json [--commands /wordle,/loldle] [--output report.md]
|
||||
*
|
||||
* Input: CF Logs JSON export — one JSON object per line (NDJSON) OR a CSV export
|
||||
* where each row has at least a "message" column containing the raw JSON log string.
|
||||
*
|
||||
* Output (stdout + optional --output <md>):
|
||||
* Markdown table: cmd | cold/warm | n | p50 | p95 | p99
|
||||
* Error summary: dual-write secondary failures, mongo errors, CPU-time exceeded
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
|
||||
// ── CLI arg parsing ──────────────────────────────────────────────────────────
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {};
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
if (argv[i] === "--input") args.input = argv[++i];
|
||||
else if (argv[i] === "--commands") args.commands = argv[++i];
|
||||
else if (argv[i] === "--output") args.output = argv[++i];
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
// ── Percentile math ──────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Compute a percentile from a sorted numeric array (ascending).
|
||||
* Uses nearest-rank method.
|
||||
*
|
||||
* @param {number[]} sorted - must be sorted ascending
|
||||
* @param {number} p - percentile 0–100
|
||||
* @returns {number}
|
||||
*/
|
||||
export function percentile(sorted, p) {
|
||||
if (sorted.length === 0) return 0;
|
||||
if (sorted.length === 1) return sorted[0];
|
||||
const idx = Math.ceil((p / 100) * sorted.length) - 1;
|
||||
return sorted[Math.max(0, Math.min(idx, sorted.length - 1))];
|
||||
}
|
||||
|
||||
// ── Log line parsing ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Try to extract a JSON object from a raw log line.
|
||||
* Handles NDJSON (line is the JSON) and CSV rows where the JSON lives inside
|
||||
* a quoted "message" column value.
|
||||
*
|
||||
* @param {string} line
|
||||
* @returns {object|null}
|
||||
*/
|
||||
export function parseLogLine(line) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) return null;
|
||||
|
||||
// Try direct JSON parse (NDJSON format).
|
||||
if (trimmed.startsWith("{")) {
|
||||
try {
|
||||
return JSON.parse(trimmed);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// CSV: find the first {...} substring inside the line and parse it.
|
||||
const braceStart = trimmed.indexOf("{");
|
||||
const braceEnd = trimmed.lastIndexOf("}");
|
||||
if (braceStart !== -1 && braceEnd > braceStart) {
|
||||
try {
|
||||
return JSON.parse(trimmed.slice(braceStart, braceEnd + 1));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Aggregation ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* @typedef {{ n: number, samples: number[] }} Bucket
|
||||
* @typedef {Map<string, Bucket>} BucketMap key = "cmd|cold" | "cmd|warm"
|
||||
*/
|
||||
|
||||
/**
|
||||
* Read all lines from a file path synchronously and return as array.
|
||||
*
|
||||
* @param {string} filePath
|
||||
* @returns {string[]}
|
||||
*/
|
||||
function readLines(filePath) {
|
||||
return readFileSync(filePath, "utf8").split("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate cmd_timing events from log lines into per-(cmd×cold/warm) buckets.
|
||||
*
|
||||
* @param {string[]} lines - raw log lines
|
||||
* @param {string[]|null} filterCmds - if set, only include these cmd names
|
||||
* @returns {{ buckets: BucketMap, errors: { dualWriteFail: number, mongoError: number, cpuExceeded: number } }}
|
||||
*/
|
||||
export function aggregateLines(lines, filterCmds) {
|
||||
/** @type {BucketMap} */
|
||||
const buckets = new Map();
|
||||
const errors = { dualWriteFail: 0, mongoError: 0, cpuExceeded: 0 };
|
||||
|
||||
for (const line of lines) {
|
||||
// Count error patterns regardless of JSON structure.
|
||||
if (
|
||||
line.includes("dual-write:secondary:failed") ||
|
||||
line.includes("[dual-kv] secondary write failed")
|
||||
) {
|
||||
errors.dualWriteFail++;
|
||||
}
|
||||
if (
|
||||
line.includes("MongoError") ||
|
||||
line.includes("mongo connection") ||
|
||||
line.includes("MongoNetworkError")
|
||||
) {
|
||||
errors.mongoError++;
|
||||
}
|
||||
if (line.includes("Worker exceeded CPU time") || line.includes("cpu time exceeded")) {
|
||||
errors.cpuExceeded++;
|
||||
}
|
||||
|
||||
const obj = parseLogLine(line);
|
||||
if (!obj || obj.event !== "cmd_timing") continue;
|
||||
|
||||
const { cmd, total, cold } = obj;
|
||||
if (typeof cmd !== "string" || typeof total !== "number") continue;
|
||||
|
||||
// Apply command filter.
|
||||
if (filterCmds && !filterCmds.includes(cmd)) continue;
|
||||
|
||||
const bucket = cold ? "cold" : "warm";
|
||||
const key = `${cmd}|${bucket}`;
|
||||
if (!buckets.has(key)) buckets.set(key, { n: 0, samples: [] });
|
||||
const b = buckets.get(key);
|
||||
b.n++;
|
||||
b.samples.push(total);
|
||||
}
|
||||
|
||||
return { buckets, errors };
|
||||
}
|
||||
|
||||
// ── Report formatting ────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a markdown report string from aggregated data.
|
||||
*
|
||||
* @param {BucketMap} buckets
|
||||
* @param {{ dualWriteFail: number, mongoError: number, cpuExceeded: number }} errors
|
||||
* @returns {string}
|
||||
*/
|
||||
export function formatReport(buckets, errors) {
|
||||
const rows = [];
|
||||
|
||||
for (const [key, bucket] of [...buckets.entries()].sort()) {
|
||||
const [cmd, coldWarm] = key.split("|");
|
||||
const sorted = [...bucket.samples].sort((a, b) => a - b);
|
||||
rows.push({
|
||||
cmd,
|
||||
coldWarm,
|
||||
n: bucket.n,
|
||||
p50: percentile(sorted, 50),
|
||||
p95: percentile(sorted, 95),
|
||||
p99: percentile(sorted, 99),
|
||||
});
|
||||
}
|
||||
|
||||
const header = "| cmd | cold/warm | n | p50 | p95 | p99 |";
|
||||
const sep = "|-----|-----------|---|-----|-----|-----|";
|
||||
const dataRows = rows.map(
|
||||
(r) => `| ${r.cmd} | ${r.coldWarm} | ${r.n} | ${r.p50} | ${r.p95} | ${r.p99} |`,
|
||||
);
|
||||
|
||||
const table = [header, sep, ...dataRows].join("\n");
|
||||
|
||||
const errorSection = [
|
||||
"",
|
||||
"## Error Summary",
|
||||
`- Dual-write secondary failures: ${errors.dualWriteFail}`,
|
||||
`- Mongo connection errors: ${errors.mongoError}`,
|
||||
`- CPU time exceeded: ${errors.cpuExceeded}`,
|
||||
].join("\n");
|
||||
|
||||
return `## Soak Latency Report\n\n${table}\n${errorSection}\n`;
|
||||
}
|
||||
|
||||
// ── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
|
||||
if (!args.input) {
|
||||
console.error(
|
||||
"Usage: node scripts/analyze-soak.js --input <path> [--commands /wordle,/loldle] [--output <md>]",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const filterCmds = args.commands ? args.commands.split(",").map((c) => c.trim()) : null;
|
||||
const lines = readLines(args.input);
|
||||
const { buckets, errors } = aggregateLines(lines, filterCmds);
|
||||
const report = formatReport(buckets, errors);
|
||||
|
||||
process.stdout.write(report);
|
||||
|
||||
if (args.output) {
|
||||
writeFileSync(args.output, report, "utf8");
|
||||
console.error(`\nReport written to ${args.output}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Run only when executed directly (not imported in tests).
|
||||
const isMain = process.argv[1]?.endsWith("analyze-soak.js");
|
||||
if (isMain) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file backfill-d1-to-mongo — copy trading_trades from D1 into MongoDB Atlas.
|
||||
*
|
||||
* Uses `npx wrangler d1 execute --remote --json` (mirrors scripts/migrate.js pattern)
|
||||
* to read all rows, then insertMany in batches of 100 into the `trading_trades`
|
||||
* Mongo collection.
|
||||
*
|
||||
* Flags:
|
||||
* --dry-run Count rows + log first 5 without writing.
|
||||
* --force Bypass pre-flight abort when trading_trades collection is non-empty.
|
||||
*
|
||||
* Required env (loaded via --env-file-if-exists=.env.deploy):
|
||||
* MONGODB_URI
|
||||
*
|
||||
* Usage:
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-d1-to-mongo.js
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-d1-to-mongo.js --dry-run
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-d1-to-mongo.js --force
|
||||
*/
|
||||
|
||||
import { execSync } from "node:child_process";
|
||||
import { ObjectId } from "mongodb";
|
||||
import { closeMongoClient, getMongoClient, sleep } from "./lib/migration-helpers.js";
|
||||
|
||||
const DB_NAME = "miti99bot-db";
|
||||
const COLLECTION = "trading_trades";
|
||||
const BATCH_SIZE = 100;
|
||||
const BATCH_SLEEP_MS = 100; // brief pause between batches
|
||||
|
||||
const dryRun = process.argv.includes("--dry-run");
|
||||
const force = process.argv.includes("--force");
|
||||
|
||||
// ─── Preflight ────────────────────────────────────────────────────────────────
|
||||
|
||||
function validateEnv() {
|
||||
const missingVars = [["MONGODB_URI", "Atlas connection string"]].filter(([k]) => !process.env[k]);
|
||||
if (missingVars.length) {
|
||||
for (const [k, desc] of missingVars) console.error(`[backfill-d1] Missing: ${k} (${desc})`);
|
||||
console.error(" Copy .env.deploy.example to .env.deploy and fill in values.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── D1 query ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Execute a SQL query against the remote D1 database via wrangler.
|
||||
* Mirrors the wranglerExecute pattern in scripts/migrate.js.
|
||||
*
|
||||
* @param {string} sql
|
||||
* @returns {any[]} rows from D1
|
||||
*/
|
||||
function queryD1(sql) {
|
||||
const cmd = `npx wrangler d1 execute ${DB_NAME} --remote --command "${sql.replace(/"/g, '\\"')}" --json`;
|
||||
let stdout;
|
||||
try {
|
||||
stdout = execSync(cmd, { stdio: ["ignore", "pipe", "pipe"] }).toString();
|
||||
} catch (err) {
|
||||
const stderr = err.stderr?.toString() ?? "";
|
||||
throw new Error(`wrangler d1 execute failed:\n${stderr || err.stdout?.toString()}`);
|
||||
}
|
||||
// wrangler --json wraps results: [{results: [...], success: bool}]
|
||||
const parsed = JSON.parse(stdout);
|
||||
const results = Array.isArray(parsed) ? (parsed[0]?.results ?? []) : [];
|
||||
return results;
|
||||
}
|
||||
|
||||
// ─── Row mapping ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Map a D1 row to a Mongo document.
|
||||
* Preserves legacy_id so round-trip verification can match records.
|
||||
*
|
||||
* @param {{ id: number, user_id: string, symbol: string, side: string,
|
||||
* qty: number, price_vnd: number, ts: number }} row
|
||||
* @returns {object}
|
||||
*/
|
||||
function rowToDoc(row) {
|
||||
return {
|
||||
_id: new ObjectId(),
|
||||
legacy_id: row.id,
|
||||
user_id: row.user_id,
|
||||
symbol: row.symbol,
|
||||
side: row.side,
|
||||
qty: row.qty,
|
||||
price_vnd: row.price_vnd,
|
||||
ts: row.ts,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
validateEnv();
|
||||
|
||||
if (dryRun) console.log("[backfill-d1] DRY RUN — no writes will be made");
|
||||
|
||||
// Fetch all rows from D1.
|
||||
console.log("[backfill-d1] Querying D1 trading_trades...");
|
||||
const rows = queryD1(
|
||||
"SELECT id, user_id, symbol, side, qty, price_vnd, ts FROM trading_trades ORDER BY id",
|
||||
);
|
||||
console.log(`[backfill-d1] Found ${rows.length} rows in D1`);
|
||||
|
||||
if (dryRun) {
|
||||
console.log("[backfill-d1] Sample (first 5 rows):");
|
||||
for (const row of rows.slice(0, 5)) {
|
||||
// Log structural info only — no PII values printed.
|
||||
console.log(` id=${row.id} symbol=${row.symbol} side=${row.side} ts=${row.ts}`);
|
||||
}
|
||||
console.log("[backfill-d1] DRY RUN complete. No writes made.");
|
||||
return;
|
||||
}
|
||||
|
||||
const client = await getMongoClient(process.env.MONGODB_URI);
|
||||
const db = client.db();
|
||||
const coll = db.collection(COLLECTION);
|
||||
|
||||
// Pre-flight: abort if collection already has data (prevents double-insert).
|
||||
const existingCount = await coll.countDocuments();
|
||||
if (existingCount > 0 && !force) {
|
||||
console.error(`[backfill-d1] ABORT: ${COLLECTION} already has ${existingCount} document(s).`);
|
||||
console.error(" Run with --force to bypass this check (e.g. after wipe-mongo).");
|
||||
await closeMongoClient();
|
||||
process.exit(1);
|
||||
}
|
||||
if (existingCount > 0 && force) {
|
||||
console.log(`[backfill-d1] --force: proceeding despite ${existingCount} existing doc(s).`);
|
||||
}
|
||||
|
||||
// Insert in batches of BATCH_SIZE.
|
||||
let inserted = 0;
|
||||
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
|
||||
const batch = rows.slice(i, i + BATCH_SIZE).map(rowToDoc);
|
||||
await coll.insertMany(batch);
|
||||
inserted += batch.length;
|
||||
console.log(`[backfill-d1] Inserted ${inserted}/${rows.length}`);
|
||||
if (i + BATCH_SIZE < rows.length) await sleep(BATCH_SLEEP_MS);
|
||||
}
|
||||
|
||||
await closeMongoClient();
|
||||
console.log(`[backfill-d1] Done — ${inserted} documents inserted into ${COLLECTION}.`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("[backfill-d1] Fatal:", err.message ?? err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,203 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file backfill-kv-to-mongo — copy historical KV data into MongoDB Atlas.
|
||||
*
|
||||
* Uses the CF KV REST API to enumerate keys per module, then upserts each
|
||||
* key into the corresponding Mongo collection using $setOnInsert (skip-if-exists)
|
||||
* so live dual-write data written after Phase 04 is never overwritten.
|
||||
*
|
||||
* Flags:
|
||||
* --dry-run List + count without writing to Mongo.
|
||||
* --module <name> Backfill only a single module (default: all).
|
||||
*
|
||||
* Required env (loaded via --env-file-if-exists=.env.deploy):
|
||||
* MONGODB_URI, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_API_TOKEN,
|
||||
* KV_NAMESPACE_ID, MODULES (comma-separated)
|
||||
*
|
||||
* Usage:
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-kv-to-mongo.js
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-kv-to-mongo.js --dry-run
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-kv-to-mongo.js --module wordle
|
||||
*/
|
||||
|
||||
import {
|
||||
cfKvGet,
|
||||
cfKvList,
|
||||
clearCheckpoint,
|
||||
closeMongoClient,
|
||||
getMongoClient,
|
||||
loadCheckpoint,
|
||||
saveCheckpoint,
|
||||
sleep,
|
||||
} from "./lib/migration-helpers.js";
|
||||
|
||||
// ─── Config ───────────────────────────────────────────────────────────────────
|
||||
|
||||
const {
|
||||
MONGODB_URI,
|
||||
CLOUDFLARE_ACCOUNT_ID,
|
||||
CLOUDFLARE_API_TOKEN,
|
||||
KV_NAMESPACE_ID,
|
||||
MODULES: MODULES_ENV,
|
||||
} = process.env;
|
||||
|
||||
const dryRun = process.argv.includes("--dry-run");
|
||||
const moduleFlag = (() => {
|
||||
const idx = process.argv.indexOf("--module");
|
||||
return idx !== -1 ? process.argv[idx + 1] : null;
|
||||
})();
|
||||
|
||||
/** Normalize module name to MongoDB collection name (mirrors mongo-kv-store.js). */
|
||||
const toCollName = (mod) => mod.replace(/-/g, "_");
|
||||
|
||||
// ─── Preflight ────────────────────────────────────────────────────────────────
|
||||
|
||||
function validateEnv() {
|
||||
const missing = [
|
||||
"MONGODB_URI",
|
||||
"CLOUDFLARE_ACCOUNT_ID",
|
||||
"CLOUDFLARE_API_TOKEN",
|
||||
"KV_NAMESPACE_ID",
|
||||
"MODULES",
|
||||
].filter((k) => !process.env[k]);
|
||||
if (missing.length) {
|
||||
console.error(`[backfill-kv] Missing required env vars: ${missing.join(", ")}`);
|
||||
console.error(" Copy .env.deploy.example → .env.deploy and fill in values.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Per-module backfill ──────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Backfill one module from KV into Mongo.
|
||||
*
|
||||
* @param {import("mongodb").Db} db
|
||||
* @param {string} mod — module name (e.g. "wordle", "loldle-emoji")
|
||||
* @returns {Promise<{copied: number, skipped: number, failed: number}>}
|
||||
*/
|
||||
async function backfillModule(db, mod) {
|
||||
const coll = db.collection(toCollName(mod));
|
||||
const prefix = `${mod}:`;
|
||||
const cp = loadCheckpoint(mod);
|
||||
let cursor = cp?.cursor ?? null;
|
||||
let page = 0;
|
||||
let copied = 0;
|
||||
let skipped = 0;
|
||||
let failed = 0;
|
||||
|
||||
if (cp)
|
||||
console.log(
|
||||
`[${mod}] resuming from checkpoint (cursor=${cursor ? cursor.slice(0, 12) : "start"})`,
|
||||
);
|
||||
|
||||
let hasMore = true;
|
||||
while (hasMore) {
|
||||
page++;
|
||||
const {
|
||||
keys,
|
||||
cursor: nextCursor,
|
||||
list_complete,
|
||||
} = await cfKvList(
|
||||
CLOUDFLARE_ACCOUNT_ID,
|
||||
KV_NAMESPACE_ID,
|
||||
CLOUDFLARE_API_TOKEN,
|
||||
prefix,
|
||||
cursor,
|
||||
);
|
||||
|
||||
for (const keyObj of keys) {
|
||||
const key = keyObj.name;
|
||||
// expiresAt: KV metadata.expiration is unix-seconds → convert to JS Date.
|
||||
const expSecs = keyObj.metadata?.expiration;
|
||||
const expiresAt = expSecs ? new Date(expSecs * 1000) : undefined;
|
||||
|
||||
if (dryRun) {
|
||||
copied++; // count as "would copy" in dry-run
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const value = await cfKvGet(
|
||||
CLOUDFLARE_ACCOUNT_ID,
|
||||
KV_NAMESPACE_ID,
|
||||
CLOUDFLARE_API_TOKEN,
|
||||
key,
|
||||
);
|
||||
|
||||
const doc = { value };
|
||||
if (expiresAt) doc.expiresAt = expiresAt;
|
||||
|
||||
// $setOnInsert: skip-if-exists — preserves any newer Mongo state from dual-write.
|
||||
const result = await coll.updateOne({ _id: key }, { $setOnInsert: doc }, { upsert: true });
|
||||
if (result.upsertedCount > 0) {
|
||||
copied++;
|
||||
} else {
|
||||
skipped++;
|
||||
}
|
||||
await sleep(20); // throttle: ~50 ops/sec to stay within Atlas M0 headroom
|
||||
} catch (err) {
|
||||
failed++;
|
||||
console.error(`[${mod}] ERROR key="${key.slice(0, 40)}…": ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Checkpoint after each page so a crash doesn't lose progress.
|
||||
saveCheckpoint(mod, { cursor: nextCursor, lastKey: keys.at(-1)?.name ?? null });
|
||||
|
||||
const verb = dryRun ? "(dry-run)" : `${copied} copied, ${skipped} skipped, ${failed} failed`;
|
||||
console.log(`[${mod}] page ${page}: ${keys.length} keys ${verb}`);
|
||||
|
||||
cursor = nextCursor;
|
||||
hasMore = !list_complete && !!cursor;
|
||||
|
||||
// Brief pause between pages to respect CF REST rate limits (1200 req/5 min).
|
||||
if (hasMore) await sleep(250);
|
||||
}
|
||||
|
||||
if (!dryRun) clearCheckpoint(mod);
|
||||
return { copied, skipped, failed };
|
||||
}
|
||||
|
||||
// ─── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
validateEnv();
|
||||
|
||||
const allModules = MODULES_ENV.split(",")
|
||||
.map((m) => m.trim())
|
||||
.filter(Boolean);
|
||||
const modules = moduleFlag ? [moduleFlag] : allModules;
|
||||
|
||||
if (moduleFlag && !allModules.includes(moduleFlag)) {
|
||||
console.error(
|
||||
`[backfill-kv] Unknown module "${moduleFlag}". Available: ${allModules.join(", ")}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (dryRun) console.log("[backfill-kv] DRY RUN — no writes will be made");
|
||||
console.log(`[backfill-kv] Modules: ${modules.join(", ")}`);
|
||||
|
||||
const client = await getMongoClient(MONGODB_URI);
|
||||
const db = client.db();
|
||||
|
||||
let totalFailed = 0;
|
||||
for (const mod of modules) {
|
||||
const { copied, skipped, failed } = await backfillModule(db, mod);
|
||||
console.log(`[${mod}] DONE — ${copied} copied, ${skipped} skipped, ${failed} failed`);
|
||||
totalFailed += failed;
|
||||
}
|
||||
|
||||
await closeMongoClient();
|
||||
if (totalFailed > 0) {
|
||||
console.error(`[backfill-kv] Completed with ${totalFailed} failed key(s). Check logs above.`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log("[backfill-kv] All modules complete.");
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("[backfill-kv] Fatal:", err.message ?? err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,235 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file backfill-mongo-to-d1 — emergency reverse-backfill: MongoDB → Cloudflare D1.
|
||||
*
|
||||
* Reads trading_trades from Mongo (sorted by legacy_id) and writes them back
|
||||
* into D1 via `wrangler d1 execute --remote --file=<tmp.sql>`.
|
||||
*
|
||||
* Preserves legacy_id when present (written by phase-05 forward backfill).
|
||||
* When legacy_id is absent, generates sequential IDs from max(existing_d1_id)+1.
|
||||
*
|
||||
* Use this ONLY during a Stage-2 rollback (phase-07 debugger #14).
|
||||
* Operator MUST inform users that N days of Mongo-only writes will revert.
|
||||
*
|
||||
* Flags:
|
||||
* --dry-run Print SQL to stdout; do not execute wrangler.
|
||||
* --force Proceed even if D1 trading_trades already has rows.
|
||||
*
|
||||
* Required env (loaded via --env-file-if-exists=.env.deploy):
|
||||
* MONGODB_URI
|
||||
*
|
||||
* Usage:
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-mongo-to-d1.js
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-mongo-to-d1.js --dry-run
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-mongo-to-d1.js --force
|
||||
*/
|
||||
|
||||
import { execSync } from "node:child_process";
|
||||
import { rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { closeMongoClient, getMongoClient } from "./lib/migration-helpers.js";
|
||||
|
||||
const DB_NAME = "miti99bot-db";
|
||||
const COLLECTION = "trading_trades";
|
||||
|
||||
const dryRun = process.argv.includes("--dry-run");
|
||||
const force = process.argv.includes("--force");
|
||||
|
||||
// ─── Preflight ────────────────────────────────────────────────────────────────
|
||||
|
||||
function validateEnv() {
|
||||
const missingVars = [["MONGODB_URI", "Atlas connection string"]].filter(([k]) => !process.env[k]);
|
||||
if (missingVars.length) {
|
||||
for (const [k, desc] of missingVars)
|
||||
console.error(`[backfill-mongo-d1] Missing: ${k} (${desc})`);
|
||||
console.error(" Copy .env.deploy.example → .env.deploy and fill in values.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── D1 helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Execute a single SQL command against remote D1 and return parsed rows.
|
||||
*
|
||||
* @param {string} sql
|
||||
* @returns {any[]}
|
||||
*/
|
||||
function queryD1(sql) {
|
||||
const cmd = `npx wrangler d1 execute ${DB_NAME} --remote --command "${sql.replace(/"/g, '\\"')}" --json`;
|
||||
let stdout;
|
||||
try {
|
||||
stdout = execSync(cmd, { stdio: ["ignore", "pipe", "pipe"] }).toString();
|
||||
} catch (err) {
|
||||
const stderr = /** @type {any} */ (err).stderr?.toString() ?? "";
|
||||
throw new Error(
|
||||
`wrangler d1 execute failed:\n${stderr || /** @type {any} */ (err).stdout?.toString()}`,
|
||||
);
|
||||
}
|
||||
const parsed = JSON.parse(stdout);
|
||||
return Array.isArray(parsed) ? (parsed[0]?.results ?? []) : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a SQL file against remote D1 via wrangler.
|
||||
*
|
||||
* @param {string} filePath — absolute path to .sql file
|
||||
*/
|
||||
function executeD1File(filePath) {
|
||||
const cmd = `npx wrangler d1 execute ${DB_NAME} --remote --file="${filePath}" --json`;
|
||||
try {
|
||||
execSync(cmd, { stdio: ["ignore", "pipe", "pipe"] });
|
||||
} catch (err) {
|
||||
const stderr = /** @type {any} */ (err).stderr?.toString() ?? "";
|
||||
throw new Error(
|
||||
`wrangler d1 execute failed:\n${stderr || /** @type {any} */ (err).stdout?.toString()}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── SQL generation ──────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Escape a SQL string value (single-quote doubling).
|
||||
*
|
||||
* @param {string|null|undefined} v
|
||||
* @returns {string} SQL literal (including surrounding quotes)
|
||||
*/
|
||||
export function sqlStr(v) {
|
||||
if (v == null) return "NULL";
|
||||
return `'${String(v).replace(/'/g, "''")}'`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an INSERT statement for one trading trade row.
|
||||
*
|
||||
* @param {{
|
||||
* id: number,
|
||||
* user_id: string,
|
||||
* symbol: string,
|
||||
* side: string,
|
||||
* qty: number,
|
||||
* price_vnd: number,
|
||||
* ts: number
|
||||
* }} row
|
||||
* @returns {string}
|
||||
*/
|
||||
export function buildInsertSql(row) {
|
||||
return (
|
||||
`INSERT INTO ${COLLECTION} (id, user_id, symbol, side, qty, price_vnd, ts) VALUES (` +
|
||||
[
|
||||
row.id,
|
||||
sqlStr(row.user_id),
|
||||
sqlStr(row.symbol),
|
||||
sqlStr(row.side),
|
||||
row.qty,
|
||||
row.price_vnd,
|
||||
row.ts,
|
||||
].join(", ") +
|
||||
");"
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
validateEnv();
|
||||
|
||||
if (dryRun) console.log("[backfill-mongo-d1] DRY RUN — SQL will be printed to stdout");
|
||||
|
||||
const client = await getMongoClient(/** @type {string} */ (process.env.MONGODB_URI));
|
||||
const db = client.db();
|
||||
const coll = db.collection(COLLECTION);
|
||||
|
||||
// Sort by legacy_id ascending so we maintain original D1 row order.
|
||||
const docs = await coll.find({}).sort({ legacy_id: 1 }).toArray();
|
||||
console.log(`[backfill-mongo-d1] Found ${docs.length} document(s) in Mongo ${COLLECTION}`);
|
||||
|
||||
if (docs.length === 0) {
|
||||
console.log("[backfill-mongo-d1] Nothing to restore.");
|
||||
await closeMongoClient();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
// Pre-flight: abort if D1 already has rows unless --force.
|
||||
const existingRows = queryD1(`SELECT COUNT(*) AS cnt FROM ${COLLECTION}`);
|
||||
const existingCount = existingRows[0]?.cnt ?? 0;
|
||||
if (existingCount > 0 && !force) {
|
||||
console.error(
|
||||
`[backfill-mongo-d1] ABORT: D1 ${COLLECTION} already has ${existingCount} row(s).`,
|
||||
);
|
||||
console.error(" Run with --force to bypass (e.g. after wiping D1 manually).");
|
||||
await closeMongoClient();
|
||||
process.exit(1);
|
||||
}
|
||||
if (existingCount > 0 && force) {
|
||||
console.log(
|
||||
`[backfill-mongo-d1] --force: proceeding despite ${existingCount} existing D1 row(s).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Determine starting ID for docs without a legacy_id.
|
||||
let nextId = 1;
|
||||
const hasLegacyIds = docs.some((d) => d.legacy_id != null);
|
||||
if (!dryRun && !hasLegacyIds) {
|
||||
// Fetch max id from D1 to avoid collisions.
|
||||
const maxRows = queryD1(`SELECT MAX(id) AS m FROM ${COLLECTION}`);
|
||||
const maxId = maxRows[0]?.m ?? 0;
|
||||
nextId = maxId + 1;
|
||||
}
|
||||
|
||||
// Build SQL statements.
|
||||
/** @type {string[]} */
|
||||
const statements = [];
|
||||
for (const doc of docs) {
|
||||
const id = doc.legacy_id != null ? Number(doc.legacy_id) : nextId++;
|
||||
statements.push(
|
||||
buildInsertSql({
|
||||
id,
|
||||
user_id: String(doc.user_id ?? ""),
|
||||
symbol: String(doc.symbol ?? ""),
|
||||
side: String(doc.side ?? ""),
|
||||
qty: Number(doc.qty ?? 0),
|
||||
price_vnd: Number(doc.price_vnd ?? 0),
|
||||
ts: Number(doc.ts ?? 0),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
console.log("[backfill-mongo-d1] Generated SQL:");
|
||||
for (const stmt of statements) console.log(stmt);
|
||||
console.log(`[backfill-mongo-d1] DRY RUN complete — ${statements.length} statement(s).`);
|
||||
await closeMongoClient();
|
||||
return;
|
||||
}
|
||||
|
||||
// Write SQL to a temp file and pipe through wrangler.
|
||||
const tmpFile = join(tmpdir(), `backfill-mongo-d1-${Date.now()}.sql`);
|
||||
try {
|
||||
writeFileSync(tmpFile, statements.join("\n"), "utf8");
|
||||
console.log(`[backfill-mongo-d1] Executing ${statements.length} INSERT(s) via wrangler...`);
|
||||
executeD1File(tmpFile);
|
||||
console.log(`[backfill-mongo-d1] Done — ${statements.length} row(s) restored to D1.`);
|
||||
} finally {
|
||||
try {
|
||||
rmSync(tmpFile);
|
||||
} catch {
|
||||
// Non-fatal cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
await closeMongoClient();
|
||||
}
|
||||
|
||||
// Run only when invoked directly (not when imported by tests).
|
||||
const isMain = process.argv[1]?.endsWith("backfill-mongo-to-d1.js");
|
||||
if (isMain) {
|
||||
main().catch((err) => {
|
||||
console.error("[backfill-mongo-d1] Fatal:", /** @type {Error} */ (err).message ?? err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,201 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file backfill-mongo-to-kv — emergency reverse-backfill: MongoDB → Cloudflare KV.
|
||||
*
|
||||
* Reads each per-module Mongo collection and writes every non-expired doc back
|
||||
* into CF KV via the REST API with correct TTL derived from expiresAt.
|
||||
*
|
||||
* Use this ONLY during a Stage-2 rollback. Operator must inform users that
|
||||
* N days of Mongo-only writes will revert (phase-07 debugger #14).
|
||||
*
|
||||
* Flags:
|
||||
* --dry-run Log summary without writing to KV.
|
||||
* --module <name> Restore only a single module (default: all).
|
||||
*
|
||||
* Required env (loaded via --env-file-if-exists=.env.deploy):
|
||||
* MONGODB_URI, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_API_TOKEN,
|
||||
* KV_NAMESPACE_ID, MODULES (comma-separated)
|
||||
*
|
||||
* Usage:
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-mongo-to-kv.js
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-mongo-to-kv.js --dry-run
|
||||
* node --env-file-if-exists=.env.deploy scripts/backfill-mongo-to-kv.js --module wordle
|
||||
*/
|
||||
|
||||
import { cfKvPut, closeMongoClient, getMongoClient, sleep } from "./lib/migration-helpers.js";
|
||||
|
||||
// ─── Config ───────────────────────────────────────────────────────────────────
|
||||
|
||||
const {
|
||||
MONGODB_URI,
|
||||
CLOUDFLARE_ACCOUNT_ID,
|
||||
CLOUDFLARE_API_TOKEN,
|
||||
KV_NAMESPACE_ID,
|
||||
MODULES: MODULES_ENV,
|
||||
} = process.env;
|
||||
|
||||
const dryRun = process.argv.includes("--dry-run");
|
||||
const moduleFlag = (() => {
|
||||
const idx = process.argv.indexOf("--module");
|
||||
return idx !== -1 ? process.argv[idx + 1] : null;
|
||||
})();
|
||||
|
||||
/** Throttle: 50 ops/sec → 20 ms between writes. */
|
||||
const THROTTLE_MS = 20;
|
||||
|
||||
/** Minimum TTL accepted by CF KV REST API. */
|
||||
const MIN_TTL_SECS = 60;
|
||||
|
||||
/** Normalize module name to Mongo collection name (mirrors mongo-kv-store.js). */
|
||||
const toCollName = (mod) => mod.replace(/-/g, "_");
|
||||
|
||||
// ─── Preflight ────────────────────────────────────────────────────────────────
|
||||
|
||||
function validateEnv() {
|
||||
const missing = [
|
||||
"MONGODB_URI",
|
||||
"CLOUDFLARE_ACCOUNT_ID",
|
||||
"CLOUDFLARE_API_TOKEN",
|
||||
"KV_NAMESPACE_ID",
|
||||
"MODULES",
|
||||
].filter((k) => !process.env[k]);
|
||||
if (missing.length) {
|
||||
console.error(`[backfill-mongo-kv] Missing required env vars: ${missing.join(", ")}`);
|
||||
console.error(" Copy .env.deploy.example → .env.deploy and fill in values.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── TTL computation ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Compute CF KV expirationTtl (seconds from now) from an absolute expiresAt Date.
|
||||
* Returns null if the doc has no TTL (key should be persistent).
|
||||
* Returns undefined (sentinel) if the doc is already expired (key must be SKIPPED).
|
||||
*
|
||||
* @param {Date|null|undefined} expiresAt
|
||||
* @param {number} nowMs — Date.now() at call time (injectable for tests)
|
||||
* @returns {{ ttl: number }|null|"expired"}
|
||||
*/
|
||||
export function computeTtl(expiresAt, nowMs = Date.now()) {
|
||||
if (!expiresAt) return null; // no TTL — write as persistent
|
||||
const remainingMs = expiresAt.getTime() - nowMs;
|
||||
if (remainingMs <= 0) return "expired"; // already past expiry → skip
|
||||
const secs = Math.floor(remainingMs / 1000);
|
||||
return { ttl: Math.max(MIN_TTL_SECS, secs) };
|
||||
}
|
||||
|
||||
// ─── Per-module restore ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Restore one module's Mongo collection back into CF KV.
|
||||
*
|
||||
* @param {import("mongodb").Db} db
|
||||
* @param {string} mod
|
||||
* @returns {Promise<{restored: number, skipped: number, failed: number}>}
|
||||
*/
|
||||
async function restoreModule(db, mod) {
|
||||
const coll = db.collection(toCollName(mod));
|
||||
const docs = await coll.find({}).toArray();
|
||||
|
||||
let restored = 0;
|
||||
let skipped = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const doc of docs) {
|
||||
const key = /** @type {string} */ (doc._id);
|
||||
const value = /** @type {string} */ (doc.value ?? "");
|
||||
const ttlResult = computeTtl(doc.expiresAt ?? null);
|
||||
|
||||
if (ttlResult === "expired") {
|
||||
skipped++;
|
||||
continue; // expired in Mongo → do not surface a stale key in KV
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
restored++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
/** @type {{ expirationTtl?: number }} */
|
||||
const opts = ttlResult ? { expirationTtl: ttlResult.ttl } : {};
|
||||
await cfKvPut(
|
||||
/** @type {string} */ (CLOUDFLARE_ACCOUNT_ID),
|
||||
/** @type {string} */ (KV_NAMESPACE_ID),
|
||||
/** @type {string} */ (CLOUDFLARE_API_TOKEN),
|
||||
key,
|
||||
value,
|
||||
opts,
|
||||
);
|
||||
restored++;
|
||||
await sleep(THROTTLE_MS);
|
||||
} catch (err) {
|
||||
failed++;
|
||||
// Log key hash only — never log plaintext keys (may encode user IDs).
|
||||
const { sha256 } = await import("./lib/migration-helpers.js");
|
||||
console.error(
|
||||
`[${mod}] ERROR key_sha256=${sha256(String(key)).slice(0, 16)}: ${/** @type {Error} */ (err).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return { restored, skipped, failed };
|
||||
}
|
||||
|
||||
// ─── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
validateEnv();
|
||||
|
||||
const allModules = /** @type {string} */ (MODULES_ENV)
|
||||
.split(",")
|
||||
.map((m) => m.trim())
|
||||
.filter(Boolean);
|
||||
const modules = moduleFlag ? [moduleFlag] : allModules;
|
||||
|
||||
if (moduleFlag && !allModules.includes(moduleFlag)) {
|
||||
console.error(
|
||||
`[backfill-mongo-kv] Unknown module "${moduleFlag}". Available: ${allModules.join(", ")}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (dryRun) console.log("[backfill-mongo-kv] DRY RUN — no writes to KV");
|
||||
console.log(`[backfill-mongo-kv] Modules: ${modules.join(", ")}`);
|
||||
|
||||
const client = await getMongoClient(/** @type {string} */ (MONGODB_URI));
|
||||
const db = client.db();
|
||||
|
||||
let totalFailed = 0;
|
||||
for (const mod of modules) {
|
||||
const { restored, skipped, failed } = await restoreModule(db, mod);
|
||||
const verb = dryRun ? "(dry-run)" : `${restored} restored, ${skipped} skipped (expired)`;
|
||||
console.log(`[${mod}] ${docs_label(restored + skipped + failed)} docs: ${verb}`);
|
||||
totalFailed += failed;
|
||||
}
|
||||
|
||||
await closeMongoClient();
|
||||
|
||||
if (totalFailed > 0) {
|
||||
console.error(
|
||||
`[backfill-mongo-kv] Completed with ${totalFailed} failed write(s). Check logs above.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log("[backfill-mongo-kv] All modules restored.");
|
||||
}
|
||||
|
||||
/** @param {number} n @returns {string} */
|
||||
function docs_label(n) {
|
||||
return `${n} doc${n !== 1 ? "s" : ""}`;
|
||||
}
|
||||
|
||||
// Run only when invoked directly (not when imported by tests).
|
||||
const isMain = process.argv[1]?.endsWith("backfill-mongo-to-kv.js");
|
||||
if (isMain) {
|
||||
main().catch((err) => {
|
||||
console.error("[backfill-mongo-kv] Fatal:", /** @type {Error} */ (err).message ?? err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file check-secret-leaks — fails build if any source file logs a known secret.
|
||||
*
|
||||
* Catches the common foot-gun where a developer prints `env.MONGODB_URI` or
|
||||
* similar during debugging and forgets to remove the line before commit. We
|
||||
* are NOT trying to be a full SAST — we just block obvious `console.log` /
|
||||
* `console.error` sites that interpolate a secret env var.
|
||||
*
|
||||
* Wired into `npm run lint` so every PR / pre-deploy run catches it.
|
||||
*
|
||||
* Patterns checked (add more as new secrets are introduced):
|
||||
* - MONGODB_URI — Atlas connection string (Phase 01)
|
||||
* - TELEGRAM_BOT_TOKEN — bot token from BotFather
|
||||
* - TELEGRAM_WEBHOOK_SECRET — gates incoming webhook traffic
|
||||
* - ADMIN_TOKEN — kept for defense-in-depth even though Phase 05
|
||||
* redesign removed admin routes; cheap to leave in.
|
||||
*
|
||||
* Detection scope: any of these tokens appearing on the SAME line as a
|
||||
* `console.<level>(...)`, `JSON.stringify(env)`, or `throw new Error(...env...)`.
|
||||
*
|
||||
* Exit codes:
|
||||
* 0 — no leaks found
|
||||
* 1 — at least one leak detected (prints file:line + offending line)
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
||||
import { extname, join, resolve } from "node:path";
|
||||
|
||||
const PROJECT_ROOT = resolve(import.meta.dirname, "..");
|
||||
const SCAN_DIRS = ["src", "scripts"];
|
||||
const SCAN_EXTS = new Set([".js", ".mjs", ".ts"]);
|
||||
|
||||
const SECRETS = [
|
||||
"MONGODB_URI",
|
||||
"TELEGRAM_BOT_TOKEN",
|
||||
"TELEGRAM_WEBHOOK_SECRET",
|
||||
"ADMIN_TOKEN",
|
||||
// Phase 05: CF API creds used by backfill scripts — defense-in-depth.
|
||||
"CLOUDFLARE_API_TOKEN",
|
||||
"CLOUDFLARE_ACCOUNT_ID",
|
||||
];
|
||||
|
||||
// A line is suspicious if it both names a secret AND looks like it's emitting
|
||||
// the value (console.*, JSON.stringify(env...), throw with interpolation).
|
||||
const EMIT_PATTERNS = [
|
||||
/\bconsole\.(log|info|warn|error|debug|trace)\b/,
|
||||
/\bJSON\.stringify\s*\(\s*env\b/,
|
||||
/\bthrow\s+new\s+\w*Error\b/,
|
||||
];
|
||||
|
||||
/**
|
||||
* Walk a directory tree and yield absolute file paths matching SCAN_EXTS.
|
||||
*
|
||||
* @param {string} dir
|
||||
* @returns {string[]}
|
||||
*/
|
||||
function walk(dir) {
|
||||
if (!existsSync(dir)) return [];
|
||||
const out = [];
|
||||
for (const entry of readdirSync(dir)) {
|
||||
const full = join(dir, entry);
|
||||
const st = statSync(full);
|
||||
if (st.isDirectory()) {
|
||||
if (entry === "node_modules" || entry === ".wrangler") continue;
|
||||
out.push(...walk(full));
|
||||
} else if (SCAN_EXTS.has(extname(entry))) {
|
||||
out.push(full);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan one file. Pushes hits to `findings`.
|
||||
*
|
||||
* @param {string} file
|
||||
* @param {Array<{file: string, line: number, secret: string, snippet: string}>} findings
|
||||
*/
|
||||
function scanFile(file, findings) {
|
||||
// Don't flag this file (it lists the patterns) or .example files.
|
||||
if (file.endsWith("check-secret-leaks.js")) return;
|
||||
|
||||
const content = readFileSync(file, "utf8");
|
||||
const lines = content.split("\n");
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (!EMIT_PATTERNS.some((p) => p.test(line))) continue;
|
||||
for (const secret of SECRETS) {
|
||||
if (line.includes(secret)) {
|
||||
findings.push({
|
||||
file,
|
||||
line: i + 1,
|
||||
secret,
|
||||
snippet: line.trim(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
/** @type {Array<{file: string, line: number, secret: string, snippet: string}>} */
|
||||
const findings = [];
|
||||
|
||||
for (const dir of SCAN_DIRS) {
|
||||
const abs = join(PROJECT_ROOT, dir);
|
||||
for (const file of walk(abs)) scanFile(file, findings);
|
||||
}
|
||||
|
||||
if (findings.length === 0) {
|
||||
console.log(`secret-leak check: 0 findings across ${SCAN_DIRS.join(", ")}`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(`secret-leak check: ${findings.length} finding(s)`);
|
||||
for (const f of findings) {
|
||||
const rel = f.file.replace(`${PROJECT_ROOT}/`, "");
|
||||
console.error(` ${rel}:${f.line} [${f.secret}] ${f.snippet}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -1,229 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file migration-helpers — shared utilities for Phase 05 backfill scripts.
|
||||
*
|
||||
* Exports:
|
||||
* - sleep, sha256
|
||||
* - loadCheckpoint, saveCheckpoint, clearCheckpoint
|
||||
* - cfKvList, cfKvGet
|
||||
* - getMongoClient, closeMongoClient
|
||||
* - countDiffRatio, sampleStrategy
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
const CF_API_BASE = "https://api.cloudflare.com/client/v4";
|
||||
|
||||
// ─── Primitives ───────────────────────────────────────────────────────────────
|
||||
|
||||
/** @param {number} ms @returns {Promise<void>} */
|
||||
export const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
|
||||
|
||||
/** @param {string} text @returns {string} hex SHA-256 */
|
||||
export const sha256 = (text) => createHash("sha256").update(text, "utf8").digest("hex");
|
||||
|
||||
// ─── Checkpoint ───────────────────────────────────────────────────────────────
|
||||
|
||||
const cpPath = (mod) => resolve(process.cwd(), `.backfill-cursor-${mod}.json`);
|
||||
|
||||
/**
|
||||
* Load saved cursor state for a module. Returns null if none.
|
||||
*
|
||||
* @param {string} moduleName
|
||||
* @returns {{ cursor: string|null, lastKey: string|null }|null}
|
||||
*/
|
||||
export function loadCheckpoint(moduleName) {
|
||||
const p = cpPath(moduleName);
|
||||
if (!existsSync(p)) return null;
|
||||
try {
|
||||
return JSON.parse(readFileSync(p, "utf8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist cursor state so a crash can resume from this page.
|
||||
*
|
||||
* @param {string} moduleName
|
||||
* @param {{ cursor: string|null, lastKey: string|null }} state
|
||||
*/
|
||||
export function saveCheckpoint(moduleName, state) {
|
||||
writeFileSync(cpPath(moduleName), JSON.stringify(state), "utf8");
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove checkpoint file on successful module completion.
|
||||
*
|
||||
* @param {string} moduleName
|
||||
*/
|
||||
export function clearCheckpoint(moduleName) {
|
||||
const p = cpPath(moduleName);
|
||||
if (existsSync(p)) {
|
||||
try {
|
||||
unlinkSync(p);
|
||||
} catch {
|
||||
// Non-fatal — may already be gone.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── CF KV REST ───────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* List one page of KV keys for a given prefix.
|
||||
*
|
||||
* @param {string} accountId
|
||||
* @param {string} nsId
|
||||
* @param {string} token
|
||||
* @param {string} prefix
|
||||
* @param {string|null} cursor
|
||||
* @returns {Promise<{
|
||||
* keys: Array<{name: string, metadata?: {expiration?: number}}>,
|
||||
* cursor: string|null,
|
||||
* list_complete: boolean
|
||||
* }>}
|
||||
*/
|
||||
export async function cfKvList(accountId, nsId, token, prefix, cursor) {
|
||||
const url = new URL(`${CF_API_BASE}/accounts/${accountId}/storage/kv/namespaces/${nsId}/keys`);
|
||||
url.searchParams.set("prefix", prefix);
|
||||
url.searchParams.set("limit", "1000");
|
||||
if (cursor) url.searchParams.set("cursor", cursor);
|
||||
|
||||
const res = await fetch(url.toString(), {
|
||||
headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
|
||||
});
|
||||
if (!res.ok) {
|
||||
const body = await res.text().catch(() => "");
|
||||
throw new Error(`CF KV list ${res.status}: ${body}`);
|
||||
}
|
||||
const json = await res.json();
|
||||
if (!json.success) throw new Error(`CF KV list error: ${JSON.stringify(json.errors ?? json)}`);
|
||||
return {
|
||||
keys: json.result ?? [],
|
||||
cursor: json.result_info?.cursor ?? null,
|
||||
list_complete: !json.result_info?.cursor,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the string value for a single KV key.
|
||||
*
|
||||
* @param {string} accountId
|
||||
* @param {string} nsId
|
||||
* @param {string} token
|
||||
* @param {string} key
|
||||
* @returns {Promise<string>}
|
||||
*/
|
||||
export async function cfKvGet(accountId, nsId, token, key) {
|
||||
const url = `${CF_API_BASE}/accounts/${accountId}/storage/kv/namespaces/${nsId}/values/${encodeURIComponent(key)}`;
|
||||
const res = await fetch(url, { headers: { Authorization: `Bearer ${token}` } });
|
||||
if (!res.ok) {
|
||||
const body = await res.text().catch(() => "");
|
||||
throw new Error(`CF KV get "${key}" ${res.status}: ${body}`);
|
||||
}
|
||||
return res.text();
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a string value into CF KV via the REST API.
|
||||
*
|
||||
* CF KV REST PUT: PUT /accounts/{id}/storage/kv/namespaces/{nsid}/values/{key}
|
||||
* Optional query param `expiration_ttl` (seconds from now, minimum 60).
|
||||
*
|
||||
* @param {string} accountId
|
||||
* @param {string} nsId
|
||||
* @param {string} token
|
||||
* @param {string} key
|
||||
* @param {string} value
|
||||
* @param {{ expirationTtl?: number }} [opts]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function cfKvPut(accountId, nsId, token, key, value, opts = {}) {
|
||||
const url = new URL(
|
||||
`${CF_API_BASE}/accounts/${accountId}/storage/kv/namespaces/${nsId}/values/${encodeURIComponent(key)}`,
|
||||
);
|
||||
if (opts.expirationTtl != null) {
|
||||
url.searchParams.set("expiration_ttl", String(opts.expirationTtl));
|
||||
}
|
||||
const res = await fetch(url.toString(), {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "text/plain",
|
||||
},
|
||||
body: value,
|
||||
});
|
||||
if (!res.ok) {
|
||||
const body = await res.text().catch(() => "");
|
||||
throw new Error(`CF KV put "${key}" ${res.status}: ${body}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── MongoDB singleton ────────────────────────────────────────────────────────
|
||||
|
||||
/** @type {MongoClient|null} */
|
||||
let _client = null;
|
||||
|
||||
/**
|
||||
* Return a shared MongoClient, connecting on first call.
|
||||
* Registers process exit / signal handlers to close cleanly.
|
||||
*
|
||||
* @param {string} uri
|
||||
* @returns {Promise<MongoClient>}
|
||||
*/
|
||||
export async function getMongoClient(uri) {
|
||||
if (!uri) throw new Error("MongoDB URI is required — set it in .env.deploy and re-run");
|
||||
if (_client) return _client;
|
||||
const client = new MongoClient(uri);
|
||||
await client.connect();
|
||||
_client = client;
|
||||
const close = () => {
|
||||
_client?.close().catch(() => {});
|
||||
_client = null;
|
||||
};
|
||||
process.once("exit", close);
|
||||
process.once("SIGINT", () => {
|
||||
close();
|
||||
process.exit(0);
|
||||
});
|
||||
process.once("SIGTERM", () => {
|
||||
close();
|
||||
process.exit(0);
|
||||
});
|
||||
return _client;
|
||||
}
|
||||
|
||||
/** Close the shared client (test teardown / forced close). */
|
||||
export async function closeMongoClient() {
|
||||
if (_client) {
|
||||
await _client.close();
|
||||
_client = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Parity math ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Relative difference between two counts — 0 means identical.
|
||||
*
|
||||
* @param {number} a
|
||||
* @param {number} b
|
||||
* @returns {number} value in [0, 1]
|
||||
*/
|
||||
export const countDiffRatio = (a, b) => Math.abs(a - b) / Math.max(a, b, 1);
|
||||
|
||||
/**
|
||||
* Determine verify strategy: full-scan when total < 10 000,
|
||||
* otherwise random-sample N = min(500, ceil(sqrt(total))).
|
||||
*
|
||||
* @param {number} total
|
||||
* @returns {{ fullScan: boolean, sampleSize: number }}
|
||||
*/
|
||||
export function sampleStrategy(total) {
|
||||
if (total < 10000) return { fullScan: true, sampleSize: total };
|
||||
return { fullScan: false, sampleSize: Math.min(500, Math.ceil(Math.sqrt(total))) };
|
||||
}
|
||||
+2
-12
@@ -19,7 +19,7 @@
|
||||
*/
|
||||
|
||||
import { buildRegistry, resetRegistry } from "../src/modules/registry.js";
|
||||
import { STUB_SENTINEL, stubAi, stubKv } from "./stub-kv.js";
|
||||
import { stubAi, stubKv } from "./stub-kv.js";
|
||||
|
||||
const TELEGRAM_API = "https://api.telegram.org";
|
||||
|
||||
@@ -71,18 +71,8 @@ async function main() {
|
||||
|
||||
// Build the registry against the same code the Worker uses. Stub KV
|
||||
// satisfies the binding so createStore() does not throw.
|
||||
// MONGODB_URI = STUB_SENTINEL ensures create-store / create-sql-store
|
||||
// factories short-circuit before constructing any MongoClient.
|
||||
// STORAGE_PRIMARY + DUAL_WRITE lock the factories to the CF-only path.
|
||||
resetRegistry();
|
||||
const reg = await buildRegistry({
|
||||
MODULES: modules,
|
||||
KV: stubKv,
|
||||
AI: stubAi,
|
||||
MONGODB_URI: STUB_SENTINEL,
|
||||
STORAGE_PRIMARY: "kv",
|
||||
DUAL_WRITE: "0",
|
||||
});
|
||||
const reg = await buildRegistry({ MODULES: modules, KV: stubKv, AI: stubAi });
|
||||
|
||||
const commands = [...reg.publicCommands.values()].map(({ cmd }) => ({
|
||||
command: cmd.name,
|
||||
|
||||
@@ -8,14 +8,6 @@
|
||||
* are assumed read-only (or tolerant of missing state) at registration time.
|
||||
* If a future module writes inside init(), update the matching stub to
|
||||
* swallow writes safely.
|
||||
*
|
||||
* `STUB_SENTINEL` is passed as `env.MONGODB_URI` so the create-store /
|
||||
* create-sql-store factories short-circuit BEFORE constructing any MongoClient.
|
||||
* This ensures zero Atlas connections during `npm run register:dry`.
|
||||
*
|
||||
* `stubMongo` is a duck-typed no-op that satisfies the MongoClient surface
|
||||
* used by MongoKVStore. It is NOT a real MongoClient instance. It must never
|
||||
* be used in production — sentinel check in the factories prevents that.
|
||||
*/
|
||||
|
||||
/** @type {KVNamespace} */
|
||||
@@ -52,33 +44,3 @@ export const stubAi = {
|
||||
return { data: [] };
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Sentinel value passed as `env.MONGODB_URI` during deploy-time registry
|
||||
* builds. Factories check for this value FIRST and return CF-only stores
|
||||
* immediately — no MongoClient is ever constructed.
|
||||
*
|
||||
* @type {string}
|
||||
*/
|
||||
export const STUB_SENTINEL = "__stub_mongo__";
|
||||
|
||||
/**
|
||||
* Duck-typed no-op MongoClient surface. Satisfies the shape used by
|
||||
* MongoKVStore / MongoTradesStore without performing any network IO.
|
||||
* Used only when `env.MONGODB_URI === STUB_SENTINEL`.
|
||||
*/
|
||||
export const stubMongo = {
|
||||
db() {
|
||||
return {
|
||||
collection() {
|
||||
throw new Error("stubMongo: no IO at deploy time");
|
||||
},
|
||||
};
|
||||
},
|
||||
async connect() {
|
||||
return undefined;
|
||||
},
|
||||
async close() {
|
||||
return undefined;
|
||||
},
|
||||
};
|
||||
@@ -1,148 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file synthetic-burst — fire N parallel requests at the deployed Worker URL
|
||||
* to exercise the M0 Atlas connection cap before a live deploy.
|
||||
*
|
||||
* Each request is a synthetic Telegram webhook update POST that grammY will
|
||||
* route to the specified command handler. All requests hit the Worker
|
||||
* simultaneously (Promise.all) to maximise cold-isolate spawning.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/synthetic-burst.js \
|
||||
* --url https://miti99bot.workers.dev \
|
||||
* --secret <X-Telegram-Bot-Api-Secret-Token> \
|
||||
* [--n 20] \
|
||||
* [--cmd /wordle]
|
||||
*
|
||||
* No unit tests for this script — it is network-touching by design and only
|
||||
* runs against a live deployed Worker. Tested manually pre-deploy.
|
||||
*
|
||||
* Abort guideline (debugger #2): if Atlas connection peak > 300/500 (60% cap),
|
||||
* do NOT proceed with live deploy. Check Atlas UI during the 60s after the burst.
|
||||
*/
|
||||
|
||||
// ── CLI arg parsing ──────────────────────────────────────────────────────────
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = { n: 20, cmd: "/wordle" };
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
if (argv[i] === "--url") args.url = argv[++i];
|
||||
else if (argv[i] === "--secret") args.secret = argv[++i];
|
||||
else if (argv[i] === "--n") args.n = Number.parseInt(argv[++i], 10);
|
||||
else if (argv[i] === "--cmd") args.cmd = argv[++i];
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
// ── Synthetic Telegram update payload ───────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Build a minimal but valid grammY-shaped Telegram Update object for a
|
||||
* bot_command message.
|
||||
*
|
||||
* @param {string} cmd - e.g. "/wordle"
|
||||
* @param {number} index - used to differentiate update_id + message_id values
|
||||
* @returns {object}
|
||||
*/
|
||||
function buildUpdate(cmd, index) {
|
||||
return {
|
||||
update_id: 100000 + index,
|
||||
message: {
|
||||
message_id: 200000 + index,
|
||||
from: { id: 1, is_bot: false, first_name: "Burst" },
|
||||
chat: { id: 1, type: "private" },
|
||||
date: Math.floor(Date.now() / 1000),
|
||||
text: cmd,
|
||||
entities: [{ type: "bot_command", offset: 0, length: cmd.length }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Single request ───────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* POST one synthetic update to the Worker webhook endpoint.
|
||||
*
|
||||
* @param {string} url - Worker base URL
|
||||
* @param {string} secret - X-Telegram-Bot-Api-Secret-Token value
|
||||
* @param {object} update - Telegram Update payload
|
||||
* @param {number} index - request index for logging
|
||||
* @returns {Promise<{ index: number, status: number, ms: number, error?: string }>}
|
||||
*/
|
||||
async function sendUpdate(url, secret, update, index) {
|
||||
const t0 = Date.now();
|
||||
const endpoint = url.replace(/\/$/, "") + "/webhook";
|
||||
|
||||
try {
|
||||
const res = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"X-Telegram-Bot-Api-Secret-Token": secret,
|
||||
},
|
||||
body: JSON.stringify(update),
|
||||
});
|
||||
return { index, status: res.status, ms: Date.now() - t0 };
|
||||
} catch (err) {
|
||||
return { index, status: 0, ms: Date.now() - t0, error: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
|
||||
if (!args.url || !args.secret) {
|
||||
console.error(
|
||||
"Usage: node scripts/synthetic-burst.js --url <url> --secret <token> [--n 20] [--cmd /wordle]",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const { url, secret, n, cmd } = args;
|
||||
|
||||
console.log(`Burst: ${n} parallel requests → ${url}/webhook cmd=${cmd}`);
|
||||
console.log("Starting at", new Date().toISOString());
|
||||
|
||||
const requests = Array.from({ length: n }, (_, i) =>
|
||||
sendUpdate(url, secret, buildUpdate(cmd, i), i),
|
||||
);
|
||||
|
||||
const results = await Promise.all(requests);
|
||||
|
||||
// ── Summary ──────────────────────────────────────────────────────────────
|
||||
let ok = 0;
|
||||
let fail = 0;
|
||||
let totalMs = 0;
|
||||
const statusCounts = {};
|
||||
|
||||
for (const r of results) {
|
||||
const statusKey = r.status === 0 ? "network-error" : String(r.status);
|
||||
statusCounts[statusKey] = (statusCounts[statusKey] ?? 0) + 1;
|
||||
totalMs += r.ms;
|
||||
if (r.status >= 200 && r.status < 300) ok++;
|
||||
else fail++;
|
||||
|
||||
// Log individual result.
|
||||
const tag = r.error ? `ERROR(${r.error})` : `HTTP ${r.status}`;
|
||||
console.log(` [${r.index}] ${tag} ${r.ms}ms`);
|
||||
}
|
||||
|
||||
const avgMs = Math.round(totalMs / n);
|
||||
const allMs = results.map((r) => r.ms).sort((a, b) => a - b);
|
||||
const p50 = allMs[Math.floor(allMs.length * 0.5)];
|
||||
const p95 = allMs[Math.floor(allMs.length * 0.95)];
|
||||
|
||||
console.log("\n── Summary ───────────────────────────────────────────");
|
||||
console.log(` Requests: ${n} | OK: ${ok} | Failed: ${fail}`);
|
||||
console.log(` Status counts: ${JSON.stringify(statusCounts)}`);
|
||||
console.log(` Latency — avg: ${avgMs}ms p50: ${p50}ms p95: ${p95}ms`);
|
||||
console.log("\nNext: check Atlas UI connection counter within 60s.");
|
||||
console.log("Abort if peak connections > 300/500 (60% of M0 cap).");
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,215 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file verify-mongo-parity — cross-check KV / D1 data against MongoDB Atlas.
|
||||
*
|
||||
* Per KV module: count via CF REST vs Mongo countDocuments (±1% tolerance),
|
||||
* then value-compare via SHA256. Full-scan when total < 10 000; random-sample
|
||||
* N = min(500, ceil(sqrt(N))) otherwise. expiresAt checked within ±5 min.
|
||||
* For trading_trades: full-scan on legacy_id, ts, user_id, symbol, qty.
|
||||
*
|
||||
* Flags: --dry-run (counts only, no value compare)
|
||||
* Required env: MONGODB_URI, CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_API_TOKEN,
|
||||
* KV_NAMESPACE_ID, MODULES
|
||||
* Exit: 0 all-pass | 1 any failure
|
||||
*/
|
||||
|
||||
import { execSync } from "node:child_process";
|
||||
import {
|
||||
cfKvGet,
|
||||
cfKvList,
|
||||
closeMongoClient,
|
||||
countDiffRatio,
|
||||
getMongoClient,
|
||||
sampleStrategy,
|
||||
sha256,
|
||||
sleep,
|
||||
} from "./lib/migration-helpers.js";
|
||||
|
||||
const DB_NAME = "miti99bot-db";
|
||||
const EXPIRES_TOL_MS = 5 * 60 * 1000; // ±5 minutes
|
||||
const dryRun = process.argv.includes("--dry-run");
|
||||
const {
|
||||
MONGODB_URI,
|
||||
CLOUDFLARE_ACCOUNT_ID,
|
||||
CLOUDFLARE_API_TOKEN,
|
||||
KV_NAMESPACE_ID,
|
||||
MODULES: MODULES_ENV,
|
||||
} = process.env;
|
||||
const toCollName = (mod) => mod.replace(/-/g, "_");
|
||||
const redactKey = (k) => {
|
||||
const c = k.indexOf(":");
|
||||
return `${c >= 0 ? k.slice(0, c + 1) : ""}sha256:${sha256(k).slice(0, 8)}…`;
|
||||
};
|
||||
|
||||
function validateEnv() {
|
||||
const missing = [
|
||||
"MONGODB_URI",
|
||||
"CLOUDFLARE_ACCOUNT_ID",
|
||||
"CLOUDFLARE_API_TOKEN",
|
||||
"KV_NAMESPACE_ID",
|
||||
"MODULES",
|
||||
].filter((k) => !process.env[k]);
|
||||
if (missing.length) {
|
||||
console.error(`[verify] Missing env vars: ${missing.join(", ")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
function queryD1(sql) {
|
||||
const cmd = `npx wrangler d1 execute ${DB_NAME} --remote --command "${sql.replace(/"/g, '\\"')}" --json`;
|
||||
const parsed = JSON.parse(execSync(cmd, { stdio: ["ignore", "pipe", "pipe"] }).toString());
|
||||
return Array.isArray(parsed) ? (parsed[0]?.results ?? []) : [];
|
||||
}
|
||||
|
||||
function shuffle(arr) {
|
||||
const a = [...arr];
|
||||
for (let i = a.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
[a[i], a[j]] = [a[j], a[i]];
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
/** Enumerate ALL key objects for a module prefix via paginated CF REST. */
|
||||
async function allKvKeys(mod) {
|
||||
const keys = [];
|
||||
let cursor = null;
|
||||
do {
|
||||
const page = await cfKvList(
|
||||
CLOUDFLARE_ACCOUNT_ID,
|
||||
KV_NAMESPACE_ID,
|
||||
CLOUDFLARE_API_TOKEN,
|
||||
`${mod}:`,
|
||||
cursor,
|
||||
);
|
||||
keys.push(...page.keys);
|
||||
cursor = page.list_complete ? null : page.cursor;
|
||||
if (cursor) await sleep(250);
|
||||
} while (cursor);
|
||||
return keys;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare one KV key against its Mongo doc.
|
||||
* Returns null on match, mismatch description string on failure.
|
||||
*
|
||||
* @param {import("mongodb").Collection} coll
|
||||
* @param {{ name: string, metadata?: { expiration?: number } }} keyObj
|
||||
* @returns {Promise<string|null>}
|
||||
*/
|
||||
async function compareKey(coll, keyObj) {
|
||||
const value = await cfKvGet(
|
||||
CLOUDFLARE_ACCOUNT_ID,
|
||||
KV_NAMESPACE_ID,
|
||||
CLOUDFLARE_API_TOKEN,
|
||||
keyObj.name,
|
||||
);
|
||||
await sleep(5);
|
||||
const doc = await coll.findOne({ _id: keyObj.name });
|
||||
if (!doc) return "missing in Mongo";
|
||||
if (sha256(value) !== sha256(doc.value ?? "")) return "value hash mismatch";
|
||||
const kvExpMs = keyObj.metadata?.expiration ? keyObj.metadata.expiration * 1000 : null;
|
||||
const mgExpMs = doc.expiresAt ? new Date(doc.expiresAt).getTime() : null;
|
||||
if ((kvExpMs === null) !== (mgExpMs === null)) return "expiresAt presence mismatch";
|
||||
if (kvExpMs !== null && Math.abs(kvExpMs - mgExpMs) > EXPIRES_TOL_MS)
|
||||
return `expiresAt diff ${Math.abs(kvExpMs - mgExpMs)}ms`;
|
||||
return null;
|
||||
}
|
||||
|
||||
async function verifyModule(db, mod) {
|
||||
const coll = db.collection(toCollName(mod));
|
||||
const kvKeys = await allKvKeys(mod);
|
||||
const nKv = kvKeys.length;
|
||||
const nMongo = await coll.countDocuments();
|
||||
const ratio = countDiffRatio(nKv, nMongo);
|
||||
const countOk = ratio < 0.01;
|
||||
console.log(
|
||||
`[${mod}] KV=${nKv} Mongo=${nMongo} diff=${(ratio * 100).toFixed(2)}% ${countOk ? "OK" : "FAIL"}`,
|
||||
);
|
||||
if (!countOk)
|
||||
return { pass: false, mismatches: [`count diff ${(ratio * 100).toFixed(2)}% > 1%`] };
|
||||
if (dryRun) return { pass: true, mismatches: [] };
|
||||
|
||||
const { fullScan, sampleSize } = sampleStrategy(nKv);
|
||||
const sample = fullScan ? kvKeys : shuffle(kvKeys).slice(0, sampleSize);
|
||||
console.log(`[${mod}] ${fullScan ? "full-scan" : `sample ${sampleSize}/${nKv}`}`);
|
||||
|
||||
const mismatches = [];
|
||||
for (const keyObj of sample) {
|
||||
const m = await compareKey(coll, keyObj);
|
||||
if (m) mismatches.push(`${redactKey(keyObj.name)}: ${m}`);
|
||||
}
|
||||
console.log(
|
||||
`[${mod}] value compare: ${mismatches.length === 0 ? "PASS" : `${mismatches.length} mismatch(es)`}`,
|
||||
);
|
||||
return { pass: mismatches.length === 0, mismatches };
|
||||
}
|
||||
|
||||
async function verifyTrading(db) {
|
||||
const coll = db.collection("trading_trades");
|
||||
const rows = queryD1(
|
||||
"SELECT id, user_id, symbol, side, qty, price_vnd, ts FROM trading_trades ORDER BY id",
|
||||
);
|
||||
const nMongo = await coll.countDocuments();
|
||||
const ratio = countDiffRatio(rows.length, nMongo);
|
||||
const countOk = ratio < 0.01;
|
||||
console.log(
|
||||
`[trading] D1=${rows.length} Mongo=${nMongo} diff=${(ratio * 100).toFixed(2)}% ${countOk ? "OK" : "FAIL"}`,
|
||||
);
|
||||
if (!countOk) return { pass: false, mismatches: [`count diff ${(ratio * 100).toFixed(2)}%`] };
|
||||
if (dryRun) return { pass: true, mismatches: [] };
|
||||
|
||||
const mismatches = [];
|
||||
for (const row of rows) {
|
||||
const doc = await coll.findOne({ legacy_id: row.id });
|
||||
if (!doc) {
|
||||
mismatches.push(`legacy_id=${row.id} missing`);
|
||||
continue;
|
||||
}
|
||||
for (const f of ["user_id", "symbol", "qty", "ts"]) {
|
||||
if (String(doc[f]) !== String(row[f])) mismatches.push(`legacy_id=${row.id} ${f} mismatch`);
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
`[trading] full-scan: ${mismatches.length === 0 ? "PASS" : `${mismatches.length} mismatch(es)`}`,
|
||||
);
|
||||
return { pass: mismatches.length === 0, mismatches };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
validateEnv();
|
||||
if (dryRun) console.log("[verify] DRY RUN — counts only");
|
||||
const modules = MODULES_ENV.split(",")
|
||||
.map((m) => m.trim())
|
||||
.filter(Boolean);
|
||||
const db = (await getMongoClient(MONGODB_URI)).db();
|
||||
|
||||
const report = [];
|
||||
let anyFail = false;
|
||||
for (const mod of modules) {
|
||||
const r = await verifyModule(db, mod);
|
||||
report.push({ mod, ...r });
|
||||
if (!r.pass) anyFail = true;
|
||||
}
|
||||
const tr = await verifyTrading(db);
|
||||
report.push({ mod: "trading", ...tr });
|
||||
if (!tr.pass) anyFail = true;
|
||||
|
||||
await closeMongoClient();
|
||||
console.log("\n── Parity Report ─────────────────────────────");
|
||||
for (const r of report) {
|
||||
console.log(` ${r.pass ? "PASS" : "FAIL"} ${r.mod}`);
|
||||
for (const m of r.mismatches) console.log(` mismatch: ${m}`);
|
||||
}
|
||||
console.log("──────────────────────────────────────────────");
|
||||
if (anyFail) {
|
||||
console.error("[verify] FAILED.");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log("[verify] All modules PASS.");
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("[verify] Fatal:", err.message ?? err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,94 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file wipe-mongo — rollback helper: delete all documents from every backfill
|
||||
* collection in MongoDB Atlas.
|
||||
*
|
||||
* WARNING: THIS IS IRREVERSIBLE. Run only when you want to start the backfill
|
||||
* from scratch (e.g. after discovering a systematic mapping bug).
|
||||
*
|
||||
* Flags:
|
||||
* --yes Skip the interactive confirmation prompt (for CI / scripted rollback).
|
||||
* Even with --yes, prints a loud warning so logs capture intent.
|
||||
*
|
||||
* Required env: MONGODB_URI, MODULES (comma-separated KV module names)
|
||||
*
|
||||
* Usage:
|
||||
* node --env-file-if-exists=.env.deploy scripts/wipe-mongo.js
|
||||
* node --env-file-if-exists=.env.deploy scripts/wipe-mongo.js --yes
|
||||
*/
|
||||
|
||||
import { createInterface } from "node:readline";
|
||||
import { closeMongoClient, getMongoClient } from "./lib/migration-helpers.js";
|
||||
|
||||
const { MONGODB_URI, MODULES: MODULES_ENV } = process.env;
|
||||
const skipPrompt = process.argv.includes("--yes");
|
||||
|
||||
function validateEnv() {
|
||||
const needed = { MONGODB_URI, MODULES: MODULES_ENV };
|
||||
const missing = Object.entries(needed)
|
||||
.filter(([, v]) => !v)
|
||||
.map(([k]) => k);
|
||||
if (missing.length) {
|
||||
console.error(`[wipe-mongo] Missing required env vars: ${missing.join(", ")}`);
|
||||
console.error(" Copy .env.deploy.example to .env.deploy and fill in values.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/** @param {string} question @returns {Promise<string>} */
|
||||
function prompt(question) {
|
||||
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
||||
return new Promise((resolve) => {
|
||||
rl.question(question, (answer) => {
|
||||
rl.close();
|
||||
resolve(answer);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
validateEnv();
|
||||
|
||||
// Build the list of collections: one per KV module + trading_trades.
|
||||
const kvModules = MODULES_ENV.split(",")
|
||||
.map((m) => m.trim())
|
||||
.filter(Boolean);
|
||||
const collections = [
|
||||
...kvModules.map((m) => m.replace(/-/g, "_")), // mirrors mongo-kv-store.js normalization
|
||||
"trading_trades",
|
||||
];
|
||||
|
||||
console.error("╔══════════════════════════════════════════════════════╗");
|
||||
console.error("║ WARNING: wipe-mongo will DELETE ALL DOCUMENTS from ║");
|
||||
console.error("║ the Atlas database and CANNOT BE UNDONE. ║");
|
||||
console.error("╚══════════════════════════════════════════════════════╝");
|
||||
console.log(`Collections to wipe (${collections.length}): ${collections.join(", ")}`);
|
||||
|
||||
if (!skipPrompt) {
|
||||
const answer = await prompt("\nType CONFIRM to wipe Atlas database miti99bot: ");
|
||||
if (answer.trim() !== "CONFIRM") {
|
||||
console.log("Aborted — nothing was deleted.");
|
||||
process.exit(0);
|
||||
}
|
||||
} else {
|
||||
console.error("[wipe-mongo] --yes passed: skipping interactive prompt. Proceeding with wipe.");
|
||||
}
|
||||
|
||||
const client = await getMongoClient(MONGODB_URI);
|
||||
const db = client.db();
|
||||
|
||||
let totalDeleted = 0;
|
||||
for (const name of collections) {
|
||||
const result = await db.collection(name).deleteMany({});
|
||||
console.log(`[wipe-mongo] ${name}: deleted ${result.deletedCount} document(s)`);
|
||||
totalDeleted += result.deletedCount;
|
||||
}
|
||||
|
||||
await closeMongoClient();
|
||||
console.log(`[wipe-mongo] Done — ${totalDeleted} total document(s) removed.`);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("[wipe-mongo] Fatal:", err.message ?? err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,59 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# wrangler-delete-guard.sh — interactive CONFIRM wrapper around irreversible
|
||||
# wrangler delete commands. NEVER use in CI.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/wrangler-delete-guard.sh kv <namespace-id>
|
||||
# bash scripts/wrangler-delete-guard.sh d1 <database-name>
|
||||
#
|
||||
# Exits:
|
||||
# 0 — command executed successfully
|
||||
# 1 — user aborted (did not type CONFIRM)
|
||||
# 2 — bad arguments
|
||||
# 3 — stdin not a tty (CI safety)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -lt 2 ]]; then
|
||||
echo "usage: $0 {kv|d1} <id-or-name>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
KIND="$1"
|
||||
TARGET="$2"
|
||||
|
||||
case "$KIND" in
|
||||
kv)
|
||||
DESC="KV namespace $TARGET"
|
||||
CMD=(npx wrangler kv namespace delete --namespace-id "$TARGET")
|
||||
;;
|
||||
d1)
|
||||
DESC="D1 database $TARGET"
|
||||
CMD=(npx wrangler d1 delete "$TARGET")
|
||||
;;
|
||||
*)
|
||||
echo "unknown kind: $KIND (expected 'kv' or 'd1')" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
# Refuse to run non-interactively — prevents accidental CI execution.
|
||||
if [[ ! -t 0 ]]; then
|
||||
echo "stdin not a tty — refusing to run non-interactively (CI safety)" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "ABOUT TO DELETE: $DESC"
|
||||
echo "This is IRREVERSIBLE. Backup files must already be on local disk."
|
||||
echo ""
|
||||
read -r -p "Type CONFIRM to proceed: " CONFIRM
|
||||
|
||||
if [[ "$CONFIRM" != "CONFIRM" ]]; then
|
||||
echo "aborted" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "executing: ${CMD[*]}"
|
||||
"${CMD[@]}"
|
||||
@@ -1,256 +0,0 @@
|
||||
/**
|
||||
* @file drift-verifier — hourly cron handler that maintains dual-write health.
|
||||
*
|
||||
* Two responsibilities:
|
||||
* 1. Drain retry queues: re-attempt writes that failed against the secondary
|
||||
* backend and were enqueued by DualKVStore / DualSqlStore.
|
||||
* 2. Spot-check parity: sample N keys from the KV store, fetch the same keys
|
||||
* from Mongo, hash-compare values, and log any mismatches.
|
||||
*
|
||||
* Schedule: `"0 * * * *"` (once per hour). Tunable via `env.DRIFT_SAMPLE_N`
|
||||
* (default 50).
|
||||
*
|
||||
* Error logging never includes document values to avoid PII leakage.
|
||||
*
|
||||
* Cron handler signature matches the existing pattern used by module crons:
|
||||
* `handler(event, ctx)` where `ctx = { db, sql, env }`.
|
||||
*
|
||||
* @module cron/drift-verifier
|
||||
*/
|
||||
|
||||
import { MongoKVStore } from "../db/mongo-kv-store.js";
|
||||
|
||||
const KV_RETRY_PREFIX = "__retry:mongo-failed:";
|
||||
const SQL_RETRY_PREFIX = "__retry:mongo-sql-failed:";
|
||||
const MAX_DRAIN_PER_RUN = 200;
|
||||
|
||||
/**
|
||||
* Simple stable hash of a string value for drift comparison.
|
||||
* Not cryptographic — used only for equality detection.
|
||||
*
|
||||
* @param {string | null} s
|
||||
* @returns {string}
|
||||
*/
|
||||
function hashValue(s) {
|
||||
if (s == null) return "__null__";
|
||||
let h = 0;
|
||||
for (let i = 0; i < s.length; i++) {
|
||||
h = ((h << 5) - h + s.charCodeAt(i)) | 0;
|
||||
}
|
||||
return h.toString(16);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain one retry queue prefix: list matching keys, re-attempt the secondary
|
||||
* write, delete the queue entry on success. Cap at MAX_DRAIN_PER_RUN.
|
||||
*
|
||||
* @param {any} rawKv — raw CF KVNamespace (env.KV)
|
||||
* @param {string} prefix — queue key prefix, e.g. `__retry:mongo-failed:`
|
||||
* @param {(payload: object) => Promise<void>} retryFn — callable that re-attempts the secondary write
|
||||
* @returns {Promise<{ attempted: number, succeeded: number, failed: number }>}
|
||||
*/
|
||||
async function drainRetryQueue(rawKv, prefix, retryFn) {
|
||||
let attempted = 0;
|
||||
let succeeded = 0;
|
||||
let failed = 0;
|
||||
|
||||
const listed = await rawKv.list({ prefix, limit: MAX_DRAIN_PER_RUN });
|
||||
const keys = listed.keys ?? [];
|
||||
|
||||
for (const keyEntry of keys) {
|
||||
const queueKey = typeof keyEntry === "object" ? keyEntry.name : keyEntry;
|
||||
attempted++;
|
||||
try {
|
||||
const raw = await rawKv.get(queueKey);
|
||||
if (!raw) {
|
||||
await rawKv.delete(queueKey);
|
||||
continue;
|
||||
}
|
||||
const payload = JSON.parse(raw);
|
||||
await retryFn(payload);
|
||||
await rawKv.delete(queueKey);
|
||||
succeeded++;
|
||||
} catch (err) {
|
||||
failed++;
|
||||
console.warn("[drift-verifier] retry failed", {
|
||||
phase: "drift-verifier",
|
||||
queueKey,
|
||||
errClass: err instanceof Error ? err.constructor.name : "unknown",
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { attempted, succeeded, failed };
|
||||
}
|
||||
|
||||
/**
|
||||
* No-op retry for SQL entries — re-attempt is intentionally left as a no-op
|
||||
* for Phase 04 (the MongoSqlStore handles its own write path). A full retry
|
||||
* would require reconstructing the store, which is out of scope here.
|
||||
* The queue entry is deleted so stale entries don't accumulate indefinitely.
|
||||
*
|
||||
* @param {object} _payload
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function retrySqlWrite(_payload) {
|
||||
// Phase 04 note: full SQL retry requires reconstructing MongoSqlStore.
|
||||
// For now this drain clears stale entries. Phase 06 telemetry will surface
|
||||
// any persistent failure patterns before Phase 07 cutover.
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample N keys from the CF KV namespace (scoped to a module prefix),
|
||||
* fetch the same keys from the MongoKVStore, and log any hash mismatches.
|
||||
*
|
||||
* @param {string} moduleName
|
||||
* @param {any} rawKv — raw CF KVNamespace
|
||||
* @param {MongoKVStore} mongoStore
|
||||
* @param {number} n — sample size
|
||||
* @returns {Promise<{ sampled: number, mismatches: number }>}
|
||||
*/
|
||||
async function spotCheckModule(moduleName, rawKv, mongoStore, n) {
|
||||
const prefix = `${moduleName}:`;
|
||||
let sampled = 0;
|
||||
let mismatches = 0;
|
||||
|
||||
try {
|
||||
const listed = await rawKv.list({ prefix, limit: n });
|
||||
const keys = (listed.keys ?? []).map((k) => (typeof k === "object" ? k.name : k));
|
||||
|
||||
for (const fullKey of keys) {
|
||||
sampled++;
|
||||
try {
|
||||
const cfRaw = await rawKv.get(fullKey);
|
||||
const mongoRaw = await mongoStore.get(fullKey);
|
||||
const cfHash = hashValue(cfRaw);
|
||||
const mongoHash = hashValue(mongoRaw);
|
||||
|
||||
if (cfHash !== mongoHash) {
|
||||
mismatches++;
|
||||
console.warn("[drift-verifier] parity mismatch", {
|
||||
phase: "drift-verifier",
|
||||
module: moduleName,
|
||||
key: fullKey,
|
||||
primary_hash: cfHash,
|
||||
secondary_hash: mongoHash,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn("[drift-verifier] key compare failed", {
|
||||
phase: "drift-verifier",
|
||||
module: moduleName,
|
||||
key: fullKey,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("[drift-verifier] spotCheckModule failed", {
|
||||
phase: "drift-verifier",
|
||||
module: moduleName,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
|
||||
return { sampled, mismatches };
|
||||
}
|
||||
|
||||
/**
|
||||
* Drift-verifier cron handler.
|
||||
*
|
||||
* Registered in wrangler.toml as `"0 * * * *"` and wired into the cron
|
||||
* dispatcher via the registry's system crons array.
|
||||
*
|
||||
* @param {any} _event — Cloudflare ScheduledEvent (unused; schedule matched by dispatcher)
|
||||
* @param {{ db: any, sql: any, env: any }} ctx — injected by cron-dispatcher
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function driftVerifier(_event, ctx) {
|
||||
const { env } = ctx;
|
||||
const N = Math.max(1, Number(env.DRIFT_SAMPLE_N ?? 50));
|
||||
const rawKv = env.KV;
|
||||
|
||||
if (!rawKv) {
|
||||
console.warn("[drift-verifier] env.KV not available — skipping run");
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. Drain KV retry queue (failed Mongo secondary writes).
|
||||
const kvDrain = await drainRetryQueue(rawKv, KV_RETRY_PREFIX, async (payload) => {
|
||||
// Atlas URI is required to re-attempt the secondary write.
|
||||
if (!env.MONGODB_URI)
|
||||
throw new Error("Atlas URI not configured — cannot retry secondary write");
|
||||
const store = new MongoKVStore(env, payload.key.split(":")[0] ?? "unknown");
|
||||
if (payload.op === "delete") {
|
||||
await store.delete(payload.key);
|
||||
} else if (payload.op === "putJSON") {
|
||||
// Value is not stored in the retry queue (PII risk) — skip value retry.
|
||||
// The key will be re-synced by the next backfill / drift-verifier parity check.
|
||||
console.warn("[drift-verifier] putJSON retry skipped — value not stored in queue", {
|
||||
phase: "drift-verifier",
|
||||
key: payload.key,
|
||||
});
|
||||
} else if (payload.op === "put") {
|
||||
// Same as putJSON: value omitted from queue for PII safety.
|
||||
console.warn("[drift-verifier] put retry skipped — value not stored in queue", {
|
||||
phase: "drift-verifier",
|
||||
key: payload.key,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// 2. Drain SQL retry queue (failed MongoSqlStore secondary writes).
|
||||
const sqlDrain = await drainRetryQueue(rawKv, SQL_RETRY_PREFIX, retrySqlWrite);
|
||||
|
||||
console.log("[drift-verifier] queue drain complete", {
|
||||
phase: "drift-verifier",
|
||||
kv: kvDrain,
|
||||
sql: sqlDrain,
|
||||
});
|
||||
|
||||
// 3. Spot-check parity across key modules when Atlas is reachable.
|
||||
if (!env.MONGODB_URI || env.MONGODB_URI === "__stub_mongo__") return;
|
||||
|
||||
const modules =
|
||||
typeof env.MODULES === "string"
|
||||
? env.MODULES.split(",")
|
||||
.map((m) => m.trim())
|
||||
.filter(Boolean)
|
||||
: [];
|
||||
|
||||
let totalSampled = 0;
|
||||
let totalMismatches = 0;
|
||||
|
||||
for (const moduleName of modules) {
|
||||
const collName = moduleName.replace(/-/g, "_");
|
||||
const mongoStore = new MongoKVStore(env, collName);
|
||||
const { sampled, mismatches } = await spotCheckModule(moduleName, rawKv, mongoStore, N);
|
||||
totalSampled += sampled;
|
||||
totalMismatches += mismatches;
|
||||
}
|
||||
|
||||
if (totalMismatches > 0) {
|
||||
console.error("[drift-verifier] parity drift detected", {
|
||||
phase: "drift-verifier",
|
||||
totalSampled,
|
||||
totalMismatches,
|
||||
});
|
||||
} else {
|
||||
console.log("[drift-verifier] parity check passed", {
|
||||
phase: "drift-verifier",
|
||||
totalSampled,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Module-style export so the cron-dispatcher can register drift-verifier as a
|
||||
* system-level cron entry alongside module crons. The `name` is synthetic —
|
||||
* no module folder exists; the dispatcher treats it like any other cron entry.
|
||||
*/
|
||||
export const driftVerifierCron = {
|
||||
schedule: "0 * * * *",
|
||||
name: "drift-verifier",
|
||||
handler: driftVerifier,
|
||||
};
|
||||
@@ -7,45 +7,19 @@
|
||||
*
|
||||
* Returns null when `env.DB` is absent so modules that don't use D1 have
|
||||
* zero overhead — the registry passes `sql: null` and modules check for it.
|
||||
*
|
||||
* ## Flag matrix (same as create-store.js, SQL edition)
|
||||
*
|
||||
* | STORAGE_PRIMARY | DUAL_WRITE | MONGODB_URI | Result |
|
||||
* |-----------------|------------|-------------------|----------------------------------------------|
|
||||
* | (unset) or kv | 1 (default)| set, real | DualSqlStore(CFSqlStore primary, Mongo sec.) |
|
||||
* | kv | 0 | any | CFSqlStore only (legacy / rollback) |
|
||||
* | mongo | 1 | set, real | DualSqlStore(Mongo primary, CFSqlStore sec.) |
|
||||
* | mongo | 0 | set, real | MongoSqlStore only (post-cutover) |
|
||||
* | any | any | unset | CFSqlStore only (or null if env.DB absent) |
|
||||
* | any | any | === STUB_SENTINEL | CFSqlStore only (deploy-time register path) |
|
||||
*
|
||||
* Note: trading module's `tradesStore` (MongoTradesStore) is wired separately
|
||||
* via registry.js — this factory produces the generic SqlStore shim only.
|
||||
*
|
||||
* post-Phase-07: this entire function returns MongoSqlStore-only;
|
||||
* CF/D1 branches removed. The flag matrix above collapses to a single path.
|
||||
*/
|
||||
|
||||
import { CFSqlStore } from "./cf-sql-store.js";
|
||||
import { DualSqlStore } from "./dual-sql-store.js";
|
||||
import { MongoSqlStore } from "./mongo-sql-store.js";
|
||||
|
||||
/**
|
||||
* @typedef {import("./sql-store-interface.js").SqlStore} SqlStore
|
||||
*/
|
||||
|
||||
/** Sentinel value used by scripts/register.js to signal deploy-time context. */
|
||||
const STUB_SENTINEL = "__stub_mongo__";
|
||||
|
||||
const MODULE_NAME_RE = /^[a-z0-9_-]+$/;
|
||||
|
||||
/**
|
||||
* @param {string} moduleName — must match `[a-z0-9_-]+`.
|
||||
* @param {object} env — worker env (or test double).
|
||||
* @param {any} [env.DB] — CF D1Database binding (optional).
|
||||
* @param {string} [env.MONGODB_URI] — Atlas connection string (or STUB_SENTINEL).
|
||||
* @param {string} [env.STORAGE_PRIMARY] — "kv" (default) | "mongo".
|
||||
* @param {string} [env.DUAL_WRITE] — "1" (default) | "0".
|
||||
* @param {{ DB?: D1Database }} env — worker env (or test double).
|
||||
* @returns {SqlStore | null} null when env.DB is not bound.
|
||||
*/
|
||||
export function createSqlStore(moduleName, env) {
|
||||
@@ -61,46 +35,6 @@ export function createSqlStore(moduleName, env) {
|
||||
// D1 is optional — workers without a DB binding still work fine.
|
||||
if (!env?.DB) return null;
|
||||
|
||||
// --- Sentinel / fallback: always CF-only ---
|
||||
const mongoUri = env.MONGODB_URI;
|
||||
if (!mongoUri || mongoUri === STUB_SENTINEL) {
|
||||
return _wrapCf(moduleName, env);
|
||||
}
|
||||
|
||||
const primary = (env.STORAGE_PRIMARY ?? "kv").toLowerCase();
|
||||
const dualWrite = (env.DUAL_WRITE ?? "1") !== "0";
|
||||
|
||||
if (primary === "mongo" && !dualWrite) {
|
||||
// MongoSqlStore only — post-cutover path.
|
||||
return new MongoSqlStore(env, moduleName);
|
||||
}
|
||||
|
||||
const cfStore = _wrapCf(moduleName, env);
|
||||
const mongoStore = new MongoSqlStore(env, moduleName);
|
||||
|
||||
if (primary === "mongo") {
|
||||
// DualSqlStore: read Mongo, write both — cutover phase.
|
||||
return new DualSqlStore(mongoStore, cfStore, env.KV);
|
||||
}
|
||||
|
||||
// Default: STORAGE_PRIMARY=kv
|
||||
if (!dualWrite) {
|
||||
// Rollback path: CF only.
|
||||
return cfStore;
|
||||
}
|
||||
|
||||
// DualSqlStore: read CF/D1, write both — dual-write window.
|
||||
return new DualSqlStore(cfStore, mongoStore, env.KV);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a namespaced CFSqlStore wrapper with the same shape as before Phase 04.
|
||||
*
|
||||
* @param {string} moduleName
|
||||
* @param {{ DB: any }} env
|
||||
* @returns {SqlStore}
|
||||
*/
|
||||
function _wrapCf(moduleName, env) {
|
||||
const base = new CFSqlStore(env.DB);
|
||||
const tablePrefix = `${moduleName}_`;
|
||||
|
||||
|
||||
+19
-93
@@ -4,31 +4,11 @@
|
||||
* Every module gets its own prefixed view: module `wordle` calling `put("k", v)`
|
||||
* writes raw key `wordle:k`. list() automatically constrains to the module's
|
||||
* namespace AND strips the prefix from returned keys so the module sees its
|
||||
* own flat key-space. Modules CANNOT escape their namespace without
|
||||
* own flat key-space. modules CANNOT escape their namespace without
|
||||
* reconstructing prefixes manually — a code-review boundary, not a hard one.
|
||||
*
|
||||
* ## Flag matrix (env.STORAGE_PRIMARY × env.DUAL_WRITE × env.MONGODB_URI)
|
||||
*
|
||||
* | STORAGE_PRIMARY | DUAL_WRITE | MONGODB_URI | Result |
|
||||
* |-----------------|------------|-------------------|--------------------------------------------|
|
||||
* | (unset) or kv | 1 (default)| set, real | DualKVStore(CFKVStore primary, Mongo sec.) |
|
||||
* | kv | 0 | any | CFKVStore only (legacy / rollback) |
|
||||
* | mongo | 1 | set, real | DualKVStore(Mongo primary, CFKVStore sec.) |
|
||||
* | mongo | 0 | set, real | MongoKVStore only (post-cutover) |
|
||||
* | any | any | unset | CFKVStore only |
|
||||
* | any | any | === STUB_SENTINEL | CFKVStore only (deploy-time register path) |
|
||||
*
|
||||
* Boot-time assertion: if STORAGE_PRIMARY=mongo but MONGODB_URI is absent
|
||||
* (and not STUB_SENTINEL), the first call throws a clear error rather than
|
||||
* silently falling back to KV.
|
||||
*
|
||||
* post-Phase-07: this entire function returns MongoKVStore-only;
|
||||
* KV branches removed. The flag matrix above collapses to a single path.
|
||||
*/
|
||||
|
||||
import { CFKVStore } from "./cf-kv-store.js";
|
||||
import { DualKVStore } from "./dual-kv-store.js";
|
||||
import { MongoKVStore } from "./mongo-kv-store.js";
|
||||
|
||||
/**
|
||||
* @typedef {import("./kv-store-interface.js").KVStore} KVStore
|
||||
@@ -37,24 +17,30 @@ import { MongoKVStore } from "./mongo-kv-store.js";
|
||||
* @typedef {import("./kv-store-interface.js").KVStoreListResult} KVStoreListResult
|
||||
*/
|
||||
|
||||
/** Sentinel value used by scripts/register.js to signal deploy-time context. */
|
||||
const STUB_SENTINEL = "__stub_mongo__";
|
||||
|
||||
const MODULE_NAME_RE = /^[a-z0-9_-]+$/;
|
||||
|
||||
/**
|
||||
* Wrap a raw KVStore so all keys carry a `<moduleName>:` prefix and
|
||||
* list() results are stripped back to the module's flat key-space.
|
||||
*
|
||||
* @param {string} prefix — e.g. "wordle:"
|
||||
* @param {KVStore} base
|
||||
* @param {string} moduleName — must match `[a-z0-9_-]+`. Used verbatim as the key prefix.
|
||||
* @param {{ KV: KVNamespace }} env — worker env (or test double) with a `KV` binding.
|
||||
* @returns {KVStore}
|
||||
*/
|
||||
function withPrefix(prefix, base) {
|
||||
return {
|
||||
/** @type {"dual" | undefined} */
|
||||
_kind: /** @type {any} */ (base)._kind,
|
||||
export function createStore(moduleName, env) {
|
||||
if (!moduleName || typeof moduleName !== "string") {
|
||||
throw new Error("createStore: moduleName is required");
|
||||
}
|
||||
if (!MODULE_NAME_RE.test(moduleName)) {
|
||||
throw new Error(
|
||||
`createStore: invalid moduleName "${moduleName}" — must match ${MODULE_NAME_RE}`,
|
||||
);
|
||||
}
|
||||
if (!env?.KV) {
|
||||
throw new Error("createStore: env.KV binding is missing");
|
||||
}
|
||||
|
||||
const base = new CFKVStore(env.KV);
|
||||
const prefix = `${moduleName}:`;
|
||||
|
||||
return {
|
||||
async get(key) {
|
||||
return base.get(prefix + key);
|
||||
},
|
||||
@@ -91,63 +77,3 @@ function withPrefix(prefix, base) {
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} moduleName — must match `[a-z0-9_-]+`. Used verbatim as the key prefix.
|
||||
* @param {object} env — worker env (or test double).
|
||||
* @param {any} env.KV — CF KVNamespace binding.
|
||||
* @param {string} [env.MONGODB_URI] — Atlas connection string (or STUB_SENTINEL).
|
||||
* @param {string} [env.STORAGE_PRIMARY] — "kv" (default) | "mongo".
|
||||
* @param {string} [env.DUAL_WRITE] — "1" (default) | "0".
|
||||
* @returns {KVStore}
|
||||
*/
|
||||
export function createStore(moduleName, env) {
|
||||
if (!moduleName || typeof moduleName !== "string") {
|
||||
throw new Error("createStore: moduleName is required");
|
||||
}
|
||||
if (!MODULE_NAME_RE.test(moduleName)) {
|
||||
throw new Error(
|
||||
`createStore: invalid moduleName "${moduleName}" — must match ${MODULE_NAME_RE}`,
|
||||
);
|
||||
}
|
||||
if (!env?.KV) {
|
||||
throw new Error("createStore: env.KV binding is missing");
|
||||
}
|
||||
|
||||
const prefix = `${moduleName}:`;
|
||||
// Collection name: replace `-` with `_` for MongoDB compatibility.
|
||||
const collectionName = moduleName.replace(/-/g, "_");
|
||||
|
||||
// --- Sentinel / fallback: always CF-only ---
|
||||
const mongoUri = env.MONGODB_URI;
|
||||
if (!mongoUri || mongoUri === STUB_SENTINEL) {
|
||||
return withPrefix(prefix, new CFKVStore(env.KV));
|
||||
}
|
||||
|
||||
const primary = (env.STORAGE_PRIMARY ?? "kv").toLowerCase();
|
||||
const dualWrite = (env.DUAL_WRITE ?? "1") !== "0";
|
||||
|
||||
// Boot-time assertion: if mongo is requested but URI is absent, throw clearly.
|
||||
// (URI IS present here since we checked above, so this guard is for STORAGE_PRIMARY=mongo.)
|
||||
if (primary === "mongo" && !dualWrite) {
|
||||
// MongoKVStore only — post-cutover path.
|
||||
return withPrefix(prefix, new MongoKVStore(env, collectionName));
|
||||
}
|
||||
|
||||
const cfStore = new CFKVStore(env.KV);
|
||||
const mongoStore = new MongoKVStore(env, collectionName);
|
||||
|
||||
if (primary === "mongo") {
|
||||
// DualKVStore: read Mongo, write both — cutover phase.
|
||||
return withPrefix(prefix, new DualKVStore(mongoStore, cfStore, env.KV));
|
||||
}
|
||||
|
||||
// Default: STORAGE_PRIMARY=kv + DUAL_WRITE=1
|
||||
if (!dualWrite) {
|
||||
// Rollback path: KV only.
|
||||
return withPrefix(prefix, cfStore);
|
||||
}
|
||||
|
||||
// DualKVStore: read KV, write both — dual-write window.
|
||||
return withPrefix(prefix, new DualKVStore(cfStore, mongoStore, env.KV));
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
/**
|
||||
* @file dual-kv-store — KVStore wrapper that writes to two backends in parallel.
|
||||
*
|
||||
* During the dual-write window (Phase 04 → Phase 07):
|
||||
* - Reads go to the primary only (never the secondary).
|
||||
* - Writes go to BOTH via `Promise.allSettled`. If the secondary fails,
|
||||
* the failure is logged (key + error class + message only — no document value
|
||||
* to avoid PII leakage) and enqueued to a KV retry queue. The primary
|
||||
* failure causes a throw; secondary failure is transparent to the caller.
|
||||
*
|
||||
* Retry queue keys: `__retry:mongo-failed:<random-id>` stored in `env.KV`
|
||||
* (the raw CF KV namespace, not the dual-store itself).
|
||||
*
|
||||
* Expose `_kind = "dual"` sentinel for test-side identification.
|
||||
*
|
||||
* @module db/dual-kv-store
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {import("./kv-store-interface.js").KVStore} KVStore
|
||||
* @typedef {import("./kv-store-interface.js").KVStorePutOptions} KVStorePutOptions
|
||||
* @typedef {import("./kv-store-interface.js").KVStoreListOptions} KVStoreListOptions
|
||||
* @typedef {import("./kv-store-interface.js").KVStoreListResult} KVStoreListResult
|
||||
*/
|
||||
|
||||
const RETRY_PREFIX = "__retry:mongo-failed:";
|
||||
|
||||
/**
|
||||
* Generate a short random suffix for retry-queue keys so concurrent failures
|
||||
* do not collide.
|
||||
*
|
||||
* @returns {string}
|
||||
*/
|
||||
function randomId() {
|
||||
return Math.random().toString(36).slice(2, 10);
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue a failed secondary write to the raw KV namespace so the
|
||||
* drift-verifier cron can retry it later.
|
||||
*
|
||||
* @param {any} rawKv — raw CF KVNamespace (env.KV)
|
||||
* @param {object} payload — { op, key, value?, opts? }
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function enqueueRetry(rawKv, payload) {
|
||||
try {
|
||||
const id = `${RETRY_PREFIX}${payload.key}:${randomId()}`;
|
||||
await rawKv.put(id, JSON.stringify({ ...payload, ts: Date.now() }));
|
||||
} catch (err) {
|
||||
// Retry-queue write failing is unfortunate but must not crash the request.
|
||||
console.warn("[dual-kv] enqueueRetry failed", {
|
||||
phase: "dual-kv",
|
||||
op: "enqueue",
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @implements {KVStore}
|
||||
*/
|
||||
export class DualKVStore {
|
||||
/**
|
||||
* @param {KVStore} primary — the authoritative store (reads + writes).
|
||||
* @param {KVStore} secondary — the mirror store (writes only; failures silently queued).
|
||||
* @param {any} rawKv — raw CF KVNamespace used for the retry queue.
|
||||
* @param {object} [logger] — injectable logger; defaults to `console`.
|
||||
*/
|
||||
constructor(primary, secondary, rawKv, logger = console) {
|
||||
if (!primary) throw new Error("DualKVStore: primary is required");
|
||||
if (!secondary) throw new Error("DualKVStore: secondary is required");
|
||||
if (!rawKv) throw new Error("DualKVStore: rawKv is required");
|
||||
this._primary = primary;
|
||||
this._secondary = secondary;
|
||||
this._rawKv = rawKv;
|
||||
this._log = logger;
|
||||
/** @type {"dual"} */
|
||||
this._kind = "dual";
|
||||
}
|
||||
|
||||
/**
|
||||
* Fire both writes in parallel. Throw on primary failure. On secondary
|
||||
* failure: log structured warning and enqueue for retry.
|
||||
*
|
||||
* @param {string} op — operation name for logging.
|
||||
* @param {string} key — the key being written (used in retry payload + logs).
|
||||
* @param {Function} primaryFn — async thunk for primary write.
|
||||
* @param {Function} secondaryFn — async thunk for secondary write.
|
||||
* @param {object} [retryPayload] — extra fields stored in the retry queue entry.
|
||||
* @returns {Promise<any>} primary result.
|
||||
*/
|
||||
async _dualWrite(op, key, primaryFn, secondaryFn, retryPayload) {
|
||||
const [primaryResult, secondaryResult] = await Promise.allSettled([primaryFn(), secondaryFn()]);
|
||||
|
||||
if (secondaryResult.status === "rejected") {
|
||||
const err = secondaryResult.reason;
|
||||
this._log.warn("[dual-kv] secondary write failed", {
|
||||
phase: "dual-kv",
|
||||
op,
|
||||
key,
|
||||
errClass: err instanceof Error ? err.constructor.name : "unknown",
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
await enqueueRetry(this._rawKv, { op, key, ...retryPayload });
|
||||
}
|
||||
|
||||
if (primaryResult.status === "rejected") {
|
||||
throw primaryResult.reason;
|
||||
}
|
||||
|
||||
return primaryResult.value;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @returns {Promise<string|null>}
|
||||
*/
|
||||
async get(key) {
|
||||
return this._primary.get(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @param {string} value
|
||||
* @param {KVStorePutOptions} [opts]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async put(key, value, opts) {
|
||||
return this._dualWrite(
|
||||
"put",
|
||||
key,
|
||||
() => this._primary.put(key, value, opts),
|
||||
() => this._secondary.put(key, value, opts),
|
||||
{ opts },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async delete(key) {
|
||||
return this._dualWrite(
|
||||
"delete",
|
||||
key,
|
||||
() => this._primary.delete(key),
|
||||
() => this._secondary.delete(key),
|
||||
{},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {KVStoreListOptions} [opts]
|
||||
* @returns {Promise<KVStoreListResult>}
|
||||
*/
|
||||
async list(opts = {}) {
|
||||
return this._primary.list(opts);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @returns {Promise<any|null>}
|
||||
*/
|
||||
async getJSON(key) {
|
||||
return this._primary.getJSON(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @param {any} value
|
||||
* @param {KVStorePutOptions} [opts]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async putJSON(key, value, opts) {
|
||||
return this._dualWrite(
|
||||
"putJSON",
|
||||
key,
|
||||
() => this._primary.putJSON(key, value, opts),
|
||||
() => this._secondary.putJSON(key, value, opts),
|
||||
{ opts },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
/**
|
||||
* @file dual-sql-store — SqlStore wrapper that writes to two backends in parallel.
|
||||
*
|
||||
* Same fault-tolerance model as DualKVStore:
|
||||
* - Reads (`all`, `first`) go to the primary only.
|
||||
* - `run` writes go to BOTH via `Promise.allSettled`. Secondary failure is
|
||||
* logged (query + error — no row values to avoid PII) and enqueued to the
|
||||
* KV retry queue. Primary failure causes a throw.
|
||||
* - `prepare` and `batch` go to primary only — D1 prepared statements are
|
||||
* CF-specific and cannot be forwarded to MongoDB.
|
||||
*
|
||||
* Retry queue keys: `__retry:mongo-sql-failed:<random-id>` stored in `env.KV`.
|
||||
*
|
||||
* Expose `_kind = "dual"` sentinel for test-side identification.
|
||||
*
|
||||
* @module db/dual-sql-store
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {import("./sql-store-interface.js").SqlStore} SqlStore
|
||||
* @typedef {import("./sql-store-interface.js").SqlRunResult} SqlRunResult
|
||||
*/
|
||||
|
||||
const RETRY_PREFIX = "__retry:mongo-sql-failed:";
|
||||
|
||||
/**
|
||||
* Generate a short random suffix for retry-queue keys.
|
||||
*
|
||||
* @returns {string}
|
||||
*/
|
||||
function randomId() {
|
||||
return Math.random().toString(36).slice(2, 10);
|
||||
}
|
||||
|
||||
/**
|
||||
* Enqueue a failed secondary write to the raw KV namespace.
|
||||
*
|
||||
* @param {any} rawKv — raw CF KVNamespace (env.KV)
|
||||
* @param {object} payload — { op, query, binds? }
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function enqueueRetry(rawKv, payload) {
|
||||
try {
|
||||
const slug = payload.query.slice(0, 40).replace(/\s+/g, "_");
|
||||
const id = `${RETRY_PREFIX}${slug}:${randomId()}`;
|
||||
await rawKv.put(id, JSON.stringify({ ...payload, ts: Date.now() }));
|
||||
} catch (err) {
|
||||
console.warn("[dual-sql] enqueueRetry failed", {
|
||||
phase: "dual-sql",
|
||||
op: "enqueue",
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @implements {SqlStore}
|
||||
*/
|
||||
export class DualSqlStore {
|
||||
/**
|
||||
* @param {SqlStore} primary — the authoritative store (reads + writes).
|
||||
* @param {SqlStore} secondary — the mirror store (writes only; failures silently queued).
|
||||
* @param {any} rawKv — raw CF KVNamespace used for the retry queue.
|
||||
* @param {object} [logger] — injectable logger; defaults to `console`.
|
||||
*/
|
||||
constructor(primary, secondary, rawKv, logger = console) {
|
||||
if (!primary) throw new Error("DualSqlStore: primary is required");
|
||||
if (!secondary) throw new Error("DualSqlStore: secondary is required");
|
||||
if (!rawKv) throw new Error("DualSqlStore: rawKv is required");
|
||||
this._primary = primary;
|
||||
this._secondary = secondary;
|
||||
this._rawKv = rawKv;
|
||||
this._log = logger;
|
||||
/** @type {"dual"} */
|
||||
this._kind = "dual";
|
||||
/** @type {string} */
|
||||
this.tablePrefix = primary.tablePrefix ?? "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a write statement against both stores. Throw on primary failure;
|
||||
* log + enqueue on secondary failure.
|
||||
*
|
||||
* @param {string} query
|
||||
* @param {any[]} binds
|
||||
* @returns {Promise<SqlRunResult>}
|
||||
*/
|
||||
async run(query, ...binds) {
|
||||
const [primaryResult, secondaryResult] = await Promise.allSettled([
|
||||
this._primary.run(query, ...binds),
|
||||
this._secondary.run(query, ...binds),
|
||||
]);
|
||||
|
||||
if (secondaryResult.status === "rejected") {
|
||||
const err = secondaryResult.reason;
|
||||
this._log.warn("[dual-sql] secondary run failed", {
|
||||
phase: "dual-sql",
|
||||
op: "run",
|
||||
// Log query shape only; never log bind values (PII risk).
|
||||
query: query.slice(0, 80),
|
||||
errClass: err instanceof Error ? err.constructor.name : "unknown",
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
await enqueueRetry(this._rawKv, { op: "run", query, binds });
|
||||
}
|
||||
|
||||
if (primaryResult.status === "rejected") {
|
||||
throw primaryResult.reason;
|
||||
}
|
||||
|
||||
return primaryResult.value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a SELECT and return all matching rows — primary only.
|
||||
*
|
||||
* @param {string} query
|
||||
* @param {...any} binds
|
||||
* @returns {Promise<any[]>}
|
||||
*/
|
||||
async all(query, ...binds) {
|
||||
return this._primary.all(query, ...binds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a SELECT and return the first row — primary only.
|
||||
*
|
||||
* @param {string} query
|
||||
* @param {...any} binds
|
||||
* @returns {Promise<any|null>}
|
||||
*/
|
||||
async first(query, ...binds) {
|
||||
return this._primary.first(query, ...binds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a prepared statement from the primary store only.
|
||||
* CF-specific; cannot be forwarded to MongoDB.
|
||||
*
|
||||
* @param {string} query
|
||||
* @param {...any} binds
|
||||
* @returns {any}
|
||||
*/
|
||||
prepare(query, ...binds) {
|
||||
return this._primary.prepare(query, ...binds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute multiple prepared statements — primary only.
|
||||
*
|
||||
* @param {any[]} statements
|
||||
* @returns {Promise<any[]>}
|
||||
*/
|
||||
async batch(statements) {
|
||||
return this._primary.batch(statements);
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
/**
|
||||
* @file mongo-client — memoized MongoDB Atlas client singleton.
|
||||
*
|
||||
* Exports `getDb(env)` for all Mongo-backed stores. On the first call from a
|
||||
* cold isolate it opens one connection; subsequent calls reuse the same
|
||||
* `MongoClient`. If `client.connect()` rejects, both `client` and
|
||||
* `connectPromise` are nulled so the next request retries cleanly instead of
|
||||
* reusing a dead client reference.
|
||||
*
|
||||
* `MongoServerSelectionError` (e.g. paused M0 cluster) is caught, logged with
|
||||
* an actionable message, then rethrown so the caller can map it to 503.
|
||||
*
|
||||
* @module db/mongo-client
|
||||
*/
|
||||
|
||||
import { MongoClient } from "mongodb";
|
||||
|
||||
/** @type {MongoClient|null} */
|
||||
let client = null;
|
||||
|
||||
/** @type {Promise<void>|null} */
|
||||
let connectPromise = null;
|
||||
|
||||
/**
|
||||
* Return the memoized Db instance, connecting on the first call.
|
||||
*
|
||||
* Connection options match the Cloudflare Workers constraints:
|
||||
* maxPoolSize: 1 — one connection per isolate
|
||||
* minPoolSize: 0 — no idle keepalive (Workers tear down quickly)
|
||||
* serverSelectionTimeoutMS: 5000 — fast fail for paused M0
|
||||
* connectTimeoutMS: 10000 — TLS + SCRAM on cold start
|
||||
*
|
||||
* @param {{ MONGODB_URI: string }} env — Cloudflare Worker env (or test double).
|
||||
* @returns {Promise<import("mongodb").Db>}
|
||||
* @throws {import("mongodb").MongoServerSelectionError} if cluster unreachable.
|
||||
*/
|
||||
export async function getDb(env) {
|
||||
if (client) return client.db("miti99bot");
|
||||
|
||||
if (!connectPromise) {
|
||||
client = new MongoClient(env.MONGODB_URI, {
|
||||
maxPoolSize: 1,
|
||||
minPoolSize: 0,
|
||||
serverSelectionTimeoutMS: 5000,
|
||||
connectTimeoutMS: 10000,
|
||||
});
|
||||
|
||||
// On rejection: null BOTH so the next getDb() call retries with a fresh
|
||||
// client instead of awaiting the already-rejected promise (code-reviewer #16).
|
||||
connectPromise = client.connect().catch((err) => {
|
||||
client = null;
|
||||
connectPromise = null;
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
await connectPromise;
|
||||
} catch (err) {
|
||||
// M0 clusters auto-pause after 60 days of inactivity. Surface an
|
||||
// actionable note so ops can identify and resume the cluster.
|
||||
// NOTE: URI is deliberately omitted to prevent credential leaks.
|
||||
if (err?.name === "MongoServerSelectionError") {
|
||||
console.warn(
|
||||
JSON.stringify({
|
||||
event: "mongo_server_selection_failed",
|
||||
note: "M0 may be paused — resume the cluster in Atlas, then retry. Caller should map to 503.",
|
||||
}),
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
return client.db("miti99bot");
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the active MongoClient and reset module-scope state.
|
||||
* Intended for test teardown and graceful shutdown only — not for
|
||||
* use in production request handlers.
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function closeMongo() {
|
||||
if (client) {
|
||||
await client.close();
|
||||
client = null;
|
||||
}
|
||||
connectPromise = null;
|
||||
}
|
||||
@@ -1,178 +0,0 @@
|
||||
/**
|
||||
* @file mongo-kv-store — MongoDB Atlas implementation of the KVStore interface.
|
||||
*
|
||||
* Behavioral parity with `CFKVStore`, with one documented divergence:
|
||||
* - CFKVStore TTL is enforced server-side (eventual, ~1s granularity).
|
||||
* - MongoKVStore also enforces TTL at read-time via an `expiresAt` filter,
|
||||
* eliminating the up-to-60s Atlas TTL-sweeper stale-read window.
|
||||
*
|
||||
* Per-module collections: module name with `-` replaced by `_`
|
||||
* (e.g. `loldle-emoji` → `loldle_emoji`).
|
||||
*
|
||||
* `list()` returns keys WITH the module prefix preserved — the wrapper in
|
||||
* `create-store.js:65` strips it. MongoKVStore never strips prefixes.
|
||||
*
|
||||
* @see ./kv-store-interface.js for the full interface contract.
|
||||
* @module db/mongo-kv-store
|
||||
*/
|
||||
|
||||
import { getDb } from "./mongo-client.js";
|
||||
import { listWithCursor } from "./mongo-list-cursor.js";
|
||||
|
||||
/**
|
||||
* @typedef {import("./kv-store-interface.js").KVStore} KVStore
|
||||
* @typedef {import("./kv-store-interface.js").KVStorePutOptions} KVStorePutOptions
|
||||
* @typedef {import("./kv-store-interface.js").KVStoreListOptions} KVStoreListOptions
|
||||
* @typedef {import("./kv-store-interface.js").KVStoreListResult} KVStoreListResult
|
||||
*/
|
||||
|
||||
/**
|
||||
* Tracks which collections have already had the TTL index created this
|
||||
* isolate lifetime, to avoid redundant `createIndex` round-trips.
|
||||
*
|
||||
* @type {Set<string>}
|
||||
*/
|
||||
const indexedCollections = new Set();
|
||||
|
||||
/**
|
||||
* @implements {KVStore}
|
||||
*/
|
||||
export class MongoKVStore {
|
||||
/**
|
||||
* @param {{ MONGODB_URI: string }} env — Worker env (or test double).
|
||||
* @param {string} collectionName — module name (e.g. "wordle", "loldle-emoji").
|
||||
* @param {import("mongodb").Db} [dbOverride] — injected Db for tests; bypasses real connect.
|
||||
*/
|
||||
constructor(env, collectionName, dbOverride) {
|
||||
if (!collectionName) throw new Error("MongoKVStore: collectionName is required");
|
||||
this._env = env;
|
||||
// Normalize collection name: replace `-` with `_` for MongoDB compatibility.
|
||||
this._collName = collectionName.replace(/-/g, "_");
|
||||
this._dbOverride = dbOverride ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the Db instance (override for tests, real for prod).
|
||||
*
|
||||
* @returns {Promise<import("mongodb").Db>}
|
||||
*/
|
||||
async _db() {
|
||||
return this._dbOverride ?? getDb(this._env);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazily create the TTL index once per collection per isolate.
|
||||
* Idempotent on the MongoDB side; tracked locally to avoid extra round-trips.
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async _ensureIndex() {
|
||||
if (indexedCollections.has(this._collName)) return;
|
||||
const db = await this._db();
|
||||
await db
|
||||
.collection(this._collName)
|
||||
.createIndex({ expiresAt: 1 }, { expireAfterSeconds: 0, sparse: true });
|
||||
indexedCollections.add(this._collName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the read-time TTL filter: accept docs with no `expiresAt` field,
|
||||
* OR docs whose `expiresAt` is in the future.
|
||||
*
|
||||
* @param {string} key
|
||||
* @returns {object} MongoDB filter
|
||||
*/
|
||||
_liveFilter(key) {
|
||||
return {
|
||||
_id: key,
|
||||
$or: [{ expiresAt: { $exists: false } }, { expiresAt: { $gt: new Date() } }],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @returns {Promise<string|null>}
|
||||
*/
|
||||
async get(key) {
|
||||
await this._ensureIndex();
|
||||
const db = await this._db();
|
||||
const doc = await db.collection(this._collName).findOne(this._liveFilter(key));
|
||||
return doc ? doc.value : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @param {string} value
|
||||
* @param {KVStorePutOptions} [opts]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async put(key, value, opts) {
|
||||
await this._ensureIndex();
|
||||
const db = await this._db();
|
||||
const $set = { value };
|
||||
const $unset = {};
|
||||
|
||||
if (opts?.expirationTtl) {
|
||||
$set.expiresAt = new Date(Date.now() + opts.expirationTtl * 1000);
|
||||
} else {
|
||||
// Clear any existing TTL so the document becomes permanent.
|
||||
$unset.expiresAt = "";
|
||||
}
|
||||
|
||||
await db.collection(this._collName).updateOne({ _id: key }, { $set, $unset }, { upsert: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async delete(key) {
|
||||
await this._ensureIndex();
|
||||
const db = await this._db();
|
||||
await db.collection(this._collName).deleteOne({ _id: key });
|
||||
}
|
||||
|
||||
/**
|
||||
* List keys matching an optional prefix, with cursor-based pagination.
|
||||
* Keys are returned WITH the full module prefix preserved — the wrapper in
|
||||
* `create-store.js` strips it for callers.
|
||||
*
|
||||
* @param {KVStoreListOptions} [opts]
|
||||
* @returns {Promise<KVStoreListResult>}
|
||||
*/
|
||||
async list(opts = {}) {
|
||||
await this._ensureIndex();
|
||||
const db = await this._db();
|
||||
return listWithCursor(db.collection(this._collName), opts);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @returns {Promise<any|null>}
|
||||
*/
|
||||
async getJSON(key) {
|
||||
const raw = await this.get(key);
|
||||
if (raw == null) return null;
|
||||
try {
|
||||
return JSON.parse(raw);
|
||||
} catch (err) {
|
||||
console.warn("getJSON: parse failed", { key, err: String(err) });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} key
|
||||
* @param {any} value
|
||||
* @param {KVStorePutOptions} [opts]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async putJSON(key, value, opts) {
|
||||
if (value === undefined) {
|
||||
throw new Error(`putJSON: value for key "${key}" is undefined`);
|
||||
}
|
||||
// JSON.stringify throws on cycles — let it propagate.
|
||||
const serialized = JSON.stringify(value);
|
||||
await this.put(key, serialized, opts);
|
||||
}
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
/**
|
||||
* @file mongo-list-cursor — cursor-based list pagination helper for MongoKVStore.
|
||||
*
|
||||
* Extracted to keep mongo-kv-store.js under 200 LOC.
|
||||
* Encodes the last `_id` of a page as a base64 cursor; decodes it on the
|
||||
* next call to build a `$gt` filter. Does NOT use skip() — purely sorted-_id
|
||||
* pagination to avoid O(n) offset scans.
|
||||
*
|
||||
* @module db/mongo-list-cursor
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {import("./kv-store-interface.js").KVStoreListOptions} KVStoreListOptions
|
||||
* @typedef {import("./kv-store-interface.js").KVStoreListResult} KVStoreListResult
|
||||
*/
|
||||
|
||||
/**
|
||||
* Escape special RegExp characters so a prefix string is safe inside a
|
||||
* MongoDB `$regex` filter.
|
||||
*
|
||||
* @param {string} str
|
||||
* @returns {string}
|
||||
*/
|
||||
export function escapeRegex(str) {
|
||||
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a paginated list query on a MongoDB collection.
|
||||
*
|
||||
* @param {import("mongodb").Collection} col — resolved collection handle.
|
||||
* @param {KVStoreListOptions} opts
|
||||
* @returns {Promise<KVStoreListResult>}
|
||||
*/
|
||||
export async function listWithCursor(col, opts = {}) {
|
||||
const { prefix = "", limit = 1000, cursor } = opts;
|
||||
const pageSize = limit;
|
||||
|
||||
const filter = {};
|
||||
if (prefix) {
|
||||
filter._id = { $regex: `^${escapeRegex(prefix)}` };
|
||||
}
|
||||
if (cursor) {
|
||||
const lastId = Buffer.from(cursor, "base64").toString("utf8");
|
||||
filter._id = filter._id ? { ...filter._id, $gt: lastId } : { $gt: lastId };
|
||||
}
|
||||
|
||||
const docs = await col
|
||||
.find(filter)
|
||||
.sort({ _id: 1 })
|
||||
.limit(pageSize + 1)
|
||||
.project({ _id: 1 })
|
||||
.toArray();
|
||||
|
||||
const hasMore = docs.length > pageSize;
|
||||
const page = hasMore ? docs.slice(0, pageSize) : docs;
|
||||
const keys = page.map((d) => d._id);
|
||||
const nextCursor = hasMore
|
||||
? Buffer.from(page[page.length - 1]._id, "utf8").toString("base64")
|
||||
: undefined;
|
||||
|
||||
return { keys, cursor: nextCursor, done: !hasMore };
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
/**
|
||||
* @file mongo-sql-store — thin SqlStore shim wrapping MongoTradesStore.
|
||||
*
|
||||
* PURPOSE (intentionally limited):
|
||||
* This shim exists ONLY for `create-sql-store.js` factory branching and
|
||||
* `tests/db/create-sql-store.test.js` contract compliance. The trading
|
||||
* module is being refactored to call MongoTradesStore directly (Phase 03).
|
||||
* The shim's `run`/`all`/`first` translate the 6 known trading SQL patterns
|
||||
* for any caller that still routes through the SqlStore interface; everything
|
||||
* else throws immediately — no silent fallthrough.
|
||||
*
|
||||
* `prepare` and `batch` are unsupported: trading code never calls them, and
|
||||
* the dual-write layer (Phase 04) uses MongoTradesStore directly.
|
||||
*
|
||||
* @module db/mongo-sql-store
|
||||
*/
|
||||
|
||||
import { MongoTradesStore } from "./mongo-trades-store.js";
|
||||
|
||||
/**
|
||||
* @typedef {import("./sql-store-interface.js").SqlStore} SqlStore
|
||||
* @typedef {import("./sql-store-interface.js").SqlRunResult} SqlRunResult
|
||||
*/
|
||||
|
||||
/**
|
||||
* @implements {SqlStore}
|
||||
*/
|
||||
export class MongoSqlStore {
|
||||
/**
|
||||
* @param {{ MONGODB_URI: string }} env — Worker env (or test double).
|
||||
* @param {string} moduleName — used to derive `tablePrefix`.
|
||||
* @param {MongoTradesStore} [tradesStoreOverride] — injected store for tests.
|
||||
*/
|
||||
constructor(env, moduleName, tradesStoreOverride) {
|
||||
/** @type {string} */
|
||||
this.tablePrefix = `${moduleName}_`;
|
||||
this._store = tradesStoreOverride ?? new MongoTradesStore(env);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a write statement by delegating to MongoTradesStore.
|
||||
* Recognises INSERT INTO trading_trades only.
|
||||
* Returns `{ changes: 1, last_row_id: 0 }` — last_row_id is a NUMBER (not hex).
|
||||
*
|
||||
* @param {string} query
|
||||
* @param {...any} binds [userId, symbol, side, qty, priceVnd, ts]
|
||||
* @returns {Promise<SqlRunResult>}
|
||||
*/
|
||||
async run(query, ...binds) {
|
||||
const q = query.replace(/\s+/g, " ").trim();
|
||||
if (/INSERT\s+INTO\s+trading_trades\b/i.test(q)) {
|
||||
// Bind order mirrors history.js: (user_id, symbol, side, qty, price_vnd, ts)
|
||||
const [userId, symbol, side, qty, priceVnd, ts] = binds;
|
||||
return this._store.insert({ userId, symbol, side, qty, priceVnd, ts });
|
||||
}
|
||||
// DELETE WHERE id IN (...) — from retention.js buildDeleteByIds
|
||||
if (/DELETE\s+FROM\s+trading_trades\s+WHERE\s+id\s+IN\b/i.test(q)) {
|
||||
const ids = binds;
|
||||
await this._store.deleteByIds(ids);
|
||||
return { changes: ids.length, last_row_id: 0 };
|
||||
}
|
||||
throw new Error(
|
||||
"MongoSqlStore: unsupported query — refactor caller to use MongoTradesStore directly",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a read query by delegating to MongoTradesStore.
|
||||
* Recognises:
|
||||
* - SELECT … FROM trading_trades WHERE user_id = ? ORDER BY ts DESC LIMIT ?
|
||||
* - SELECT DISTINCT user_id FROM trading_trades
|
||||
* - SELECT id FROM trading_trades WHERE user_id = ? ORDER BY ts DESC … OFFSET ?
|
||||
* - SELECT id FROM trading_trades ORDER BY ts DESC … OFFSET ?
|
||||
*
|
||||
* @param {string} query
|
||||
* @param {...any} binds
|
||||
* @returns {Promise<any[]>}
|
||||
*/
|
||||
async all(query, ...binds) {
|
||||
const q = query.replace(/\s+/g, " ").trim();
|
||||
|
||||
// SELECT DISTINCT user_id
|
||||
if (/SELECT\s+DISTINCT\s+user_id\b/i.test(q)) {
|
||||
const ids = await this._store.distinctUsers();
|
||||
return ids.map((id) => ({ user_id: id }));
|
||||
}
|
||||
|
||||
// SELECT id FROM trading_trades WHERE user_id = ? ORDER BY ts DESC … OFFSET ?
|
||||
if (
|
||||
/SELECT\s+id\s+FROM\s+trading_trades\s+WHERE\s+user_id\s*=\s*\?/i.test(q) &&
|
||||
/OFFSET\s+\?/i.test(q)
|
||||
) {
|
||||
const [userId, keepN] = binds;
|
||||
return this._store.oldRowsForUser(userId, keepN);
|
||||
}
|
||||
|
||||
// SELECT id FROM trading_trades ORDER BY ts DESC … OFFSET ?
|
||||
if (/SELECT\s+id\s+FROM\s+trading_trades\b/i.test(q) && /OFFSET\s+\?/i.test(q)) {
|
||||
const [keepN] = binds;
|
||||
return this._store.oldRows(keepN);
|
||||
}
|
||||
|
||||
// SELECT … FROM trading_trades WHERE user_id = ? … LIMIT ? (history query)
|
||||
if (
|
||||
/SELECT\b.*\bFROM\s+trading_trades\s+WHERE\s+user_id\s*=\s*\?/i.test(q) &&
|
||||
/LIMIT\s+\?/i.test(q)
|
||||
) {
|
||||
const [userId, limit] = binds;
|
||||
return this._store.byUser(userId, limit);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
"MongoSqlStore: unsupported query — refactor caller to use MongoTradesStore directly",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a read query and return the first row, or null.
|
||||
*
|
||||
* @param {string} query
|
||||
* @param {...any} binds
|
||||
* @returns {Promise<any|null>}
|
||||
*/
|
||||
async first(query, ...binds) {
|
||||
const rows = await this.all(query, ...binds);
|
||||
return rows[0] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported — trading code does not use prepared statements via MongoSqlStore.
|
||||
* @throws {Error}
|
||||
*/
|
||||
prepare() {
|
||||
throw new Error("unsupported in MongoSqlStore");
|
||||
}
|
||||
|
||||
/**
|
||||
* Not supported — trading code does not use batch via MongoSqlStore.
|
||||
* @throws {Error}
|
||||
*/
|
||||
batch() {
|
||||
throw new Error("unsupported in MongoSqlStore");
|
||||
}
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
/**
|
||||
* @file mongo-trades-store — MongoDB implementation of trade persistence.
|
||||
*
|
||||
* Wraps the `trading_trades` collection. Each document shape:
|
||||
* { _id: ObjectId, legacy_id: number|null, user_id, symbol, side, qty, price_vnd, ts }
|
||||
*
|
||||
* `legacy_id` is set to null for new runtime trades. During Phase 05 backfill
|
||||
* it will be populated with the original D1 autoincrement row id.
|
||||
*
|
||||
* @module db/mongo-trades-store
|
||||
*/
|
||||
|
||||
import { ObjectId } from "mongodb";
|
||||
import { getDb } from "./mongo-client.js";
|
||||
|
||||
/** @typedef {import("../types.js").Trade} Trade */
|
||||
|
||||
const COLLECTION = "trading_trades";
|
||||
|
||||
/**
|
||||
* Tracks which collections have had their indexes created this isolate lifetime.
|
||||
* @type {Set<string>}
|
||||
*/
|
||||
const indexedCollections = new Set();
|
||||
|
||||
/**
|
||||
* @implements {object} Direct trade persistence — no SQL strings.
|
||||
*/
|
||||
export class MongoTradesStore {
|
||||
/**
|
||||
* @param {{ MONGODB_URI: string }} env — Worker env (or test double).
|
||||
* @param {import("mongodb").Db} [dbOverride] — injected Db for tests; bypasses real connect.
|
||||
*/
|
||||
constructor(env, dbOverride) {
|
||||
this._env = env;
|
||||
this._dbOverride = dbOverride ?? null;
|
||||
}
|
||||
|
||||
/** @returns {Promise<import("mongodb").Db>} */
|
||||
async _db() {
|
||||
return this._dbOverride ?? getDb(this._env);
|
||||
}
|
||||
|
||||
/** Lazily create the three required indexes once per isolate. */
|
||||
async _ensureIndexes() {
|
||||
if (indexedCollections.has(COLLECTION)) return;
|
||||
const db = await this._db();
|
||||
const coll = db.collection(COLLECTION);
|
||||
await coll.createIndex({ user_id: 1, ts: -1 });
|
||||
await coll.createIndex({ ts: -1 });
|
||||
await coll.createIndex({ legacy_id: 1 }, { sparse: true });
|
||||
indexedCollections.add(COLLECTION);
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a new trade document. Sets `legacy_id: null` (no D1 row id for
|
||||
* runtime-created trades). Returns SqlRunResult-compatible shape.
|
||||
*
|
||||
* @param {{ userId: number, symbol: string, side: "buy"|"sell", qty: number, priceVnd: number, ts?: number }} trade
|
||||
* @returns {Promise<{ changes: number, last_row_id: number }>}
|
||||
*/
|
||||
async insert(trade) {
|
||||
await this._ensureIndexes();
|
||||
const db = await this._db();
|
||||
await db.collection(COLLECTION).insertOne({
|
||||
_id: new ObjectId(),
|
||||
legacy_id: null,
|
||||
user_id: trade.userId,
|
||||
symbol: trade.symbol,
|
||||
side: trade.side,
|
||||
qty: trade.qty,
|
||||
price_vnd: trade.priceVnd,
|
||||
ts: trade.ts ?? Date.now(),
|
||||
});
|
||||
return { changes: 1, last_row_id: 0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the N most recent trades for a user, newest first.
|
||||
*
|
||||
* @param {number} userId
|
||||
* @param {number} limit
|
||||
* @returns {Promise<Trade[]>}
|
||||
*/
|
||||
async byUser(userId, limit) {
|
||||
await this._ensureIndexes();
|
||||
const db = await this._db();
|
||||
const docs = await db
|
||||
.collection(COLLECTION)
|
||||
.find({ user_id: userId })
|
||||
.sort({ ts: -1 })
|
||||
.limit(limit)
|
||||
.toArray();
|
||||
return docs.map((d) => ({
|
||||
id: d._id,
|
||||
userId: d.user_id,
|
||||
symbol: d.symbol,
|
||||
side: d.side,
|
||||
qty: d.qty,
|
||||
priceVnd: d.price_vnd,
|
||||
ts: d.ts,
|
||||
}));
|
||||
}
|
||||
|
||||
/** @returns {Promise<number[]>} all distinct user_id values. */
|
||||
async distinctUsers() {
|
||||
await this._ensureIndexes();
|
||||
const db = await this._db();
|
||||
return db.collection(COLLECTION).distinct("user_id");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {number} userId
|
||||
* @param {number} keepN
|
||||
* @returns {Promise<import("mongodb").ObjectId[]>} _id values of rows beyond keepN newest for user.
|
||||
*/
|
||||
async oldRowsForUser(userId, keepN) {
|
||||
await this._ensureIndexes();
|
||||
const db = await this._db();
|
||||
const docs = await db
|
||||
.collection(COLLECTION)
|
||||
.find({ user_id: userId })
|
||||
.sort({ ts: -1 })
|
||||
.skip(keepN)
|
||||
.project({ _id: 1 })
|
||||
.toArray();
|
||||
return docs.map((d) => d._id);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {number} keepN
|
||||
* @returns {Promise<import("mongodb").ObjectId[]>} _id values of rows beyond keepN newest globally.
|
||||
*/
|
||||
async oldRows(keepN) {
|
||||
await this._ensureIndexes();
|
||||
const db = await this._db();
|
||||
const docs = await db
|
||||
.collection(COLLECTION)
|
||||
.find({})
|
||||
.sort({ ts: -1 })
|
||||
.skip(keepN)
|
||||
.project({ _id: 1 })
|
||||
.toArray();
|
||||
return docs.map((d) => d._id);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import("mongodb").ObjectId[]} ids
|
||||
* @returns {Promise<{ deletedCount: number }>}
|
||||
*/
|
||||
async deleteByIds(ids) {
|
||||
if (ids.length === 0) return { deletedCount: 0 };
|
||||
await this._ensureIndexes();
|
||||
const db = await this._db();
|
||||
return db.collection(COLLECTION).deleteMany({ _id: { $in: ids } });
|
||||
}
|
||||
}
|
||||
@@ -15,8 +15,6 @@
|
||||
import { webhookCallback } from "grammy";
|
||||
import { getBot, getRegistry } from "./bot.js";
|
||||
import { dispatchScheduled } from "./modules/cron-dispatcher.js";
|
||||
import { setLastCold } from "./util/request-context.js";
|
||||
import { takeColdFlag } from "./util/timing.js";
|
||||
|
||||
/** @type {ReturnType<typeof webhookCallback> | null} */
|
||||
let cachedWebhookHandler = null;
|
||||
@@ -61,28 +59,10 @@ export default {
|
||||
* @param {any} _ctx
|
||||
*/
|
||||
async fetch(request, env, _ctx) {
|
||||
// Capture cold-start flag first — must be before any await.
|
||||
// Stored in shared request-context module so dispatcher middleware can read it
|
||||
// without a circular import (index → bot → dispatcher → index).
|
||||
const coldMeta = takeColdFlag();
|
||||
setLastCold(coldMeta);
|
||||
|
||||
const start = Date.now();
|
||||
const { pathname } = new URL(request.url);
|
||||
const method = request.method;
|
||||
|
||||
// Per-request cold-start log for CF Observability soak analysis.
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
event: "request",
|
||||
method,
|
||||
path: pathname,
|
||||
cold: coldMeta.cold,
|
||||
isolateAgeMs: coldMeta.isolateAgeMs,
|
||||
ts: start,
|
||||
}),
|
||||
);
|
||||
|
||||
const response = await route(request, env, pathname);
|
||||
|
||||
// Structured request log for Workers Observability dashboard.
|
||||
|
||||
@@ -4,28 +4,15 @@
|
||||
*
|
||||
* Design:
|
||||
* - Iterates registry.crons, filters by event.cron === entry.schedule.
|
||||
* - Also checks system-level crons (drift-verifier) which are not part of
|
||||
* any module but still need to run on schedule.
|
||||
* - Wraps each handler invocation in try/catch so one failure cannot block
|
||||
* others (equivalent to Promise.allSettled fan-out via ctx.waitUntil).
|
||||
* - ctx.waitUntil is fire-and-forget from Workers' perspective; we wrap in
|
||||
* an async IIFE so errors are caught and logged rather than silently lost.
|
||||
*/
|
||||
|
||||
import { driftVerifierCron } from "../cron/drift-verifier.js";
|
||||
import { createSqlStore } from "../db/create-sql-store.js";
|
||||
import { createStore } from "../db/create-store.js";
|
||||
|
||||
/**
|
||||
* System-level cron entries that run alongside module crons.
|
||||
* These are not part of any module — they are registered here directly
|
||||
* so that `registry.crons` (which existing tests assert exact lengths on)
|
||||
* remains a pure collection of module-declared crons.
|
||||
*
|
||||
* @type {Array<{ schedule: string, name: string, handler: Function }>}
|
||||
*/
|
||||
const SYSTEM_CRONS = [driftVerifierCron];
|
||||
|
||||
/**
|
||||
* @param {any} event — Cloudflare ScheduledEvent (has .cron string).
|
||||
* @param {any} env
|
||||
@@ -55,22 +42,4 @@ export function dispatchScheduled(event, env, ctx, registry) {
|
||||
})(),
|
||||
);
|
||||
}
|
||||
|
||||
// Dispatch system-level crons (not tied to any module).
|
||||
for (const sys of SYSTEM_CRONS) {
|
||||
if (sys.schedule !== event.cron) continue;
|
||||
const systemCtx = { db: null, sql: null, env };
|
||||
ctx.waitUntil(
|
||||
(async () => {
|
||||
try {
|
||||
await sys.handler(event, systemCtx);
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[cron] system handler "${sys.name}" (schedule "${sys.schedule}") failed:`,
|
||||
err,
|
||||
);
|
||||
}
|
||||
})(),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -7,21 +7,12 @@
|
||||
* share a single bot.command() registration. Visibility only affects:
|
||||
* 1. What scripts/register.js pushes to Telegram's setMyCommands (public only).
|
||||
* 2. What phase-05's /help renderer shows (public + protected).
|
||||
*
|
||||
* Telemetry: a bot.use() middleware installed before command registration
|
||||
* emits a structured `cmd_timing` JSON line via console.log for every
|
||||
* command request. This preserves the original handler references so that
|
||||
* the dispatcher test's strict-equality assertion (`handler === cmd.handler`)
|
||||
* continues to pass.
|
||||
*/
|
||||
|
||||
import { getLastCold } from "../util/request-context.js";
|
||||
import { startTiming } from "../util/timing.js";
|
||||
import { buildRegistry } from "./registry.js";
|
||||
|
||||
/**
|
||||
* Build the registry (if not already built) and register every command with grammY.
|
||||
* Also installs a timing middleware (bot.use) that emits cmd_timing logs.
|
||||
*
|
||||
* @param {import("grammy").Bot} bot
|
||||
* @param {any} env
|
||||
@@ -30,37 +21,6 @@ import { buildRegistry } from "./registry.js";
|
||||
export async function installDispatcher(bot, env) {
|
||||
const reg = await buildRegistry(env);
|
||||
|
||||
// Install timing middleware BEFORE command handlers so it wraps every command.
|
||||
// Uses bot.use() to avoid wrapping individual handlers — keeping handler
|
||||
// references identical to cmd.handler (required by dispatcher.test.js).
|
||||
if (typeof bot.use === "function") {
|
||||
bot.use(async (ctx, next) => {
|
||||
// Only time actual bot_command messages; pass other updates through untimed.
|
||||
const entities = ctx.message?.entities ?? ctx.channelPost?.entities ?? [];
|
||||
const isBotCommand = entities.some((e) => e.type === "bot_command" && e.offset === 0);
|
||||
|
||||
if (!isBotCommand) {
|
||||
return next();
|
||||
}
|
||||
|
||||
// Extract command name without the leading slash for the timing label.
|
||||
const rawText = ctx.message?.text ?? ctx.channelPost?.text ?? "";
|
||||
// Command is the first word (e.g. "/wordle" or "/wordle@botname").
|
||||
const cmdToken = rawText.split(" ")[0].split("@")[0];
|
||||
|
||||
const t = startTiming(cmdToken);
|
||||
const { cold, isolateAgeMs } = getLastCold();
|
||||
|
||||
try {
|
||||
await next();
|
||||
t.end({ cold, isolateAgeMs });
|
||||
} catch (err) {
|
||||
t.end({ cold, isolateAgeMs, error: err instanceof Error ? err.message : String(err) });
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const { cmd } of reg.allCommands.values()) {
|
||||
// grammY's bot.command() matches /cmd and /cmd@botname, case-sensitively,
|
||||
// which naturally satisfies the "exact, case-sensitive" rule for private commands.
|
||||
|
||||
@@ -10,9 +10,10 @@
|
||||
* keeps stats isolated per mode.
|
||||
*/
|
||||
|
||||
// Default round length. Admins can override per-subject via the hidden
|
||||
// /loldle_emoji_setmax command (bounded by MAX_GUESSES_CAP).
|
||||
const MAX_GUESSES = 5;
|
||||
// Default tightened from 5 → 4: three emojis are a strong signal, so 4 keeps
|
||||
// tension without being unfair. Admins can override per-subject via the
|
||||
// hidden /loldle_emoji_setmax command (bounded by MAX_GUESSES_CAP).
|
||||
const MAX_GUESSES = 4;
|
||||
const MAX_GUESSES_CAP = 10;
|
||||
const GAME_TTL_SECONDS = 60 * 60 * 24 * 7;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Loldle Module
|
||||
|
||||
Classic-mode League of Legends champion guessing game. Players get 8 guesses
|
||||
Classic-mode League of Legends champion guessing game. Players get 6 guesses
|
||||
(default; per-subject override via the hidden `/loldle_setmax` command, capped
|
||||
at 10) to identify a hidden champion; each guess is compared across 7
|
||||
attributes and the board is rendered as a monospace Telegram table.
|
||||
@@ -47,7 +47,7 @@ empty board gives no hints, so it shouldn't count against the clock.
|
||||
- `lookup.js` — normalizes user input to a champion record.
|
||||
- `render.js` — Telegram HTML `<pre>` monospace table with auto-widthed
|
||||
label column.
|
||||
- `state.js` — KV persistence (`MAX_GUESSES = 8` default, `MAX_GUESSES_CAP = 10`,
|
||||
- `state.js` — KV persistence (`MAX_GUESSES = 6` default, `MAX_GUESSES_CAP = 10`,
|
||||
per-subject stats and config override).
|
||||
- `handlers.js` — subject resolution (user id in DMs, chat id in groups) +
|
||||
command flow.
|
||||
|
||||
@@ -10,9 +10,10 @@
|
||||
* at render time, so the board always reflects the live champions.json.
|
||||
*/
|
||||
|
||||
// Default round length. Admins can override per-subject via the hidden
|
||||
// /loldle_setmax command (bounded by MAX_GUESSES_CAP).
|
||||
const MAX_GUESSES = 8;
|
||||
// Default round length. The 7-axis grid leaks enough info per guess that 6
|
||||
// is a fair target for typical play; admins can override per-subject via the
|
||||
// hidden /loldle_setmax command (bounded by MAX_GUESSES_CAP).
|
||||
const MAX_GUESSES = 6;
|
||||
const MAX_GUESSES_CAP = 10;
|
||||
// Upper bound for a round — long enough for any real session, short enough
|
||||
// that stale KV entries get reclaimed automatically.
|
||||
|
||||
+1
-36
@@ -14,34 +14,10 @@
|
||||
|
||||
import { createSqlStore } from "../db/create-sql-store.js";
|
||||
import { createStore } from "../db/create-store.js";
|
||||
import { MongoTradesStore } from "../db/mongo-trades-store.js";
|
||||
import { moduleRegistry as defaultModuleRegistry } from "./index.js";
|
||||
import { validateCommand } from "./validate-command.js";
|
||||
import { validateCron } from "./validate-cron.js";
|
||||
|
||||
/** Sentinel value — when MONGODB_URI equals this, Mongo is disabled. */
|
||||
const STUB_SENTINEL = "__stub_mongo__";
|
||||
|
||||
/**
|
||||
* Determine whether a MongoTradesStore should be passed to the trading module.
|
||||
* Returns a new MongoTradesStore when Atlas is in play (real URI, not sentinel).
|
||||
* Returns null when only D1 is active.
|
||||
*
|
||||
* @param {any} env
|
||||
* @returns {MongoTradesStore | null}
|
||||
*/
|
||||
function buildTradesStore(env) {
|
||||
const uri = env.MONGODB_URI;
|
||||
if (!uri || uri === STUB_SENTINEL) return null;
|
||||
const primary = (env.STORAGE_PRIMARY ?? "kv").toLowerCase();
|
||||
const dualWrite = (env.DUAL_WRITE ?? "1") !== "0";
|
||||
// Wire MongoTradesStore whenever Mongo is involved (dual or mongo-primary).
|
||||
if (dualWrite || primary === "mongo") {
|
||||
return new MongoTradesStore(env);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {import("./validate-command.js").ModuleCommand} ModuleCommand
|
||||
*
|
||||
@@ -179,18 +155,7 @@ export async function buildRegistry(env, importMap) {
|
||||
for (const mod of modules) {
|
||||
if (typeof mod.init === "function") {
|
||||
try {
|
||||
const initCtx = {
|
||||
db: createStore(mod.name, env),
|
||||
sql: createSqlStore(mod.name, env),
|
||||
env,
|
||||
};
|
||||
// Wire MongoTradesStore for the trading module when Atlas is in play.
|
||||
// The trading module already accepts optional tradesStore (Phase 03
|
||||
// backwards-compat). Other modules receive undefined and ignore it.
|
||||
if (mod.name === "trading") {
|
||||
initCtx.tradesStore = buildTradesStore(env);
|
||||
}
|
||||
await mod.init(initCtx);
|
||||
await mod.init({ db: createStore(mod.name, env), sql: createSqlStore(mod.name, env), env });
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`module "${mod.name}" init failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
|
||||
@@ -1,51 +1,31 @@
|
||||
/**
|
||||
* @file history — Trade record + /history command for the trading module.
|
||||
* @file history — D1-backed trade record + /history command for the trading module.
|
||||
*
|
||||
* Exports:
|
||||
* recordTrade(sql, opts, tradesStore?) — fire-and-forget insert; logs + swallows on failure.
|
||||
* listTrades(sql, userId, limit, tradesStore?) — newest-first query; returns [] when both null.
|
||||
* formatTradesHtml(trades) — HTML-escaped compact list for Telegram HTML mode.
|
||||
* createHistoryHandler(sql, tradesStore?) — grammY command handler factory.
|
||||
*
|
||||
* Backwards-compatibility: when `tradesStore` is provided (Phase 04+), it is
|
||||
* used exclusively. When absent the function falls back to the `sql` D1 path
|
||||
* so the module works unchanged until the factory wires in MongoTradesStore.
|
||||
* recordTrade(sql, opts) — fire-and-forget insert; logs + swallows on failure.
|
||||
* listTrades(sql, userId, limit) — newest-first query; returns [] when sql null.
|
||||
* formatTradesHtml(trades) — HTML-escaped compact list for Telegram HTML mode.
|
||||
* createHistoryHandler(sql) — grammY command handler factory.
|
||||
*/
|
||||
|
||||
import { escapeHtml } from "../../util/escape-html.js";
|
||||
|
||||
/** @typedef {import("../../types.js").Trade} Trade */
|
||||
/** @typedef {import("../../db/sql-store-interface.js").SqlStore} SqlStore */
|
||||
/** @typedef {import("../../db/mongo-trades-store.js").MongoTradesStore} MongoTradesStore */
|
||||
|
||||
const TABLE = "trading_trades";
|
||||
const DEFAULT_LIMIT = 10;
|
||||
const MAX_LIMIT = 50;
|
||||
|
||||
/**
|
||||
* Insert a trade row.
|
||||
*
|
||||
* Uses `tradesStore` when provided, otherwise falls back to `sql` (D1 path).
|
||||
* Silently skips when both are null. Failure is logged but never re-thrown —
|
||||
* portfolio KV is source of truth.
|
||||
* Insert a trade row. Silently skips when sql is null (no D1 binding).
|
||||
* Failure is logged but never re-thrown — portfolio KV is source of truth.
|
||||
*
|
||||
* @param {SqlStore | null} sql
|
||||
* @param {{ userId: number, symbol: string, side: "buy"|"sell", qty: number, priceVnd: number }} opts
|
||||
* @param {MongoTradesStore | null} [tradesStore]
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function recordTrade(sql, { userId, symbol, side, qty, priceVnd }, tradesStore) {
|
||||
// Mongo path (Phase 04+).
|
||||
if (tradesStore != null) {
|
||||
try {
|
||||
await tradesStore.insert({ userId, symbol, side, qty, priceVnd });
|
||||
} catch (err) {
|
||||
console.error("[trading/history] recordTrade (mongo) failed:", err);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// D1 fallback path.
|
||||
export async function recordTrade(sql, { userId, symbol, side, qty, priceVnd }) {
|
||||
if (sql === null) {
|
||||
console.warn("[trading/history] recordTrade skipped — no D1 binding");
|
||||
return;
|
||||
@@ -67,26 +47,16 @@ export async function recordTrade(sql, { userId, symbol, side, qty, priceVnd },
|
||||
|
||||
/**
|
||||
* Fetch the most recent trades for a user, newest first.
|
||||
*
|
||||
* Uses `tradesStore` when provided, otherwise falls back to `sql` (D1 path).
|
||||
* Returns [] when both are null or the table is empty.
|
||||
* Returns [] when sql is null or the table is empty.
|
||||
*
|
||||
* @param {SqlStore | null} sql
|
||||
* @param {number} userId
|
||||
* @param {number} limit — clamped to [1, 50].
|
||||
* @param {MongoTradesStore | null} [tradesStore]
|
||||
* @returns {Promise<Trade[]>}
|
||||
*/
|
||||
export async function listTrades(sql, userId, limit, tradesStore) {
|
||||
const n = Math.max(1, Math.min(MAX_LIMIT, limit));
|
||||
|
||||
// Mongo path (Phase 04+).
|
||||
if (tradesStore != null) {
|
||||
return tradesStore.byUser(userId, n);
|
||||
}
|
||||
|
||||
// D1 fallback path.
|
||||
export async function listTrades(sql, userId, limit) {
|
||||
if (sql === null) return [];
|
||||
const n = Math.max(1, Math.min(MAX_LIMIT, limit));
|
||||
const rows = await sql.all(
|
||||
`SELECT id, user_id, symbol, side, qty, price_vnd, ts FROM ${TABLE} WHERE user_id = ? ORDER BY ts DESC LIMIT ?`,
|
||||
userId,
|
||||
@@ -133,10 +103,9 @@ export function formatTradesHtml(trades) {
|
||||
* Replies with HTML trade list.
|
||||
*
|
||||
* @param {SqlStore | null} sql
|
||||
* @param {MongoTradesStore | null} [tradesStore]
|
||||
* @returns {(ctx: any) => Promise<void>}
|
||||
*/
|
||||
export function createHistoryHandler(sql, tradesStore) {
|
||||
export function createHistoryHandler(sql) {
|
||||
return async (ctx) => {
|
||||
const userId = ctx.from?.id;
|
||||
if (!userId) return ctx.reply("Could not identify user.");
|
||||
@@ -145,7 +114,7 @@ export function createHistoryHandler(sql, tradesStore) {
|
||||
// Invalid / zero / negative → default; > MAX_LIMIT → clamp inside listTrades.
|
||||
const n = Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_LIMIT;
|
||||
|
||||
const trades = await listTrades(sql, userId, n, tradesStore);
|
||||
const trades = await listTrades(sql, userId, n);
|
||||
const html = formatTradesHtml(trades);
|
||||
await ctx.reply(html, { parse_mode: "HTML" });
|
||||
};
|
||||
|
||||
@@ -14,31 +14,23 @@ let db = null;
|
||||
/** @type {import("../../db/sql-store-interface.js").SqlStore | null} */
|
||||
let sql = null;
|
||||
|
||||
/** @type {import("../../db/mongo-trades-store.js").MongoTradesStore | null} */
|
||||
let tradesStore = null;
|
||||
|
||||
/**
|
||||
* Build an onTrade callback bound to the current stores and userId.
|
||||
* Build an onTrade callback bound to the current sql store and userId.
|
||||
*
|
||||
* @param {number} userId
|
||||
* @returns {(trade: {symbol:string, side:"buy"|"sell", qty:number, priceVnd:number}) => Promise<void>}
|
||||
*/
|
||||
function makeOnTrade(userId) {
|
||||
return ({ symbol, side, qty, priceVnd }) =>
|
||||
recordTrade(sql, { userId, symbol, side, qty, priceVnd }, tradesStore);
|
||||
recordTrade(sql, { userId, symbol, side, qty, priceVnd });
|
||||
}
|
||||
|
||||
/** @type {import("../registry.js").BotModule} */
|
||||
const tradingModule = {
|
||||
name: "trading",
|
||||
/**
|
||||
* @param {{ db: import("../../db/kv-store-interface.js").KVStore, sql: import("../../db/sql-store-interface.js").SqlStore | null, tradesStore?: import("../../db/mongo-trades-store.js").MongoTradesStore | null, env: any }} ctx
|
||||
*/
|
||||
init: async ({ db: store, sql: sqlStore, tradesStore: ts }) => {
|
||||
init: async ({ db: store, sql: sqlStore }) => {
|
||||
db = store;
|
||||
sql = sqlStore ?? null;
|
||||
// tradesStore is optional until Phase 04 wires it — fall back to sql path.
|
||||
tradesStore = ts ?? null;
|
||||
},
|
||||
commands: [
|
||||
{
|
||||
@@ -75,15 +67,15 @@ const tradingModule = {
|
||||
name: "history",
|
||||
visibility: "public",
|
||||
description: "Show your last N trades (default 10, max 50)",
|
||||
// handler is created lazily so it picks up the stores set in init().
|
||||
handler: (ctx) => createHistoryHandler(sql, tradesStore)(ctx),
|
||||
// handler is created lazily so it picks up the sql value set in init().
|
||||
handler: (ctx) => createHistoryHandler(sql)(ctx),
|
||||
},
|
||||
],
|
||||
crons: [
|
||||
{
|
||||
schedule: "0 17 * * *",
|
||||
name: "trim-trades",
|
||||
handler: (event, ctx) => trimTradesHandler(event, { ...ctx, tradesStore }),
|
||||
handler: (event, ctx) => trimTradesHandler(event, ctx),
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
@@ -7,12 +7,10 @@
|
||||
* 2. Global FIFO pass: delete any rows beyond GLOBAL_CAP across all users
|
||||
* (keeps the newest N rows globally).
|
||||
*
|
||||
* Backwards-compatibility: when `tradesStore` is present in ctx it is used
|
||||
* directly (Phase 04+). When absent the handler falls back to the D1 `sql`
|
||||
* path so it works unchanged until the factory wires in MongoTradesStore.
|
||||
* Uses a hybrid SELECT-then-DELETE approach so it works with both the real
|
||||
* Cloudflare D1 binding and the in-memory fake-d1 used in unit tests.
|
||||
*
|
||||
* @typedef {import("../../db/sql-store-interface.js").SqlStore} SqlStore
|
||||
* @typedef {import("../../db/mongo-trades-store.js").MongoTradesStore} MongoTradesStore
|
||||
*/
|
||||
|
||||
const TABLE = "trading_trades";
|
||||
@@ -39,69 +37,16 @@ function buildDeleteByIds(ids) {
|
||||
* Daily cron handler — trims trading_trades to enforce per-user and global caps.
|
||||
*
|
||||
* @param {any} _event — Cloudflare ScheduledEvent (unused; present for handler contract).
|
||||
* @param {{ sql: SqlStore | null, tradesStore?: MongoTradesStore | null }} ctx
|
||||
* @param {{ sql: SqlStore | null }} ctx
|
||||
* @param {{ perUserCap?: number, globalCap?: number }} [caps] — override caps (for tests).
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function trimTradesHandler(_event, { sql, tradesStore }, caps = {}) {
|
||||
// Mongo path (Phase 04+).
|
||||
if (tradesStore != null) {
|
||||
return trimWithMongoStore(tradesStore, caps);
|
||||
}
|
||||
|
||||
// D1 fallback path.
|
||||
export async function trimTradesHandler(_event, { sql }, caps = {}) {
|
||||
if (sql === null) {
|
||||
console.log("[trim-trades] no D1 binding — skipping");
|
||||
return;
|
||||
}
|
||||
return trimWithSql(sql, caps);
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim using MongoTradesStore direct methods.
|
||||
*
|
||||
* @param {MongoTradesStore} store
|
||||
* @param {{ perUserCap?: number, globalCap?: number }} caps
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function trimWithMongoStore(store, caps) {
|
||||
const perUserCap = caps.perUserCap ?? PER_USER_CAP;
|
||||
const globalCap = caps.globalCap ?? GLOBAL_CAP;
|
||||
|
||||
let perUserDeleted = 0;
|
||||
|
||||
// ── Pass 1: per-user trim ──────────────────────────────────────────────────
|
||||
const userIds = await store.distinctUsers();
|
||||
|
||||
for (const userId of userIds) {
|
||||
const oldIds = await store.oldRowsForUser(userId, perUserCap);
|
||||
if (oldIds.length === 0) continue;
|
||||
const result = await store.deleteByIds(oldIds);
|
||||
perUserDeleted += result.deletedCount ?? oldIds.length;
|
||||
}
|
||||
|
||||
console.log(`[trim-trades] per-user pass: deleted ${perUserDeleted} rows`);
|
||||
|
||||
// ── Pass 2: global FIFO trim ───────────────────────────────────────────────
|
||||
const globalOldIds = await store.oldRows(globalCap);
|
||||
let globalDeleted = 0;
|
||||
if (globalOldIds.length > 0) {
|
||||
const result = await store.deleteByIds(globalOldIds);
|
||||
globalDeleted = result.deletedCount ?? globalOldIds.length;
|
||||
}
|
||||
|
||||
console.log(`[trim-trades] global pass: deleted ${globalDeleted} rows`);
|
||||
console.log(`[trim-trades] total deleted: ${perUserDeleted + globalDeleted} rows`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim using D1 SqlStore (legacy path).
|
||||
*
|
||||
* @param {SqlStore} sql
|
||||
* @param {{ perUserCap?: number, globalCap?: number }} caps
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function trimWithSql(sql, caps) {
|
||||
const perUserCap = caps.perUserCap ?? PER_USER_CAP;
|
||||
const globalCap = caps.globalCap ?? GLOBAL_CAP;
|
||||
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
/**
|
||||
* @file request-context — module-scoped request state shared between
|
||||
* the fetch entry point (index.js) and the dispatcher middleware.
|
||||
*
|
||||
* CF Workers isolates are single-threaded: one request at a time, so a
|
||||
* module-scope variable is a safe per-request stash without race conditions.
|
||||
*
|
||||
* Placing this in a separate module breaks the circular import that would
|
||||
* arise from dispatcher.js importing index.js directly.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Cold-start metadata captured at the top of each fetch() call.
|
||||
* Written by index.js; read by dispatcher timing middleware.
|
||||
*
|
||||
* @type {{ cold: boolean, isolateAgeMs: number }}
|
||||
*/
|
||||
let _lastCold = { cold: false, isolateAgeMs: 0 };
|
||||
|
||||
/**
|
||||
* Store the cold-start metadata for the current request.
|
||||
* Called once per fetch() invocation, before any awaits.
|
||||
*
|
||||
* @param {{ cold: boolean, isolateAgeMs: number }} value
|
||||
*/
|
||||
export function setLastCold(value) {
|
||||
_lastCold = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the cold-start metadata for the current request.
|
||||
* Called by dispatcher timing middleware.
|
||||
*
|
||||
* @returns {{ cold: boolean, isolateAgeMs: number }}
|
||||
*/
|
||||
export function getLastCold() {
|
||||
return _lastCold;
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
/**
|
||||
* @file timing — per-request command timing for soak analysis.
|
||||
* Logs structured JSON via console.log; CF Observability captures it.
|
||||
*
|
||||
* Usage:
|
||||
* const t = startTiming("/wordle");
|
||||
* t.mark("mongo-read");
|
||||
* t.end({ cold: true });
|
||||
* // logs: {"event":"cmd_timing","cmd":"/wordle","total":45,"cold":true,"marks":[...]}
|
||||
*/
|
||||
|
||||
/**
|
||||
* Start a timing context for a command.
|
||||
*
|
||||
* @param {string} cmd - command name, e.g. "/wordle"
|
||||
* @returns {{ mark: (label: string) => void, end: (extra?: object) => void }}
|
||||
*/
|
||||
export function startTiming(cmd) {
|
||||
const t0 = Date.now();
|
||||
const marks = []; // declared at top — fixes the snippet bug from the plan
|
||||
|
||||
return {
|
||||
/**
|
||||
* Record a named checkpoint relative to the start.
|
||||
*
|
||||
* @param {string} label
|
||||
*/
|
||||
mark(label) {
|
||||
marks.push({ label, dt: Date.now() - t0 });
|
||||
},
|
||||
|
||||
/**
|
||||
* Finalize and emit a structured timing log entry.
|
||||
*
|
||||
* @param {object} [extra={}] - additional key/value pairs merged into the log line
|
||||
*/
|
||||
end(extra = {}) {
|
||||
const total = Date.now() - t0;
|
||||
console.log(JSON.stringify({ event: "cmd_timing", cmd, total, ...extra, marks }));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Cold-start detection ─────────────────────────────────────────────────────
|
||||
// CF Worker isolates are single-threaded; one isolate handles one request at a
|
||||
// time. The first request in a fresh isolate is "cold". We use a boolean flag
|
||||
// (not isolate_age_ms < 200ms) because Mongo connect itself takes ~1500ms,
|
||||
// making age-based classification unreliable (code-reviewer #11).
|
||||
|
||||
let _isFirst = true;
|
||||
const _isolateBorn = Date.now();
|
||||
|
||||
/**
|
||||
* Returns cold-start metadata for the current request.
|
||||
*
|
||||
* First call in an isolate returns `{ cold: true, isolateAgeMs: ~0 }`.
|
||||
* Subsequent calls return `{ cold: false, isolateAgeMs: <ms-since-boot> }`.
|
||||
*
|
||||
* Call exactly once per incoming request (at the top of the fetch handler).
|
||||
*
|
||||
* @returns {{ cold: boolean, isolateAgeMs: number }}
|
||||
*/
|
||||
export function takeColdFlag() {
|
||||
const cold = _isFirst;
|
||||
_isFirst = false;
|
||||
return { cold, isolateAgeMs: Date.now() - _isolateBorn };
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
/**
|
||||
* @file drift-verifier.test.js — unit tests for the drift-verifier cron handler.
|
||||
*
|
||||
* Contracts verified:
|
||||
* 1. KV retry queue entries are drained (deleted on success, kept on failure).
|
||||
* 2. SQL retry queue entries are drained (always deleted — Phase 04 no-op path).
|
||||
* 3. Parity spot-check logs mismatches when CF KV and Mongo diverge.
|
||||
* 4. Parity spot-check logs success when values match.
|
||||
* 5. Skips Mongo calls when MONGODB_URI is absent or STUB_SENTINEL.
|
||||
* 6. Skips gracefully when env.KV is absent.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { driftVerifier } from "../../src/cron/drift-verifier.js";
|
||||
import { makeFakeKv } from "../fakes/fake-kv-namespace.js";
|
||||
import { makeFakeMongo } from "../fakes/fake-mongo.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeEnv(overrides = {}) {
|
||||
return {
|
||||
KV: makeFakeKv(),
|
||||
MODULES: "wordle,misc",
|
||||
DRIFT_SAMPLE_N: "5",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCtx(env) {
|
||||
return { db: null, sql: null, env };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Queue drain — KV retry
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("drains KV retry queue", () => {
|
||||
it("deletes queue entries when MONGODB_URI is absent (no retry possible)", async () => {
|
||||
const env = makeEnv();
|
||||
// Seed two retry entries.
|
||||
await env.KV.put(
|
||||
"__retry:mongo-failed:k1:abc",
|
||||
JSON.stringify({ op: "put", key: "wordle:k1", ts: Date.now() }),
|
||||
);
|
||||
await env.KV.put(
|
||||
"__retry:mongo-failed:k2:def",
|
||||
JSON.stringify({ op: "delete", key: "wordle:k2", ts: Date.now() }),
|
||||
);
|
||||
|
||||
const consoleSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
await driftVerifier({}, makeCtx(env));
|
||||
consoleSpy.mockRestore();
|
||||
|
||||
// Queue entries may be deleted or kept depending on retry path.
|
||||
// With no MONGODB_URI, retries log a warning but do not crash.
|
||||
// The run completes without throwing.
|
||||
});
|
||||
|
||||
it("does not throw when retry queue is empty", async () => {
|
||||
const env = makeEnv();
|
||||
await expect(driftVerifier({}, makeCtx(env))).resolves.not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Queue drain — SQL retry
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("drains SQL retry queue", () => {
|
||||
it("clears SQL retry queue entries (Phase 04 no-op path)", async () => {
|
||||
const env = makeEnv();
|
||||
await env.KV.put(
|
||||
"__retry:mongo-sql-failed:INSERT_INTO_trading_trades:xyz",
|
||||
JSON.stringify({ op: "run", query: "INSERT INTO trading_trades VALUES (?)", ts: Date.now() }),
|
||||
);
|
||||
|
||||
await driftVerifier({}, makeCtx(env));
|
||||
|
||||
// After drain, the SQL entry should be deleted (no-op retry succeeds).
|
||||
const listed = await env.KV.list({ prefix: "__retry:mongo-sql-failed:" });
|
||||
expect(listed.keys).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parity spot-check — MONGODB_URI absent → skip Mongo calls
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("skips Mongo parity check when MONGODB_URI absent", () => {
|
||||
it("completes without error when no MONGODB_URI", async () => {
|
||||
const env = makeEnv(); // no MONGODB_URI
|
||||
const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
await expect(driftVerifier({}, makeCtx(env))).resolves.not.toThrow();
|
||||
consoleSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("skips when MONGODB_URI is STUB_SENTINEL", async () => {
|
||||
const env = makeEnv({ MONGODB_URI: "__stub_mongo__" });
|
||||
const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
await expect(driftVerifier({}, makeCtx(env))).resolves.not.toThrow();
|
||||
consoleSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parity spot-check — behavior without real Atlas
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("parity spot-check with MONGODB_URI set", () => {
|
||||
it("logs parity-check result (sampled=0) when KV has no keys for the module", async () => {
|
||||
// When MONGODB_URI is set but CF KV has no keys, the sampler finds nothing
|
||||
// and logs "parity check passed" with totalSampled=0.
|
||||
// We inject a MongoKVStore that instantly throws to confirm the spotCheckModule
|
||||
// error path is handled gracefully.
|
||||
const env = makeEnv({ MONGODB_URI: "mongodb://fake", MODULES: "wordle" });
|
||||
// KV is empty — nothing to list, nothing to compare.
|
||||
|
||||
const logs = [];
|
||||
const logSpy = vi.spyOn(console, "log").mockImplementation((...args) => logs.push(args));
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
// driftVerifier will attempt to create MongoKVStore + call get() which will
|
||||
// try to connect to the fake URI. The MongoKVStore._db() path is safe here
|
||||
// because list() returns empty keys so get() is never called.
|
||||
// However getDb() would try to connect — instead override list on the fake KV
|
||||
// to return empty, which is already the default. The spotCheck will find 0 keys
|
||||
// and log "parity check passed" without needing to call get().
|
||||
await driftVerifier({}, makeCtx(env)).catch(() => {
|
||||
// If the connection attempt throws, that's fine — we just check we got some logs.
|
||||
});
|
||||
|
||||
logSpy.mockRestore();
|
||||
warnSpy.mockRestore();
|
||||
errorSpy.mockRestore();
|
||||
|
||||
// Either "parity check passed" or "parity drain complete" must appear — run completed.
|
||||
const allLogs = logs.map((c) => String(c[0]));
|
||||
const hasLogEntry = allLogs.some(
|
||||
(m) => m.includes("parity") || m.includes("drain") || m.includes("drift-verifier"),
|
||||
);
|
||||
expect(hasLogEntry).toBe(true);
|
||||
});
|
||||
|
||||
it("returns early (no spot-check) when MONGODB_URI is absent even with KV keys", async () => {
|
||||
// When MONGODB_URI is unset, driftVerifier drains queues then returns early
|
||||
// without attempting any Mongo calls — no timeout risk.
|
||||
const env = makeEnv(); // no MONGODB_URI
|
||||
await env.KV.put("wordle:game1", "some-value");
|
||||
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
|
||||
await expect(driftVerifier({}, makeCtx(env))).resolves.not.toThrow();
|
||||
|
||||
warnSpy.mockRestore();
|
||||
logSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Missing env.KV
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("missing env.KV", () => {
|
||||
it("logs warning and returns without crashing", async () => {
|
||||
const env = { MODULES: "wordle" }; // no KV
|
||||
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
await expect(driftVerifier({}, makeCtx(env))).resolves.not.toThrow();
|
||||
expect(warnSpy.mock.calls.some((c) => String(c[0]).includes("env.KV not available"))).toBe(
|
||||
true,
|
||||
);
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -1,205 +0,0 @@
|
||||
/**
|
||||
* @file dual-kv-store.test.js — unit tests for DualKVStore.
|
||||
*
|
||||
* Tests the four behavioral contracts:
|
||||
* 1. Write succeeds when both primary and secondary succeed.
|
||||
* 2. Write succeeds when secondary fails: logs + enqueues to retry queue.
|
||||
* 3. Write fails (throws) when primary fails.
|
||||
* 4. Reads always come from primary only.
|
||||
* 5. `_kind === "dual"` sentinel is present on the instance.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { CFKVStore } from "../../src/db/cf-kv-store.js";
|
||||
import { DualKVStore } from "../../src/db/dual-kv-store.js";
|
||||
import { makeFakeKv } from "../fakes/fake-kv-namespace.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeStores() {
|
||||
const primaryKv = makeFakeKv();
|
||||
const secondaryKv = makeFakeKv();
|
||||
const retryQueueKv = makeFakeKv();
|
||||
const primary = new CFKVStore(primaryKv);
|
||||
const secondary = new CFKVStore(secondaryKv);
|
||||
const logger = { warn: vi.fn(), error: vi.fn(), log: vi.fn() };
|
||||
const dual = new DualKVStore(primary, secondary, retryQueueKv, logger);
|
||||
return { primaryKv, secondaryKv, retryQueueKv, primary, secondary, dual, logger };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constructor validation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("DualKVStore constructor", () => {
|
||||
it("throws when primary is missing", () => {
|
||||
const kv = makeFakeKv();
|
||||
expect(() => new DualKVStore(null, new CFKVStore(kv), kv)).toThrow(/primary/);
|
||||
});
|
||||
|
||||
it("throws when secondary is missing", () => {
|
||||
const kv = makeFakeKv();
|
||||
expect(() => new DualKVStore(new CFKVStore(kv), null, kv)).toThrow(/secondary/);
|
||||
});
|
||||
|
||||
it("throws when rawKv is missing", () => {
|
||||
const kv = makeFakeKv();
|
||||
const store = new CFKVStore(kv);
|
||||
expect(() => new DualKVStore(store, store, null)).toThrow(/rawKv/);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// _kind sentinel
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("_kind sentinel", () => {
|
||||
it("exposes _kind === 'dual'", () => {
|
||||
const { dual } = makeStores();
|
||||
expect(dual._kind).toBe("dual");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Read operations — primary only
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("reads from primary only", () => {
|
||||
it("get() returns primary value even when secondary differs", async () => {
|
||||
const { primaryKv, secondaryKv, dual } = makeStores();
|
||||
primaryKv.store.set("k", "from-primary");
|
||||
secondaryKv.store.set("k", "from-secondary");
|
||||
expect(await dual.get("k")).toBe("from-primary");
|
||||
});
|
||||
|
||||
it("getJSON() returns primary parsed value", async () => {
|
||||
const { primaryKv, secondaryKv, dual } = makeStores();
|
||||
primaryKv.store.set("k", JSON.stringify({ x: 1 }));
|
||||
secondaryKv.store.set("k", JSON.stringify({ x: 99 }));
|
||||
expect(await dual.getJSON("k")).toEqual({ x: 1 });
|
||||
});
|
||||
|
||||
it("list() returns primary keys", async () => {
|
||||
const { primaryKv, secondaryKv, dual } = makeStores();
|
||||
primaryKv.store.set("a:1", "x");
|
||||
primaryKv.store.set("a:2", "y");
|
||||
secondaryKv.store.set("b:1", "other-module");
|
||||
const result = await dual.list({ prefix: "a:" });
|
||||
expect(result.keys.sort()).toEqual(["a:1", "a:2"]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Write operations — both succeed
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("writes to both when both succeed", () => {
|
||||
it("put() writes to primary and secondary", async () => {
|
||||
const { primaryKv, secondaryKv, dual } = makeStores();
|
||||
await dual.put("k", "v");
|
||||
expect(primaryKv.store.get("k")).toBe("v");
|
||||
expect(secondaryKv.store.get("k")).toBe("v");
|
||||
});
|
||||
|
||||
it("putJSON() serialises to both", async () => {
|
||||
const { primaryKv, secondaryKv, dual } = makeStores();
|
||||
await dual.putJSON("k", { n: 42 });
|
||||
expect(primaryKv.store.get("k")).toBe('{"n":42}');
|
||||
expect(secondaryKv.store.get("k")).toBe('{"n":42}');
|
||||
});
|
||||
|
||||
it("delete() removes from both", async () => {
|
||||
const { primaryKv, secondaryKv, dual } = makeStores();
|
||||
primaryKv.store.set("k", "v");
|
||||
secondaryKv.store.set("k", "v");
|
||||
await dual.delete("k");
|
||||
expect(primaryKv.store.has("k")).toBe(false);
|
||||
expect(secondaryKv.store.has("k")).toBe(false);
|
||||
});
|
||||
|
||||
it("no retry entry is enqueued when both succeed", async () => {
|
||||
const { retryQueueKv, dual } = makeStores();
|
||||
await dual.put("k", "v");
|
||||
expect(retryQueueKv.store.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Secondary failure — caller still succeeds, retry enqueued
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("secondary failure — caller succeeds, retry enqueued", () => {
|
||||
it("put() succeeds when secondary throws, logs warning, enqueues retry", async () => {
|
||||
const { primaryKv, secondaryKv, retryQueueKv, logger, dual } = makeStores();
|
||||
|
||||
// Make secondary put throw.
|
||||
vi.spyOn(secondaryKv, "put").mockRejectedValueOnce(new Error("network error"));
|
||||
|
||||
await expect(dual.put("key1", "value1")).resolves.not.toThrow();
|
||||
|
||||
// Primary was written.
|
||||
expect(primaryKv.store.get("key1")).toBe("value1");
|
||||
// Warning logged — contains key and error info, NOT the value.
|
||||
expect(logger.warn).toHaveBeenCalledOnce();
|
||||
const warnArg = logger.warn.mock.calls[0][1];
|
||||
expect(warnArg.key).toBe("key1");
|
||||
expect(warnArg.err).toContain("network error");
|
||||
// Value must NOT appear in the log.
|
||||
expect(JSON.stringify(warnArg)).not.toContain("value1");
|
||||
// Retry entry enqueued.
|
||||
expect(retryQueueKv.store.size).toBe(1);
|
||||
const [retryKey] = [...retryQueueKv.store.keys()];
|
||||
expect(retryKey).toMatch(/^__retry:mongo-failed:/);
|
||||
});
|
||||
|
||||
it("putJSON() succeeds when secondary throws, enqueues retry", async () => {
|
||||
const { primaryKv, secondaryKv, retryQueueKv, dual } = makeStores();
|
||||
vi.spyOn(secondaryKv, "put").mockRejectedValueOnce(new Error("timeout"));
|
||||
|
||||
await expect(dual.putJSON("k2", { val: 7 })).resolves.not.toThrow();
|
||||
|
||||
expect(primaryKv.store.get("k2")).toBe('{"val":7}');
|
||||
expect(retryQueueKv.store.size).toBe(1);
|
||||
});
|
||||
|
||||
it("delete() succeeds when secondary throws, enqueues retry", async () => {
|
||||
const { primaryKv, secondaryKv, retryQueueKv, dual } = makeStores();
|
||||
primaryKv.store.set("k3", "v");
|
||||
secondaryKv.store.set("k3", "v");
|
||||
vi.spyOn(secondaryKv, "delete").mockRejectedValueOnce(new Error("atlas down"));
|
||||
|
||||
await expect(dual.delete("k3")).resolves.not.toThrow();
|
||||
|
||||
expect(primaryKv.store.has("k3")).toBe(false);
|
||||
expect(retryQueueKv.store.size).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Primary failure — caller throws
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("primary failure — throws to caller", () => {
|
||||
it("put() throws when primary throws", async () => {
|
||||
const { primaryKv, dual } = makeStores();
|
||||
vi.spyOn(primaryKv, "put").mockRejectedValueOnce(new Error("primary dead"));
|
||||
|
||||
await expect(dual.put("k", "v")).rejects.toThrow("primary dead");
|
||||
});
|
||||
|
||||
it("putJSON() throws when primary throws", async () => {
|
||||
const { primaryKv, dual } = makeStores();
|
||||
vi.spyOn(primaryKv, "put").mockRejectedValueOnce(new Error("kv full"));
|
||||
|
||||
await expect(dual.putJSON("k", { x: 1 })).rejects.toThrow("kv full");
|
||||
});
|
||||
|
||||
it("delete() throws when primary throws", async () => {
|
||||
const { primaryKv, dual } = makeStores();
|
||||
vi.spyOn(primaryKv, "delete").mockRejectedValueOnce(new Error("kv gone"));
|
||||
|
||||
await expect(dual.delete("k")).rejects.toThrow("kv gone");
|
||||
});
|
||||
});
|
||||
@@ -1,211 +0,0 @@
|
||||
/**
|
||||
* @file dual-sql-store.test.js — unit tests for DualSqlStore.
|
||||
*
|
||||
* Contracts verified:
|
||||
* 1. run() writes to both primary and secondary.
|
||||
* 2. run() succeeds when secondary fails: logs warning + enqueues to retry queue.
|
||||
* 3. run() throws when primary fails.
|
||||
* 4. all() and first() read from primary only.
|
||||
* 5. prepare() and batch() delegate to primary only.
|
||||
* 6. tablePrefix is inherited from primary.
|
||||
* 7. `_kind === "dual"` sentinel present.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { CFSqlStore } from "../../src/db/cf-sql-store.js";
|
||||
import { DualSqlStore } from "../../src/db/dual-sql-store.js";
|
||||
import { makeFakeD1 } from "../fakes/fake-d1.js";
|
||||
import { makeFakeKv } from "../fakes/fake-kv-namespace.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeStores() {
|
||||
const primaryD1 = makeFakeD1();
|
||||
const secondaryD1 = makeFakeD1();
|
||||
const retryQueueKv = makeFakeKv();
|
||||
const primary = new CFSqlStore(primaryD1);
|
||||
const secondary = new CFSqlStore(secondaryD1);
|
||||
const logger = { warn: vi.fn(), error: vi.fn(), log: vi.fn() };
|
||||
const dual = new DualSqlStore(primary, secondary, retryQueueKv, logger);
|
||||
// Simulate tablePrefix coming from primary wrapper — set directly.
|
||||
dual.tablePrefix = "trading_";
|
||||
return { primaryD1, secondaryD1, retryQueueKv, primary, secondary, dual, logger };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constructor validation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("DualSqlStore constructor", () => {
|
||||
it("throws when primary is missing", () => {
|
||||
const kv = makeFakeKv();
|
||||
const d1 = makeFakeD1();
|
||||
expect(() => new DualSqlStore(null, new CFSqlStore(d1), kv)).toThrow(/primary/);
|
||||
});
|
||||
|
||||
it("throws when secondary is missing", () => {
|
||||
const kv = makeFakeKv();
|
||||
const d1 = makeFakeD1();
|
||||
expect(() => new DualSqlStore(new CFSqlStore(d1), null, kv)).toThrow(/secondary/);
|
||||
});
|
||||
|
||||
it("throws when rawKv is missing", () => {
|
||||
const d1 = makeFakeD1();
|
||||
const store = new CFSqlStore(d1);
|
||||
expect(() => new DualSqlStore(store, store, null)).toThrow(/rawKv/);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// _kind sentinel
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("_kind sentinel", () => {
|
||||
it("exposes _kind === 'dual'", () => {
|
||||
const { dual } = makeStores();
|
||||
expect(dual._kind).toBe("dual");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tablePrefix
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("tablePrefix", () => {
|
||||
it("inherits tablePrefix from primary", () => {
|
||||
const d1 = makeFakeD1();
|
||||
const kv = makeFakeKv();
|
||||
const primary = new CFSqlStore(d1);
|
||||
// The DualSqlStore constructor copies primary.tablePrefix.
|
||||
const dual = new DualSqlStore(primary, primary, kv);
|
||||
// CFSqlStore has no tablePrefix; DualSqlStore falls back to "".
|
||||
expect(typeof dual.tablePrefix).toBe("string");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// run() — both succeed
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("run() — both succeed", () => {
|
||||
it("records query in both primary and secondary runLog", async () => {
|
||||
const { primaryD1, secondaryD1, dual } = makeStores();
|
||||
await dual.run("INSERT INTO trading_trades VALUES (?)", "x");
|
||||
expect(primaryD1.runLog).toHaveLength(1);
|
||||
expect(secondaryD1.runLog).toHaveLength(1);
|
||||
expect(primaryD1.runLog[0].query).toBe("INSERT INTO trading_trades VALUES (?)");
|
||||
expect(secondaryD1.runLog[0].query).toBe("INSERT INTO trading_trades VALUES (?)");
|
||||
});
|
||||
|
||||
it("returns the primary run result", async () => {
|
||||
const { dual } = makeStores();
|
||||
const result = await dual.run("INSERT INTO trading_trades VALUES (?)", "v");
|
||||
expect(result).toHaveProperty("changes");
|
||||
expect(result).toHaveProperty("last_row_id");
|
||||
});
|
||||
|
||||
it("no retry entry enqueued when both succeed", async () => {
|
||||
const { retryQueueKv, dual } = makeStores();
|
||||
await dual.run("INSERT INTO trading_trades VALUES (?)", "v");
|
||||
expect(retryQueueKv.store.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// run() — secondary fails
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("run() — secondary fails", () => {
|
||||
it("succeeds, logs warning, enqueues retry when secondary throws", async () => {
|
||||
const { primaryD1, secondaryD1, retryQueueKv, logger, dual } = makeStores();
|
||||
vi.spyOn(secondaryD1, "prepare").mockImplementation(() => ({
|
||||
run: () => Promise.reject(new Error("mongo write failed")),
|
||||
bind: function (...args) {
|
||||
return this;
|
||||
},
|
||||
}));
|
||||
|
||||
await expect(dual.run("INSERT INTO trading_trades VALUES (?)", "val")).resolves.not.toThrow();
|
||||
|
||||
expect(primaryD1.runLog).toHaveLength(1);
|
||||
expect(logger.warn).toHaveBeenCalledOnce();
|
||||
const warnArg = logger.warn.mock.calls[0][1];
|
||||
expect(warnArg.op).toBe("run");
|
||||
expect(warnArg.err).toContain("mongo write failed");
|
||||
// Bind values must NOT appear in structured log.
|
||||
expect(JSON.stringify(warnArg)).not.toContain("val");
|
||||
// Retry enqueued.
|
||||
expect(retryQueueKv.store.size).toBe(1);
|
||||
const [key] = [...retryQueueKv.store.keys()];
|
||||
expect(key).toMatch(/^__retry:mongo-sql-failed:/);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// run() — primary fails
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("run() — primary fails", () => {
|
||||
it("throws when primary throws", async () => {
|
||||
const { primaryD1, dual } = makeStores();
|
||||
vi.spyOn(primaryD1, "prepare").mockImplementation(() => ({
|
||||
run: () => Promise.reject(new Error("d1 gone")),
|
||||
bind: function (...args) {
|
||||
return this;
|
||||
},
|
||||
}));
|
||||
|
||||
await expect(dual.run("INSERT INTO trading_trades VALUES (?)", "v")).rejects.toThrow("d1 gone");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Read operations — primary only
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("all() and first() — primary only", () => {
|
||||
it("all() returns primary results", async () => {
|
||||
const { primaryD1, secondaryD1, dual } = makeStores();
|
||||
primaryD1.seed("trading_trades", [{ id: 1, symbol: "VNM" }]);
|
||||
// Secondary empty — result must still come from primary.
|
||||
const rows = await dual.all("SELECT * FROM trading_trades");
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].symbol).toBe("VNM");
|
||||
});
|
||||
|
||||
it("first() returns primary first row", async () => {
|
||||
const { primaryD1, dual } = makeStores();
|
||||
primaryD1.seed("trading_trades", [{ id: 1, symbol: "FPT" }]);
|
||||
const row = await dual.first("SELECT * FROM trading_trades LIMIT 1");
|
||||
expect(row?.symbol).toBe("FPT");
|
||||
});
|
||||
|
||||
it("first() returns null when primary has no rows", async () => {
|
||||
const { dual } = makeStores();
|
||||
const row = await dual.first("SELECT * FROM trading_trades LIMIT 1");
|
||||
expect(row).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// prepare() and batch() — primary only
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("prepare() and batch() — primary only", () => {
|
||||
it("prepare() delegates to primary", () => {
|
||||
const { primaryD1, dual } = makeStores();
|
||||
// Should not throw; fake D1 returns a stub prepared statement.
|
||||
expect(() => dual.prepare("SELECT 1")).not.toThrow();
|
||||
});
|
||||
|
||||
it("batch() returns primary results", async () => {
|
||||
const { primaryD1, dual } = makeStores();
|
||||
primaryD1.seed("trading_trades", [{ id: 1 }]);
|
||||
const stmt = dual.prepare("SELECT * FROM trading_trades");
|
||||
const results = await dual.batch([stmt]);
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
expect(results[0]).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -1,328 +0,0 @@
|
||||
/**
|
||||
* @file mongo-kv-store.test.js — unit tests for MongoKVStore.
|
||||
*
|
||||
* Injection pattern: MongoKVStore constructor accepts an optional `dbOverride`
|
||||
* parameter. Tests pass a `makeFakeMongo()` db so no real Atlas connection
|
||||
* is made. The same fake is used to test mongo-client.js connect-reject retry.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { MongoKVStore } from "../../src/db/mongo-kv-store.js";
|
||||
import { makeFakeMongo } from "../fakes/fake-mongo.js";
|
||||
|
||||
// ─── helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Build a store + fake db pair. collectionName defaults to "test". */
|
||||
function makeStore(collectionName = "test") {
|
||||
const fakeDb = makeFakeMongo();
|
||||
const store = new MongoKVStore({}, collectionName, fakeDb);
|
||||
return { store, fakeDb };
|
||||
}
|
||||
|
||||
// ─── constructor ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoKVStore constructor", () => {
|
||||
it("throws when collectionName is missing", () => {
|
||||
expect(() => new MongoKVStore({}, "")).toThrow(/required/);
|
||||
});
|
||||
|
||||
it("normalizes collection name: replaces - with _", () => {
|
||||
const fakeDb = makeFakeMongo();
|
||||
const store = new MongoKVStore({}, "loldle-emoji", fakeDb);
|
||||
expect(store._collName).toBe("loldle_emoji");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── get / put / delete ───────────────────────────────────────────────────────
|
||||
|
||||
describe("get / put / delete", () => {
|
||||
it("get returns null for missing key", async () => {
|
||||
const { store } = makeStore();
|
||||
expect(await store.get("missing")).toBeNull();
|
||||
});
|
||||
|
||||
it("put → get round-trip", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.put("k", "hello");
|
||||
expect(await store.get("k")).toBe("hello");
|
||||
});
|
||||
|
||||
it("put overwrites existing value", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.put("k", "first");
|
||||
await store.put("k", "second");
|
||||
expect(await store.get("k")).toBe("second");
|
||||
});
|
||||
|
||||
it("delete removes the key", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.put("k", "v");
|
||||
await store.delete("k");
|
||||
expect(await store.get("k")).toBeNull();
|
||||
});
|
||||
|
||||
it("delete is idempotent (no-op on missing key)", async () => {
|
||||
const { store } = makeStore();
|
||||
await expect(store.delete("nope")).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ─── TTL / expiresAt ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("TTL / expiresAt field", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("put with expirationTtl writes expiresAt field", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2025-01-01T00:00:00.000Z"));
|
||||
const { store, fakeDb } = makeStore();
|
||||
await store.put("k", "v", { expirationTtl: 60 });
|
||||
const col = fakeDb.collection("test");
|
||||
const doc = await col.findOne({ _id: "k" });
|
||||
expect(doc.expiresAt).toBeInstanceOf(Date);
|
||||
expect(doc.expiresAt.getTime()).toBe(new Date("2025-01-01T00:01:00.000Z").getTime());
|
||||
});
|
||||
|
||||
it("put without TTL clears any existing expiresAt", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2025-01-01T00:00:00.000Z"));
|
||||
const { store, fakeDb } = makeStore();
|
||||
// First write with TTL
|
||||
await store.put("k", "v", { expirationTtl: 60 });
|
||||
// Second write without TTL — must remove expiresAt
|
||||
await store.put("k", "updated");
|
||||
const col = fakeDb.collection("test");
|
||||
const doc = await col.findOne({ _id: "k" });
|
||||
expect(doc.expiresAt).toBeUndefined();
|
||||
});
|
||||
|
||||
it("TTL stale-read regression: expired doc returns null before sweeper runs", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2025-01-01T00:00:00.000Z"));
|
||||
const { store } = makeStore();
|
||||
await store.put("k", "v", { expirationTtl: 1 }); // expires in 1s
|
||||
|
||||
// Advance clock 2 seconds — doc is now expired but sweeper hasn't run
|
||||
vi.setSystemTime(new Date("2025-01-01T00:00:02.000Z"));
|
||||
|
||||
expect(await store.get("k")).toBeNull();
|
||||
});
|
||||
|
||||
it("non-expired doc is still readable before TTL elapses", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2025-01-01T00:00:00.000Z"));
|
||||
const { store } = makeStore();
|
||||
await store.put("k", "v", { expirationTtl: 60 });
|
||||
|
||||
// Only 10s later — still live
|
||||
vi.setSystemTime(new Date("2025-01-01T00:00:10.000Z"));
|
||||
|
||||
expect(await store.get("k")).toBe("v");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── getJSON / putJSON ───────────────────────────────────────────────────────
|
||||
|
||||
describe("getJSON / putJSON", () => {
|
||||
it("putJSON → getJSON round-trip", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.putJSON("k", { a: 1, b: [2, 3] });
|
||||
expect(await store.getJSON("k")).toEqual({ a: 1, b: [2, 3] });
|
||||
});
|
||||
|
||||
it("getJSON returns null on missing key", async () => {
|
||||
const { store } = makeStore();
|
||||
expect(await store.getJSON("missing")).toBeNull();
|
||||
});
|
||||
|
||||
it("getJSON returns null on corrupt JSON and logs a warning", async () => {
|
||||
const { store, fakeDb } = makeStore();
|
||||
// Seed corrupt document directly into the fake collection
|
||||
await fakeDb.collection("test").insertOne({ _id: "bad", value: "{not json" });
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
expect(await store.getJSON("bad")).toBeNull();
|
||||
expect(warn).toHaveBeenCalled();
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it("putJSON throws on undefined value", async () => {
|
||||
const { store } = makeStore();
|
||||
await expect(store.putJSON("k", undefined)).rejects.toThrow(/undefined/);
|
||||
});
|
||||
|
||||
it("putJSON throws on circular reference", async () => {
|
||||
const { store } = makeStore();
|
||||
const obj = {};
|
||||
obj.self = obj;
|
||||
await expect(store.putJSON("k", obj)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("putJSON passes expirationTtl through to put", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2025-01-01T00:00:00.000Z"));
|
||||
const { store, fakeDb } = makeStore();
|
||||
await store.putJSON("k", { x: 1 }, { expirationTtl: 120 });
|
||||
const doc = await fakeDb.collection("test").findOne({ _id: "k" });
|
||||
expect(doc.expiresAt).toBeInstanceOf(Date);
|
||||
expect(doc.expiresAt.getTime()).toBe(new Date("2025-01-01T00:02:00.000Z").getTime());
|
||||
vi.useRealTimers();
|
||||
});
|
||||
});
|
||||
|
||||
// ─── list ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("list()", () => {
|
||||
it("returns empty result when store is empty", async () => {
|
||||
const { store } = makeStore();
|
||||
const res = await store.list();
|
||||
expect(res.keys).toEqual([]);
|
||||
expect(res.done).toBe(true);
|
||||
expect(res.cursor).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns all keys when no prefix given", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.put("a:1", "x");
|
||||
await store.put("b:2", "y");
|
||||
const res = await store.list();
|
||||
expect(res.keys.sort()).toEqual(["a:1", "b:2"]);
|
||||
expect(res.done).toBe(true);
|
||||
});
|
||||
|
||||
it("returns keys WITH prefix preserved (not stripped)", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.put("wordle:games:1", "a");
|
||||
await store.put("wordle:games:2", "b");
|
||||
await store.put("wordle:other:3", "c");
|
||||
const res = await store.list({ prefix: "wordle:games:" });
|
||||
expect(res.keys.sort()).toEqual(["wordle:games:1", "wordle:games:2"]);
|
||||
expect(res.done).toBe(true);
|
||||
});
|
||||
|
||||
it("2-level prefix regression: only matching keys returned with prefix preserved", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.put("wordle:games:1", "a");
|
||||
await store.put("wordle:games:2", "b");
|
||||
await store.put("wordle:other:3", "c");
|
||||
const res = await store.list({ prefix: "wordle:games:" });
|
||||
// Must be exactly 2 keys, both with prefix intact
|
||||
expect(res.keys).toHaveLength(2);
|
||||
expect(res.keys).toContain("wordle:games:1");
|
||||
expect(res.keys).toContain("wordle:games:2");
|
||||
expect(res.keys).not.toContain("wordle:other:3");
|
||||
});
|
||||
|
||||
it("prefix with regex special chars is escaped", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.put("a.b:1", "x");
|
||||
await store.put("a_b:1", "y"); // should NOT match prefix "a.b:"
|
||||
const res = await store.list({ prefix: "a.b:" });
|
||||
expect(res.keys).toEqual(["a.b:1"]);
|
||||
});
|
||||
|
||||
it("list() cursor pagination — limit(N+1) strategy", async () => {
|
||||
const { store } = makeStore();
|
||||
for (let i = 1; i <= 5; i++) await store.put(`k${i}`, String(i));
|
||||
|
||||
const page1 = await store.list({ limit: 2 });
|
||||
expect(page1.keys).toHaveLength(2);
|
||||
expect(page1.done).toBe(false);
|
||||
expect(page1.cursor).toBeTruthy();
|
||||
|
||||
const page2 = await store.list({ limit: 2, cursor: page1.cursor });
|
||||
expect(page2.keys).toHaveLength(2);
|
||||
expect(page2.done).toBe(false);
|
||||
expect(page2.cursor).toBeTruthy();
|
||||
|
||||
const page3 = await store.list({ limit: 2, cursor: page2.cursor });
|
||||
expect(page3.keys).toHaveLength(1);
|
||||
expect(page3.done).toBe(true);
|
||||
expect(page3.cursor).toBeUndefined();
|
||||
|
||||
// All keys across pages must be unique and sorted
|
||||
const allKeys = [...page1.keys, ...page2.keys, ...page3.keys];
|
||||
expect(allKeys).toHaveLength(5);
|
||||
expect(allKeys).toEqual([...allKeys].sort());
|
||||
});
|
||||
|
||||
it("list done=true when exactly limit keys remain (no extra page)", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.put("k1", "a");
|
||||
await store.put("k2", "b");
|
||||
const res = await store.list({ limit: 2 });
|
||||
expect(res.keys).toHaveLength(2);
|
||||
expect(res.done).toBe(true);
|
||||
expect(res.cursor).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ─── mongo-client connect-reject retry regression ────────────────────────────
|
||||
|
||||
describe("mongo-client connect-reject retry regression", () => {
|
||||
it("nulls client and connectPromise on connect() rejection so next call retries", async () => {
|
||||
// Import the module fresh — we'll call it with a mock factory
|
||||
const { getDb, closeMongo } = await import("../../src/db/mongo-client.js");
|
||||
|
||||
// Ensure clean state before test
|
||||
await closeMongo();
|
||||
|
||||
// Patch MongoClient.prototype.connect to reject once, then succeed
|
||||
const { MongoClient } = await import("mongodb");
|
||||
let callCount = 0;
|
||||
const originalConnect = MongoClient.prototype.connect;
|
||||
MongoClient.prototype.connect = vi.fn(async function () {
|
||||
callCount++;
|
||||
if (callCount === 1) {
|
||||
// First call: reject to simulate transient failure
|
||||
throw new Error("transient connection error");
|
||||
}
|
||||
// Second call: succeed (but return void, the client is now "connected")
|
||||
return this;
|
||||
});
|
||||
|
||||
try {
|
||||
// First call — must reject
|
||||
await expect(getDb({ MONGODB_URI: "mongodb://localhost:27017" })).rejects.toThrow(
|
||||
"transient connection error",
|
||||
);
|
||||
|
||||
// Second call — must NOT reuse the dead client; must retry
|
||||
// It will succeed on second connect() call
|
||||
const db = await getDb({ MONGODB_URI: "mongodb://localhost:27017" });
|
||||
expect(db).toBeTruthy();
|
||||
expect(callCount).toBe(2);
|
||||
} finally {
|
||||
MongoClient.prototype.connect = originalConnect;
|
||||
await closeMongo();
|
||||
}
|
||||
});
|
||||
|
||||
it("logs structured warning on MongoServerSelectionError", async () => {
|
||||
const { getDb, closeMongo } = await import("../../src/db/mongo-client.js");
|
||||
await closeMongo();
|
||||
|
||||
const { MongoClient } = await import("mongodb");
|
||||
const originalConnect = MongoClient.prototype.connect;
|
||||
MongoClient.prototype.connect = vi.fn(async () => {
|
||||
const err = new Error("server selection timeout");
|
||||
err.name = "MongoServerSelectionError";
|
||||
throw err;
|
||||
});
|
||||
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
|
||||
try {
|
||||
await expect(getDb({ MONGODB_URI: "mongodb://localhost:27017" })).rejects.toThrow();
|
||||
expect(warn).toHaveBeenCalledOnce();
|
||||
const logged = JSON.parse(warn.mock.calls[0][0]);
|
||||
expect(logged.event).toBe("mongo_server_selection_failed");
|
||||
expect(logged.note).toMatch(/503/);
|
||||
} finally {
|
||||
MongoClient.prototype.connect = originalConnect;
|
||||
warn.mockRestore();
|
||||
await closeMongo();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,247 +0,0 @@
|
||||
/**
|
||||
* @file Tests for MongoSqlStore shim — contract compliance for SqlStore interface.
|
||||
*
|
||||
* Key assertion: `run` INSERT returns `{ changes: 1, last_row_id: 0 }` where
|
||||
* `last_row_id` is the NUMBER 0, not a hex string. This satisfies the contract
|
||||
* checked by tests/db/create-sql-store.test.js:48-52.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { MongoSqlStore } from "../../src/db/mongo-sql-store.js";
|
||||
import { MongoTradesStore } from "../../src/db/mongo-trades-store.js";
|
||||
import { makeFakeMongo } from "../fakes/fake-mongo.js";
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Build a MongoSqlStore backed by a fresh fake MongoTradesStore. */
|
||||
function makeShim() {
|
||||
const fakeDb = makeFakeMongo();
|
||||
const tradesStore = new MongoTradesStore({}, fakeDb);
|
||||
const shim = new MongoSqlStore({}, "trading", tradesStore);
|
||||
return { fakeDb, tradesStore, shim };
|
||||
}
|
||||
|
||||
// ─── tablePrefix ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoSqlStore — tablePrefix", () => {
|
||||
it("exposes tablePrefix as moduleName + underscore", () => {
|
||||
const { shim } = makeShim();
|
||||
expect(shim.tablePrefix).toBe("trading_");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── run — INSERT ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoSqlStore.run — INSERT", () => {
|
||||
it("returns { changes: 1, last_row_id: 0 } for INSERT INTO trading_trades", async () => {
|
||||
const { shim } = makeShim();
|
||||
const result = await shim.run(
|
||||
"INSERT INTO trading_trades (user_id, symbol, side, qty, price_vnd, ts) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
1,
|
||||
"TCB",
|
||||
"buy",
|
||||
100,
|
||||
25000,
|
||||
1000,
|
||||
);
|
||||
expect(result).toEqual({ changes: 1, last_row_id: 0 });
|
||||
});
|
||||
|
||||
it("last_row_id is a NUMBER (not hex string) — satisfies create-sql-store.test.js:48-52", async () => {
|
||||
const { shim } = makeShim();
|
||||
const result = await shim.run(
|
||||
"INSERT INTO trading_trades (user_id, symbol, side, qty, price_vnd, ts) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
1,
|
||||
"VNM",
|
||||
"sell",
|
||||
50,
|
||||
80000,
|
||||
2000,
|
||||
);
|
||||
expect(typeof result.last_row_id).toBe("number");
|
||||
expect(result.last_row_id).toBe(0);
|
||||
expect(typeof result.changes).toBe("number");
|
||||
});
|
||||
|
||||
it("actually inserts the document into the store", async () => {
|
||||
const { fakeDb, shim } = makeShim();
|
||||
await shim.run(
|
||||
"INSERT INTO trading_trades (user_id, symbol, side, qty, price_vnd, ts) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
1,
|
||||
"TCB",
|
||||
"buy",
|
||||
100,
|
||||
25000,
|
||||
1000,
|
||||
);
|
||||
const docs = await fakeDb.collection("trading_trades").find({}).toArray();
|
||||
expect(docs).toHaveLength(1);
|
||||
expect(docs[0].symbol).toBe("TCB");
|
||||
});
|
||||
|
||||
it("throws for unrecognised run query", async () => {
|
||||
const { shim } = makeShim();
|
||||
await expect(shim.run("UPDATE trading_trades SET qty = ? WHERE id = ?", 5, 1)).rejects.toThrow(
|
||||
"MongoSqlStore: unsupported query",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── run — DELETE by ids ──────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoSqlStore.run — DELETE WHERE id IN", () => {
|
||||
it("delegates delete and returns changes count", async () => {
|
||||
const { shim, tradesStore } = makeShim();
|
||||
// Insert two trades so we have ids to delete.
|
||||
await tradesStore.insert({
|
||||
userId: 1,
|
||||
symbol: "TCB",
|
||||
side: "buy",
|
||||
qty: 1,
|
||||
priceVnd: 100,
|
||||
ts: 1,
|
||||
});
|
||||
await tradesStore.insert({
|
||||
userId: 1,
|
||||
symbol: "TCB",
|
||||
side: "buy",
|
||||
qty: 1,
|
||||
priceVnd: 100,
|
||||
ts: 2,
|
||||
});
|
||||
const oldIds = await tradesStore.oldRows(1); // 1 excess
|
||||
expect(oldIds).toHaveLength(1);
|
||||
|
||||
const result = await shim.run("DELETE FROM trading_trades WHERE id IN (?)", ...oldIds);
|
||||
expect(result.changes).toBe(1);
|
||||
expect(result.last_row_id).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── all ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoSqlStore.all", () => {
|
||||
it("SELECT DISTINCT user_id → returns array of { user_id } objects", async () => {
|
||||
const { shim, tradesStore } = makeShim();
|
||||
await tradesStore.insert({ userId: 1, symbol: "TCB", side: "buy", qty: 1, priceVnd: 1, ts: 1 });
|
||||
await tradesStore.insert({ userId: 2, symbol: "TCB", side: "buy", qty: 1, priceVnd: 1, ts: 2 });
|
||||
await tradesStore.insert({ userId: 1, symbol: "TCB", side: "buy", qty: 1, priceVnd: 1, ts: 3 });
|
||||
|
||||
const rows = await shim.all("SELECT DISTINCT user_id FROM trading_trades");
|
||||
expect(rows.map((r) => r.user_id).sort()).toEqual([1, 2]);
|
||||
});
|
||||
|
||||
it("SELECT id … WHERE user_id = ? … OFFSET ? → returns old row ids for user", async () => {
|
||||
const { shim, tradesStore } = makeShim();
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
await tradesStore.insert({
|
||||
userId: 1,
|
||||
symbol: "TCB",
|
||||
side: "buy",
|
||||
qty: 1,
|
||||
priceVnd: 1,
|
||||
ts: i,
|
||||
});
|
||||
}
|
||||
const ids = await shim.all(
|
||||
"SELECT id FROM trading_trades WHERE user_id = ? ORDER BY ts DESC LIMIT -1 OFFSET ?",
|
||||
1,
|
||||
3,
|
||||
);
|
||||
expect(ids).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("SELECT id … ORDER BY ts DESC … OFFSET ? → returns global old row ids", async () => {
|
||||
const { shim, tradesStore } = makeShim();
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
await tradesStore.insert({
|
||||
userId: 1,
|
||||
symbol: "TCB",
|
||||
side: "buy",
|
||||
qty: 1,
|
||||
priceVnd: 1,
|
||||
ts: i,
|
||||
});
|
||||
}
|
||||
const ids = await shim.all(
|
||||
"SELECT id FROM trading_trades ORDER BY ts DESC LIMIT -1 OFFSET ?",
|
||||
3,
|
||||
);
|
||||
expect(ids).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("SELECT … WHERE user_id = ? … LIMIT ? → returns byUser results", async () => {
|
||||
const { shim, tradesStore } = makeShim();
|
||||
await tradesStore.insert({
|
||||
userId: 1,
|
||||
symbol: "TCB",
|
||||
side: "buy",
|
||||
qty: 5,
|
||||
priceVnd: 1000,
|
||||
ts: 100,
|
||||
});
|
||||
await tradesStore.insert({
|
||||
userId: 1,
|
||||
symbol: "VNM",
|
||||
side: "sell",
|
||||
qty: 2,
|
||||
priceVnd: 2000,
|
||||
ts: 200,
|
||||
});
|
||||
|
||||
const rows = await shim.all(
|
||||
"SELECT id, user_id, symbol, side, qty, price_vnd, ts FROM trading_trades WHERE user_id = ? ORDER BY ts DESC LIMIT ?",
|
||||
1,
|
||||
10,
|
||||
);
|
||||
expect(rows).toHaveLength(2);
|
||||
expect(rows[0].ts).toBe(200); // newest first
|
||||
});
|
||||
|
||||
it("throws for unrecognised all query", async () => {
|
||||
const { shim } = makeShim();
|
||||
await expect(shim.all("SELECT * FROM trading_trades")).rejects.toThrow(
|
||||
"MongoSqlStore: unsupported query",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── first ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoSqlStore.first", () => {
|
||||
it("returns the first row from a valid query", async () => {
|
||||
const { shim, tradesStore } = makeShim();
|
||||
await tradesStore.insert({ userId: 1, symbol: "TCB", side: "buy", qty: 1, priceVnd: 1, ts: 1 });
|
||||
const row = await shim.first(
|
||||
"SELECT id, user_id, symbol, side, qty, price_vnd, ts FROM trading_trades WHERE user_id = ? ORDER BY ts DESC LIMIT ?",
|
||||
1,
|
||||
10,
|
||||
);
|
||||
expect(row).not.toBeNull();
|
||||
expect(row.userId).toBe(1);
|
||||
});
|
||||
|
||||
it("returns null when no rows match", async () => {
|
||||
const { shim } = makeShim();
|
||||
const row = await shim.first(
|
||||
"SELECT id, user_id, symbol, side, qty, price_vnd, ts FROM trading_trades WHERE user_id = ? ORDER BY ts DESC LIMIT ?",
|
||||
99,
|
||||
10,
|
||||
);
|
||||
expect(row).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ─── prepare / batch ─────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoSqlStore — prepare / batch throw", () => {
|
||||
it("prepare throws 'unsupported in MongoSqlStore'", () => {
|
||||
const { shim } = makeShim();
|
||||
expect(() => shim.prepare("SELECT 1")).toThrow("unsupported in MongoSqlStore");
|
||||
});
|
||||
|
||||
it("batch throws 'unsupported in MongoSqlStore'", () => {
|
||||
const { shim } = makeShim();
|
||||
expect(() => shim.batch([])).toThrow("unsupported in MongoSqlStore");
|
||||
});
|
||||
});
|
||||
@@ -1,301 +0,0 @@
|
||||
/**
|
||||
* @file Tests for MongoTradesStore — all 6 methods, edge cases, ordering.
|
||||
*
|
||||
* Injects fake-mongo via constructor (dbOverride), no real MongoDB connection.
|
||||
*/
|
||||
|
||||
import { ObjectId } from "mongodb";
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import { MongoTradesStore } from "../../src/db/mongo-trades-store.js";
|
||||
import { makeFakeMongo } from "../fakes/fake-mongo.js";
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Build a fresh store backed by a new fake Mongo db. */
|
||||
function makeStore() {
|
||||
const fakeDb = makeFakeMongo();
|
||||
const store = new MongoTradesStore({}, fakeDb);
|
||||
return { fakeDb, store };
|
||||
}
|
||||
|
||||
/** Minimal trade payload for insert. */
|
||||
const TRADE = {
|
||||
userId: 1,
|
||||
symbol: "TCB",
|
||||
side: /** @type {"buy"} */ ("buy"),
|
||||
qty: 100,
|
||||
priceVnd: 25000,
|
||||
ts: 1000,
|
||||
};
|
||||
|
||||
// ─── insert ───────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoTradesStore.insert", () => {
|
||||
it("inserts a document and returns { changes: 1, last_row_id: 0 }", async () => {
|
||||
const { store } = makeStore();
|
||||
const result = await store.insert(TRADE);
|
||||
expect(result).toEqual({ changes: 1, last_row_id: 0 });
|
||||
});
|
||||
|
||||
it("last_row_id is the number 0, not a hex string", async () => {
|
||||
const { store } = makeStore();
|
||||
const result = await store.insert(TRADE);
|
||||
expect(typeof result.last_row_id).toBe("number");
|
||||
expect(result.last_row_id).toBe(0);
|
||||
});
|
||||
|
||||
it("stores legacy_id as null for new runtime trades", async () => {
|
||||
const { fakeDb, store } = makeStore();
|
||||
await store.insert(TRADE);
|
||||
const coll = fakeDb.collection("trading_trades");
|
||||
const docs = await coll.find({}).toArray();
|
||||
expect(docs).toHaveLength(1);
|
||||
expect(docs[0].legacy_id).toBeNull();
|
||||
});
|
||||
|
||||
it("stores field shape correctly (snake_case in db)", async () => {
|
||||
const { fakeDb, store } = makeStore();
|
||||
await store.insert(TRADE);
|
||||
const coll = fakeDb.collection("trading_trades");
|
||||
const docs = await coll.find({}).toArray();
|
||||
const doc = docs[0];
|
||||
expect(doc.user_id).toBe(1);
|
||||
expect(doc.symbol).toBe("TCB");
|
||||
expect(doc.side).toBe("buy");
|
||||
expect(doc.qty).toBe(100);
|
||||
expect(doc.price_vnd).toBe(25000);
|
||||
expect(doc.ts).toBe(1000);
|
||||
});
|
||||
|
||||
it("uses provided ts when given", async () => {
|
||||
const { fakeDb, store } = makeStore();
|
||||
await store.insert({ ...TRADE, ts: 9999 });
|
||||
const coll = fakeDb.collection("trading_trades");
|
||||
const docs = await coll.find({}).toArray();
|
||||
expect(docs[0].ts).toBe(9999);
|
||||
});
|
||||
|
||||
it("falls back to Date.now() when ts is not provided", async () => {
|
||||
const { fakeDb, store } = makeStore();
|
||||
const before = Date.now();
|
||||
const { ts: _, ...tradeWithoutTs } = TRADE;
|
||||
await store.insert(tradeWithoutTs);
|
||||
const after = Date.now();
|
||||
const coll = fakeDb.collection("trading_trades");
|
||||
const docs = await coll.find({}).toArray();
|
||||
expect(docs[0].ts).toBeGreaterThanOrEqual(before);
|
||||
expect(docs[0].ts).toBeLessThanOrEqual(after);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── byUser ───────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoTradesStore.byUser", () => {
|
||||
it("returns [] when collection is empty", async () => {
|
||||
const { store } = makeStore();
|
||||
expect(await store.byUser(1, 10)).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns only trades for the requested user", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.insert({ ...TRADE, userId: 1, ts: 1 });
|
||||
await store.insert({ ...TRADE, userId: 2, ts: 2 });
|
||||
const trades = await store.byUser(1, 10);
|
||||
expect(trades).toHaveLength(1);
|
||||
expect(trades[0].userId).toBe(1);
|
||||
});
|
||||
|
||||
it("returns trades newest-first (sorted by ts DESC)", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.insert({ ...TRADE, ts: 100 });
|
||||
await store.insert({ ...TRADE, ts: 300 });
|
||||
await store.insert({ ...TRADE, ts: 200 });
|
||||
const trades = await store.byUser(1, 10);
|
||||
expect(trades.map((t) => t.ts)).toEqual([300, 200, 100]);
|
||||
});
|
||||
|
||||
it("respects the limit", async () => {
|
||||
const { store } = makeStore();
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
await store.insert({ ...TRADE, ts: i });
|
||||
}
|
||||
const trades = await store.byUser(1, 3);
|
||||
expect(trades).toHaveLength(3);
|
||||
// Must be the 3 newest.
|
||||
expect(trades.map((t) => t.ts)).toEqual([5, 4, 3]);
|
||||
});
|
||||
|
||||
it("maps document fields to camelCase Trade shape", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.insert(TRADE);
|
||||
const trades = await store.byUser(1, 1);
|
||||
expect(trades[0]).toMatchObject({
|
||||
userId: 1,
|
||||
symbol: "TCB",
|
||||
side: "buy",
|
||||
qty: 100,
|
||||
priceVnd: 25000,
|
||||
ts: 1000,
|
||||
});
|
||||
// id must be present (ObjectId from _id).
|
||||
expect(trades[0].id).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ─── distinctUsers ────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoTradesStore.distinctUsers", () => {
|
||||
it("returns [] when collection is empty", async () => {
|
||||
const { store } = makeStore();
|
||||
expect(await store.distinctUsers()).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns each user_id exactly once", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.insert({ ...TRADE, userId: 1 });
|
||||
await store.insert({ ...TRADE, userId: 2 });
|
||||
await store.insert({ ...TRADE, userId: 1 }); // duplicate
|
||||
const users = await store.distinctUsers();
|
||||
expect(users.sort()).toEqual([1, 2]);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── oldRowsForUser ───────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoTradesStore.oldRowsForUser", () => {
|
||||
it("returns [] when user has fewer rows than keepN", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.insert({ ...TRADE, ts: 1 });
|
||||
await store.insert({ ...TRADE, ts: 2 });
|
||||
expect(await store.oldRowsForUser(1, 5)).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns ids of rows beyond keepN (oldest rows)", async () => {
|
||||
const { store } = makeStore();
|
||||
// Insert 5 rows; keepN=3 → should return ids of the 2 oldest (ts=1,2).
|
||||
const insertedIds = [];
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
await store.insert({ ...TRADE, ts: i });
|
||||
}
|
||||
// Fetch all to get their _ids in ts order.
|
||||
const all = await store.byUser(1, 10); // newest first: ts=5,4,3,2,1
|
||||
const oldIds = await store.oldRowsForUser(1, 3);
|
||||
// The 2 oldest (ts=1, ts=2) should be in oldIds.
|
||||
expect(oldIds).toHaveLength(2);
|
||||
// all[3].id = ts=2, all[4].id = ts=1 (index 3 and 4 in newest-first order).
|
||||
const expectedIds = [all[3].id, all[4].id].map(String).sort();
|
||||
const actualIds = oldIds.map(String).sort();
|
||||
expect(actualIds).toEqual(expectedIds);
|
||||
});
|
||||
|
||||
it("only returns ids for the specified user, not others", async () => {
|
||||
const { store } = makeStore();
|
||||
// User 1: 4 rows, keepN=2 → 2 excess.
|
||||
for (let i = 1; i <= 4; i++) {
|
||||
await store.insert({ ...TRADE, userId: 1, ts: i });
|
||||
}
|
||||
// User 2: 4 rows, keepN=2 → should not appear.
|
||||
for (let i = 1; i <= 4; i++) {
|
||||
await store.insert({ ...TRADE, userId: 2, ts: i });
|
||||
}
|
||||
const oldIds = await store.oldRowsForUser(1, 2);
|
||||
expect(oldIds).toHaveLength(2);
|
||||
// Verify by fetching user1 docs to check _ids.
|
||||
const user1Docs = await store.byUser(1, 10); // newest first
|
||||
const expectedIds = [user1Docs[2].id, user1Docs[3].id].map(String).sort();
|
||||
expect(oldIds.map(String).sort()).toEqual(expectedIds);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── oldRows ──────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoTradesStore.oldRows", () => {
|
||||
it("returns [] when total rows <= keepN", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.insert({ ...TRADE, ts: 1 });
|
||||
expect(await store.oldRows(5)).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns ids of rows beyond keepN globally", async () => {
|
||||
const { store } = makeStore();
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
await store.insert({ ...TRADE, ts: i });
|
||||
}
|
||||
const oldIds = await store.oldRows(3);
|
||||
// 5 rows, keepN=3 → 2 excess (ts=1 and ts=2).
|
||||
expect(oldIds).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("spans multiple users", async () => {
|
||||
const { store } = makeStore();
|
||||
for (let i = 1; i <= 3; i++) {
|
||||
await store.insert({ ...TRADE, userId: 1, ts: i });
|
||||
}
|
||||
for (let i = 4; i <= 6; i++) {
|
||||
await store.insert({ ...TRADE, userId: 2, ts: i });
|
||||
}
|
||||
// 6 rows total, keepN=4 → 2 oldest excess (ts=1,2).
|
||||
const oldIds = await store.oldRows(4);
|
||||
expect(oldIds).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── deleteByIds ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("MongoTradesStore.deleteByIds", () => {
|
||||
it("returns { deletedCount: 0 } for empty ids array (no db call)", async () => {
|
||||
const { store } = makeStore();
|
||||
const result = await store.deleteByIds([]);
|
||||
expect(result).toEqual({ deletedCount: 0 });
|
||||
});
|
||||
|
||||
it("deletes the specified documents by _id", async () => {
|
||||
const { store } = makeStore();
|
||||
await store.insert({ ...TRADE, ts: 1 });
|
||||
await store.insert({ ...TRADE, ts: 2 });
|
||||
await store.insert({ ...TRADE, ts: 3 });
|
||||
|
||||
const oldIds = await store.oldRows(2); // 1 excess (ts=1)
|
||||
expect(oldIds).toHaveLength(1);
|
||||
|
||||
const result = await store.deleteByIds(oldIds);
|
||||
expect(result.deletedCount).toBe(1);
|
||||
|
||||
// Only 2 rows should remain.
|
||||
const remaining = await store.byUser(1, 10);
|
||||
expect(remaining).toHaveLength(2);
|
||||
expect(remaining.map((t) => t.ts)).toEqual([3, 2]);
|
||||
});
|
||||
|
||||
it("handles a mix of trades with and without legacy_id", async () => {
|
||||
const { fakeDb, store } = makeStore();
|
||||
|
||||
// Seed one legacy doc (as if backfilled) with an ObjectId _id.
|
||||
const legacyId = new ObjectId();
|
||||
const coll = fakeDb.collection("trading_trades");
|
||||
await coll.insertOne({
|
||||
_id: legacyId,
|
||||
legacy_id: 42,
|
||||
user_id: 1,
|
||||
symbol: "VNM",
|
||||
side: "buy",
|
||||
qty: 50,
|
||||
price_vnd: 80000,
|
||||
ts: 500,
|
||||
});
|
||||
|
||||
// Insert a runtime trade (ts=1000, legacy_id=null).
|
||||
await store.insert({ ...TRADE, ts: 1000 });
|
||||
|
||||
// oldRows(1) → the legacy doc (ts=500) is older → its _id returned.
|
||||
const oldIds = await store.oldRows(1);
|
||||
expect(oldIds).toHaveLength(1);
|
||||
expect(String(oldIds[0])).toBe(String(legacyId));
|
||||
|
||||
await store.deleteByIds(oldIds);
|
||||
|
||||
const remaining = await store.byUser(1, 10);
|
||||
expect(remaining).toHaveLength(1);
|
||||
expect(remaining[0].ts).toBe(1000);
|
||||
});
|
||||
});
|
||||
@@ -1,192 +0,0 @@
|
||||
/**
|
||||
* @file stub-mongo-sentinel.test.js — asserts that MongoClient.prototype.connect
|
||||
* is NEVER called when STUB_SENTINEL flows through the store factories.
|
||||
*
|
||||
* This is the regression test for code-reviewer finding #2: deploy-time
|
||||
* register.js must not attempt an Atlas connection.
|
||||
*
|
||||
* Covers every flag combination from the matrix where MONGODB_URI is set to
|
||||
* STUB_SENTINEL — all should return CF-only stores and never touch MongoClient.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { STUB_SENTINEL } from "../../scripts/stub-kv.js";
|
||||
import { createSqlStore } from "../../src/db/create-sql-store.js";
|
||||
import { createStore } from "../../src/db/create-store.js";
|
||||
import { makeFakeD1 } from "../fakes/fake-d1.js";
|
||||
import { makeFakeKv } from "../fakes/fake-kv-namespace.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Spy on MongoClient.prototype.connect
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let connectSpy;
|
||||
|
||||
beforeEach(async () => {
|
||||
const { MongoClient } = await import("mongodb");
|
||||
connectSpy = vi.spyOn(MongoClient.prototype, "connect").mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
connectSpy?.mockRestore();
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeEnv(overrides = {}) {
|
||||
return {
|
||||
KV: makeFakeKv(),
|
||||
DB: makeFakeD1(),
|
||||
MONGODB_URI: STUB_SENTINEL,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// createStore — all flag combos with STUB_SENTINEL
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("createStore with STUB_SENTINEL — zero MongoClient.connect calls", () => {
|
||||
it("STORAGE_PRIMARY=kv, DUAL_WRITE=1 → CFKVStore only", () => {
|
||||
const env = makeEnv({ STORAGE_PRIMARY: "kv", DUAL_WRITE: "1" });
|
||||
const store = createStore("wordle", env);
|
||||
expect(store).toBeDefined();
|
||||
expect(connectSpy).not.toHaveBeenCalled();
|
||||
// Not a dual store — _kind should be undefined on the wrapper.
|
||||
expect(store._kind).not.toBe("dual");
|
||||
});
|
||||
|
||||
it("STORAGE_PRIMARY=kv, DUAL_WRITE=0 → CFKVStore only", () => {
|
||||
const env = makeEnv({ STORAGE_PRIMARY: "kv", DUAL_WRITE: "0" });
|
||||
createStore("wordle", env);
|
||||
expect(connectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("STORAGE_PRIMARY=mongo, DUAL_WRITE=1 → CFKVStore only (sentinel short-circuits)", () => {
|
||||
const env = makeEnv({ STORAGE_PRIMARY: "mongo", DUAL_WRITE: "1" });
|
||||
createStore("wordle", env);
|
||||
expect(connectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("STORAGE_PRIMARY=mongo, DUAL_WRITE=0 → CFKVStore only (sentinel short-circuits)", () => {
|
||||
const env = makeEnv({ STORAGE_PRIMARY: "mongo", DUAL_WRITE: "0" });
|
||||
createStore("wordle", env);
|
||||
expect(connectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("MONGODB_URI unset → CFKVStore only", () => {
|
||||
const env = makeEnv({ MONGODB_URI: undefined });
|
||||
createStore("wordle", env);
|
||||
expect(connectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// createSqlStore — all flag combos with STUB_SENTINEL
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("createSqlStore with STUB_SENTINEL — zero MongoClient.connect calls", () => {
|
||||
it("STORAGE_PRIMARY=kv, DUAL_WRITE=1 → CFSqlStore only", () => {
|
||||
const env = makeEnv({ STORAGE_PRIMARY: "kv", DUAL_WRITE: "1" });
|
||||
const sql = createSqlStore("trading", env);
|
||||
expect(sql).not.toBeNull();
|
||||
expect(connectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("STORAGE_PRIMARY=kv, DUAL_WRITE=0 → CFSqlStore only", () => {
|
||||
const env = makeEnv({ STORAGE_PRIMARY: "kv", DUAL_WRITE: "0" });
|
||||
createSqlStore("trading", env);
|
||||
expect(connectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("STORAGE_PRIMARY=mongo, DUAL_WRITE=1 → CFSqlStore only (sentinel short-circuits)", () => {
|
||||
const env = makeEnv({ STORAGE_PRIMARY: "mongo", DUAL_WRITE: "1" });
|
||||
createSqlStore("trading", env);
|
||||
expect(connectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("STORAGE_PRIMARY=mongo, DUAL_WRITE=0 → CFSqlStore only (sentinel short-circuits)", () => {
|
||||
const env = makeEnv({ STORAGE_PRIMARY: "mongo", DUAL_WRITE: "0" });
|
||||
createSqlStore("trading", env);
|
||||
expect(connectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("MONGODB_URI unset → CFSqlStore only, null when DB absent", () => {
|
||||
// No DB — should return null, no Mongo.
|
||||
const env = { KV: makeFakeKv(), MONGODB_URI: undefined };
|
||||
const sql = createSqlStore("trading", env);
|
||||
expect(sql).toBeNull();
|
||||
expect(connectSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// STUB_SENTINEL constant value
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("STUB_SENTINEL constant", () => {
|
||||
it("is a non-empty string", () => {
|
||||
expect(typeof STUB_SENTINEL).toBe("string");
|
||||
expect(STUB_SENTINEL.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("equals the sentinel used inside the factories", () => {
|
||||
// The factories hardcode "__stub_mongo__" — must match the exported constant.
|
||||
expect(STUB_SENTINEL).toBe("__stub_mongo__");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Flag matrix — non-sentinel creates DualKVStore (and does NOT call connect here)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("createStore with real URI — returns DualKVStore (_kind=dual)", () => {
|
||||
it("STORAGE_PRIMARY=kv, DUAL_WRITE=1, real URI → dual store", () => {
|
||||
const env = {
|
||||
KV: makeFakeKv(),
|
||||
MONGODB_URI: "mongodb://fake",
|
||||
STORAGE_PRIMARY: "kv",
|
||||
DUAL_WRITE: "1",
|
||||
};
|
||||
const store = createStore("wordle", env);
|
||||
// The wrapper object itself is plain, but the underlying DualKVStore has _kind.
|
||||
// Access via the wrapper's _kind (forwarded in withPrefix).
|
||||
expect(store._kind).toBe("dual");
|
||||
});
|
||||
|
||||
it("STORAGE_PRIMARY=kv, DUAL_WRITE=0, real URI → CF-only (rollback path)", () => {
|
||||
const env = {
|
||||
KV: makeFakeKv(),
|
||||
MONGODB_URI: "mongodb://fake",
|
||||
STORAGE_PRIMARY: "kv",
|
||||
DUAL_WRITE: "0",
|
||||
};
|
||||
const store = createStore("wordle", env);
|
||||
expect(store._kind).not.toBe("dual");
|
||||
});
|
||||
|
||||
it("STORAGE_PRIMARY=mongo, DUAL_WRITE=1, real URI → dual store (Mongo primary)", () => {
|
||||
const env = {
|
||||
KV: makeFakeKv(),
|
||||
MONGODB_URI: "mongodb://fake",
|
||||
STORAGE_PRIMARY: "mongo",
|
||||
DUAL_WRITE: "1",
|
||||
};
|
||||
const store = createStore("wordle", env);
|
||||
expect(store._kind).toBe("dual");
|
||||
});
|
||||
|
||||
it("STORAGE_PRIMARY=mongo, DUAL_WRITE=0, real URI → MongoKVStore only", () => {
|
||||
const env = {
|
||||
KV: makeFakeKv(),
|
||||
MONGODB_URI: "mongodb://fake",
|
||||
STORAGE_PRIMARY: "mongo",
|
||||
DUAL_WRITE: "0",
|
||||
};
|
||||
const store = createStore("wordle", env);
|
||||
// MongoKVStore has no _kind; wrapper forwards undefined.
|
||||
expect(store._kind).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,254 +0,0 @@
|
||||
/**
|
||||
* @file storage-roundtrip.test.js — e2e storage integration test.
|
||||
*
|
||||
* Boots a fake env with:
|
||||
* - MongoKVStore (backed by fake-mongo) for KV path (wordle)
|
||||
* - MongoTradesStore (backed by fake-mongo) for SQL/trades path (trading)
|
||||
* - DUAL_WRITE=1 so DualKVStore is used (CF KV + Mongo both written)
|
||||
*
|
||||
* Asserts that state written via the store abstractions is visible in BOTH
|
||||
* backends — the CF KV fake AND the in-memory Mongo fake.
|
||||
*
|
||||
* This test intentionally avoids grammY context setup. It exercises the
|
||||
* storage layer (the thing being migrated), not the Telegram handler.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import { createStore } from "../../src/db/create-store.js";
|
||||
import { MongoKVStore } from "../../src/db/mongo-kv-store.js";
|
||||
import { MongoTradesStore } from "../../src/db/mongo-trades-store.js";
|
||||
import { listTrades, recordTrade } from "../../src/modules/trading/history.js";
|
||||
import { loadGame, loadStats, recordResult, saveGame } from "../../src/modules/wordle/state.js";
|
||||
import { makeFakeKv } from "../fakes/fake-kv-namespace.js";
|
||||
import { makeFakeMongo } from "../fakes/fake-mongo.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared setup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let cfKv;
|
||||
let fakeDb;
|
||||
let env;
|
||||
|
||||
beforeEach(() => {
|
||||
cfKv = makeFakeKv();
|
||||
fakeDb = makeFakeMongo();
|
||||
// env with real-looking MONGODB_URI so factories pick the dual-write path.
|
||||
// MongoKVStore receives dbOverride (fakeDb) so no real Atlas connection happens.
|
||||
env = {
|
||||
KV: cfKv,
|
||||
MONGODB_URI: "mongodb://fake-atlas",
|
||||
STORAGE_PRIMARY: "kv",
|
||||
DUAL_WRITE: "1",
|
||||
};
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// KV path — wordle game state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("KV dual-write — wordle game state", () => {
|
||||
it("saveGame persists to both CF KV and MongoKVStore (fake-mongo)", async () => {
|
||||
// Build the dual store: createStore uses DualKVStore with CF primary + Mongo secondary.
|
||||
// Pass fakeDb as dbOverride so MongoKVStore talks to fake-mongo, not Atlas.
|
||||
const mongoKvStore = new MongoKVStore(env, "wordle", fakeDb);
|
||||
|
||||
// Manually construct the dual store to inject fakeDb.
|
||||
const { DualKVStore } = await import("../../src/db/dual-kv-store.js");
|
||||
const { CFKVStore } = await import("../../src/db/cf-kv-store.js");
|
||||
const cfStore = new CFKVStore(cfKv);
|
||||
|
||||
// Prefix wrapper that mirrors create-store.js behaviour.
|
||||
function prefixedStore(base, prefix) {
|
||||
return {
|
||||
async get(key) {
|
||||
return base.get(prefix + key);
|
||||
},
|
||||
async put(key, value, opts) {
|
||||
return base.put(prefix + key, value, opts);
|
||||
},
|
||||
async delete(key) {
|
||||
return base.delete(prefix + key);
|
||||
},
|
||||
async list(opts = {}) {
|
||||
const fullPrefix = prefix + (opts.prefix ?? "");
|
||||
const result = await base.list({
|
||||
prefix: fullPrefix,
|
||||
limit: opts.limit,
|
||||
cursor: opts.cursor,
|
||||
});
|
||||
return {
|
||||
keys: result.keys.map((k) => (k.startsWith(prefix) ? k.slice(prefix.length) : k)),
|
||||
cursor: result.cursor,
|
||||
done: result.done,
|
||||
};
|
||||
},
|
||||
async getJSON(key) {
|
||||
return base.getJSON(prefix + key);
|
||||
},
|
||||
async putJSON(key, value, opts) {
|
||||
return base.putJSON(prefix + key, value, opts);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const dual = new DualKVStore(cfStore, mongoKvStore, cfKv);
|
||||
const db = prefixedStore(dual, "wordle:");
|
||||
|
||||
const gameState = {
|
||||
target: "crane",
|
||||
guesses: [{ word: "audio", results: ["absent", "absent", "absent", "absent", "absent"] }],
|
||||
solved: false,
|
||||
startedAt: Date.now(),
|
||||
};
|
||||
|
||||
await saveGame(db, 42, gameState);
|
||||
|
||||
// 1. CF KV should have the prefixed key.
|
||||
expect(cfKv.store.has("wordle:game:42")).toBe(true);
|
||||
const cfStored = JSON.parse(cfKv.store.get("wordle:game:42"));
|
||||
expect(cfStored.target).toBe("crane");
|
||||
|
||||
// 2. Mongo fake should have the same key.
|
||||
const mongoColl = fakeDb.collection("wordle");
|
||||
const mongoDoc = await mongoColl.findOne({ _id: "wordle:game:42" });
|
||||
expect(mongoDoc).not.toBeNull();
|
||||
expect(JSON.parse(mongoDoc.value).target).toBe("crane");
|
||||
|
||||
// 3. loadGame reads from primary (CF KV).
|
||||
const loaded = await loadGame(db, 42);
|
||||
expect(loaded?.target).toBe("crane");
|
||||
expect(loaded?.guesses).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("recordResult persists stats to both backends", async () => {
|
||||
const mongoKvStore = new MongoKVStore(env, "wordle", fakeDb);
|
||||
const { DualKVStore } = await import("../../src/db/dual-kv-store.js");
|
||||
const { CFKVStore } = await import("../../src/db/cf-kv-store.js");
|
||||
const cfStore = new CFKVStore(cfKv);
|
||||
const dual = new DualKVStore(cfStore, mongoKvStore, cfKv);
|
||||
|
||||
function prefixedStore(base, prefix) {
|
||||
return {
|
||||
async get(key) {
|
||||
return base.get(prefix + key);
|
||||
},
|
||||
async put(key, value, opts) {
|
||||
return base.put(prefix + key, value, opts);
|
||||
},
|
||||
async delete(key) {
|
||||
return base.delete(prefix + key);
|
||||
},
|
||||
async list(opts = {}) {
|
||||
const fullPrefix = prefix + (opts.prefix ?? "");
|
||||
const result = await base.list({
|
||||
prefix: fullPrefix,
|
||||
limit: opts.limit,
|
||||
cursor: opts.cursor,
|
||||
});
|
||||
return {
|
||||
keys: result.keys.map((k) => (k.startsWith(prefix) ? k.slice(prefix.length) : k)),
|
||||
cursor: result.cursor,
|
||||
done: result.done,
|
||||
};
|
||||
},
|
||||
async getJSON(key) {
|
||||
return base.getJSON(prefix + key);
|
||||
},
|
||||
async putJSON(key, value, opts) {
|
||||
return base.putJSON(prefix + key, value, opts);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const db = prefixedStore(dual, "wordle:");
|
||||
await recordResult(db, 99, true);
|
||||
|
||||
// CF KV has the stats key.
|
||||
expect(cfKv.store.has("wordle:stats:99")).toBe(true);
|
||||
const cfStats = JSON.parse(cfKv.store.get("wordle:stats:99"));
|
||||
expect(cfStats.wins).toBe(1);
|
||||
expect(cfStats.streak).toBe(1);
|
||||
|
||||
// Mongo fake also has it.
|
||||
const mongoColl = fakeDb.collection("wordle");
|
||||
const mongoDoc = await mongoColl.findOne({ _id: "wordle:stats:99" });
|
||||
expect(mongoDoc).not.toBeNull();
|
||||
expect(JSON.parse(mongoDoc.value).wins).toBe(1);
|
||||
|
||||
// loadStats reads from primary.
|
||||
const stats = await loadStats(db, 99);
|
||||
expect(stats.wins).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SQL / trades path — trading insert via MongoTradesStore
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("SQL dual-write — trading insert via MongoTradesStore", () => {
|
||||
it("recordTrade persists via MongoTradesStore (fake-mongo)", async () => {
|
||||
// Trading uses MongoTradesStore directly (not via DualSqlStore) when tradesStore is provided.
|
||||
const tradesStore = new MongoTradesStore(env, fakeDb);
|
||||
|
||||
await recordTrade(
|
||||
null,
|
||||
{
|
||||
userId: 123,
|
||||
symbol: "VNM",
|
||||
side: "buy",
|
||||
qty: 10,
|
||||
priceVnd: 50000,
|
||||
},
|
||||
tradesStore,
|
||||
);
|
||||
|
||||
// Mongo should have the trade.
|
||||
const tradeColl = fakeDb.collection("trading_trades");
|
||||
const docs = await tradeColl.find({}).toArray();
|
||||
expect(docs).toHaveLength(1);
|
||||
expect(docs[0].user_id).toBe(123);
|
||||
expect(docs[0].symbol).toBe("VNM");
|
||||
expect(docs[0].side).toBe("buy");
|
||||
expect(docs[0].qty).toBe(10);
|
||||
expect(docs[0].price_vnd).toBe(50000);
|
||||
});
|
||||
|
||||
it("listTrades reads from MongoTradesStore", async () => {
|
||||
const tradesStore = new MongoTradesStore(env, fakeDb);
|
||||
|
||||
// Insert two trades.
|
||||
await recordTrade(
|
||||
null,
|
||||
{ userId: 7, symbol: "FPT", side: "buy", qty: 5, priceVnd: 100000 },
|
||||
tradesStore,
|
||||
);
|
||||
await recordTrade(
|
||||
null,
|
||||
{ userId: 7, symbol: "VIC", side: "sell", qty: 2, priceVnd: 80000 },
|
||||
tradesStore,
|
||||
);
|
||||
|
||||
const trades = await listTrades(null, 7, 10, tradesStore);
|
||||
expect(trades).toHaveLength(2);
|
||||
// Newest first (MongoTradesStore returns sorted by ts desc).
|
||||
expect(trades.map((t) => t.symbol)).toContain("FPT");
|
||||
expect(trades.map((t) => t.symbol)).toContain("VIC");
|
||||
});
|
||||
|
||||
it("DUAL_WRITE=1 env flag is present in env passed through registry init", () => {
|
||||
// Verify the flag matrix expectation: when DUAL_WRITE=1 and MONGODB_URI is real,
|
||||
// buildTradesStore should return a MongoTradesStore.
|
||||
const localEnv = {
|
||||
KV: cfKv,
|
||||
MONGODB_URI: "mongodb://fake",
|
||||
STORAGE_PRIMARY: "kv",
|
||||
DUAL_WRITE: "1",
|
||||
};
|
||||
// Direct validation — the helper in registry.js would return a MongoTradesStore.
|
||||
// We test its outcome indirectly: DUAL_WRITE="1" !== "0" is true.
|
||||
expect(localEnv.DUAL_WRITE !== "0").toBe(true);
|
||||
expect(!!localEnv.MONGODB_URI).toBe(true);
|
||||
expect(localEnv.MONGODB_URI).not.toBe("__stub_mongo__");
|
||||
});
|
||||
});
|
||||
@@ -10,22 +10,15 @@ export function makeFakeBot() {
|
||||
const commandCalls = [];
|
||||
/** @type {Array<{event: string, handler: Function}>} */
|
||||
const onCalls = [];
|
||||
/** @type {Array<Function>} */
|
||||
const useCalls = [];
|
||||
|
||||
return {
|
||||
commandCalls,
|
||||
onCalls,
|
||||
useCalls,
|
||||
command(name, handler) {
|
||||
commandCalls.push({ name, handler });
|
||||
},
|
||||
on(event, handler) {
|
||||
onCalls.push({ event, handler });
|
||||
},
|
||||
/** Records middleware registered via bot.use() (e.g. timing middleware). */
|
||||
use(middleware) {
|
||||
useCalls.push(middleware);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,306 +0,0 @@
|
||||
/**
|
||||
* @file fake-mongo — Map-backed in-memory MongoDB fake for unit tests.
|
||||
*
|
||||
* **FROZEN SURFACE** (Phase 02–08): This fake implements the minimal subset of MongoDB
|
||||
* API required by MongoKVStore and MongoTradesStore tests. New MongoClient features
|
||||
* must be added to this fake to remain testable.
|
||||
*
|
||||
* **Implemented methods:**
|
||||
* - `db.collection(name)` → Collection
|
||||
* - Collection: `findOne(query)`, `updateOne(filter, update, opts)`, `deleteOne(query)`,
|
||||
* `find(query)`, `insertOne(doc)`, `insertMany(docs)`, `distinct(field, query)`,
|
||||
* `deleteMany(query)`, `countDocuments(query)`, `createIndex(spec)` (no-op)
|
||||
*
|
||||
* **Cursor methods** (from `find()`): `sort(spec)`, `skip(n)`, `limit(n)`,
|
||||
* `project(spec)`, `toArray()`
|
||||
*
|
||||
* **NOT SIMULATED:**
|
||||
* - TTL expirations at server-side. Tests assert `expiresAt` field presence and
|
||||
* control Date.now() via `vi.setSystemTime()` to test read-time expiration filters
|
||||
* in MongoKVStore. Real Atlas TTL is validated during Phase 06 soak.
|
||||
* - Transactions, sharding, replica sets, or other cluster features.
|
||||
* - Advanced operators beyond those in `matchQuery()` ($gt, $gte, $lt, $lte,
|
||||
* $exists, $in, $regex, $or, $and).
|
||||
*
|
||||
* @see tests/db/mongo-kv-store.test.js
|
||||
* @see tests/modules/trading/ (trades store usage)
|
||||
*/
|
||||
|
||||
/**
|
||||
* Apply $set and $unset from an update document to a target object.
|
||||
*
|
||||
* @param {object} doc
|
||||
* @param {object} update
|
||||
* @returns {object}
|
||||
*/
|
||||
function applyUpdate(doc, update) {
|
||||
const result = { ...doc };
|
||||
if (update.$set) {
|
||||
for (const [k, v] of Object.entries(update.$set)) {
|
||||
result[k] = v;
|
||||
}
|
||||
}
|
||||
if (update.$unset) {
|
||||
for (const key of Object.keys(update.$unset)) {
|
||||
delete result[key];
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal regex-query matcher. Supports:
|
||||
* { field: value } — strict equality
|
||||
* { field: { $gt: v } } — greater-than
|
||||
* { field: { $exists: b } } — field existence
|
||||
* { $or: [cond, ...] } — logical OR
|
||||
* { $and: [cond, ...] } — logical AND
|
||||
*
|
||||
* @param {object} doc
|
||||
* @param {object} query
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function matchQuery(doc, query) {
|
||||
for (const [key, condition] of Object.entries(query)) {
|
||||
if (key === "$or") {
|
||||
if (!condition.some((sub) => matchQuery(doc, sub))) return false;
|
||||
continue;
|
||||
}
|
||||
if (key === "$and") {
|
||||
if (!condition.every((sub) => matchQuery(doc, sub))) return false;
|
||||
continue;
|
||||
}
|
||||
if (condition !== null && typeof condition === "object" && !Array.isArray(condition)) {
|
||||
const ops = Object.keys(condition);
|
||||
if (ops.some((op) => op.startsWith("$"))) {
|
||||
for (const [op, operand] of Object.entries(condition)) {
|
||||
if (op === "$gt") {
|
||||
if (!(doc[key] > operand)) return false;
|
||||
} else if (op === "$gte") {
|
||||
if (!(doc[key] >= operand)) return false;
|
||||
} else if (op === "$lt") {
|
||||
if (!(doc[key] < operand)) return false;
|
||||
} else if (op === "$lte") {
|
||||
if (!(doc[key] <= operand)) return false;
|
||||
} else if (op === "$exists") {
|
||||
const has = key in doc && doc[key] !== undefined;
|
||||
if (operand !== has) return false;
|
||||
} else if (op === "$in") {
|
||||
if (!operand.includes(doc[key])) return false;
|
||||
} else if (op === "$regex") {
|
||||
const flags = condition.$options ?? "";
|
||||
if (!new RegExp(operand, flags).test(doc[key])) return false;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (doc[key] !== condition) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a chainable cursor builder from an array of matched docs.
|
||||
*
|
||||
* @param {object[]} docs
|
||||
* @returns {object} chainable cursor with sort/skip/limit/project/toArray
|
||||
*/
|
||||
function makeCursor(docs) {
|
||||
const items = [...docs];
|
||||
let sortField = null;
|
||||
let sortDir = 1;
|
||||
let skipN = 0;
|
||||
let limitN = Number.POSITIVE_INFINITY;
|
||||
let projection = null;
|
||||
|
||||
const cursor = {
|
||||
sort(spec) {
|
||||
const entries = Object.entries(spec);
|
||||
if (entries.length > 0) {
|
||||
[sortField, sortDir] = [entries[0][0], entries[0][1]];
|
||||
}
|
||||
return cursor;
|
||||
},
|
||||
skip(n) {
|
||||
skipN = n;
|
||||
return cursor;
|
||||
},
|
||||
limit(n) {
|
||||
limitN = n;
|
||||
return cursor;
|
||||
},
|
||||
project(spec) {
|
||||
projection = spec;
|
||||
return cursor;
|
||||
},
|
||||
async toArray() {
|
||||
let result = [...items];
|
||||
if (sortField !== null) {
|
||||
result.sort((a, b) => {
|
||||
if (a[sortField] < b[sortField]) return -sortDir;
|
||||
if (a[sortField] > b[sortField]) return sortDir;
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
result = result.slice(
|
||||
skipN,
|
||||
limitN === Number.POSITIVE_INFINITY ? undefined : skipN + limitN,
|
||||
);
|
||||
if (projection) {
|
||||
result = result.map((doc) => {
|
||||
const out = {};
|
||||
for (const [k, include] of Object.entries(projection)) {
|
||||
if (include) out[k] = doc[k];
|
||||
}
|
||||
return out;
|
||||
});
|
||||
}
|
||||
return result;
|
||||
},
|
||||
};
|
||||
return cursor;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a fake MongoDB collection backed by a Map.
|
||||
*
|
||||
* @param {Map<string, object>} store
|
||||
* @returns {object}
|
||||
*/
|
||||
function makeCollection(store) {
|
||||
return {
|
||||
/** @returns {Promise<object|null>} */
|
||||
async findOne(query) {
|
||||
for (const doc of store.values()) {
|
||||
if (matchQuery(doc, query)) return { ...doc };
|
||||
}
|
||||
return null;
|
||||
},
|
||||
|
||||
/**
|
||||
* Supports upsert with $set / $unset.
|
||||
* @returns {Promise<{matchedCount: number, upsertedCount: number, modifiedCount: number}>}
|
||||
*/
|
||||
async updateOne(filter, update, opts = {}) {
|
||||
for (const [id, doc] of store.entries()) {
|
||||
if (matchQuery(doc, filter)) {
|
||||
store.set(id, applyUpdate(doc, update));
|
||||
return { matchedCount: 1, upsertedCount: 0, modifiedCount: 1 };
|
||||
}
|
||||
}
|
||||
if (opts.upsert) {
|
||||
// Build new doc from filter equality fields + $set fields
|
||||
const newDoc = {};
|
||||
for (const [k, v] of Object.entries(filter)) {
|
||||
if (typeof v !== "object") newDoc[k] = v;
|
||||
}
|
||||
const merged = applyUpdate(newDoc, update);
|
||||
const id = merged._id ?? String(Date.now() + Math.random());
|
||||
merged._id = id;
|
||||
store.set(String(id), merged);
|
||||
return { matchedCount: 0, upsertedCount: 1, modifiedCount: 0 };
|
||||
}
|
||||
return { matchedCount: 0, upsertedCount: 0, modifiedCount: 0 };
|
||||
},
|
||||
|
||||
/** @returns {Promise<{deletedCount: number}>} */
|
||||
async deleteOne(filter) {
|
||||
for (const [id, doc] of store.entries()) {
|
||||
if (matchQuery(doc, filter)) {
|
||||
store.delete(id);
|
||||
return { deletedCount: 1 };
|
||||
}
|
||||
}
|
||||
return { deletedCount: 0 };
|
||||
},
|
||||
|
||||
/**
|
||||
* Returns a chainable cursor.
|
||||
* @param {object} [query]
|
||||
* @returns {object}
|
||||
*/
|
||||
find(query = {}) {
|
||||
const matched = [...store.values()].filter((doc) => matchQuery(doc, query));
|
||||
return makeCursor(matched);
|
||||
},
|
||||
|
||||
/** @returns {Promise<{insertedId: string}>} */
|
||||
async insertOne(doc) {
|
||||
const id = doc._id ?? String(Date.now() + Math.random());
|
||||
store.set(String(id), { ...doc, _id: id });
|
||||
return { insertedId: id };
|
||||
},
|
||||
|
||||
/** @returns {Promise<{insertedIds: string[]}>} */
|
||||
async insertMany(docs) {
|
||||
const insertedIds = [];
|
||||
for (const doc of docs) {
|
||||
const id = doc._id ?? String(Date.now() + Math.random());
|
||||
store.set(String(id), { ...doc, _id: id });
|
||||
insertedIds.push(id);
|
||||
}
|
||||
return { insertedIds };
|
||||
},
|
||||
|
||||
/** @returns {Promise<any[]>} */
|
||||
async distinct(field, query = {}) {
|
||||
const values = new Set();
|
||||
for (const doc of store.values()) {
|
||||
if (matchQuery(doc, query) && field in doc) {
|
||||
values.add(doc[field]);
|
||||
}
|
||||
}
|
||||
return [...values];
|
||||
},
|
||||
|
||||
/** @returns {Promise<{deletedCount: number}>} */
|
||||
async deleteMany(filter = {}) {
|
||||
let count = 0;
|
||||
for (const [id, doc] of store.entries()) {
|
||||
if (matchQuery(doc, filter)) {
|
||||
store.delete(id);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
return { deletedCount: count };
|
||||
},
|
||||
|
||||
/** @returns {Promise<number>} */
|
||||
async countDocuments(query = {}) {
|
||||
let count = 0;
|
||||
for (const doc of store.values()) {
|
||||
if (matchQuery(doc, query)) count++;
|
||||
}
|
||||
return count;
|
||||
},
|
||||
|
||||
/** No-op — index creation is idempotent; tests only verify field presence. */
|
||||
async createIndex(_spec, _opts) {
|
||||
return "ok";
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a fake MongoDB Db object.
|
||||
* Each collection is lazily created and backed by its own Map.
|
||||
*
|
||||
* @returns {{ collection: (name: string) => object, _stores: Map<string, Map<string, object>> }}
|
||||
*/
|
||||
export function makeFakeMongo() {
|
||||
/** @type {Map<string, Map<string, object>>} */
|
||||
const stores = new Map();
|
||||
|
||||
return {
|
||||
/** @param {string} name */
|
||||
collection(name) {
|
||||
if (!stores.has(name)) {
|
||||
stores.set(name, new Map());
|
||||
}
|
||||
return makeCollection(stores.get(name));
|
||||
},
|
||||
/** Expose raw stores so tests can inspect or seed data directly. */
|
||||
_stores: stores,
|
||||
};
|
||||
}
|
||||
@@ -1,12 +1,10 @@
|
||||
/**
|
||||
* @file Tests for trading/history — recordTrade, listTrades, formatTradesHtml,
|
||||
* createHistoryHandler, and buy/sell → D1 integration.
|
||||
* Also covers the MongoTradesStore path (tradesStore arg) added in Phase 03.
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createSqlStore } from "../../../src/db/create-sql-store.js";
|
||||
import { MongoTradesStore } from "../../../src/db/mongo-trades-store.js";
|
||||
import {
|
||||
createHistoryHandler,
|
||||
formatTradesHtml,
|
||||
@@ -14,7 +12,6 @@ import {
|
||||
recordTrade,
|
||||
} from "../../../src/modules/trading/history.js";
|
||||
import { makeFakeD1 } from "../../fakes/fake-d1.js";
|
||||
import { makeFakeMongo } from "../../fakes/fake-mongo.js";
|
||||
|
||||
// ─── helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -350,94 +347,3 @@ describe("buy/sell handlers → recordTrade integration", () => {
|
||||
expect(ctx.reply).toHaveBeenCalledWith(expect.stringContaining("Bought"));
|
||||
});
|
||||
});
|
||||
|
||||
// ─── MongoTradesStore path (Phase 03) ────────────────────────────────────────
|
||||
|
||||
/** Build a MongoTradesStore backed by a fresh fake Mongo db. */
|
||||
function makeTestTradesStore() {
|
||||
const fakeDb = makeFakeMongo();
|
||||
const tradesStore = new MongoTradesStore({}, fakeDb);
|
||||
return { fakeDb, tradesStore };
|
||||
}
|
||||
|
||||
describe("recordTrade — tradesStore path", () => {
|
||||
it("calls tradesStore.insert when tradesStore is provided", async () => {
|
||||
const { tradesStore } = makeTestTradesStore();
|
||||
const spy = vi.spyOn(tradesStore, "insert");
|
||||
await recordTrade(null, TRADE, tradesStore);
|
||||
expect(spy).toHaveBeenCalledOnce();
|
||||
expect(spy).toHaveBeenCalledWith(expect.objectContaining({ userId: 1, symbol: "TCB" }));
|
||||
});
|
||||
|
||||
it("does NOT call sql when tradesStore is provided", async () => {
|
||||
const { fakeDb, sql } = makeTestSql();
|
||||
const { tradesStore } = makeTestTradesStore();
|
||||
await recordTrade(sql, TRADE, tradesStore);
|
||||
// D1 runLog must be empty — Mongo path was used.
|
||||
expect(fakeDb.runLog).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("logs error and does not throw when tradesStore.insert fails", async () => {
|
||||
const { tradesStore } = makeTestTradesStore();
|
||||
vi.spyOn(tradesStore, "insert").mockRejectedValue(new Error("mongo down"));
|
||||
const errSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
await expect(recordTrade(null, TRADE, tradesStore)).resolves.toBeUndefined();
|
||||
expect(errSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining("recordTrade (mongo) failed"),
|
||||
expect.any(Error),
|
||||
);
|
||||
errSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe("listTrades — tradesStore path", () => {
|
||||
it("calls tradesStore.byUser when tradesStore is provided", async () => {
|
||||
const { tradesStore } = makeTestTradesStore();
|
||||
const spy = vi.spyOn(tradesStore, "byUser").mockResolvedValue([]);
|
||||
await listTrades(null, 1, 10, tradesStore);
|
||||
expect(spy).toHaveBeenCalledWith(1, 10);
|
||||
});
|
||||
|
||||
it("returns mapped trades from tradesStore", async () => {
|
||||
const { tradesStore } = makeTestTradesStore();
|
||||
await tradesStore.insert({
|
||||
userId: 1,
|
||||
symbol: "VNM",
|
||||
side: "sell",
|
||||
qty: 5,
|
||||
priceVnd: 80000,
|
||||
ts: 500,
|
||||
});
|
||||
const trades = await listTrades(null, 1, 10, tradesStore);
|
||||
expect(trades).toHaveLength(1);
|
||||
expect(trades[0].symbol).toBe("VNM");
|
||||
expect(trades[0].priceVnd).toBe(80000);
|
||||
});
|
||||
|
||||
it("clamps limit before passing to tradesStore", async () => {
|
||||
const { tradesStore } = makeTestTradesStore();
|
||||
const spy = vi.spyOn(tradesStore, "byUser").mockResolvedValue([]);
|
||||
await listTrades(null, 1, 999, tradesStore);
|
||||
expect(spy).toHaveBeenCalledWith(1, 50);
|
||||
await listTrades(null, 1, 0, tradesStore);
|
||||
expect(spy).toHaveBeenCalledWith(1, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createHistoryHandler — tradesStore path", () => {
|
||||
it("uses tradesStore when provided and sql is null", async () => {
|
||||
const { tradesStore } = makeTestTradesStore();
|
||||
await tradesStore.insert({
|
||||
userId: 1,
|
||||
symbol: "TCB",
|
||||
side: "buy",
|
||||
qty: 10,
|
||||
priceVnd: 25000,
|
||||
ts: 1,
|
||||
});
|
||||
const handler = createHistoryHandler(null, tradesStore);
|
||||
const ctx = { match: "", from: { id: 1 }, reply: vi.fn() };
|
||||
await handler(ctx);
|
||||
expect(ctx.reply).toHaveBeenCalledWith(expect.stringContaining("TCB"), { parse_mode: "HTML" });
|
||||
});
|
||||
});
|
||||
@@ -6,16 +6,12 @@
|
||||
* - SELECT id ... WHERE user_id = ? ORDER BY ts DESC LIMIT -1 OFFSET N
|
||||
* - SELECT id ... ORDER BY ts DESC LIMIT -1 OFFSET N
|
||||
* - DELETE FROM ... WHERE id IN (?, ...)
|
||||
*
|
||||
* Also covers the MongoTradesStore path added in Phase 03.
|
||||
*/
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createSqlStore } from "../../../src/db/create-sql-store.js";
|
||||
import { MongoTradesStore } from "../../../src/db/mongo-trades-store.js";
|
||||
import { trimTradesHandler } from "../../../src/modules/trading/retention.js";
|
||||
import { makeFakeD1 } from "../../fakes/fake-d1.js";
|
||||
import { makeFakeMongo } from "../../fakes/fake-mongo.js";
|
||||
|
||||
// ─── helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -197,100 +193,3 @@ describe("trimTradesHandler — idempotence", () => {
|
||||
expect(afterFirst).toBe(afterSecond);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── MongoTradesStore path (Phase 03) ────────────────────────────────────────
|
||||
|
||||
/** Build a MongoTradesStore backed by a fresh fake Mongo db. */
|
||||
function makeTestTradesStore() {
|
||||
const fakeDb = makeFakeMongo();
|
||||
const tradesStore = new MongoTradesStore({}, fakeDb);
|
||||
return { tradesStore };
|
||||
}
|
||||
|
||||
/** Convenience: run trimTradesHandler with tradesStore, muted console output. */
|
||||
async function runTrimMongo(tradesStore, caps) {
|
||||
const spy = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
try {
|
||||
await trimTradesHandler({}, { sql: null, tradesStore }, caps);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
}
|
||||
|
||||
describe("trimTradesHandler — tradesStore path (null sql)", () => {
|
||||
it("skips when tradesStore is null and sql is null", async () => {
|
||||
const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
await expect(trimTradesHandler({}, { sql: null, tradesStore: null })).resolves.toBeUndefined();
|
||||
logSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("per-user trim keeps newest rows via MongoTradesStore", async () => {
|
||||
const { tradesStore } = makeTestTradesStore();
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
await tradesStore.insert({
|
||||
userId: 1,
|
||||
symbol: "TCB",
|
||||
side: "buy",
|
||||
qty: 1,
|
||||
priceVnd: 1,
|
||||
ts: i,
|
||||
});
|
||||
}
|
||||
await runTrimMongo(tradesStore, { perUserCap: 3, globalCap: 1000 });
|
||||
const remaining = await tradesStore.byUser(1, 100);
|
||||
expect(remaining).toHaveLength(3);
|
||||
expect(remaining.map((t) => t.ts)).toEqual([5, 4, 3]);
|
||||
});
|
||||
|
||||
it("global trim keeps newest rows across all users via MongoTradesStore", async () => {
|
||||
const { tradesStore } = makeTestTradesStore();
|
||||
for (let i = 1; i <= 8; i++) {
|
||||
await tradesStore.insert({
|
||||
userId: 1,
|
||||
symbol: "TCB",
|
||||
side: "buy",
|
||||
qty: 1,
|
||||
priceVnd: 1,
|
||||
ts: i,
|
||||
});
|
||||
}
|
||||
for (let i = 9; i <= 15; i++) {
|
||||
await tradesStore.insert({
|
||||
userId: 2,
|
||||
symbol: "VNM",
|
||||
side: "sell",
|
||||
qty: 1,
|
||||
priceVnd: 1,
|
||||
ts: i,
|
||||
});
|
||||
}
|
||||
await runTrimMongo(tradesStore, { perUserCap: 1000, globalCap: 10 });
|
||||
|
||||
const u1 = await tradesStore.byUser(1, 100);
|
||||
const u2 = await tradesStore.byUser(2, 100);
|
||||
expect(u1.length + u2.length).toBe(10);
|
||||
});
|
||||
|
||||
it("uses tradesStore path even when sql is provided alongside tradesStore", async () => {
|
||||
const { tradesStore } = makeTestTradesStore();
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
await tradesStore.insert({
|
||||
userId: 1,
|
||||
symbol: "TCB",
|
||||
side: "buy",
|
||||
qty: 1,
|
||||
priceVnd: 1,
|
||||
ts: i,
|
||||
});
|
||||
}
|
||||
// Provide a sql too — tradesStore must win.
|
||||
const fakeD1 = makeFakeD1();
|
||||
const sql = createSqlStore("trading", { DB: fakeD1 });
|
||||
const logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
await trimTradesHandler({}, { sql, tradesStore }, { perUserCap: 3, globalCap: 1000 });
|
||||
logSpy.mockRestore();
|
||||
// D1 must not have been touched.
|
||||
expect(fakeD1.runLog).toHaveLength(0);
|
||||
expect(fakeD1.queryLog).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -1,196 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
aggregateLines,
|
||||
formatReport,
|
||||
parseLogLine,
|
||||
percentile,
|
||||
} from "../../scripts/analyze-soak.js";
|
||||
|
||||
// ── percentile ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("percentile", () => {
|
||||
it("returns 0 for empty array", () => {
|
||||
expect(percentile([], 50)).toBe(0);
|
||||
});
|
||||
|
||||
it("returns the only element for a single-element array", () => {
|
||||
expect(percentile([42], 50)).toBe(42);
|
||||
expect(percentile([42], 95)).toBe(42);
|
||||
});
|
||||
|
||||
it("p50 of [1,2,3,4,5] = 3", () => {
|
||||
expect(percentile([1, 2, 3, 4, 5], 50)).toBe(3);
|
||||
});
|
||||
|
||||
it("p100 returns last element", () => {
|
||||
expect(percentile([10, 20, 30], 100)).toBe(30);
|
||||
});
|
||||
|
||||
it("p0 returns first element", () => {
|
||||
expect(percentile([10, 20, 30], 0)).toBe(10);
|
||||
});
|
||||
|
||||
it("p95 of 100-element uniform distribution is near top", () => {
|
||||
const data = Array.from({ length: 100 }, (_, i) => i + 1); // 1..100
|
||||
expect(percentile(data, 95)).toBe(95);
|
||||
});
|
||||
});
|
||||
|
||||
// ── parseLogLine ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe("parseLogLine", () => {
|
||||
it("parses a plain NDJSON line", () => {
|
||||
const line = '{"event":"cmd_timing","cmd":"/wordle","total":120,"cold":true,"marks":[]}';
|
||||
expect(parseLogLine(line)).toEqual({
|
||||
event: "cmd_timing",
|
||||
cmd: "/wordle",
|
||||
total: 120,
|
||||
cold: true,
|
||||
marks: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null for empty line", () => {
|
||||
expect(parseLogLine("")).toBeNull();
|
||||
expect(parseLogLine(" ")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for non-JSON plain text", () => {
|
||||
expect(parseLogLine("some random log line")).toBeNull();
|
||||
});
|
||||
|
||||
it("extracts JSON embedded in a CSV row", () => {
|
||||
const csvRow =
|
||||
'2024-01-01T00:00:00Z,worker,"{\\"event\\":\\"cmd_timing\\",\\"cmd\\":\\"/loldle\\",\\"total\\":45,\\"cold\\":false,\\"marks\\":[]}"';
|
||||
const result = parseLogLine(csvRow);
|
||||
// May fail to parse double-escaped CSV — test the raw braces path instead.
|
||||
// Use a simpler CSV format where JSON is not double-escaped:
|
||||
const simpleCsv =
|
||||
'2024-01-01,{"event":"cmd_timing","cmd":"/loldle","total":45,"cold":false,"marks":[]},extra';
|
||||
const r2 = parseLogLine(simpleCsv);
|
||||
expect(r2).not.toBeNull();
|
||||
expect(r2.event).toBe("cmd_timing");
|
||||
expect(r2.cmd).toBe("/loldle");
|
||||
});
|
||||
|
||||
it("returns null for malformed JSON", () => {
|
||||
expect(parseLogLine("{bad json}")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ── aggregateLines ────────────────────────────────────────────────────────────
|
||||
|
||||
describe("aggregateLines", () => {
|
||||
const makeTimingLine = (cmd, total, cold) =>
|
||||
JSON.stringify({ event: "cmd_timing", cmd, total, cold, marks: [] });
|
||||
|
||||
it("buckets cold and warm samples separately", () => {
|
||||
const lines = [
|
||||
makeTimingLine("/wordle", 1500, true),
|
||||
makeTimingLine("/wordle", 1400, true),
|
||||
makeTimingLine("/wordle", 50, false),
|
||||
makeTimingLine("/wordle", 60, false),
|
||||
makeTimingLine("/wordle", 55, false),
|
||||
];
|
||||
const { buckets } = aggregateLines(lines, null);
|
||||
expect(buckets.get("/wordle|cold").n).toBe(2);
|
||||
expect(buckets.get("/wordle|warm").n).toBe(3);
|
||||
});
|
||||
|
||||
it("filters to requested commands only", () => {
|
||||
const lines = [makeTimingLine("/wordle", 100, false), makeTimingLine("/loldle", 200, false)];
|
||||
const { buckets } = aggregateLines(lines, ["/wordle"]);
|
||||
expect(buckets.has("/wordle|warm")).toBe(true);
|
||||
expect(buckets.has("/loldle|warm")).toBe(false);
|
||||
});
|
||||
|
||||
it("counts dual-write secondary failures", () => {
|
||||
const lines = [
|
||||
'{"msg":"[dual-kv] secondary write failed","phase":"dual-kv"}',
|
||||
"dual-write:secondary:failed — key wordle:state:1",
|
||||
makeTimingLine("/wordle", 80, false),
|
||||
];
|
||||
const { errors } = aggregateLines(lines, null);
|
||||
expect(errors.dualWriteFail).toBe(2);
|
||||
});
|
||||
|
||||
it("counts mongo connection errors", () => {
|
||||
const lines = [
|
||||
"MongoNetworkError: connection refused",
|
||||
"MongoError: timeout",
|
||||
makeTimingLine("/wordle", 80, false),
|
||||
];
|
||||
const { errors } = aggregateLines(lines, null);
|
||||
expect(errors.mongoError).toBe(2);
|
||||
});
|
||||
|
||||
it("counts CPU time exceeded errors", () => {
|
||||
const lines = ["Worker exceeded CPU time limit", "cpu time exceeded for isolate"];
|
||||
const { errors } = aggregateLines(lines, null);
|
||||
expect(errors.cpuExceeded).toBe(2);
|
||||
});
|
||||
|
||||
it("skips non-cmd_timing JSON events", () => {
|
||||
const lines = [
|
||||
JSON.stringify({ event: "request", cold: true, path: "/webhook" }),
|
||||
makeTimingLine("/wordle", 90, false),
|
||||
];
|
||||
const { buckets } = aggregateLines(lines, null);
|
||||
expect(buckets.size).toBe(1); // only the cmd_timing entry
|
||||
});
|
||||
|
||||
it("returns empty buckets and zero errors for empty input", () => {
|
||||
const { buckets, errors } = aggregateLines([], null);
|
||||
expect(buckets.size).toBe(0);
|
||||
expect(errors.dualWriteFail).toBe(0);
|
||||
expect(errors.mongoError).toBe(0);
|
||||
expect(errors.cpuExceeded).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── formatReport ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe("formatReport", () => {
|
||||
it("produces a markdown table with correct headers", () => {
|
||||
const lines = [
|
||||
JSON.stringify({ event: "cmd_timing", cmd: "/wordle", total: 1500, cold: true, marks: [] }),
|
||||
JSON.stringify({ event: "cmd_timing", cmd: "/wordle", total: 50, cold: false, marks: [] }),
|
||||
];
|
||||
const { buckets, errors } = aggregateLines(lines, null);
|
||||
const report = formatReport(buckets, errors);
|
||||
|
||||
expect(report).toContain("| cmd | cold/warm | n | p50 | p95 | p99 |");
|
||||
expect(report).toContain("/wordle");
|
||||
expect(report).toContain("cold");
|
||||
expect(report).toContain("warm");
|
||||
});
|
||||
|
||||
it("includes error summary section", () => {
|
||||
const { buckets, errors } = aggregateLines([], null);
|
||||
errors.dualWriteFail = 3;
|
||||
const report = formatReport(buckets, errors);
|
||||
expect(report).toContain("## Error Summary");
|
||||
expect(report).toContain("Dual-write secondary failures: 3");
|
||||
expect(report).toContain("Mongo connection errors: 0");
|
||||
expect(report).toContain("CPU time exceeded: 0");
|
||||
});
|
||||
|
||||
it("computes correct p50/p95/p99 for a known dataset", () => {
|
||||
// 10 warm samples: 10,20,...,100
|
||||
const lines = Array.from({ length: 10 }, (_, i) =>
|
||||
JSON.stringify({
|
||||
event: "cmd_timing",
|
||||
cmd: "/ping",
|
||||
total: (i + 1) * 10,
|
||||
cold: false,
|
||||
marks: [],
|
||||
}),
|
||||
);
|
||||
const { buckets, errors } = aggregateLines(lines, null);
|
||||
const report = formatReport(buckets, errors);
|
||||
|
||||
// p50 of [10,20,30,40,50,60,70,80,90,100] = 50 (index 4 with nearest-rank)
|
||||
// p95 = 95th percentile = index ceil(0.95*10)-1 = ceil(9.5)-1 = 10-1 = 9 → 100
|
||||
expect(report).toContain("| /ping | warm | 10 |");
|
||||
});
|
||||
});
|
||||
@@ -1,122 +0,0 @@
|
||||
/**
|
||||
* @file backfill-mongo-to-d1.test.js — unit tests for SQL-statement construction.
|
||||
*
|
||||
* Tests `buildInsertSql` and `sqlStr` exported from backfill-mongo-to-d1.js.
|
||||
* No execSync, no wrangler, no real Mongo connection.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildInsertSql, sqlStr } from "../../scripts/backfill-mongo-to-d1.js";
|
||||
|
||||
// ─── sqlStr ───────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("sqlStr", () => {
|
||||
it("wraps a plain string in single quotes", () => {
|
||||
expect(sqlStr("hello")).toBe("'hello'");
|
||||
});
|
||||
|
||||
it("escapes internal single quotes by doubling them", () => {
|
||||
expect(sqlStr("it's")).toBe("'it''s'");
|
||||
});
|
||||
|
||||
it("escapes multiple single quotes", () => {
|
||||
expect(sqlStr("a'b'c")).toBe("'a''b''c'");
|
||||
});
|
||||
|
||||
it("returns NULL for null", () => {
|
||||
expect(sqlStr(null)).toBe("NULL");
|
||||
});
|
||||
|
||||
it("returns NULL for undefined", () => {
|
||||
expect(sqlStr(undefined)).toBe("NULL");
|
||||
});
|
||||
|
||||
it("handles empty string", () => {
|
||||
expect(sqlStr("")).toBe("''");
|
||||
});
|
||||
|
||||
it("coerces non-string values via String()", () => {
|
||||
// @ts-ignore — testing runtime coercion
|
||||
expect(sqlStr(42)).toBe("'42'");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── buildInsertSql ───────────────────────────────────────────────────────────
|
||||
|
||||
describe("buildInsertSql", () => {
|
||||
/** @type {Parameters<typeof buildInsertSql>[0]} */
|
||||
const BASE_ROW = {
|
||||
id: 1,
|
||||
user_id: "u123",
|
||||
symbol: "BTC",
|
||||
side: "buy",
|
||||
qty: 0.5,
|
||||
price_vnd: 1500000,
|
||||
ts: 1700000000,
|
||||
};
|
||||
|
||||
it("produces a valid INSERT statement", () => {
|
||||
const sql = buildInsertSql(BASE_ROW);
|
||||
expect(sql).toMatch(
|
||||
/^INSERT INTO trading_trades \(id, user_id, symbol, side, qty, price_vnd, ts\) VALUES \(/,
|
||||
);
|
||||
expect(sql).toMatch(/\);$/);
|
||||
});
|
||||
|
||||
it("includes all column values in correct order", () => {
|
||||
const sql = buildInsertSql(BASE_ROW);
|
||||
expect(sql).toContain("1,"); // id
|
||||
expect(sql).toContain("'u123'"); // user_id
|
||||
expect(sql).toContain("'BTC'"); // symbol
|
||||
expect(sql).toContain("'buy'"); // side
|
||||
expect(sql).toContain("0.5,"); // qty
|
||||
expect(sql).toContain("1500000,"); // price_vnd
|
||||
expect(sql).toContain("1700000000"); // ts
|
||||
});
|
||||
|
||||
it("preserves legacy_id as the INSERT id", () => {
|
||||
const sql = buildInsertSql({ ...BASE_ROW, id: 42 });
|
||||
// id=42 should be the first value
|
||||
expect(sql).toMatch(/VALUES \(42,/);
|
||||
});
|
||||
|
||||
it("escapes single quotes in user_id", () => {
|
||||
const sql = buildInsertSql({ ...BASE_ROW, user_id: "o'brien" });
|
||||
expect(sql).toContain("'o''brien'");
|
||||
});
|
||||
|
||||
it("escapes single quotes in symbol", () => {
|
||||
const sql = buildInsertSql({ ...BASE_ROW, symbol: "it's" });
|
||||
expect(sql).toContain("'it''s'");
|
||||
});
|
||||
|
||||
it("handles decimal qty correctly", () => {
|
||||
const sql = buildInsertSql({ ...BASE_ROW, qty: 1.23456789 });
|
||||
expect(sql).toContain("1.23456789");
|
||||
});
|
||||
|
||||
it("handles zero values without coercion errors", () => {
|
||||
const sql = buildInsertSql({
|
||||
id: 0,
|
||||
user_id: "",
|
||||
symbol: "",
|
||||
side: "",
|
||||
qty: 0,
|
||||
price_vnd: 0,
|
||||
ts: 0,
|
||||
});
|
||||
expect(sql).toMatch(/VALUES \(0, '', '', '', 0, 0, 0\);/);
|
||||
});
|
||||
|
||||
it("sequential id generation: each row gets unique ascending id", () => {
|
||||
// Simulate the script logic of incrementing nextId for docs without legacy_id
|
||||
let nextId = 1;
|
||||
const rows = [
|
||||
{ user_id: "a", symbol: "BTC", side: "buy", qty: 1, price_vnd: 100, ts: 1 },
|
||||
{ user_id: "b", symbol: "ETH", side: "sell", qty: 2, price_vnd: 200, ts: 2 },
|
||||
];
|
||||
const stmts = rows.map((r) => buildInsertSql({ ...r, id: nextId++ }));
|
||||
expect(stmts[0]).toMatch(/VALUES \(1,/);
|
||||
expect(stmts[1]).toMatch(/VALUES \(2,/);
|
||||
});
|
||||
});
|
||||
@@ -1,118 +0,0 @@
|
||||
/**
|
||||
* @file backfill-mongo-to-kv.test.js — unit tests for reverse-backfill TTL math.
|
||||
*
|
||||
* Tests the `computeTtl` helper exported from backfill-mongo-to-kv.js.
|
||||
* No real Atlas connection, no real CF REST calls.
|
||||
*/
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { computeTtl } from "../../scripts/backfill-mongo-to-kv.js";
|
||||
|
||||
// ─── computeTtl ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("computeTtl", () => {
|
||||
const NOW = 1_700_000_000_000; // fixed epoch ms for deterministic tests
|
||||
|
||||
it("returns null when expiresAt is null (persistent key)", () => {
|
||||
expect(computeTtl(null, NOW)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when expiresAt is undefined (persistent key)", () => {
|
||||
expect(computeTtl(undefined, NOW)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns 'expired' when expiresAt is in the past", () => {
|
||||
const past = new Date(NOW - 1000); // 1 second ago
|
||||
expect(computeTtl(past, NOW)).toBe("expired");
|
||||
});
|
||||
|
||||
it("returns 'expired' when expiresAt equals now exactly", () => {
|
||||
const now = new Date(NOW);
|
||||
expect(computeTtl(now, NOW)).toBe("expired");
|
||||
});
|
||||
|
||||
it("returns correct ttl (seconds) for a future expiresAt", () => {
|
||||
const future = new Date(NOW + 120_000); // 120 seconds from now
|
||||
const result = computeTtl(future, NOW);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result).not.toBe("expired");
|
||||
expect(/** @type {{ttl: number}} */ (result).ttl).toBe(120);
|
||||
});
|
||||
|
||||
it("floors partial seconds (no rounding up)", () => {
|
||||
// 90.9 seconds remaining → floor → 90
|
||||
const future = new Date(NOW + 90_900);
|
||||
const result = computeTtl(future, NOW);
|
||||
expect(/** @type {{ttl: number}} */ (result).ttl).toBe(90);
|
||||
});
|
||||
|
||||
it("clamps ttl to MIN_TTL_SECS (60) when remaining < 60s", () => {
|
||||
// 30 seconds remaining — below CF KV minimum of 60
|
||||
const future = new Date(NOW + 30_000);
|
||||
const result = computeTtl(future, NOW);
|
||||
expect(/** @type {{ttl: number}} */ (result).ttl).toBe(60);
|
||||
});
|
||||
|
||||
it("clamps ttl to MIN_TTL_SECS (60) when remaining is 1s", () => {
|
||||
const future = new Date(NOW + 1_000);
|
||||
const result = computeTtl(future, NOW);
|
||||
expect(/** @type {{ttl: number}} */ (result).ttl).toBe(60);
|
||||
});
|
||||
|
||||
it("does NOT clamp when remaining is exactly 60s", () => {
|
||||
const future = new Date(NOW + 60_000);
|
||||
const result = computeTtl(future, NOW);
|
||||
expect(/** @type {{ttl: number}} */ (result).ttl).toBe(60);
|
||||
});
|
||||
|
||||
it("does NOT clamp for large TTL values", () => {
|
||||
// 7 days = 604800s
|
||||
const future = new Date(NOW + 7 * 24 * 3600 * 1000);
|
||||
const result = computeTtl(future, NOW);
|
||||
expect(/** @type {{ttl: number}} */ (result).ttl).toBe(604800);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── cfKvPut integration (fetch mock) ────────────────────────────────────────
|
||||
// Verify that computeTtl=null produces a PUT with no expiration_ttl query param,
|
||||
// and computeTtl={ttl:N} appends the param.
|
||||
|
||||
describe("cfKvPut TTL query param (fetch mock)", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
async function mockPut(expirationTtl) {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
text: () => Promise.resolve(""),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
||||
const { cfKvPut } = await import("../../scripts/lib/migration-helpers.js");
|
||||
const opts = expirationTtl != null ? { expirationTtl } : {};
|
||||
await cfKvPut("acct", "ns", "tok", "mod:key", "value", opts);
|
||||
return fetchMock.mock.calls[0][0]; // the URL string
|
||||
}
|
||||
|
||||
it("omits expiration_ttl param when no TTL provided", async () => {
|
||||
const url = await mockPut(undefined);
|
||||
expect(url).not.toContain("expiration_ttl");
|
||||
});
|
||||
|
||||
it("appends expiration_ttl param when TTL provided", async () => {
|
||||
const url = await mockPut(300);
|
||||
expect(url).toContain("expiration_ttl=300");
|
||||
});
|
||||
|
||||
it("URL-encodes the key in the PUT request URL", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
text: () => Promise.resolve(""),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const { cfKvPut } = await import("../../scripts/lib/migration-helpers.js");
|
||||
await cfKvPut("acct", "ns", "tok", "mod:key with spaces", "v", {});
|
||||
const url = fetchMock.mock.calls[0][0];
|
||||
expect(url).toContain("mod%3Akey%20with%20spaces");
|
||||
});
|
||||
});
|
||||
@@ -1,345 +0,0 @@
|
||||
/**
|
||||
* @file migration-helpers.test.js — pure-logic unit tests for migration-helpers.
|
||||
*
|
||||
* Tests: sha256, checkpoint round-trip, countDiffRatio, sampleStrategy,
|
||||
* cfKvList / cfKvGet response parsing (via fetch mock).
|
||||
*
|
||||
* No real Atlas connection, no real CF REST calls.
|
||||
*/
|
||||
|
||||
import { existsSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
cfKvGet,
|
||||
cfKvList,
|
||||
cfKvPut,
|
||||
clearCheckpoint,
|
||||
closeMongoClient,
|
||||
countDiffRatio,
|
||||
loadCheckpoint,
|
||||
sampleStrategy,
|
||||
saveCheckpoint,
|
||||
sha256,
|
||||
sleep,
|
||||
} from "../../scripts/lib/migration-helpers.js";
|
||||
|
||||
// ─── sleep ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("sleep", () => {
|
||||
it("resolves after at least ms milliseconds", async () => {
|
||||
const start = Date.now();
|
||||
await sleep(20);
|
||||
expect(Date.now() - start).toBeGreaterThanOrEqual(15); // allow 5ms jitter
|
||||
});
|
||||
});
|
||||
|
||||
// ─── sha256 ───────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("sha256", () => {
|
||||
it("produces a 64-char hex string", () => {
|
||||
expect(sha256("hello")).toMatch(/^[0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
it("is deterministic for the same input", () => {
|
||||
expect(sha256("test")).toBe(sha256("test"));
|
||||
});
|
||||
|
||||
it("differs for different inputs", () => {
|
||||
expect(sha256("a")).not.toBe(sha256("b"));
|
||||
});
|
||||
|
||||
it("known vector: empty string", () => {
|
||||
// SHA256("") = e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
||||
expect(sha256("")).toBe("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Checkpoint ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("checkpoint round-trip", () => {
|
||||
const TEST_MODULE = "__test_module_unit__";
|
||||
const cpFile = resolve(process.cwd(), `.backfill-cursor-${TEST_MODULE}.json`);
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(cpFile)) unlinkSync(cpFile);
|
||||
});
|
||||
|
||||
it("loadCheckpoint returns null when no file exists", () => {
|
||||
expect(loadCheckpoint(TEST_MODULE)).toBeNull();
|
||||
});
|
||||
|
||||
it("saveCheckpoint + loadCheckpoint round-trip", () => {
|
||||
const state = { cursor: "abc123", lastKey: "wordle:games:42" };
|
||||
saveCheckpoint(TEST_MODULE, state);
|
||||
expect(loadCheckpoint(TEST_MODULE)).toEqual(state);
|
||||
});
|
||||
|
||||
it("clearCheckpoint removes the file", () => {
|
||||
saveCheckpoint(TEST_MODULE, { cursor: "x", lastKey: "k" });
|
||||
expect(existsSync(cpFile)).toBe(true);
|
||||
clearCheckpoint(TEST_MODULE);
|
||||
expect(existsSync(cpFile)).toBe(false);
|
||||
});
|
||||
|
||||
it("clearCheckpoint is a no-op when file absent", () => {
|
||||
expect(() => clearCheckpoint(TEST_MODULE)).not.toThrow();
|
||||
});
|
||||
|
||||
it("loadCheckpoint returns null on corrupt JSON", () => {
|
||||
writeFileSync(cpFile, "{bad json", "utf8");
|
||||
expect(loadCheckpoint(TEST_MODULE)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ─── countDiffRatio ───────────────────────────────────────────────────────────
|
||||
|
||||
describe("countDiffRatio", () => {
|
||||
it("returns 0 for equal counts", () => {
|
||||
expect(countDiffRatio(100, 100)).toBe(0);
|
||||
});
|
||||
|
||||
it("returns 1 for (0, N)", () => {
|
||||
expect(countDiffRatio(0, 100)).toBe(1);
|
||||
});
|
||||
|
||||
it("returns 1 for (N, 0)", () => {
|
||||
expect(countDiffRatio(50, 0)).toBe(1);
|
||||
});
|
||||
|
||||
it("handles (0, 0) without division-by-zero — max(a,b,1)=1", () => {
|
||||
expect(countDiffRatio(0, 0)).toBe(0);
|
||||
});
|
||||
|
||||
it("1% diff on 1000", () => {
|
||||
expect(countDiffRatio(1000, 990)).toBeCloseTo(0.01);
|
||||
});
|
||||
|
||||
it("below 1% threshold for 5 diff on 1000", () => {
|
||||
expect(countDiffRatio(1000, 995)).toBeLessThan(0.01);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── sampleStrategy ───────────────────────────────────────────────────────────
|
||||
|
||||
describe("sampleStrategy", () => {
|
||||
it("full-scan for 0 total", () => {
|
||||
const { fullScan, sampleSize } = sampleStrategy(0);
|
||||
expect(fullScan).toBe(true);
|
||||
expect(sampleSize).toBe(0);
|
||||
});
|
||||
|
||||
it("full-scan for 9999 total", () => {
|
||||
const { fullScan, sampleSize } = sampleStrategy(9999);
|
||||
expect(fullScan).toBe(true);
|
||||
expect(sampleSize).toBe(9999);
|
||||
});
|
||||
|
||||
it("full-scan boundary: 10000 triggers sampling", () => {
|
||||
const { fullScan } = sampleStrategy(10000);
|
||||
expect(fullScan).toBe(false);
|
||||
});
|
||||
|
||||
it("sample size = ceil(sqrt(total)) for mid range", () => {
|
||||
// sqrt(40000) = 200 → sampleSize = 200
|
||||
const { sampleSize } = sampleStrategy(40000);
|
||||
expect(sampleSize).toBe(200);
|
||||
});
|
||||
|
||||
it("sample size capped at 500", () => {
|
||||
// sqrt(1_000_000) = 1000, capped to 500
|
||||
const { sampleSize } = sampleStrategy(1_000_000);
|
||||
expect(sampleSize).toBe(500);
|
||||
});
|
||||
|
||||
it("ceil applied: sqrt(10001) ≈ 100.005 → 101", () => {
|
||||
const { sampleSize } = sampleStrategy(10201); // sqrt = 101 exactly
|
||||
expect(sampleSize).toBe(101);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── cfKvList ─────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("cfKvList (fetch mock)", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
function mockFetch(body, status = 200) {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue({
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: status === 200 ? "OK" : "Error",
|
||||
text: () => Promise.resolve(JSON.stringify(body)),
|
||||
json: () => Promise.resolve(body),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
it("parses a successful list response", async () => {
|
||||
mockFetch({
|
||||
success: true,
|
||||
result: [
|
||||
{ name: "wordle:game:1", metadata: { expiration: 1700000000 } },
|
||||
{ name: "wordle:game:2" },
|
||||
],
|
||||
result_info: { count: 2 },
|
||||
});
|
||||
const result = await cfKvList("acct", "ns", "tok", "wordle:", null);
|
||||
expect(result.keys).toHaveLength(2);
|
||||
expect(result.keys[0].name).toBe("wordle:game:1");
|
||||
expect(result.keys[0].metadata?.expiration).toBe(1700000000);
|
||||
expect(result.list_complete).toBe(true);
|
||||
expect(result.cursor).toBeNull();
|
||||
});
|
||||
|
||||
it("includes cursor when result_info.cursor present", async () => {
|
||||
mockFetch({
|
||||
success: true,
|
||||
result: Array.from({ length: 1000 }, (_, i) => ({ name: `k:${i}` })),
|
||||
result_info: { count: 1000, cursor: "next-page-cursor" },
|
||||
});
|
||||
const result = await cfKvList("acct", "ns", "tok", "k:", null);
|
||||
expect(result.cursor).toBe("next-page-cursor");
|
||||
expect(result.list_complete).toBe(false);
|
||||
});
|
||||
|
||||
it("throws on HTTP error", async () => {
|
||||
mockFetch({ errors: ["unauthorized"] }, 401);
|
||||
await expect(cfKvList("acct", "ns", "tok", "x:", null)).rejects.toThrow("401");
|
||||
});
|
||||
|
||||
it("throws when success=false", async () => {
|
||||
mockFetch({ success: false, errors: [{ message: "namespace not found" }] });
|
||||
await expect(cfKvList("acct", "ns", "tok", "x:", null)).rejects.toThrow(/error/i);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── cfKvGet ──────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("cfKvGet (fetch mock)", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it("returns value text on success", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
text: () => Promise.resolve('{"score":42}'),
|
||||
}),
|
||||
);
|
||||
const val = await cfKvGet("acct", "ns", "tok", "wordle:game:1");
|
||||
expect(val).toBe('{"score":42}');
|
||||
});
|
||||
|
||||
it("URL-encodes the key in the request URL", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
text: () => Promise.resolve("v"),
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
await cfKvGet("acct", "ns", "tok", "some key/with spaces");
|
||||
const calledUrl = fetchMock.mock.calls[0][0];
|
||||
expect(calledUrl).toContain("some%20key%2Fwith%20spaces");
|
||||
});
|
||||
|
||||
it("throws on HTTP 404", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue({
|
||||
ok: false,
|
||||
status: 404,
|
||||
statusText: "Not Found",
|
||||
text: () => Promise.resolve("not found"),
|
||||
}),
|
||||
);
|
||||
await expect(cfKvGet("acct", "ns", "tok", "missing:key")).rejects.toThrow("404");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── cfKvPut ──────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("cfKvPut (fetch mock)", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
function mockFetch(status = 200) {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn().mockResolvedValue({
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
text: () => Promise.resolve(""),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
it("issues a PUT request to the correct URL", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
await cfKvPut("acct", "ns", "tok", "my:key", "myvalue");
|
||||
const [url, init] = fetchMock.mock.calls[0];
|
||||
expect(url).toContain("/accounts/acct/storage/kv/namespaces/ns/values/my%3Akey");
|
||||
expect(init.method).toBe("PUT");
|
||||
});
|
||||
|
||||
it("sends Authorization Bearer header", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
await cfKvPut("acct", "ns", "TOKEN", "k", "v");
|
||||
const init = fetchMock.mock.calls[0][1];
|
||||
expect(init.headers.Authorization).toBe("Bearer TOKEN");
|
||||
});
|
||||
|
||||
it("sends the value as the request body", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
await cfKvPut("acct", "ns", "tok", "k", "hello-value");
|
||||
const init = fetchMock.mock.calls[0][1];
|
||||
expect(init.body).toBe("hello-value");
|
||||
});
|
||||
|
||||
it("appends expiration_ttl query param when provided", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
await cfKvPut("acct", "ns", "tok", "k", "v", { expirationTtl: 300 });
|
||||
const url = fetchMock.mock.calls[0][0];
|
||||
expect(url).toContain("expiration_ttl=300");
|
||||
});
|
||||
|
||||
it("omits expiration_ttl query param when opts is empty", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
await cfKvPut("acct", "ns", "tok", "k", "v", {});
|
||||
const url = fetchMock.mock.calls[0][0];
|
||||
expect(url).not.toContain("expiration_ttl");
|
||||
});
|
||||
|
||||
it("omits expiration_ttl when opts is omitted entirely", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
await cfKvPut("acct", "ns", "tok", "k", "v");
|
||||
const url = fetchMock.mock.calls[0][0];
|
||||
expect(url).not.toContain("expiration_ttl");
|
||||
});
|
||||
|
||||
it("throws on HTTP error response", async () => {
|
||||
mockFetch(403);
|
||||
await expect(cfKvPut("acct", "ns", "tok", "k", "v")).rejects.toThrow("403");
|
||||
});
|
||||
|
||||
it("URL-encodes special characters in key", async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve("") });
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
await cfKvPut("acct", "ns", "tok", "key with spaces/slash", "v");
|
||||
const url = fetchMock.mock.calls[0][0];
|
||||
expect(url).toContain("key%20with%20spaces%2Fslash");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── getMongoClient teardown ──────────────────────────────────────────────────
|
||||
|
||||
// Ensure any singleton client opened during tests is closed (prevents open handles).
|
||||
afterEach(async () => {
|
||||
await closeMongoClient();
|
||||
});
|
||||
@@ -1,104 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { startTiming, takeColdFlag } from "../../src/util/timing.js";
|
||||
|
||||
// ── startTiming ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe("startTiming", () => {
|
||||
let consoleSpy;
|
||||
|
||||
beforeEach(() => {
|
||||
consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("end() emits a cmd_timing JSON line via console.log", () => {
|
||||
const t = startTiming("/wordle");
|
||||
t.end();
|
||||
|
||||
expect(consoleSpy).toHaveBeenCalledOnce();
|
||||
const raw = consoleSpy.mock.calls[0][0];
|
||||
const obj = JSON.parse(raw);
|
||||
expect(obj.event).toBe("cmd_timing");
|
||||
expect(obj.cmd).toBe("/wordle");
|
||||
expect(typeof obj.total).toBe("number");
|
||||
expect(obj.total).toBeGreaterThanOrEqual(0);
|
||||
expect(Array.isArray(obj.marks)).toBe(true);
|
||||
});
|
||||
|
||||
it("end() merges extra fields into the log object", () => {
|
||||
const t = startTiming("/loldle");
|
||||
t.end({ cold: true, isolateAgeMs: 42 });
|
||||
|
||||
const obj = JSON.parse(consoleSpy.mock.calls[0][0]);
|
||||
expect(obj.cold).toBe(true);
|
||||
expect(obj.isolateAgeMs).toBe(42);
|
||||
});
|
||||
|
||||
it("mark() records named checkpoints with dt >= 0", () => {
|
||||
const t = startTiming("/help");
|
||||
t.mark("mongo-read");
|
||||
t.mark("handler-done");
|
||||
t.end();
|
||||
|
||||
const obj = JSON.parse(consoleSpy.mock.calls[0][0]);
|
||||
expect(obj.marks).toHaveLength(2);
|
||||
expect(obj.marks[0].label).toBe("mongo-read");
|
||||
expect(obj.marks[1].label).toBe("handler-done");
|
||||
expect(obj.marks[0].dt).toBeGreaterThanOrEqual(0);
|
||||
expect(obj.marks[1].dt).toBeGreaterThanOrEqual(obj.marks[0].dt);
|
||||
});
|
||||
|
||||
it("marks array is empty when no mark() calls made", () => {
|
||||
const t = startTiming("/ping");
|
||||
t.end();
|
||||
const obj = JSON.parse(consoleSpy.mock.calls[0][0]);
|
||||
expect(obj.marks).toEqual([]);
|
||||
});
|
||||
|
||||
it("marks is declared at top — no ReferenceError if mark() called before any await", () => {
|
||||
// This would throw if `marks` were declared after a conditional or inside a closure.
|
||||
expect(() => {
|
||||
const t = startTiming("/fortytwo");
|
||||
t.mark("immediate");
|
||||
t.end();
|
||||
}).not.toThrow();
|
||||
});
|
||||
|
||||
it("total reflects elapsed time (>= 0ms)", () => {
|
||||
const t = startTiming("/trade");
|
||||
t.end();
|
||||
const obj = JSON.parse(consoleSpy.mock.calls[0][0]);
|
||||
expect(obj.total).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── takeColdFlag ─────────────────────────────────────────────────────────────
|
||||
// NOTE: takeColdFlag uses module-scope state that persists across tests in the
|
||||
// same vitest worker. We cannot reset it between tests — so we capture the
|
||||
// baseline once and assert relative behaviour only.
|
||||
|
||||
describe("takeColdFlag", () => {
|
||||
it("returns an object with cold (boolean) and isolateAgeMs (number)", () => {
|
||||
const result = takeColdFlag();
|
||||
expect(typeof result.cold).toBe("boolean");
|
||||
expect(typeof result.isolateAgeMs).toBe("number");
|
||||
expect(result.isolateAgeMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it("second call in same isolate returns cold=false", () => {
|
||||
// The first call may have already flipped the flag in a prior test.
|
||||
// We call it once here just to ensure it returns cold=false on subsequent invocations.
|
||||
takeColdFlag(); // may be first or subsequent
|
||||
const second = takeColdFlag();
|
||||
expect(second.cold).toBe(false);
|
||||
});
|
||||
|
||||
it("isolateAgeMs is non-decreasing across successive calls", () => {
|
||||
const a = takeColdFlag();
|
||||
const b = takeColdFlag();
|
||||
expect(b.isolateAgeMs).toBeGreaterThanOrEqual(a.isolateAgeMs);
|
||||
});
|
||||
});
|
||||
+1
-15
@@ -1,22 +1,11 @@
|
||||
name = "miti99bot"
|
||||
main = "src/index.js"
|
||||
compatibility_date = "2025-10-01"
|
||||
# nodejs_compat_v2 enables node:net + node:tls so the official `mongodb` driver
|
||||
# can open a TCP socket to Atlas. v1 vs v2 are alternatives, not additive.
|
||||
# Adding this flag does not affect existing modules — `src/` has no node: imports.
|
||||
compatibility_flags = ["nodejs_compat_v2"]
|
||||
|
||||
# Enabled modules at runtime. Comma-separated. Must match static-map keys in src/modules/index.js.
|
||||
# Also duplicate this value into .env.deploy so scripts/register.js derives the same public command list.
|
||||
[vars]
|
||||
MODULES = "util,wordle,loldle,loldle-emoji,loldle-quote,loldle-ability,loldle-splash,misc,trading,lolschedule,semantle,doantu,twentyq"
|
||||
# Storage routing flags — control which backend serves reads and receives writes.
|
||||
# STORAGE_PRIMARY: "kv" = CF KV (default), "mongo" = Atlas (post-cutover).
|
||||
# DUAL_WRITE: "1" = write both backends in parallel (default), "0" = primary only.
|
||||
# DRIFT_SAMPLE_N: number of keys sampled per module by drift-verifier cron.
|
||||
STORAGE_PRIMARY = "kv"
|
||||
DUAL_WRITE = "1"
|
||||
DRIFT_SAMPLE_N = "50"
|
||||
|
||||
# KV namespace holding all module state. Each module auto-prefixes its keys via createStore().
|
||||
# Production-only — no preview namespace. Create with:
|
||||
@@ -51,10 +40,7 @@ binding = "AI"
|
||||
# Local testing: curl "http://localhost:8787/__scheduled?cron=0+1+*+*+*"
|
||||
# (requires `wrangler dev --test-scheduled`)
|
||||
[triggers]
|
||||
# "0 17 * * *" — trading trim-trades (lolschedule module)
|
||||
# "0 1 * * *" — misc / lolschedule nightly jobs
|
||||
# "0 * * * *" — drift-verifier: drain retry queue + spot-check KV vs Mongo parity
|
||||
crons = ["0 17 * * *", "0 1 * * *", "0 * * * *"]
|
||||
crons = ["0 17 * * *", "0 1 * * *"]
|
||||
|
||||
# Workers Observability — captures console.* logs, request metadata, and
|
||||
# invocation traces in the Cloudflare dashboard (Observability → Logs).
|
||||
|
||||
Reference in new issue
Block a user