diff --git a/.env.example b/.env.example index a276319..f481e1c 100644 --- a/.env.example +++ b/.env.example @@ -15,8 +15,8 @@ MONGO_DATABASE=tiennm99bot # Comma-separated module list. Empty = load every module, including any module # added later — so list them explicitly when a deployment should only gain a new # module deliberately. wordledaily serves /wordle too; the retired name wordle -# still loads it. -MODULES=util,misc,random,amlich,wordledaily,loldle,noitu,lol,stock,gold,coin,stats,monkeyd,sticker,alias,blacklist,weather +# still loads it. The removed module monkeyd is skipped with a warning. +MODULES=util,misc,random,amlich,wordledaily,loldle,noitu,lol,stock,gold,coin,stats,sticker,alias,blacklist,weather # Telegram user id for owner-only commands. OWNER_ID= # Comma-separated admin Telegram user ids. diff --git a/AGENTS.md b/AGENTS.md index faa9cb3..1764acc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,11 +7,6 @@ in-memory otherwise, which is what tests and local no-database runs use. Read `README.md` before implementation work. -The `monkeyd` module's crawler, PDF renderer, and export flow are an in-tree -port of `tiennm99/mttools/monkeyd-crawler` under -`internal/modules/monkeyd/{crawler,pdf,export}`. Change them here; they no -longer track the mttools copy. - `renderer/` is a separate Node 24 service (JavaScript + JSDoc, Remotion) that draws the `random` module's animations. It has its own `package.json`, tests, and `renderer/AGENTS.md`; run its npm commands from that folder. `compose.yml` diff --git a/Dockerfile b/Dockerfile index d7055d4..c72e170 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,6 @@ RUN CGO_ENABLED=0 GOOS=linux go build \ # ffmpeg comes from Alpine's own package rather than a third-party static-ffmpeg # image, keeping the supply chain to bases already trusted by this build. FROM alpine:3.22 -# No fonts are installed here: the monkeyd module's PDF renderer falls back to a -# font compiled into the binary when the host has none. RUN apk add --no-cache ffmpeg ca-certificates \ && addgroup -g 65532 -S nonroot \ && adduser -u 65532 -S -G nonroot nonroot diff --git a/README.md b/README.md index 529dafa..bde109c 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,6 @@ Atlas via long polling and an in-process cron scheduler. | `sticker` | `/addsticker` — append a replied sticker, image, video or GIF to one shared pack. See [docs/sticker-packs.md](docs/sticker-packs.md) | | `alias` | `/alias ` save a replied message under a name, then send it back with `/insert `, bare `/`, or inline `@botname `; `/aliases` lists, `/unalias` deletes. See [docs/aliases.md](docs/aliases.md) | | `blacklist` | Per-topic text deny-list with whitelist exceptions: `/blacklist_add`, `/blacklist_del`, `/whitelist_add`, `/whitelist_del`, `/blacklist_rules` lists both, `/blacklist_check` judges a text, `/blacklist` does either, `/whitelist_rnd` picks a random exception. Passive — the bot never scans chat. See [docs/blacklist.md](docs/blacklist.md) | -| `monkeyd` | `/monkeyd_crawl [font_size]` export a monkeydd.com novel as a PDF, `/monkeyd_tags ` list its tags as hashtags | | `weather` | Weather from Open-Meteo: `/thoitiet` for the next 6 hours hour by hour, `/thoitiethomnay` for current conditions and today, `/thoitietngaymai` for tomorrow, `/thoitiettuannay` for the next 7 days. Each takes `[location... | lat,long]` (Vietnamese with or without diacritics, shorthand like `hcm`/`hn`, a foreign city, or decimal coordinates such as `10.74,106.72`); the default is Tân Thuận (formerly Q.7). Flood alerts for Tân Thuận (Q.7): `/thuyvan` shows the 5-day tide-peak forecast at Phú An and Nhà Bè (Đài KTTV Nam Bộ bulletin), the rain forecast, and live VNDMS river gauges within 30 km; `/thuyvan_subscribe` / `/thuyvan_unsubscribe` toggle a 10:30 ICT push (retried at 12:30 if that run fails) sent only when a forecast peak reaches báo động I (1.40 m) or rain reaches 50 mm | Commands marked (admin) require a user ID in `ADMIN_IDS` or the owner; commands @@ -160,75 +159,6 @@ The stock-only `assets..openedAt` marker identifies the current position lifecycle, invalidates buttons after a full sale and later repurchase, and prevents a position opened after Record date from applying an older event. -### Novel PDF export - -`/monkeyd_crawl [font_size]` downloads every chapter of a monkeydd.com -novel and sends it back as a single PDF document, sized for reading on a phone. -The crawler, the PDF renderer, and the export flow live in -`internal/modules/monkeyd/{crawler,pdf,export}`, ported from -[mttools/monkeyd-crawler](https://github.com/tiennm99/mttools/tree/main/monkeyd-crawler) -and trimmed to what the bot uses: a fixed phone page and the bundled DejaVu -Sans font. - -`font_size` is the body text size in points and accepts half points. It ranges -from 6 to 24 and defaults to the crawler's own default of 10, which fits roughly -43 characters per line across 26 lines on the 90×160 mm page. Larger values -trade characters per line for legibility: 12 gives about 36. Headings and the -title page scale with it. Omitting the argument passes no size at all, so the -crawler's default applies rather than a second one defined here. The document -caption reports the size that was used. - -The command is public, so any member of a chat can run it. One invocation makes -hundreds of outbound requests spread over several minutes, so two things bound -the cost and are deliberate: only `monkeydd.com` URLs are accepted, and exactly -one export runs at a time across the whole bot. A missing scheme is filled in, -so a pasted bare hostname works. - -The host allowlist is not only about the parser — the extractor is written -against that site's markup and would find nothing elsewhere — it also stops the -command being used to make the bot fetch arbitrary URLs. - -The bot replies immediately that the export started, then sends the PDF when it -is ready; a second request while one is in flight is told which novel is -currently running. Requests are spaced out by the crawler, so a run is -deliberately slow rather than aggressive. - -Raw pages are cached under the system temp directory, so re-exporting the same -novel costs no requests. The cache is not pruned and a container restart clears -it. Finished PDFs are deleted after upload. Telegram caps bot uploads at 50 MB; -a larger book is reported instead of being sent. - -The PDF embeds a font covering Vietnamese diacritics. The crawler prefers a -system font and falls back to one compiled into the binary, so the runtime -image needs no fonts installed. - -### Novel tags - -`/monkeyd_tags ` reports a novel's tags as a hashtag line, sent as a code -block so it can be copied in one tap: - -```text -#MonkeyD #CổĐại #GiaĐình - -https://monkeydd.com/truong-an-gwem.html -``` - -Each label becomes one hashtag with spaces and punctuation removed and every -word capitalised, because Telegram ends a hashtag at the first character that is -not a letter, digit, or underscore. Diacritics are kept. A label with no letters -is dropped rather than emitted as a bare `#`. - -The tags are the novel's own genres, identified by their `itemprop="genre"` -microdata. The same page also links every genre on the site as navigation — 69 -of them against one novel's 6 in a sampled page — so matching the category URL -shape instead would return the whole menu. - -This command costs a single request and runs inline rather than in the -background, bounded by a short timeout: handlers are dispatched one at a time, so -a stalled fetch would hold up every other command. It shares the export page -cache, so a lookup for a novel that was already exported costs no request at -all. - ## Layout ``` diff --git a/cmd/server/command_menu_test.go b/cmd/server/command_menu_test.go index 302d703..ff11aee 100644 --- a/cmd/server/command_menu_test.go +++ b/cmd/server/command_menu_test.go @@ -73,8 +73,6 @@ func TestCommandDiscovery_AllPublicCommandsHaveSafeMetadata(t *testing.T) { "loldle": "[champion]", "loldle_giveup": "", "loldle_stats": "", - "monkeyd_crawl": " [font_size]", - "monkeyd_tags": "", "noitu": "", "noitubot": "", "noitutop": "", diff --git a/cmd/server/main.go b/cmd/server/main.go index 0a9fad0..da6d789 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -33,7 +33,6 @@ import ( "github.com/tiennm99/tiennm99bot/internal/modules/lol" "github.com/tiennm99/tiennm99bot/internal/modules/loldle" "github.com/tiennm99/tiennm99bot/internal/modules/misc" - "github.com/tiennm99/tiennm99bot/internal/modules/monkeyd" "github.com/tiennm99/tiennm99bot/internal/modules/noitu" "github.com/tiennm99/tiennm99bot/internal/modules/random" "github.com/tiennm99/tiennm99bot/internal/modules/stats" @@ -99,7 +98,6 @@ func factories() map[string]modules.Factory { "misc": misc.New, "random": random.New, "amlich": amlich.New, - "monkeyd": monkeyd.New, wordledaily.ShortName: wordledaily.New, noitu.ShortName: noitu.New, loldle.ShortName: loldle.New, @@ -422,12 +420,25 @@ var moduleAliases = map[string]string{ wordledaily.LegacyShortName: wordledaily.ShortName, } +// removedModules names catalog keys that were deleted without a successor. +// A MODULES value written before the removal skips them with a warning +// instead of failing startup on an unknown module. +var removedModules = map[string]bool{ + "monkeyd": true, +} + // normalizeModules replaces retired module names in a MODULES list with -// their successors, dropping duplicates that creates. +// their successors, drops removed modules, and drops duplicates that creates. +// A list naming only removed modules comes back empty and so loads every +// module, the same as an unset MODULES. func normalizeModules(names []string) []string { out := make([]string, 0, len(names)) seen := map[string]bool{} for _, name := range names { + if removedModules[name] { + log.Warn("MODULES names a removed module; skipping it", "module", name) + continue + } if to, ok := moduleAliases[name]; ok { log.Warn("MODULES names a retired module; loading its successor", "module", name, "successor", to) name = to diff --git a/cmd/server/main_test.go b/cmd/server/main_test.go index 9a4bf2d..1c56a52 100644 --- a/cmd/server/main_test.go +++ b/cmd/server/main_test.go @@ -201,6 +201,25 @@ func TestNormalizeModules_MapsRetiredWordleToWordleDaily(t *testing.T) { } } +// The monkeyd module was deleted: a MODULES value still naming it loads the +// rest of the list instead of failing on an unknown module. +func TestNormalizeModules_DropsRemovedMonkeyd(t *testing.T) { + for in, want := range map[string]string{ + "util,monkeyd,stats": "util stats", + "monkeyd,misc": "misc", + } { + if got := strings.Join(normalizeModules(splitCSV(in)), " "); got != want { + t.Errorf("normalizeModules(%q) = %q, want %q", in, got, want) + } + } + if _, ok := factories()["monkeyd"]; ok { + t.Fatal("the removed monkeyd module is still in the catalog") + } + if _, err := modules.Build(normalizeModules(splitCSV("util,monkeyd,stats")), factories(), storage.NewMemoryProvider(), modules.BuildOptions{}); err != nil { + t.Fatalf("Build with removed module: %v", err) + } +} + // An empty MODULES loads every module, so a command name that collides with an // existing module surfaces here as a test failure rather than as a startup // crash on deploy. diff --git a/docs/deploy-coolify-selfhosted.md b/docs/deploy-coolify-selfhosted.md index 483e928..8eb8dbe 100644 --- a/docs/deploy-coolify-selfhosted.md +++ b/docs/deploy-coolify-selfhosted.md @@ -39,7 +39,7 @@ Copy [`.env.example`](../.env.example) → `.env` (gitignored) and fill in. | `TELEGRAM_BOT_TOKEN` | ✅ | from @BotFather; startup fails without it | | `MONGO_URL` | ✅ | Atlas SRV string **incl. credentials** — secret, never logged | | `MONGO_DATABASE` | ✅ | e.g. `tiennm99bot` | -| `MODULES` | optional | CSV; empty = all modules, including any added later. The retired `wordle` name loads `wordledaily`, which now serves `/wordle` | +| `MODULES` | optional | CSV; empty = all modules, including any added later. The retired `wordle` name loads `wordledaily`, which now serves `/wordle`; the removed `monkeyd` name is skipped with a warning | | `OWNER_ID` | optional | Telegram user id for owner-only commands, the deploy DM, and the `/addsticker` pack owner. Unset = owner-only commands are denied and `/addsticker` refuses | | `ADMIN_IDS` | optional | CSV of Telegram user ids for admin-only commands | | `BOT_USERNAME` | optional | the bot's Telegram username, without `@`; unset = asked from Telegram (`getMe`) once at startup. Used for the default sticker pack name and the lol User-Agent | @@ -161,9 +161,10 @@ MP4, with the same text fallback. - **`coin`** stores cash as `usd` and embeds positions as `assets..{quantity,base}`. - **`system`** holds one marker per completed one-time startup migration. Keep - those records as audit history. No one-time migration runs at startup now; - the completed ones were removed from the code once production data was - verified migrated. + those records as audit history. The stats command renames and the + retirement of the removed `monkeyd` commands still run at startup and skip + once their marker is complete; earlier completed migrations were removed + from the code once production data was verified migrated. ## 2. Coolify diff --git a/go.mod b/go.mod index 6b53f9d..34f7640 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/tiennm99/tiennm99bot go 1.26.5 require ( - github.com/go-pdf/fpdf v0.9.0 github.com/go-telegram/bot v1.20.0 github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0 github.com/robfig/cron/v3 v3.0.1 @@ -11,8 +10,6 @@ require ( github.com/testcontainers/testcontainers-go/modules/mongodb v0.43.0 go.mongodb.org/mongo-driver/v2 v2.7.0 golang.org/x/image v0.45.0 - golang.org/x/net v0.57.0 - golang.org/x/sync v0.22.0 golang.org/x/text v0.41.0 ) @@ -69,6 +66,7 @@ require ( go.opentelemetry.io/otel/metric v1.41.0 // indirect go.opentelemetry.io/otel/trace v1.41.0 // indirect golang.org/x/crypto v0.54.0 // indirect + golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index dc6b15c..d959d0a 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-pdf/fpdf v0.9.0 h1:PPvSaUuo1iMi9KkaAn90NuKi+P4gwMedWPHhj8YlJQw= -github.com/go-pdf/fpdf v0.9.0/go.mod h1:oO8N111TkmKb9D7VvWGLvLJlaZUQVPM+6V42pp3iV4Y= github.com/go-telegram/bot v1.20.0 h1:4Pea/qTidSspr4WBJw9FbHUMNhYeqszBqQUfsQEyFbc= github.com/go-telegram/bot v1.20.0/go.mod h1:i2TRs7fXWIeaceF3z7KzsMt/he0TwkVC680mvdTFYeM= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -145,8 +143,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= diff --git a/internal/modules/monkeyd/crawl_args.go b/internal/modules/monkeyd/crawl_args.go deleted file mode 100644 index 44775dc..0000000 --- a/internal/modules/monkeyd/crawl_args.go +++ /dev/null @@ -1,70 +0,0 @@ -package monkeyd - -import ( - "errors" - "fmt" - "math" - "strconv" - "strings" -) - -// Font size bounds for the optional argument. The body font drives the heading -// and title sizes too, and the default page is only 78mm wide between margins, -// so the useful range is narrow: at 6pt a line holds about 72 characters, at -// 24pt about 18. -const ( - minFontSize = 6.0 - maxFontSize = 24.0 -) - -var errTooManyArgs = errors.New("too many arguments") - -// crawlArgs is a validated /monkeyd_crawl invocation. -type crawlArgs struct { - // NovelURL is normalised and known to be on an allowed host. - NovelURL string - - // FontSize is the requested body font size in points, or 0 to leave the - // crawler's default in place. - FontSize float64 -} - -// parseCrawlArgs validates the text after the command: a novel URL, optionally -// followed by a body font size. -func parseCrawlArgs(arg string) (crawlArgs, error) { - fields := strings.Fields(arg) - if len(fields) == 0 { - return crawlArgs{}, errNotAURL - } - if len(fields) > 2 { - return crawlArgs{}, errTooManyArgs - } - - novelURL, err := normalizeNovelURL(fields[0]) - if err != nil { - return crawlArgs{}, err - } - parsed := crawlArgs{NovelURL: novelURL} - - if len(fields) == 2 { - if parsed.FontSize, err = parseFontSize(fields[1]); err != nil { - return crawlArgs{}, err - } - } - return parsed, nil -} - -// parseFontSize accepts a font size in points, including half points, and -// rejects anything outside the readable range. -func parseFontSize(raw string) (float64, error) { - size, err := strconv.ParseFloat(raw, 64) - // ParseFloat accepts "NaN" and "Inf", neither of which is a font size, and - // both would reach the renderer as a silently broken layout. - if err != nil || math.IsNaN(size) || math.IsInf(size, 0) { - return 0, fmt.Errorf("font size %q is not a number", raw) - } - if size < minFontSize || size > maxFontSize { - return 0, fmt.Errorf("font size must be between %g and %g points", minFontSize, maxFontSize) - } - return size, nil -} diff --git a/internal/modules/monkeyd/crawl_args_test.go b/internal/modules/monkeyd/crawl_args_test.go deleted file mode 100644 index dffc9c7..0000000 --- a/internal/modules/monkeyd/crawl_args_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package monkeyd - -import ( - "strings" - "testing" -) - -func TestParseCrawlArgs(t *testing.T) { - tests := []struct { - name string - arg string - wantURL string - wantFontSize float64 - wantErr bool - }{ - { - name: "url only leaves the font size unset", - arg: testNovelURL, - wantURL: testNovelURL, - wantFontSize: 0, - }, - { - name: "url with font size", - arg: testNovelURL + " 14", - wantURL: testNovelURL, - wantFontSize: 14, - }, - { - name: "half point font size", - arg: testNovelURL + " 10.5", - wantURL: testNovelURL, - wantFontSize: 10.5, - }, - { - name: "extra whitespace between arguments", - arg: " " + testNovelURL + " 12 ", - wantURL: testNovelURL, - wantFontSize: 12, - }, - { - name: "bare host gets a scheme", - arg: "monkeydd.com/novel.html 9", - wantURL: "https://monkeydd.com/novel.html", - wantFontSize: 9, - }, - { - name: "font size at the lower bound", - arg: testNovelURL + " 6", - wantURL: testNovelURL, - wantFontSize: minFontSize, - }, - { - name: "font size at the upper bound", - arg: testNovelURL + " 24", - wantURL: testNovelURL, - wantFontSize: maxFontSize, - }, - { - name: "empty argument", - arg: "", - wantErr: true, - }, - { - name: "font size below the lower bound", - arg: testNovelURL + " 5", - wantErr: true, - }, - { - name: "font size above the upper bound", - arg: testNovelURL + " 25", - wantErr: true, - }, - { - name: "zero font size", - arg: testNovelURL + " 0", - wantErr: true, - }, - { - name: "negative font size", - arg: testNovelURL + " -12", - wantErr: true, - }, - { - name: "non-numeric font size", - arg: testNovelURL + " big", - wantErr: true, - }, - { - name: "NaN font size", - arg: testNovelURL + " NaN", - wantErr: true, - }, - { - name: "infinite font size", - arg: testNovelURL + " Inf", - wantErr: true, - }, - { - name: "too many arguments", - arg: testNovelURL + " 12 a5", - wantErr: true, - }, - { - name: "disallowed host", - arg: "https://example.com/novel.html 12", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := parseCrawlArgs(tt.arg) - if tt.wantErr { - if err == nil { - t.Fatalf("parseCrawlArgs(%q) = %+v, want error", tt.arg, got) - } - return - } - if err != nil { - t.Fatalf("parseCrawlArgs(%q) returned error: %v", tt.arg, err) - } - if got.NovelURL != tt.wantURL { - t.Errorf("NovelURL = %q, want %q", got.NovelURL, tt.wantURL) - } - if got.FontSize != tt.wantFontSize { - t.Errorf("FontSize = %v, want %v", got.FontSize, tt.wantFontSize) - } - }) - } -} - -// The out-of-range message must state the bounds; "invalid font size" alone -// leaves the user guessing what to type instead. -func TestParseFontSizeErrorNamesTheBounds(t *testing.T) { - _, err := parseFontSize("99") - if err == nil { - t.Fatal("expected an error for 99") - } - for _, want := range []string{"6", "24"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("error %q does not mention the bound %q", err, want) - } - } -} diff --git a/internal/modules/monkeyd/crawler/chapter.go b/internal/modules/monkeyd/crawler/chapter.go deleted file mode 100644 index bb12a1f..0000000 --- a/internal/modules/monkeyd/crawler/chapter.go +++ /dev/null @@ -1,187 +0,0 @@ -package crawler - -import ( - "bytes" - "fmt" - "strconv" - "strings" - - "golang.org/x/net/html" -) - -// contentElementID is the container holding a chapter's body text. -const contentElementID = "chapter-content-render" - -// Chapter is one fetched chapter, reduced to plain paragraphs. -type Chapter struct { - Label string - URL string - Paragraphs []string -} - -// Heading is the chapter title to print. Labels are often a bare number, which -// reads poorly as a heading, so those get the Vietnamese word for "chapter". -func (c *Chapter) Heading() string { - label := strings.TrimSpace(c.Label) - if label == "" { - return "Chương" - } - if _, err := strconv.Atoi(label); err == nil { - return "Chương " + label - } - return label -} - -// WordCount is a rough word count, used to sanity-check extraction. -func (c *Chapter) WordCount() int { - n := 0 - for _, p := range c.Paragraphs { - n += len(strings.Fields(p)) - } - return n -} - -// blockTags end the current paragraph when opened or closed. -var blockTags = map[string]bool{ - "p": true, "div": true, "br": true, "hr": true, "blockquote": true, - "h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, - "li": true, "ul": true, "ol": true, "tr": true, -} - -// skipTags never contribute prose. -// -// Anchors are included because inside a chapter body they are always site -// chrome: the prev/next chapter navigation and the sponsor call-to-action are -// both links, while novel prose never needs one. Inline emphasis tags (b, i, -// em) are deliberately absent so italics in the prose survive; the site's icons -// use but carry no text. -var skipTags = map[string]bool{ - "script": true, "style": true, "noscript": true, "iframe": true, - "ins": true, "form": true, "select": true, "button": true, "textarea": true, - "a": true, "img": true, "svg": true, -} - -// junkClasses marks containers the site injects into the chapter body. Their -// whole subtree is dropped. -// -// These are matched on class rather than position because the blocks move: the -// sponsor block opens the body on most chapters but is absent on others, and -// the watermark is planted at a different paragraph in every chapter. Only -// site-specific class names are listed; generic Bootstrap utilities such as -// "my-4" or "text-center" are not, since prose could legitimately carry them. -// -// Note the sibling class "actac" is NOT junk: it wraps the real chapter text and -// carries style="display:none", because the site gates the body behind a click -// on the sponsor link and reveals it with JavaScript. Skipping hidden elements, -// or skipping "act*" as a family, would therefore discard the whole chapter. -var junkClasses = map[string]bool{ - "actcl": true, // sponsor block shown in place of the gated chapter body - "signature": true, // "[Truyện được đăng tải duy nhất tại ...]" source watermark -} - -// hasJunkClass reports whether a node is an injected non-prose container. -func hasJunkClass(n *html.Node) bool { - for _, tok := range strings.Fields(attr(n, "class")) { - if junkClasses[tok] { - return true - } - } - return false -} - -// ParseChapter extracts a chapter's paragraphs, restoring the words the site -// serves through CSS :before rules instead of markup. -func ParseChapter(page []byte, ref ChapterRef) (*Chapter, error) { - doc, err := html.Parse(bytes.NewReader(page)) - if err != nil { - return nil, fmt.Errorf("parse chapter %s: %w", ref.URL, err) - } - content := elementByID(doc, contentElementID) - if content == nil { - return nil, fmt.Errorf("chapter %s: no #%s container (page layout may have changed)", - ref.URL, contentElementID) - } - - ch := &Chapter{ - Label: ref.Label, - URL: ref.URL, - Paragraphs: extractParagraphs(content, ParseWordClasses(page)), - } - if len(ch.Paragraphs) == 0 { - return nil, fmt.Errorf("chapter %s: extracted no text", ref.URL) - } - ch.Paragraphs = dropRepeatedTitle(ch.Paragraphs, ref.Label) - return ch, nil -} - -// extractParagraphs walks the content subtree into plain paragraphs, replacing -// each word-carrying element with the word its CSS rule injects. -func extractParagraphs(content *html.Node, words map[string]string) []string { - var b strings.Builder - - var walk func(*html.Node) - walk = func(n *html.Node) { - switch n.Type { - case html.TextNode: - // The HTML parser has already decoded entities such as ư. - b.WriteString(n.Data) - return - case html.ElementNode: - if skipTags[n.Data] || hasJunkClass(n) { - return - } - // These elements are empty in the markup; the CSS word replaces them. - if word, ok := injectedWord(n, words); ok { - b.WriteString(word) - return - } - if blockTags[n.Data] { - b.WriteByte('\n') - } - } - - for c := n.FirstChild; c != nil; c = c.NextSibling { - walk(c) - } - - if n.Type == html.ElementNode && blockTags[n.Data] { - b.WriteByte('\n') - } - } - walk(content) - - var paragraphs []string - for _, line := range strings.Split(b.String(), "\n") { - if p := collapseSpaces(line); p != "" { - paragraphs = append(paragraphs, p) - } - } - return paragraphs -} - -// injectedWord returns the word a node's class supplies via CSS, if any. -func injectedWord(n *html.Node, words map[string]string) (string, bool) { - class := attr(n, "class") - if class == "" { - return "", false - } - for _, tok := range strings.Fields(class) { - if word, ok := words[tok]; ok { - return word, true - } - } - return "", false -} - -// dropRepeatedTitle removes a leading paragraph that only repeats the chapter -// label, since the export prints its own heading. -func dropRepeatedTitle(paragraphs []string, label string) []string { - if len(paragraphs) < 2 { - return paragraphs - } - first := strings.TrimSpace(paragraphs[0]) - if strings.EqualFold(first, strings.TrimSpace(label)) { - return paragraphs[1:] - } - return paragraphs -} diff --git a/internal/modules/monkeyd/crawler/chapter_test.go b/internal/modules/monkeyd/crawler/chapter_test.go deleted file mode 100644 index f366951..0000000 --- a/internal/modules/monkeyd/crawler/chapter_test.go +++ /dev/null @@ -1,200 +0,0 @@ -package crawler - -import ( - "strings" - "testing" -) - -// chapterFixture mirrors the real page shape: words split between markup and -// CSS :before rules, HTML entities,   spacer paragraphs and an ad script -// inside the content container. -const chapterFixture = ` - -

TEN TRUYEN - 1

-
-

1

-

 

-

Nghe trưởng tử noi.

-

 

-

Một và di.

- -
` - -func TestParseWordClassesDecodesEscapes(t *testing.T) { - words := ParseWordClasses([]byte(chapterFixture)) - - for class, want := range map[string]string{ - "t-aaa": "vị", - "j-bbb": "trồ", - "z-ccc": "nàng", - } { - if got := words[class]; got != want { - t.Errorf("words[%q] = %q, want %q", class, got, want) - } - } - if len(words) != 4 { - t.Errorf("got %d rules, want 4", len(words)) - } -} - -func TestParseChapterRestoresCSSWords(t *testing.T) { - ch, err := ParseChapter([]byte(chapterFixture), ChapterRef{Label: "1", URL: "http://x/1.html"}) - if err != nil { - t.Fatalf("ParseChapter: %v", err) - } - - want := []string{ - "Nghe vị trưởng tử noi.", - "Một trồ và nàng di.", - } - if len(ch.Paragraphs) != len(want) { - t.Fatalf("got %d paragraphs %q, want %d", len(ch.Paragraphs), ch.Paragraphs, len(want)) - } - for i, w := range want { - if ch.Paragraphs[i] != w { - t.Errorf("paragraph %d = %q, want %q", i, ch.Paragraphs[i], w) - } - } -} - -// The CSS-injected words are the difference between real text and text with -// silent holes, so guard against a regression that drops them. -func TestParseChapterWithoutCSSWouldLoseWords(t *testing.T) { - withoutCSS := strings.Replace(chapterFixture, `.t-aaa:before { content: "v\1ecb "; }`, "", 1) - - ch, err := ParseChapter([]byte(withoutCSS), ChapterRef{Label: "1", URL: "http://x/1.html"}) - if err != nil { - t.Fatalf("ParseChapter: %v", err) - } - if strings.Contains(ch.Paragraphs[0], "vị") { - t.Fatal("word appeared without its CSS rule; fixture no longer proves anything") - } - if want := "Nghe trưởng tử noi."; ch.Paragraphs[0] != want { - t.Errorf("paragraph 0 = %q, want %q", ch.Paragraphs[0], want) - } -} - -func TestParseChapterDropsScriptsAndSpacers(t *testing.T) { - ch, err := ParseChapter([]byte(chapterFixture), ChapterRef{Label: "1", URL: "http://x/1.html"}) - if err != nil { - t.Fatalf("ParseChapter: %v", err) - } - for _, p := range ch.Paragraphs { - if strings.Contains(p, "ads()") { - t.Errorf("script text leaked into paragraph %q", p) - } - if strings.TrimSpace(p) == "" { - t.Error("empty spacer paragraph was kept") - } - if strings.Contains(p, " ") { - t.Errorf("non-breaking space survived in %q", p) - } - } -} - -// The leading "

1

" repeats the chapter label and would print twice. -func TestParseChapterDropsRepeatedTitle(t *testing.T) { - ch, err := ParseChapter([]byte(chapterFixture), ChapterRef{Label: "1", URL: "http://x/1.html"}) - if err != nil { - t.Fatalf("ParseChapter: %v", err) - } - if ch.Paragraphs[0] == "1" { - t.Error("repeated chapter label was kept as a paragraph") - } -} - -// gatedChapterFixture mirrors how most chapters are served: a visible sponsor -// block (div.actcl) stands in for the body, while the real text sits in a -// sibling div.actac hidden with display:none and revealed by the site's -// JavaScript. A source watermark and prev/next navigation bracket the prose. -const gatedChapterFixture = ` - -
-
-

Moi Quy doc gia CLICK vao lien ket

-

mo ung dung Shopee, sau do quay tro lai de doc!

- - CLICK -

MonkeyD va doi ngu Editor xin chan thanh cam on!

-
-
-

Doan van dau tien co tri.

-

 

-

[Truyen duoc dang tai duy nhat tai MonkeyDD.com - https://monkeydd.com/n/10.html.]

-

Doan van cuoi cung.

-
- -
` - -// The gated body is the one thing that must survive: it is hidden with -// display:none, so any rule that drops hidden or "act*" containers silently -// discards the entire chapter. -func TestParseChapterKeepsGatedBody(t *testing.T) { - ch, err := ParseChapter([]byte(gatedChapterFixture), ChapterRef{Label: "10", URL: "http://x/10.html"}) - if err != nil { - t.Fatalf("ParseChapter: %v", err) - } - - want := []string{ - "Doan van dau tien co vị tri.", - "Doan van cuoi cung.", - } - if len(ch.Paragraphs) != len(want) { - t.Fatalf("got %d paragraphs %q, want %d", len(ch.Paragraphs), ch.Paragraphs, len(want)) - } - for i, w := range want { - if ch.Paragraphs[i] != w { - t.Errorf("paragraph %d = %q, want %q", i, ch.Paragraphs[i], w) - } - } -} - -// Everything the site injects around the prose must be gone. -func TestParseChapterDropsInjectedBlocks(t *testing.T) { - ch, err := ParseChapter([]byte(gatedChapterFixture), ChapterRef{Label: "10", URL: "http://x/10.html"}) - if err != nil { - t.Fatalf("ParseChapter: %v", err) - } - body := strings.Join(ch.Paragraphs, "\n") - - for _, junk := range []string{ - "Shopee", // sponsor copy - "CLICK", // sponsor call to action - "cam on", // sponsor sign-off - "MonkeyDD.com", // source watermark - "Chương trước", // navigation - "Chương sau", // navigation - } { - if strings.Contains(body, junk) { - t.Errorf("injected text %q survived extraction in %q", junk, body) - } - } -} - -func TestParseChapterMissingContainer(t *testing.T) { - if _, err := ParseChapter([]byte(`

hi

`), - ChapterRef{URL: "http://x/1.html"}); err == nil { - t.Fatal("want an error when the content container is absent") - } -} - -func TestChapterHeading(t *testing.T) { - for _, tc := range []struct{ label, want string }{ - {"14", "Chương 14"}, - {"Chương 12", "Chương 12"}, - {"", "Chương"}, - } { - ch := &Chapter{Label: tc.label} - if got := ch.Heading(); got != tc.want { - t.Errorf("Heading(%q) = %q, want %q", tc.label, got, tc.want) - } - } -} diff --git a/internal/modules/monkeyd/crawler/client.go b/internal/modules/monkeyd/crawler/client.go deleted file mode 100644 index a0cdb16..0000000 --- a/internal/modules/monkeyd/crawler/client.go +++ /dev/null @@ -1,135 +0,0 @@ -package crawler - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "sync" - "time" -) - -// The site rejects requests without a browser-like User-Agent. -const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + - "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" - -// maxPageSize caps how much of a response we buffer; chapter pages are ~130 KB. -const maxPageSize = 8 << 20 - -// statusError reports an unexpected HTTP status so Get can decide whether -// retrying is worthwhile. -type statusError struct { - code int - status string -} - -func (e *statusError) Error() string { return "unexpected status " + e.status } - -// retryable is true for transient failures. A 404 or 403 will not fix itself, -// so those fail immediately instead of burning the retry budget. -func (e *statusError) retryable() bool { - return e.code == http.StatusTooManyRequests || e.code >= 500 -} - -// Client fetches pages from monkeydd.com. It spaces requests out by a fixed -// delay no matter how many goroutines call Get, so raising the worker count -// never raises the request rate, and it retries transient failures with -// exponential backoff. -type Client struct { - http *http.Client - ua string - delay time.Duration - retries int - - mu sync.Mutex - nextSlot time.Time -} - -func NewClient(delay time.Duration, retries int) *Client { - return &Client{ - http: &http.Client{Timeout: 45 * time.Second}, - ua: defaultUserAgent, - delay: delay, - retries: retries, - } -} - -// reserve claims the next request slot and blocks until it comes due, holding -// the global rate at one request per delay across all callers. -func (c *Client) reserve(ctx context.Context) error { - c.mu.Lock() - slot := c.nextSlot - if now := time.Now(); slot.Before(now) { - slot = now - } - c.nextSlot = slot.Add(c.delay) - c.mu.Unlock() - - wait := time.Until(slot) - if wait <= 0 { - return nil - } - timer := time.NewTimer(wait) - defer timer.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - return nil - } -} - -// Get fetches url, retrying transient failures. -func (c *Client) Get(ctx context.Context, url string) ([]byte, error) { - var lastErr error - for attempt := 0; attempt <= c.retries; attempt++ { - if attempt > 0 { - backoff := time.Duration(1< 0 { - c.logf("warning: %d chapter(s) appear only in the chapter dropdown and were not "+ - "in the landing page list; verify the export is complete", len(extra)) - } - if len(final) != len(novel.Chapters) { - c.logf("chapter list reconciled to %d chapters using the in-chapter dropdown", len(final)) - } - novel.Chapters = final - return novel, nil -} - -// NovelInfo fetches only the landing page and returns what it carries: title, -// slug, tags, and the chapter list as that page shows it. -// -// Unlike Novel it does not also fetch a chapter page to cross-check the chapter -// list, so it costs a single request. Callers that only want metadata should -// prefer it; callers about to export every chapter want Novel, whose -// reconciliation guards against a silently truncated list. -func (c *Crawler) NovelInfo(ctx context.Context, novelURL string) (*Novel, error) { - page, err := c.page(ctx, novelURL) - if err != nil { - return nil, err - } - return ParseNovelPage(page, novelURL) -} - -// Chapters fetches every chapter concurrently and returns them in reading -// order. Any chapter that cannot be fetched or parsed fails the whole run -// rather than yielding a book with a hole in it. -func (c *Crawler) Chapters(ctx context.Context, novel *Novel) ([]*Chapter, error) { - chapters := make([]*Chapter, len(novel.Chapters)) - - workers := c.Workers - if workers < 1 { - workers = 1 - } - - group, groupCtx := errgroup.WithContext(ctx) - group.SetLimit(workers) - - var mu sync.Mutex - done := 0 - - for i, ref := range novel.Chapters { - i, ref := i, ref - group.Go(func() error { - page, err := c.page(groupCtx, ref.URL) - if err != nil { - return err - } - chapter, err := ParseChapter(page, ref) - if err != nil { - return err - } - chapters[i] = chapter - - mu.Lock() - done++ - c.logf("fetched %d/%d: %s (%d words)", done, len(novel.Chapters), - chapter.Heading(), chapter.WordCount()) - mu.Unlock() - return nil - }) - } - - if err := group.Wait(); err != nil { - return nil, err - } - return chapters, nil -} - -// page returns a page from the cache when available, otherwise fetches and -// caches it. -func (c *Crawler) page(ctx context.Context, pageURL string) ([]byte, error) { - path := c.cachePath(pageURL) - if path != "" { - if body, err := os.ReadFile(path); err == nil && len(body) > 0 { //nolint:gosec // G304: cachePath strips every path separator from the URL - return body, nil - } - } - - body, err := c.Client.Get(ctx, pageURL) - if err != nil { - return nil, err - } - - if path != "" { - if err := os.MkdirAll(filepath.Dir(path), 0o750); err == nil { - // A failed cache write must not fail the crawl. - _ = os.WriteFile(path, body, 0o600) - } - } - return body, nil -} - -// unsafeFileChars matches everything not allowed in a cache file name. -var unsafeFileChars = regexp.MustCompile(`[^A-Za-z0-9._-]+`) - -// cachePath maps a page URL to a cache file, or "" when caching is disabled. -func (c *Crawler) cachePath(pageURL string) string { - if c.CacheDir == "" { - return "" - } - u, err := url.Parse(pageURL) - if err != nil { - return "" - } - name := unsafeFileChars.ReplaceAllString(strings.Trim(u.Path, "/"), "_") - if name == "" { - return "" - } - if !strings.HasSuffix(name, ".html") { - name += ".html" - } - return filepath.Join(c.CacheDir, name) -} - -// TotalWords sums the word count across chapters. -func TotalWords(chapters []*Chapter) int { - n := 0 - for _, ch := range chapters { - n += ch.WordCount() - } - return n -} - -// Describe renders a one-line summary of a crawl result. -func Describe(novel *Novel, chapters []*Chapter) string { - return fmt.Sprintf("%s — %d chapters, %d words", novel.Title, len(chapters), TotalWords(chapters)) -} diff --git a/internal/modules/monkeyd/crawler/css_words.go b/internal/modules/monkeyd/crawler/css_words.go deleted file mode 100644 index 7566e61..0000000 --- a/internal/modules/monkeyd/crawler/css_words.go +++ /dev/null @@ -1,50 +0,0 @@ -package crawler - -import ( - "regexp" - "strconv" - "strings" -) - -// The site hides part of every chapter behind CSS rather than putting it in the -// markup. Chapter HTML carries empty elements such as -// -// Nghe trưởng tử -// -// and the page stylesheet supplies the missing word: -// -// .t-3e625e...:before { content: "vị"; } -// -// Reading DOM text alone therefore drops hundreds of words per chapter without -// any visible error. wordRule finds those rules so the words can be put back. -var wordRule = regexp.MustCompile(`\.([A-Za-z0-9_-]+)\s*::?before\s*\{[^}]*?content\s*:\s*"((?:[^"\\]|\\.)*)"`) - -// cssEscape matches a CSS character escape: a hex code point, optionally -// followed by one whitespace terminator, or an escaped literal character. -var cssEscape = regexp.MustCompile(`\\([0-9A-Fa-f]{1,6})\s?|\\(.)`) - -// ParseWordClasses maps CSS class name to the word its :before rule injects. -func ParseWordClasses(page []byte) map[string]string { - words := make(map[string]string) - for _, m := range wordRule.FindAllSubmatch(page, -1) { - words[string(m[1])] = decodeCSSString(string(m[2])) - } - return words -} - -// decodeCSSString resolves the escape sequences allowed inside a CSS string. -func decodeCSSString(s string) string { - if !strings.Contains(s, `\`) { - return s - } - return cssEscape.ReplaceAllStringFunc(s, func(esc string) string { - m := cssEscape.FindStringSubmatch(esc) - if m[1] != "" { - if cp, err := strconv.ParseInt(m[1], 16, 32); err == nil && cp > 0 { - return string(rune(cp)) - } - return "" - } - return m[2] - }) -} diff --git a/internal/modules/monkeyd/crawler/html_nodes.go b/internal/modules/monkeyd/crawler/html_nodes.go deleted file mode 100644 index 2aee719..0000000 --- a/internal/modules/monkeyd/crawler/html_nodes.go +++ /dev/null @@ -1,95 +0,0 @@ -package crawler - -import ( - "strings" - - "golang.org/x/net/html" -) - -// attr returns the value of the named attribute, or "" when absent. -func attr(n *html.Node, name string) string { - for _, a := range n.Attr { - if a.Key == name { - return a.Val - } - } - return "" -} - -// hasClass reports whether the node carries the given class token. -func hasClass(n *html.Node, class string) bool { - for _, tok := range strings.Fields(attr(n, "class")) { - if tok == class { - return true - } - } - return false -} - -// findNode returns the first node in document order satisfying match. -func findNode(root *html.Node, match func(*html.Node) bool) *html.Node { - if match(root) { - return root - } - for c := root.FirstChild; c != nil; c = c.NextSibling { - if found := findNode(c, match); found != nil { - return found - } - } - return nil -} - -// findAllNodes returns every node satisfying match, in document order. -func findAllNodes(root *html.Node, match func(*html.Node) bool) []*html.Node { - var out []*html.Node - var walk func(*html.Node) - walk = func(n *html.Node) { - if match(n) { - out = append(out, n) - } - for c := n.FirstChild; c != nil; c = c.NextSibling { - walk(c) - } - } - walk(root) - return out -} - -// elementByID finds an element by its id attribute. -func elementByID(root *html.Node, id string) *html.Node { - return findNode(root, func(n *html.Node) bool { - return n.Type == html.ElementNode && attr(n, "id") == id - }) -} - -// elementByTag finds the first element with the given tag name. -func elementByTag(root *html.Node, tag string) *html.Node { - return findNode(root, func(n *html.Node) bool { - return n.Type == html.ElementNode && n.Data == tag - }) -} - -// nodeText collects the descendant text of a node with whitespace collapsed. -func nodeText(n *html.Node) string { - if n == nil { - return "" - } - var b strings.Builder - var walk func(*html.Node) - walk = func(n *html.Node) { - if n.Type == html.TextNode { - b.WriteString(n.Data) - } - for c := n.FirstChild; c != nil; c = c.NextSibling { - walk(c) - } - } - walk(n) - return collapseSpaces(b.String()) -} - -// collapseSpaces trims the string and reduces every whitespace run, including -// the non-breaking spaces the site emits as  , to a single space. -func collapseSpaces(s string) string { - return strings.Join(strings.Fields(s), " ") -} diff --git a/internal/modules/monkeyd/crawler/novel.go b/internal/modules/monkeyd/crawler/novel.go deleted file mode 100644 index e097dfd..0000000 --- a/internal/modules/monkeyd/crawler/novel.go +++ /dev/null @@ -1,223 +0,0 @@ -package crawler - -import ( - "bytes" - "fmt" - "net/url" - "strings" - - "golang.org/x/net/html" -) - -// ChapterRef points at a single chapter listed on a novel page. -type ChapterRef struct { - Label string // as shown on the site, e.g. "14" or "Chương 12" - URL string -} - -// Novel is a novel's landing page: its title and its chapters in reading order. -type Novel struct { - Title string - Slug string - URL string - - // Tags are the novel's own genres, as labelled on the site, in the order - // they appear. Empty when the page lists none. - Tags []string - Chapters []ChapterRef -} - -// ParseNovelPage reads the title and chapter list from a novel landing page. -// -// Chapter URLs are always taken from the anchors on the page. Slugs are not -// uniform across novels ("/14.html" on one, "/chuong-12.html" on another) and -// numbering has gaps, so generating URLs from a chapter count would fetch 404s -// and miss real chapters. -func ParseNovelPage(page []byte, pageURL string) (*Novel, error) { - doc, err := html.Parse(bytes.NewReader(page)) - if err != nil { - return nil, fmt.Errorf("parse novel page: %w", err) - } - base, err := url.Parse(pageURL) - if err != nil { - return nil, fmt.Errorf("parse novel url: %w", err) - } - - novel := &Novel{ - Title: novelTitle(doc), - Slug: slugFromNovelURL(base), - URL: pageURL, - Tags: tagsFromInfo(doc), - Chapters: chapterRefsFromList(doc, base), - } - if novel.Title == "" { - novel.Title = novel.Slug - } - if len(novel.Chapters) == 0 { - return nil, fmt.Errorf("no chapters found on %s (page layout may have changed)", pageURL) - } - return novel, nil -} - -// novelTitle prefers the

heading and falls back to the document title. -func novelTitle(doc *html.Node) string { - if h1 := elementByTag(doc, "h1"); h1 != nil { - if t := nodeText(h1); t != "" { - return t - } - } - return nodeText(elementByTag(doc, "title")) -} - -// tagsFromInfo reads the novel's own genres out of the info block. -// -// The anchors are matched on the schema.org itemprop="genre" microdata rather -// than on their href. The page also carries a site-wide genre menu linking every -// category — 69 distinct ones against this novel's 6 on a sampled page — so -// matching the /the-loai/ URL shape would pull in the whole navigation. -func tagsFromInfo(doc *html.Node) []string { - links := findAllNodes(doc, func(n *html.Node) bool { - return n.Type == html.ElementNode && n.Data == "a" && attr(n, "itemprop") == "genre" - }) - - var tags []string - seen := make(map[string]bool, len(links)) - for _, link := range links { - // The label appears both as the link text and as its title attribute; - // prefer the text and fall back to the attribute. - name := nodeText(link) - if name == "" { - name = collapseSpaces(attr(link, "title")) - } - if name == "" || seen[name] { - continue - } - seen[name] = true - tags = append(tags, name) - } - return tags -} - -// slugFromNovelURL turns https://host/tro-lai-nam-thang-cu.html into -// "tro-lai-nam-thang-cu". -func slugFromNovelURL(u *url.URL) string { - seg := strings.Trim(u.Path, "/") - if i := strings.LastIndex(seg, "/"); i >= 0 { - seg = seg[i+1:] - } - return strings.TrimSuffix(seg, ".html") -} - -// chapterRefsFromList reads the "list-chapters" block on the landing page. -// The site lists newest first, so the result is reversed into reading order. -func chapterRefsFromList(doc *html.Node, base *url.URL) []ChapterRef { - list := findNode(doc, func(n *html.Node) bool { - return n.Type == html.ElementNode && hasClass(n, "list-chapters") - }) - if list == nil { - return nil - } - - titles := findAllNodes(list, func(n *html.Node) bool { - return n.Type == html.ElementNode && hasClass(n, "episode-title") - }) - - var refs []ChapterRef - for _, title := range titles { - link := elementByTag(title, "a") - if link == nil { - continue - } - href := strings.TrimSpace(attr(link, "href")) - if href == "" { - continue - } - abs, err := base.Parse(href) - if err != nil { - continue - } - refs = append(refs, ChapterRef{Label: nodeText(link), URL: abs.String()}) - } - return reverseRefs(refs) -} - -// ChapterRefsFromSelect reads the chapter dropdown embedded in every chapter -// page, whose options hold "novel-slug,chapter-slug" pairs. This is a second, -// independent view of the chapter list used to cross-check the landing page. -func ChapterRefsFromSelect(page []byte, base *url.URL) ([]ChapterRef, error) { - doc, err := html.Parse(bytes.NewReader(page)) - if err != nil { - return nil, fmt.Errorf("parse chapter page: %w", err) - } - sel := elementByID(doc, "selected_chapter") - if sel == nil { - return nil, nil - } - - var refs []ChapterRef - for _, opt := range findAllNodes(sel, func(n *html.Node) bool { - return n.Type == html.ElementNode && n.Data == "option" - }) { - novelSlug, chapterSlug, ok := strings.Cut(attr(opt, "value"), ",") - if !ok || novelSlug == "" || chapterSlug == "" { - continue - } - abs, err := base.Parse("/" + novelSlug + "/" + chapterSlug + ".html") - if err != nil { - continue - } - refs = append(refs, ChapterRef{Label: nodeText(opt), URL: abs.String()}) - } - return reverseRefs(refs), nil -} - -// ReconcileChapterRefs picks the chapter list to crawl from the landing page -// list and the in-chapter dropdown. -// -// Both views come from the same site ordering, so when the dropdown is a -// superset it is preferred: that keeps the run correct even if the landing page -// ever truncates or paginates its list. Anything the dropdown alone knows about -// while disagreeing on order is returned as extra so the caller can warn rather -// than silently export a short book. -func ReconcileChapterRefs(fromList, fromSelect []ChapterRef) (final, extra []ChapterRef) { - if len(fromSelect) == 0 { - return fromList, nil - } - - inSelect := refURLSet(fromSelect) - if listIsSubsetOf(fromList, inSelect) { - return fromSelect, nil - } - - inList := refURLSet(fromList) - for _, ref := range fromSelect { - if !inList[ref.URL] { - extra = append(extra, ref) - } - } - return fromList, extra -} - -func refURLSet(refs []ChapterRef) map[string]bool { - set := make(map[string]bool, len(refs)) - for _, r := range refs { - set[r.URL] = true - } - return set -} - -func listIsSubsetOf(refs []ChapterRef, set map[string]bool) bool { - for _, r := range refs { - if !set[r.URL] { - return false - } - } - return true -} - -func reverseRefs(refs []ChapterRef) []ChapterRef { - for i, j := 0, len(refs)-1; i < j; i, j = i+1, j-1 { - refs[i], refs[j] = refs[j], refs[i] - } - return refs -} diff --git a/internal/modules/monkeyd/crawler/novel_test.go b/internal/modules/monkeyd/crawler/novel_test.go deleted file mode 100644 index 0e1cfa9..0000000 --- a/internal/modules/monkeyd/crawler/novel_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package crawler - -import ( - "net/url" - "testing" -) - -// novelFixture reproduces the two traits that break naive crawlers: the list is -// newest-first, and chapter numbering has a gap (no chapter 4). -const novelFixture = `TEN TRUYEN -

TEN TRUYEN

-
- - - - -
` - -func TestParseNovelPageOrdersChaptersForReading(t *testing.T) { - novel, err := ParseNovelPage([]byte(novelFixture), "https://monkeydd.com/n.html") - if err != nil { - t.Fatalf("ParseNovelPage: %v", err) - } - - if novel.Title != "TEN TRUYEN" { - t.Errorf("Title = %q", novel.Title) - } - if novel.Slug != "n" { - t.Errorf("Slug = %q, want %q", novel.Slug, "n") - } - - wantLabels := []string{"1", "2", "3", "5"} - if len(novel.Chapters) != len(wantLabels) { - t.Fatalf("got %d chapters, want %d", len(novel.Chapters), len(wantLabels)) - } - for i, want := range wantLabels { - if novel.Chapters[i].Label != want { - t.Errorf("chapter %d label = %q, want %q", i, novel.Chapters[i].Label, want) - } - } - // Relative hrefs must resolve against the novel URL. - if got, want := novel.Chapters[1].URL, "https://monkeydd.com/n/2.html"; got != want { - t.Errorf("chapter 2 URL = %q, want %q", got, want) - } -} - -// tagsFixture mirrors the real page: the novel's own genres carry -// itemprop="genre", while a site-wide menu links every category without it. A -// parser matching on the /the-loai/ href shape would swallow the whole menu. -const tagsFixture = `TEN TRUYEN -

TEN TRUYEN

-
-
Thể loại
-
- Trọng Sinh - Cổ Đại - - Cổ Đại -
-
- -
- -
` - -func TestParseNovelPageReadsTags(t *testing.T) { - novel, err := ParseNovelPage([]byte(tagsFixture), "https://monkeydd.com/n.html") - if err != nil { - t.Fatalf("ParseNovelPage: %v", err) - } - - // "Cổ Đại" has its inner whitespace collapsed, the empty anchor falls back - // to its title attribute, and the repeated genre appears once. - want := []string{"Trọng Sinh", "Cổ Đại", "Gia Đình"} - if len(novel.Tags) != len(want) { - t.Fatalf("Tags = %q, want %q", novel.Tags, want) - } - for i := range want { - if novel.Tags[i] != want[i] { - t.Errorf("Tags[%d] = %q, want %q", i, novel.Tags[i], want[i]) - } - } -} - -func TestParseNovelPageWithoutTags(t *testing.T) { - novel, err := ParseNovelPage([]byte(novelFixture), "https://monkeydd.com/n.html") - if err != nil { - t.Fatalf("ParseNovelPage: %v", err) - } - if len(novel.Tags) != 0 { - t.Errorf("Tags = %q, want none", novel.Tags) - } -} - -func TestParseNovelPageNoChapters(t *testing.T) { - if _, err := ParseNovelPage([]byte(`x`), - "https://monkeydd.com/n.html"); err == nil { - t.Fatal("want an error when no chapters are listed") - } -} - -const selectFixture = ` -` - -func TestChapterRefsFromSelect(t *testing.T) { - base, err := url.Parse("https://monkeydd.com/n.html") - if err != nil { - t.Fatal(err) - } - refs, err := ChapterRefsFromSelect([]byte(selectFixture), base) - if err != nil { - t.Fatalf("ChapterRefsFromSelect: %v", err) - } - - if len(refs) != 3 { - t.Fatalf("got %d refs %+v, want 3 (malformed option skipped)", len(refs), refs) - } - if got, want := refs[0].URL, "https://monkeydd.com/n/1.html"; got != want { - t.Errorf("first ref URL = %q, want %q", got, want) - } - if got, want := refs[1].URL, "https://monkeydd.com/n/chuong-3.html"; got != want { - t.Errorf("second ref URL = %q, want %q", got, want) - } -} - -func TestReconcileChapterRefs(t *testing.T) { - ref := func(u string) ChapterRef { return ChapterRef{Label: u, URL: u} } - - t.Run("dropdown superset wins", func(t *testing.T) { - list := []ChapterRef{ref("a"), ref("b")} - sel := []ChapterRef{ref("a"), ref("b"), ref("c")} - - final, extra := ReconcileChapterRefs(list, sel) - if len(final) != 3 { - t.Errorf("got %d chapters, want the 3 from the dropdown", len(final)) - } - if len(extra) != 0 { - t.Errorf("got %d extra, want 0", len(extra)) - } - }) - - t.Run("disagreement is reported not silently merged", func(t *testing.T) { - list := []ChapterRef{ref("a"), ref("z")} - sel := []ChapterRef{ref("a"), ref("b")} - - final, extra := ReconcileChapterRefs(list, sel) - if len(final) != 2 || final[1].URL != "z" { - t.Errorf("final = %+v, want the landing page list", final) - } - if len(extra) != 1 || extra[0].URL != "b" { - t.Errorf("extra = %+v, want [b] so the caller can warn", extra) - } - }) - - t.Run("empty dropdown falls back to the list", func(t *testing.T) { - list := []ChapterRef{ref("a")} - final, extra := ReconcileChapterRefs(list, nil) - if len(final) != 1 || len(extra) != 0 { - t.Errorf("final = %+v, extra = %+v", final, extra) - } - }) -} diff --git a/internal/modules/monkeyd/export/export.go b/internal/modules/monkeyd/export/export.go deleted file mode 100644 index 351197a..0000000 --- a/internal/modules/monkeyd/export/export.go +++ /dev/null @@ -1,154 +0,0 @@ -// Package export turns a novel URL into a PDF file: resolve the chapter list, -// fetch the chapters, render the PDF with the bundled font. -package export - -import ( - "context" - "fmt" - "net/url" - "path/filepath" - "regexp" - "strings" - "time" - - "github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/crawler" - "github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/pdf" -) - -// Defaults the bot advertises to users or reuses in other crawls. -const ( - DefaultFontSize = 10.0 - DefaultDelay = 400 * time.Millisecond -) - -// Fixed layout and crawl settings. The bot exposes only the font size. -const ( - lineSpacing = 1.55 // multiple of font size - margin = 6.0 // millimetres - workers = 4 - retries = 3 -) - -// Request describes one export. -type Request struct { - NovelURL string - - // OutDir receives the PDF, named after the novel title. - OutDir string - - // FontSize is in points. 0 means DefaultFontSize. - FontSize float64 - - // CacheDir stores raw pages so a re-export costs no requests. Empty - // disables the cache. - CacheDir string - - // Log receives progress messages. Optional. - Log func(format string, args ...any) -} - -// Result reports what was produced. -type Result struct { - Path string - Title string - SourceURL string - Chapters int - Words int - Page pdf.PageSize -} - -// Summary renders a one-line description of the exported book. -func (r *Result) Summary() string { - return fmt.Sprintf("%s — %d chapters, %d words", r.Title, r.Chapters, r.Words) -} - -// Export fetches the novel at req.NovelURL and writes it as a PDF, returning -// where it landed. The context bounds the whole crawl; cancelling it abandons -// the run without leaving a partial PDF behind. -func Export(ctx context.Context, req Request) (*Result, error) { - if req.FontSize == 0 { - req.FontSize = DefaultFontSize - } - if err := req.validate(); err != nil { - return nil, err - } - - c := &crawler.Crawler{ - Client: crawler.NewClient(DefaultDelay, retries), - CacheDir: req.CacheDir, - Workers: workers, - Log: req.Log, - } - - novel, err := c.Novel(ctx, req.NovelURL) - if err != nil { - return nil, err - } - - chapters, err := c.Chapters(ctx, novel) - if err != nil { - return nil, err - } - - outPath := filepath.Join(req.OutDir, SafeFileName(novel.Title, novel.Slug)+".pdf") - opts := pdf.Options{ - Page: pdf.PhonePage, - Margin: margin, - Font: pdf.BundledFont(), - FontSize: req.FontSize, - LineSpacing: lineSpacing, - Title: novel.Title, - SourceURL: novel.URL, - } - if err := pdf.Write(outPath, opts, toPDFChapters(chapters)); err != nil { - return nil, err - } - - return &Result{ - Path: outPath, - Title: novel.Title, - SourceURL: novel.URL, - Chapters: len(chapters), - Words: crawler.TotalWords(chapters), - Page: pdf.PhonePage, - }, nil -} - -// validate rejects a Request before any request is made, so a typo costs no -// fetches. -func (r *Request) validate() error { - if r.NovelURL == "" { - return fmt.Errorf("novel url is required") - } - parsed, err := url.Parse(r.NovelURL) - if err != nil { - return fmt.Errorf("invalid novel url: %w", err) - } - if parsed.Scheme != "http" && parsed.Scheme != "https" { - return fmt.Errorf("invalid novel url: want an http(s) URL, got %q", r.NovelURL) - } - if r.FontSize <= 0 { - return fmt.Errorf("font size must be positive") - } - return nil -} - -func toPDFChapters(chapters []*crawler.Chapter) []pdf.Chapter { - out := make([]pdf.Chapter, 0, len(chapters)) - for _, ch := range chapters { - out = append(out, pdf.Chapter{Heading: ch.Heading(), Paragraphs: ch.Paragraphs}) - } - return out -} - -var unsafeNameChars = regexp.MustCompile(`[^\p{L}\p{N}]+`) - -// SafeFileName builds a file name from the novel title, falling back to the -// slug when the title has no usable characters. The result has no extension. -func SafeFileName(title, fallback string) string { - name := strings.Trim(unsafeNameChars.ReplaceAllString(title, "-"), "-") - if name == "" { - return fallback - } - return name -} diff --git a/internal/modules/monkeyd/export/export_test.go b/internal/modules/monkeyd/export/export_test.go deleted file mode 100644 index 2a9aed1..0000000 --- a/internal/modules/monkeyd/export/export_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package export - -import "testing" - -func TestValidate(t *testing.T) { - tests := []struct { - name string - req Request - wantErr bool - }{ - { - name: "url and font size are valid", - req: Request{NovelURL: "https://monkeydd.com/example.html", FontSize: DefaultFontSize}, - }, - { - name: "missing url", - req: Request{FontSize: DefaultFontSize}, - wantErr: true, - }, - { - name: "non-http scheme", - req: Request{NovelURL: "ftp://monkeydd.com/example.html", FontSize: DefaultFontSize}, - wantErr: true, - }, - { - name: "scheme-less url", - req: Request{NovelURL: "monkeydd.com/example.html", FontSize: DefaultFontSize}, - wantErr: true, - }, - { - name: "negative font size", - req: Request{NovelURL: "https://monkeydd.com/example.html", FontSize: -1}, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := tt.req.validate() - if tt.wantErr && err == nil { - t.Error("validate() = nil, want error") - } - if !tt.wantErr && err != nil { - t.Errorf("validate() = %v, want nil", err) - } - }) - } -} - -// An invalid request must fail before the crawler makes any request. -func TestExportRejectsInvalidRequestWithoutFetching(t *testing.T) { - if _, err := Export(t.Context(), Request{NovelURL: "ftp://monkeydd.com/x.html", OutDir: t.TempDir()}); err == nil { - t.Fatal("Export = nil error, want an invalid url error") - } -} - -func TestSafeFileName(t *testing.T) { - tests := []struct { - title string - fallback string - want string - }{ - {"Trở Lại Năm Tháng Cũ", "slug", "Trở-Lại-Năm-Tháng-Cũ"}, - {"Chapter: One / Two", "slug", "Chapter-One-Two"}, - {" --- ", "slug", "slug"}, - {"", "slug", "slug"}, - } - for _, tt := range tests { - if got := SafeFileName(tt.title, tt.fallback); got != tt.want { - t.Errorf("SafeFileName(%q, %q) = %q, want %q", tt.title, tt.fallback, got, tt.want) - } - } -} - -func TestResultSummary(t *testing.T) { - r := &Result{Title: "Example", Chapters: 12, Words: 3400} - want := "Example — 12 chapters, 3400 words" - if got := r.Summary(); got != want { - t.Errorf("Summary() = %q, want %q", got, want) - } -} diff --git a/internal/modules/monkeyd/export_job.go b/internal/modules/monkeyd/export_job.go deleted file mode 100644 index 0bd7f4b..0000000 --- a/internal/modules/monkeyd/export_job.go +++ /dev/null @@ -1,168 +0,0 @@ -package monkeyd - -import ( - "context" - "fmt" - "os" - "path/filepath" - "runtime/debug" - "time" - - "github.com/go-telegram/bot" - "github.com/go-telegram/bot/models" - - "github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/export" - - "github.com/tiennm99/tiennm99bot/internal/log" - "github.com/tiennm99/tiennm99bot/internal/modules/util/chathelper" -) - -const ( - // crawlTimeout bounds one export. A long novel is roughly one request per - // 400ms plus retries, so even a thousand chapters fits well inside this; - // the ceiling exists to release the single export slot if the site stalls. - crawlTimeout = 30 * time.Minute - - // uploadTimeout is separate from crawlTimeout so a slow crawl cannot eat - // the budget needed to actually deliver the finished PDF. - uploadTimeout = 5 * time.Minute - - // statusTimeout bounds the short progress and failure replies. - statusTimeout = 30 * time.Second - - // maxDocumentBytes is the Bot API's upload ceiling for sendDocument. - // Checked before opening the upload so an oversized book fails with an - // explanation instead of a Telegram API error. - maxDocumentBytes = 50 << 20 -) - -// cacheDirName is the shared page cache under the system temp directory. -// Keeping it outside the per-run directory is what makes a repeat export of the -// same novel cost no requests. It is not pruned; a container restart clears it. -const cacheDirName = "tiennm99bot-monkeyd-cache" - -// export runs one crawl to completion and delivers the PDF. It is called on its -// own goroutine, detached from the Telegram handler context. -func (r *runner) export(b *bot.Bot, msg *models.Message, args crawlArgs) { - novelURL := args.NovelURL - defer r.end() - // A panic here would reach no handler recover and would take the whole - // process down with it, so contain it. - defer func() { - if p := recover(); p != nil { - log.Error("monkeyd export panicked", - "command", commandName, "url", novelURL, "panic", p, "stack", string(debug.Stack())) - r.reportFailure(b, msg, "The export crashed. Nothing was sent.") - } - }() - - ctx, cancel := context.WithTimeout(context.Background(), crawlTimeout) - defer cancel() - - // The PDF is a temporary artefact: it is uploaded and then dropped. - outDir, err := os.MkdirTemp("", "monkeyd-export-*") - if err != nil { - log.Error("monkeyd temp dir failed", "command", commandName, "err", err) - r.reportFailure(b, msg, "Could not create a working directory for the export.") - return - } - defer func() { - if err := os.RemoveAll(outDir); err != nil { - log.Warn("monkeyd temp cleanup failed", "command", commandName, "dir", outDir, "err", err) - } - }() - - result, err := r.exporter(ctx, export.Request{ - NovelURL: novelURL, - OutDir: outDir, - // Zero means the caller gave no font size, which Export reads as - // "use the default" — exactly the intent. - FontSize: args.FontSize, - CacheDir: filepath.Join(os.TempDir(), cacheDirName), - // Per-chapter progress is one line per chapter — useful when - // diagnosing a stuck export, too noisy for the default level. - Log: func(format string, args ...any) { - log.Debug("monkeyd crawl: "+fmt.Sprintf(format, args...), "command", commandName, "url", novelURL) - }, - }) - if err != nil { - log.Error("monkeyd export failed", "command", commandName, "url", novelURL, "err", err) - r.reportFailure(b, msg, "Export failed: "+err.Error()) - return - } - - log.Info("monkeyd export done", "command", commandName, "url", novelURL, - "title", result.Title, "chapters", result.Chapters, "words", result.Words) - - if err := sendPDF(b, msg, result, effectiveFontSize(args.FontSize)); err != nil { - log.Error("monkeyd delivery failed", "command", commandName, "url", novelURL, "err", err) - r.reportFailure(b, msg, "The novel was exported but the PDF could not be sent: "+err.Error()) - } -} - -// effectiveFontSize resolves what the renderer actually used, so the caption -// can report a real number rather than "default". -func effectiveFontSize(requested float64) float64 { - if requested == 0 { - return export.DefaultFontSize - } - return requested -} - -// sendPDF uploads the finished book as a Telegram document. -func sendPDF(b *bot.Bot, msg *models.Message, result *export.Result, fontSize float64) error { - info, err := os.Stat(result.Path) - if err != nil { - return fmt.Errorf("stat pdf: %w", err) - } - if info.Size() > maxDocumentBytes { - return fmt.Errorf("the PDF is %.1f MB, above Telegram's %d MB limit for bots", - float64(info.Size())/(1<<20), maxDocumentBytes>>20) - } - - file, err := os.Open(result.Path) - if err != nil { - return fmt.Errorf("open pdf: %w", err) - } - defer func() { _ = file.Close() }() - - ctx, cancel := context.WithTimeout(context.Background(), uploadTimeout) - defer cancel() - - // MessageThreadID keeps the document in the forum topic the command came - // from, matching chathelper.Reply's behaviour. - _, err = b.SendDocument(ctx, &bot.SendDocumentParams{ - ChatID: msg.Chat.ID, - MessageThreadID: msg.MessageThreadID, - Document: &models.InputFileUpload{ - Filename: filepath.Base(result.Path), - Data: file, - }, - Caption: caption(result, fontSize), - }) - return err -} - -// captionLimit is the Bot API's maximum caption length in characters. -const captionLimit = 1024 - -// caption describes the book under the document. Plain text, so a title -// containing markup characters needs no escaping. -func caption(result *export.Result, fontSize float64) string { - text := fmt.Sprintf("%s\n%s page (%.0f x %.0f mm), %gpt\n%s", - result.Summary(), result.Page.Name, result.Page.W, result.Page.H, fontSize, result.SourceURL) - if runes := []rune(text); len(runes) > captionLimit { - text = string(runes[:captionLimit]) - } - return text -} - -// reportFailure tells the chat the export did not produce a document. Delivery -// failures here are only logged — there is nothing further to fall back to. -func (r *runner) reportFailure(b *bot.Bot, msg *models.Message, text string) { - ctx, cancel := context.WithTimeout(context.Background(), statusTimeout) - defer cancel() - if err := chathelper.Reply(ctx, b, msg, text); err != nil { - log.Error("monkeyd failure reply failed", "command", commandName, "err", err) - } -} diff --git a/internal/modules/monkeyd/handlers_test.go b/internal/modules/monkeyd/handlers_test.go deleted file mode 100644 index cd06998..0000000 --- a/internal/modules/monkeyd/handlers_test.go +++ /dev/null @@ -1,386 +0,0 @@ -package monkeyd - -import ( - "context" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/export" - "github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/pdf" - - "github.com/tiennm99/tiennm99bot/internal/modules" - "github.com/tiennm99/tiennm99bot/internal/storage" - "github.com/tiennm99/tiennm99bot/internal/testutil" -) - -const testNovelURL = "https://monkeydd.com/tro-lai-nam-thang-cu.html" - -// install wires the module to a recording bot. The returned runner is the one -// behind the registered command, so tests can substitute its exporter and run -// the export synchronously instead of on a detached goroutine. -// -// ownerID becomes the bot owner. Both commands are public, so it grants no -// extra access; the public-access tests below rely on a non-owner sender. -func install(t *testing.T, ownerID int64) (*testutil.RecordingBot, *runner) { - t.Helper() - return installWith(t, ownerID, func(context.Context, string) ([]string, error) { - t.Helper() - t.Error("tags fetcher called by an export test") - return nil, nil - }) -} - -// installWith is install with an explicit tag fetcher, for the tags command. -func installWith(t *testing.T, ownerID int64, fetch tagsFetcher) (*testutil.RecordingBot, *runner) { - t.Helper() - rb := testutil.NewRecordingBot(t) - r := newRunner() - // Run the export inline so assertions do not race a detached goroutine. - r.launch = func(job func()) { job() } - mod := newModule(r, fetch) - - reg := &modules.Registry{ - Modules: []modules.Module{{Name: "monkeyd", Commands: mod.Commands}}, - AllCommands: map[string]modules.Command{}, - } - for _, c := range mod.Commands { - reg.AllCommands[c.Name] = c - } - modules.Install(rb.Bot, reg, modules.Auth{BotOwnerID: ownerID}) - return rb, r -} - -// stubPDF creates a file of the given size standing in for a rendered book and -// returns a Result describing it, as export.Export would. The file is sized by -// truncation so an over-the-limit case costs no real bytes. -func stubPDF(t *testing.T, dir, name string, size int64) *export.Result { - t.Helper() - path := filepath.Join(dir, name) - file, err := os.Create(path) - if err != nil { - t.Fatalf("create stub pdf: %v", err) - } - if err := file.Truncate(size); err != nil { - _ = file.Close() - t.Fatalf("size stub pdf: %v", err) - } - if err := file.Close(); err != nil { - t.Fatalf("close stub pdf: %v", err) - } - return &export.Result{ - Path: path, - Title: "Example Novel", - SourceURL: testNovelURL, - Chapters: 3, - Words: 1200, - Page: pdf.PhonePage, - } -} - -func TestCrawl_NoArgumentRepliesUsage(t *testing.T) { - rb, _ := install(t, 999) - rb.Bot.ProcessUpdate(context.Background(), testutil.NewPrivateMessage(999, "/monkeyd_crawl")) - - if got := rb.LastSent().Text(); !strings.Contains(got, usage) { - t.Errorf("reply = %q, want it to contain %q", got, usage) - } -} - -func TestCrawl_RejectsDisallowedHost(t *testing.T) { - rb, r := install(t, 999) - called := false - r.exporter = func(context.Context, export.Request) (*export.Result, error) { - called = true - return nil, nil - } - - rb.Bot.ProcessUpdate(context.Background(), - testutil.NewPrivateMessage(999, "/monkeyd_crawl https://example.com/novel.html")) - - if called { - t.Error("exporter ran for a disallowed host") - } - if got := rb.LastSent().Text(); !strings.Contains(got, AllowedHostsHint) { - t.Errorf("reply = %q, want it to name %q", got, AllowedHostsHint) - } -} - -func TestCrawl_SendsPDFOnSuccess(t *testing.T) { - rb, r := install(t, 999) - dir := t.TempDir() - - var gotRequest export.Request - r.exporter = func(_ context.Context, req export.Request) (*export.Result, error) { - gotRequest = req - return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil - } - - rb.Bot.ProcessUpdate(context.Background(), - testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL)) - - if gotRequest.NovelURL != testNovelURL { - t.Errorf("exporter got NovelURL %q, want %q", gotRequest.NovelURL, testNovelURL) - } - if gotRequest.OutDir == "" { - t.Error("exporter got an empty OutDir; the PDF would land in the working directory") - } - - calls := rb.Sent() - if len(calls) < 2 { - t.Fatalf("expected an acknowledgement and a document, got %d calls: %+v", len(calls), calls) - } - if first := calls[0]; !strings.Contains(first.Text(), testNovelURL) { - t.Errorf("first reply = %q, want it to name the novel URL", first.Text()) - } - last := calls[len(calls)-1] - if last.Method != "sendDocument" { - t.Fatalf("last call = %q, want sendDocument", last.Method) - } - if caption := last.Form["caption"]; !strings.Contains(caption, "Example Novel") { - t.Errorf("caption = %q, want it to name the novel", caption) - } -} - -func TestCrawl_PassesRequestedFontSizeToExporter(t *testing.T) { - rb, r := install(t, 999) - dir := t.TempDir() - - var gotRequest export.Request - r.exporter = func(_ context.Context, req export.Request) (*export.Result, error) { - gotRequest = req - return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil - } - - rb.Bot.ProcessUpdate(context.Background(), - testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL+" 14")) - - if gotRequest.FontSize != 14 { - t.Errorf("exporter got FontSize %v, want 14", gotRequest.FontSize) - } - if caption := rb.LastSent().Form["caption"]; !strings.Contains(caption, "14pt") { - t.Errorf("caption = %q, want it to report 14pt", caption) - } -} - -// Omitting the argument must leave Export to apply its own default rather than -// the bot inventing one, so the two cannot disagree. -func TestCrawl_OmittedFontSizeLeavesRequestZero(t *testing.T) { - rb, r := install(t, 999) - dir := t.TempDir() - - var gotRequest export.Request - r.exporter = func(_ context.Context, req export.Request) (*export.Result, error) { - gotRequest = req - return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil - } - - rb.Bot.ProcessUpdate(context.Background(), - testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL)) - - if gotRequest.FontSize != 0 { - t.Errorf("exporter got FontSize %v, want 0 (defer to Export)", gotRequest.FontSize) - } - // The caption still has to name a real size, not "0pt". - want := fmt.Sprintf("%gpt", export.DefaultFontSize) - if caption := rb.LastSent().Form["caption"]; !strings.Contains(caption, want) { - t.Errorf("caption = %q, want it to report %s", caption, want) - } -} - -func TestCrawl_RejectsBadFontSize(t *testing.T) { - rb, r := install(t, 999) - called := false - r.exporter = func(context.Context, export.Request) (*export.Result, error) { - called = true - return nil, nil - } - - rb.Bot.ProcessUpdate(context.Background(), - testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL+" 100")) - - if called { - t.Error("exporter ran despite an out-of-range font size") - } - if got := rb.LastSent().Text(); !strings.Contains(got, "between 6 and 24") { - t.Errorf("reply = %q, want it to state the font size bounds", got) - } -} - -func TestCrawl_ReportsExportFailure(t *testing.T) { - rb, r := install(t, 999) - r.exporter = func(context.Context, export.Request) (*export.Result, error) { - return nil, errors.New("fetch failed: 404") - } - - rb.Bot.ProcessUpdate(context.Background(), - testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL)) - - last := rb.LastSent() - if last.Method == "sendDocument" { - t.Fatal("a document was sent despite the export failing") - } - if got := last.Text(); !strings.Contains(got, "404") { - t.Errorf("failure reply = %q, want it to carry the underlying error", got) - } -} - -func TestCrawl_RefusesOversizedPDF(t *testing.T) { - rb, r := install(t, 999) - dir := t.TempDir() - r.exporter = func(context.Context, export.Request) (*export.Result, error) { - return stubPDF(t, dir, "Huge-Novel.pdf", maxDocumentBytes+1), nil - } - - rb.Bot.ProcessUpdate(context.Background(), - testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL)) - - last := rb.LastSent() - if last.Method == "sendDocument" { - t.Fatal("an oversized document was uploaded instead of being refused") - } - if got := last.Text(); !strings.Contains(got, "limit") { - t.Errorf("reply = %q, want it to explain the size limit", got) - } -} - -// The single export slot must be released whether the run succeeded or failed, -// otherwise the command is dead until the process restarts. -func TestCrawl_ReleasesSlotAfterRun(t *testing.T) { - for _, tc := range []struct { - name string - exporter func(t *testing.T) func(context.Context, export.Request) (*export.Result, error) - }{ - { - name: "after success", - exporter: func(t *testing.T) func(context.Context, export.Request) (*export.Result, error) { - dir := t.TempDir() - return func(context.Context, export.Request) (*export.Result, error) { - return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil - } - }, - }, - { - name: "after failure", - exporter: func(*testing.T) func(context.Context, export.Request) (*export.Result, error) { - return func(context.Context, export.Request) (*export.Result, error) { - return nil, errors.New("boom") - } - }, - }, - { - name: "after panic", - exporter: func(*testing.T) func(context.Context, export.Request) (*export.Result, error) { - return func(context.Context, export.Request) (*export.Result, error) { - panic("boom") - } - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - rb, r := install(t, 999) - r.exporter = tc.exporter(t) - - rb.Bot.ProcessUpdate(context.Background(), - testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL)) - - r.mu.Lock() - running := r.running - r.mu.Unlock() - if running { - t.Error("export slot still held after the run finished") - } - }) - } -} - -func TestCrawl_RepliesBusyWhileAnotherExportRuns(t *testing.T) { - rb, r := install(t, 999) - // Simulate an in-flight export rather than racing a real one. - if ok, _ := r.begin("https://monkeydd.com/other-novel.html"); !ok { - t.Fatal("could not claim the export slot") - } - - called := false - r.exporter = func(context.Context, export.Request) (*export.Result, error) { - called = true - return nil, nil - } - - rb.Bot.ProcessUpdate(context.Background(), - testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL)) - - if called { - t.Error("a second export started while one was already running") - } - got := rb.LastSent().Text() - if !strings.Contains(got, "other-novel") { - t.Errorf("busy reply = %q, want it to name the in-flight novel", got) - } -} - -// The command is public: a sender who is neither owner nor admin must still get -// a reply, including in a group. -func TestCrawl_AvailableToAnySender(t *testing.T) { - rb, r := install(t, 999) - dir := t.TempDir() - r.exporter = func(context.Context, export.Request) (*export.Result, error) { - return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil - } - - rb.Bot.ProcessUpdate(context.Background(), - testutil.NewGroupMessage(-100, 12345, "/monkeyd_crawl "+testNovelURL)) - - calls := rb.Sent() - if len(calls) == 0 { - t.Fatal("a non-admin sender got no reply from a public command") - } - if last := calls[len(calls)-1]; last.Method != "sendDocument" { - t.Errorf("last call = %q, want sendDocument", last.Method) - } -} - -func TestRegistration(t *testing.T) { - mod := New(modules.Deps{Store: storage.NewMemoryProvider().Collection("monkeyd")}) - - wantParameters := map[string]string{ - commandName: " [font_size]", - tagsCommandName: "", - } - if len(mod.Commands) != len(wantParameters) { - t.Fatalf("expected %d commands, got %d", len(wantParameters), len(mod.Commands)) - } - - for _, cmd := range mod.Commands { - want, known := wantParameters[cmd.Name] - if !known { - t.Errorf("unexpected command %q", cmd.Name) - continue - } - delete(wantParameters, cmd.Name) - - if cmd.Parameters != want { - t.Errorf("/%s parameters = %q, want %q", cmd.Name, cmd.Parameters, want) - } - if cmd.Visibility != modules.VisibilityPublic { - t.Errorf("/%s visibility = %v, want Public", cmd.Name, cmd.Visibility) - } - if cmd.Description == "" { - t.Errorf("/%s description is empty; command discovery requires one", cmd.Name) - } - if cmd.Handler == nil { - t.Errorf("/%s handler is nil", cmd.Name) - } - } - for name := range wantParameters { - t.Errorf("command %q was not registered", name) - } - - if len(mod.Crons) != 0 || len(mod.Callbacks) != 0 { - t.Errorf("expected no crons or callbacks, got %d crons and %d callbacks", - len(mod.Crons), len(mod.Callbacks)) - } -} diff --git a/internal/modules/monkeyd/hashtags.go b/internal/modules/monkeyd/hashtags.go deleted file mode 100644 index 871b4e3..0000000 --- a/internal/modules/monkeyd/hashtags.go +++ /dev/null @@ -1,77 +0,0 @@ -package monkeyd - -import ( - "strings" - "unicode" -) - -// sourceHashtag leads every tag line so the source is visible when the line is -// pasted somewhere on its own. -const sourceHashtag = "#MonkeyD" - -// tagsMessage renders the tag line followed by the novel URL, separated by a -// blank line. Plain text: the caller decides how to present it. -func tagsMessage(tags []string, novelURL string) string { - return hashtagLine(tags) + "\n\n" + novelURL -} - -// hashtagLine turns genre labels into a single space-separated line of -// hashtags, led by sourceHashtag. Labels that cannot form a usable hashtag are -// dropped, and duplicates collapse — two labels differing only in punctuation -// or spacing produce the same hashtag. -func hashtagLine(tags []string) string { - parts := []string{sourceHashtag} - seen := map[string]bool{sourceHashtag: true} - for _, tag := range tags { - body := hashtag(tag) - if body == "" { - continue - } - tag := "#" + body - if seen[tag] { - continue - } - seen[tag] = true - parts = append(parts, tag) - } - return strings.Join(parts, " ") -} - -// hashtag converts a genre label into a hashtag body: "Cổ Đại" -> "CổĐại". -// -// Telegram ends a hashtag at the first character that is not a letter, digit or -// underscore, so spaces and punctuation are removed rather than replaced, and -// each word is capitalised to keep the boundaries readable. Diacritics survive: -// Telegram accepts non-ASCII letters in hashtags. -// -// Returns "" for a label with no letters. A digits-only hashtag is not linkified -// by Telegram, so emitting one would only add noise. -func hashtag(tag string) string { - var b strings.Builder - b.Grow(len(tag)) - - hasLetter := false - wordStart := true - for _, r := range tag { - switch { - case unicode.IsLetter(r): - hasLetter = true - if wordStart { - r = unicode.ToUpper(r) - } - b.WriteRune(r) - wordStart = false - case unicode.IsDigit(r): - b.WriteRune(r) - wordStart = false - default: - // Dropped, but it still ends the word, so the next letter is - // capitalised: "sci-fi" -> "SciFi". - wordStart = true - } - } - if !hasLetter { - return "" - } - return b.String() -} diff --git a/internal/modules/monkeyd/hashtags_test.go b/internal/modules/monkeyd/hashtags_test.go deleted file mode 100644 index b73eeb2..0000000 --- a/internal/modules/monkeyd/hashtags_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package monkeyd - -import "testing" - -func TestHashtag(t *testing.T) { - tests := []struct { - tag string - want string - }{ - {"Cổ Đại", "CổĐại"}, - {"Gia Đình", "GiaĐình"}, - {"Trọng Sinh", "TrọngSinh"}, - {"Nữ Cường", "NữCường"}, - {"Vả Mặt", "VảMặt"}, - // Single word passes through untouched. - {"Ngôn", "Ngôn"}, - // A lower-case label is capitalised per word. - {"ngôn tình", "NgônTình"}, - // Punctuation is removed but still ends the word. - {"sci-fi", "SciFi"}, - {"Đông/Tây", "ĐôngTây"}, - {"Truyện (Chữ)", "TruyệnChữ"}, - // Collapsed whitespace and surrounding space. - {" Cổ Đại ", "CổĐại"}, - // Digits are kept. - {"3D", "3D"}, - {"18+ Tuổi", "18Tuổi"}, - // Nothing usable: Telegram will not linkify a digits-only hashtag. - {"18+", ""}, - {"", ""}, - {"---", ""}, - {"+", ""}, - } - for _, tt := range tests { - if got := hashtag(tt.tag); got != tt.want { - t.Errorf("hashtag(%q) = %q, want %q", tt.tag, got, tt.want) - } - } -} - -func TestHashtagLine(t *testing.T) { - got := hashtagLine([]string{"Cổ Đại", "Gia Đình"}) - want := "#MonkeyD #CổĐại #GiaĐình" - if got != want { - t.Errorf("hashtagLine() = %q, want %q", got, want) - } -} - -func TestHashtagLineWithoutTags(t *testing.T) { - if got := hashtagLine(nil); got != sourceHashtag { - t.Errorf("hashtagLine(nil) = %q, want %q", got, sourceHashtag) - } -} - -// Labels differing only in punctuation collapse to one hashtag, and an unusable -// label is dropped rather than emitted as a bare "#". -func TestHashtagLineDropsUnusableAndDuplicates(t *testing.T) { - got := hashtagLine([]string{"Cổ Đại", "Cổ-Đại", "18+", "Gia Đình"}) - want := "#MonkeyD #CổĐại #GiaĐình" - if got != want { - t.Errorf("hashtagLine() = %q, want %q", got, want) - } -} - -func TestTagsMessage(t *testing.T) { - got := tagsMessage([]string{"Cổ Đại", "Gia Đình"}, "https://monkeydd.com/truong-an-gwem.html") - want := "#MonkeyD #CổĐại #GiaĐình\n\nhttps://monkeydd.com/truong-an-gwem.html" - if got != want { - t.Errorf("tagsMessage() =\n%q\nwant\n%q", got, want) - } -} - -// The block is what makes the line copyable in one tap, and its contents must be -// escaped so a label containing markup cannot break out of it. -func TestMonospaceBlockEscapes(t *testing.T) { - got := monospaceBlock("#MonkeyD #A&") - want := "
#MonkeyD #A<b>&
" - if got != want { - t.Errorf("monospaceBlock() = %q, want %q", got, want) - } -} diff --git a/internal/modules/monkeyd/monkeyd.go b/internal/modules/monkeyd/monkeyd.go deleted file mode 100644 index b23aeb1..0000000 --- a/internal/modules/monkeyd/monkeyd.go +++ /dev/null @@ -1,160 +0,0 @@ -// Package monkeyd exports a monkeydd.com novel as a PDF and sends it back as a -// Telegram document (/monkeyd_crawl), and reports a novel's genre tags as a -// copyable hashtag line (/monkeyd_tags). The crawling and rendering live in the -// monkeyd-crawler submodule (third_party/monkeyd-crawler); this module is the -// Telegram surface around it: argument validation, one-at-a-time scheduling, -// and delivery. -package monkeyd - -import ( - "context" - "fmt" - "strings" - "sync" - - "github.com/go-telegram/bot" - "github.com/go-telegram/bot/models" - - "github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/export" - - "github.com/tiennm99/tiennm99bot/internal/modules" - "github.com/tiennm99/tiennm99bot/internal/modules/util/chathelper" -) - -// commandName is the PDF export command; tagsCommandName is the other one. -const commandName = "monkeyd_crawl" - -// parameters is the display syntax shared by the command menu, /help, and the -// usage text below, so the three cannot drift apart. -const parameters = " [font_size]" - -// usage is shown when the command arrives without usable arguments. The bounds -// and default are read from their definitions rather than restated, so the -// message stays true if either changes. -var usage = fmt.Sprintf( - "Usage: /%s %s\nfont_size is in points, %g to %g (default %g).", - commandName, parameters, minFontSize, maxFontSize, export.DefaultFontSize) - -// New is the module Factory. The module keeps no persistent state — an export -// is a one-shot job — so deps.Store is unused. -func New(_ modules.Deps) modules.Module { - return newModule(newRunner(), fetchTags) -} - -// newModule builds the module around a given runner and tag fetcher, which is -// how tests supply a stubbed exporter, a synchronous launch, and tags without -// network access. -func newModule(r *runner, fetch tagsFetcher) modules.Module { - return modules.Module{ - Commands: []modules.Command{ - { - Name: commandName, - Visibility: modules.VisibilityPublic, - // Public despite being expensive: one invocation makes - // hundreds of outbound requests over several minutes. The - // host allowlist and the single in-flight export are what - // bound the cost, so both are load-bearing here. - Description: "Export a " + AllowedHostsHint + " novel as a PDF", - Parameters: parameters, - Handler: r.handle, - }, - tagsCommand(fetch), - }, - } -} - -// runner serialises exports. The crawler spaces its own requests out per run, -// so two concurrent crawls would double the request rate against the site — -// and a novel is minutes of work, which makes queueing pointless. One at a -// time, globally, with a clear reply to anyone who asks meanwhile. -type runner struct { - mu sync.Mutex - running bool - current string // novel URL of the in-flight export, for the busy reply - - // exporter is the crawl-and-render step. It is a field so tests can - // exercise scheduling and delivery without network access. - exporter func(context.Context, export.Request) (*export.Result, error) - - // launch runs an export job. Production detaches it onto its own - // goroutine; tests substitute a synchronous run for determinism. - launch func(job func()) -} - -func newRunner() *runner { - return &runner{ - exporter: export.Export, - launch: func(job func()) { - // Detached from the handler context on purpose: handlers run one - // at a time (the bot is built WithNotAsyncHandlers), so crawling - // inline would block every other command for minutes. The job - // owns its own timeout and recovers its own panics. - go job() //nolint:gosec // G118: intentional; see runner.export - }, - } -} - -// begin claims the single export slot, reporting the in-flight URL when it is -// already taken. -func (r *runner) begin(novelURL string) (ok bool, inFlight string) { - r.mu.Lock() - defer r.mu.Unlock() - if r.running { - return false, r.current - } - r.running = true - r.current = novelURL - return true, "" -} - -func (r *runner) end() { - r.mu.Lock() - defer r.mu.Unlock() - r.running = false - r.current = "" -} - -func (r *runner) handle(ctx context.Context, b *bot.Bot, update *models.Update) error { - msg := update.Message - if msg == nil { - return nil - } - - arg := chathelper.ArgAfterCommand(msg.Text) - if arg == "" { - return chathelper.Reply(ctx, b, msg, usage) - } - - args, err := parseCrawlArgs(arg) - if err != nil { - return chathelper.Reply(ctx, b, msg, fmt.Sprintf("%s.\n%s", capitalize(err.Error()), usage)) - } - novelURL := args.NovelURL - - ok, inFlight := r.begin(novelURL) - if !ok { - return chathelper.Reply(ctx, b, msg, - "Already exporting "+inFlight+". Try again once it finishes.") - } - - // Reply before starting so the user knows the wait is expected. If the - // reply cannot be delivered, drop the slot rather than crawling for a chat - // that will never hear the result. - if err := chathelper.Reply(ctx, b, msg, - "Exporting "+novelURL+" — this takes a few minutes. I will send the PDF here when it is ready."); err != nil { - r.end() - return err - } - - r.launch(func() { r.export(b, msg, args) }) - return nil -} - -// capitalize upper-cases the first letter so a lower-case error string reads as -// a sentence in a Telegram reply. -func capitalize(s string) string { - if s == "" { - return s - } - return strings.ToUpper(s[:1]) + s[1:] -} diff --git a/internal/modules/monkeyd/novel_url.go b/internal/modules/monkeyd/novel_url.go deleted file mode 100644 index 8cf3963..0000000 --- a/internal/modules/monkeyd/novel_url.go +++ /dev/null @@ -1,71 +0,0 @@ -package monkeyd - -import ( - "errors" - "net/url" - "strings" -) - -// allowedHosts are the hostnames the crawler's extractor understands. It is -// written against monkeydd.com's specific markup — the chapter list, the -// in-chapter dropdown, and the CSS rules that supply part of the chapter text — -// so another host would parse to nothing useful. Refusing it up front also -// keeps the command from being used to make the bot fetch arbitrary URLs. -var allowedHosts = map[string]bool{ - "monkeydd.com": true, - "www.monkeydd.com": true, -} - -// AllowedHostsHint lists the accepted hosts for user-facing usage text. -const AllowedHostsHint = "monkeydd.com" - -var ( - errNotAURL = errors.New("that does not look like a URL") - errHostNotAllow = errors.New("only " + AllowedHostsHint + " novel URLs are supported") - errNoPath = errors.New("that URL has no novel path — link the novel's own page") -) - -// normalizeNovelURL validates a user-supplied novel URL and returns the form to -// crawl. A missing scheme is filled in with https, since people paste bare -// hostnames; anything else that is not a plain http(s) URL on an allowed host -// is rejected. -func normalizeNovelURL(raw string) (string, error) { - raw = strings.TrimSpace(raw) - if raw == "" { - return "", errNotAURL - } - // A bare "monkeydd.com/x.html" parses as a path with no host, so give it a - // scheme before parsing rather than trying to interpret the result. - if !strings.Contains(raw, "://") { - raw = "https://" + raw - } - - parsed, err := url.Parse(raw) - if err != nil { - return "", errNotAURL - } - if parsed.Scheme != "http" && parsed.Scheme != "https" { - return "", errNotAURL - } - // Credentials in the URL are never needed here and would be logged with the - // crawl, so treat them as malformed input. - if parsed.User != nil { - return "", errNotAURL - } - // Hostname() drops any port, which the allowlist must not be fooled by. - if !allowedHosts[strings.ToLower(parsed.Hostname())] { - return "", errHostNotAllow - } - if strings.Trim(parsed.Path, "/") == "" { - return "", errNoPath - } - // Rebuild from the parsed parts so the crawl uses a canonical host and no - // fragment; query strings are kept because the site may need them. - canonical := url.URL{ - Scheme: parsed.Scheme, - Host: strings.ToLower(parsed.Host), - Path: parsed.Path, - RawQuery: parsed.RawQuery, - } - return canonical.String(), nil -} diff --git a/internal/modules/monkeyd/novel_url_test.go b/internal/modules/monkeyd/novel_url_test.go deleted file mode 100644 index 5d173cc..0000000 --- a/internal/modules/monkeyd/novel_url_test.go +++ /dev/null @@ -1,128 +0,0 @@ -package monkeyd - -import "testing" - -func TestNormalizeNovelURL(t *testing.T) { - tests := []struct { - name string - raw string - want string - wantErr bool - }{ - { - name: "https url passes through", - raw: "https://monkeydd.com/tro-lai-nam-thang-cu.html", - want: "https://monkeydd.com/tro-lai-nam-thang-cu.html", - }, - { - name: "www host is allowed", - raw: "https://www.monkeydd.com/tro-lai-nam-thang-cu.html", - want: "https://www.monkeydd.com/tro-lai-nam-thang-cu.html", - }, - { - name: "missing scheme gets https", - raw: "monkeydd.com/tro-lai-nam-thang-cu.html", - want: "https://monkeydd.com/tro-lai-nam-thang-cu.html", - }, - { - name: "http is kept", - raw: "http://monkeydd.com/tro-lai-nam-thang-cu.html", - want: "http://monkeydd.com/tro-lai-nam-thang-cu.html", - }, - { - name: "surrounding whitespace is trimmed", - raw: " https://monkeydd.com/tro-lai-nam-thang-cu.html ", - want: "https://monkeydd.com/tro-lai-nam-thang-cu.html", - }, - { - name: "uppercase host is canonicalised", - raw: "https://MonkeyDD.com/tro-lai-nam-thang-cu.html", - want: "https://monkeydd.com/tro-lai-nam-thang-cu.html", - }, - { - name: "fragment is dropped", - raw: "https://monkeydd.com/tro-lai-nam-thang-cu.html#chuong-1", - want: "https://monkeydd.com/tro-lai-nam-thang-cu.html", - }, - { - name: "query is preserved", - raw: "https://monkeydd.com/novel.html?page=2", - want: "https://monkeydd.com/novel.html?page=2", - }, - { - name: "empty input", - raw: "", - wantErr: true, - }, - { - name: "other host", - raw: "https://example.com/novel.html", - wantErr: true, - }, - { - name: "host that merely ends with the allowed name", - raw: "https://evilmonkeydd.com/novel.html", - wantErr: true, - }, - { - name: "host that merely contains the allowed name", - raw: "https://monkeydd.com.evil.example/novel.html", - wantErr: true, - }, - { - name: "allowed host as a subdomain of another host", - raw: "https://monkeydd.com.attacker.test/novel.html", - wantErr: true, - }, - { - name: "non-http scheme", - raw: "file:///etc/passwd", - wantErr: true, - }, - { - name: "credentials in url", - raw: "https://user:pass@monkeydd.com/novel.html", - wantErr: true, - }, - { - name: "landing page with no novel path", - raw: "https://monkeydd.com", - wantErr: true, - }, - { - name: "root path only", - raw: "https://monkeydd.com/", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := normalizeNovelURL(tt.raw) - if tt.wantErr { - if err == nil { - t.Fatalf("normalizeNovelURL(%q) = %q, want error", tt.raw, got) - } - return - } - if err != nil { - t.Fatalf("normalizeNovelURL(%q) returned error: %v", tt.raw, err) - } - if got != tt.want { - t.Errorf("normalizeNovelURL(%q) = %q, want %q", tt.raw, got, tt.want) - } - }) - } -} - -// A port on an allowed host must not defeat the allowlist, and must survive -// canonicalisation so the crawl reaches the same place the user asked for. -func TestNormalizeNovelURLKeepsPort(t *testing.T) { - got, err := normalizeNovelURL("http://monkeydd.com:8080/novel.html") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if want := "http://monkeydd.com:8080/novel.html"; got != want { - t.Errorf("got %q, want %q", got, want) - } -} diff --git a/internal/modules/monkeyd/pdf/font.go b/internal/modules/monkeyd/pdf/font.go deleted file mode 100644 index 945b459..0000000 --- a/internal/modules/monkeyd/pdf/font.go +++ /dev/null @@ -1,37 +0,0 @@ -package pdf - -import ( - _ "embed" -) - -// bundledTTF is the only font the bot renders with. A minimal container has no -// system fonts, and a single embedded font keeps the PDF identical on every -// host. See fonts/NOTICE.md for provenance and licensing. -// -// Vietnamese text needs the Latin Extended Additional block (ư, ạ, ế, ộ …); -// DejaVu Sans covers it, where a basic-Latin font would silently drop the -// diacritics. -// -//go:embed fonts/DejaVuSans.ttf -var bundledTTF []byte - -// BundledFontName labels the embedded font in diagnostics. It is not a path; -// the font is compiled into the binary. -const BundledFontName = "DejaVu Sans (bundled)" - -// Font is font data ready to embed in a PDF. -// -// The data is carried as bytes rather than as a path because fpdf joins a font -// path onto its own font directory, which it defaults to "." — turning an -// absolute path into a working-directory-relative one that resolves only when -// the process happens to run from the filesystem root. -type Font struct { - // Name identifies the font for diagnostics. - Name string - Data []byte -} - -// BundledFont returns the font compiled into the binary. -func BundledFont() Font { - return Font{Name: BundledFontName, Data: bundledTTF} -} diff --git a/internal/modules/monkeyd/pdf/font_test.go b/internal/modules/monkeyd/pdf/font_test.go deleted file mode 100644 index 973d92b..0000000 --- a/internal/modules/monkeyd/pdf/font_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package pdf - -import ( - "testing" - - "golang.org/x/image/font/sfnt" -) - -func TestBundledFontIsParseable(t *testing.T) { - font := BundledFont() - if font.Name != BundledFontName { - t.Errorf("Name = %q, want %q", font.Name, BundledFontName) - } - if _, err := sfnt.Parse(font.Data); err != nil { - t.Fatalf("bundled font does not parse as a TrueType font: %v", err) - } -} - -// The bundled font exists to render Vietnamese. A replacement that lacks these -// glyphs would silently emit blanks, so check before trusting it. -func TestBundledFontCoversVietnamese(t *testing.T) { - parsed, err := sfnt.Parse(BundledFont().Data) - if err != nil { - t.Fatalf("parse bundled font: %v", err) - } - var buf sfnt.Buffer - for _, r := range []rune{'ư', 'ơ', 'đ', 'ạ', 'ế', 'ộ', 'ữ', 'ằ', 'ỷ', 'ỹ', 'Ọ', 'Ế', 'Ư', 'Đ'} { - index, err := parsed.GlyphIndex(&buf, r) - if err != nil { - t.Errorf("GlyphIndex(%q): %v", r, err) - continue - } - // Glyph 0 is .notdef — the character is absent from the font. - if index == 0 { - t.Errorf("bundled font has no glyph for %q (U+%04X)", r, r) - } - } -} diff --git a/internal/modules/monkeyd/pdf/fonts/DejaVuSans.ttf b/internal/modules/monkeyd/pdf/fonts/DejaVuSans.ttf deleted file mode 100644 index e5f7eec..0000000 Binary files a/internal/modules/monkeyd/pdf/fonts/DejaVuSans.ttf and /dev/null differ diff --git a/internal/modules/monkeyd/pdf/fonts/NOTICE.md b/internal/modules/monkeyd/pdf/fonts/NOTICE.md deleted file mode 100644 index 9394353..0000000 --- a/internal/modules/monkeyd/pdf/fonts/NOTICE.md +++ /dev/null @@ -1,24 +0,0 @@ -# Bundled font - -`DejaVuSans.ttf` is embedded into the binary and used when no font is supplied -and no suitable system font is found. It covers the Latin Extended Additional -block, which is what Vietnamese diacritics need — a font with only basic Latin -coverage silently drops them. - -The following is recorded in the font file's own name table: - -- Version: `Version 2.37` -- Copyright: `Copyright (c) 2003 by Bitstream, Inc. All Rights Reserved.` - `Copyright (c) 2006 by Tavmjong Bah. All Rights Reserved.` - `DejaVu changes are in public domain` -- License information: - -The DejaVu fonts are free and redistributable, which is why they ship with most -Linux distributions. This copy came from Alpine's `font-dejavu` package, which -does not include the license text as a separate file. For a vendored copy of the -full license text, take `LICENSE` from the upstream DejaVu release and add it to -this directory. - -To swap the bundled font, replace `DejaVuSans.ttf` and update this file. Verify -the replacement covers Vietnamese first — `TestBundledFontCoversVietnamese` in -`../font_test.go` checks a representative set of characters. diff --git a/internal/modules/monkeyd/pdf/pdf.go b/internal/modules/monkeyd/pdf/pdf.go deleted file mode 100644 index fc47a42..0000000 --- a/internal/modules/monkeyd/pdf/pdf.go +++ /dev/null @@ -1,132 +0,0 @@ -package pdf - -import ( - "fmt" - "strings" - - "github.com/go-pdf/fpdf" -) - -// pointsToMM converts typographic points to millimetres. -const pointsToMM = 25.4 / 72.0 - -// bodyFont is the internal family name registered with the PDF. -const bodyFont = "body" - -// PageSize is a page in millimetres. -type PageSize struct { - Name string - W, H float64 -} - -// PhonePage is the page every export uses. -// -// Phone reading depends far more on page shape than on font size. A phone -// viewer scales a whole page to fit the screen, so a large font on an A4 page -// still ends up tiny: the page is ~3x wider than the screen and gets shrunk to -// match. A page cut to the phone's own aspect ratio fills the screen at 100%, -// which is why this is a small 9:16 page rather than A4 with big type. -var PhonePage = PageSize{Name: "phone", W: 90, H: 160} - -// Chapter is a chapter ready to render. -type Chapter struct { - Heading string - Paragraphs []string -} - -// Options controls the exported PDF. -type Options struct { - Page PageSize - Margin float64 // mm - Font Font // BundledFont() - FontSize float64 // pt - LineSpacing float64 // multiple of font size - Title string - SourceURL string -} - -// footerReserve is the vertical space kept clear for the page number. -const footerReserve = 6.0 - -// Write renders the chapters to a PDF at path. -func Write(path string, opts Options, chapters []Chapter) error { - pdf := fpdf.NewCustom(&fpdf.InitType{ - UnitStr: "mm", - Size: fpdf.SizeType{Wd: opts.Page.W, Ht: opts.Page.H}, - }) - - pdf.SetMargins(opts.Margin, opts.Margin, opts.Margin) - pdf.SetAutoPageBreak(true, opts.Margin+footerReserve) - - // Embeds a subset of the TrueType data, which is what makes the Vietnamese - // diacritics render instead of falling back to "?". The bytes are passed - // directly rather than by path: the path-taking variant joins the name onto - // fpdf's own font directory (default "."), which mangles an absolute path - // into a working-directory-relative one. - pdf.AddUTF8FontFromBytes(bodyFont, "", opts.Font.Data) - pdf.SetFont(bodyFont, "", opts.FontSize) - pdf.SetTitle(opts.Title, true) - - lineHeight := opts.FontSize * opts.LineSpacing * pointsToMM - paragraphGap := lineHeight * 0.45 - headingSize := opts.FontSize * 1.35 - - addFooter(pdf, opts) - writeTitlePage(pdf, opts, len(chapters)) - - for _, ch := range chapters { - pdf.AddPage() - - pdf.SetFontSize(headingSize) - pdf.MultiCell(0, headingSize*1.3*pointsToMM, ch.Heading, "", "L", false) - pdf.Ln(paragraphGap * 1.6) - - pdf.SetFontSize(opts.FontSize) - for _, p := range ch.Paragraphs { - // "J" justifies, which keeps the short measure of a phone page tidy. - pdf.MultiCell(0, lineHeight, p, "", "J", false) - pdf.Ln(paragraphGap) - } - } - - if err := pdf.OutputFileAndClose(path); err != nil { - return fmt.Errorf("write pdf %s: %w", path, err) - } - return nil -} - -// addFooter prints a centred page number, restoring the body font size so the -// footer callback cannot leak its own size into the following content. -func addFooter(pdf *fpdf.Fpdf, opts Options) { - pdf.SetFooterFunc(func() { - if pdf.PageNo() <= 1 { - return - } - pdf.SetY(-(opts.Margin + footerReserve*0.6)) - pdf.SetFontSize(opts.FontSize * 0.75) - pdf.SetTextColor(120, 120, 120) - pdf.CellFormat(0, 4, fmt.Sprintf("%d", pdf.PageNo()-1), "", 0, "C", false, 0, "") - pdf.SetTextColor(0, 0, 0) - pdf.SetFontSize(opts.FontSize) - }) -} - -func writeTitlePage(pdf *fpdf.Fpdf, opts Options, chapterCount int) { - pdf.AddPage() - pdf.SetY(opts.Page.H * 0.30) - - titleSize := opts.FontSize * 1.9 - pdf.SetFontSize(titleSize) - pdf.MultiCell(0, titleSize*1.35*pointsToMM, strings.ToUpper(opts.Title), "", "C", false) - - pdf.Ln(opts.FontSize * pointsToMM * 2) - pdf.SetFontSize(opts.FontSize * 0.85) - pdf.SetTextColor(90, 90, 90) - pdf.MultiCell(0, opts.FontSize*1.4*pointsToMM, - fmt.Sprintf("%d chương", chapterCount), "", "C", false) - if opts.SourceURL != "" { - pdf.MultiCell(0, opts.FontSize*1.4*pointsToMM, opts.SourceURL, "", "C", false) - } - pdf.SetTextColor(0, 0, 0) - pdf.SetFontSize(opts.FontSize) -} diff --git a/internal/modules/monkeyd/pdf/pdf_test.go b/internal/modules/monkeyd/pdf/pdf_test.go deleted file mode 100644 index 392c5ab..0000000 --- a/internal/modules/monkeyd/pdf/pdf_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package pdf - -import ( - "os" - "path/filepath" - "testing" -) - -func testOptions(t *testing.T) Options { - t.Helper() - return Options{ - Page: PhonePage, - Margin: 6, - Font: BundledFont(), - FontSize: 12, - LineSpacing: 1.55, - Title: "TRỞ LẠI NĂM THÁNG CŨ", - SourceURL: "https://example.test/n.html", - } -} - -func TestWriteProducesReadablePDF(t *testing.T) { - path := filepath.Join(t.TempDir(), "out.pdf") - - chapters := []Chapter{ - {Heading: "Chương 1", Paragraphs: []string{ - "Nghe vị trưởng tử nói chuyện với nàng.", - "Một đoạn văn khác để kiểm tra ngắt dòng tự động trên trang nhỏ.", - }}, - {Heading: "Chương 2", Paragraphs: []string{"Đoạn cuối."}}, - } - - if err := Write(path, testOptions(t), chapters); err != nil { - t.Fatalf("Write: %v", err) - } - - info, err := os.Stat(path) - if err != nil { - t.Fatalf("stat output: %v", err) - } - if info.Size() == 0 { - t.Fatal("wrote an empty PDF") - } - - header := make([]byte, 5) - f, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - defer f.Close() - if _, err := f.Read(header); err != nil { - t.Fatal(err) - } - if string(header) != "%PDF-" { - t.Errorf("output does not start with a PDF header, got %q", header) - } -} - -// A novel with many chapters must not overflow a page; auto page break plus the -// footer reserve handles that, so a long chapter should span several pages. -func TestWriteHandlesLongChapters(t *testing.T) { - path := filepath.Join(t.TempDir(), "long.pdf") - - paragraphs := make([]string, 200) - for i := range paragraphs { - paragraphs[i] = "Một đoạn văn dài để buộc trình kết xuất phải sang trang mới nhiều lần." - } - - if err := Write(path, testOptions(t), []Chapter{{Heading: "Chương 1", Paragraphs: paragraphs}}); err != nil { - t.Fatalf("Write: %v", err) - } - info, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } - if info.Size() < 2000 { - t.Errorf("output suspiciously small (%d bytes) for 200 paragraphs", info.Size()) - } -} diff --git a/internal/modules/monkeyd/tags_command.go b/internal/modules/monkeyd/tags_command.go deleted file mode 100644 index a32f2b7..0000000 --- a/internal/modules/monkeyd/tags_command.go +++ /dev/null @@ -1,123 +0,0 @@ -package monkeyd - -import ( - "context" - "fmt" - "html" - "os" - "path/filepath" - "strings" - "time" - - "github.com/go-telegram/bot" - "github.com/go-telegram/bot/models" - - "github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/crawler" - "github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/export" - - "github.com/tiennm99/tiennm99bot/internal/log" - "github.com/tiennm99/tiennm99bot/internal/modules" - "github.com/tiennm99/tiennm99bot/internal/modules/util/chathelper" -) - -// tagsCommandName is the command that reports a novel's tags. -const tagsCommandName = "monkeyd_tags" - -// tagsParameters is the display syntax shared by the command menu, /help, and -// the usage text. -const tagsParameters = "" - -const tagsUsage = "Usage: /" + tagsCommandName + " " + tagsParameters - -const ( - // tagsFetchTimeout bounds the tag fetch. Unlike an export this is one - // request and runs inline, and handlers are dispatched one at a time, so a - // stalled fetch would hold up every other command until it gives up. - tagsFetchTimeout = 30 * time.Second - - // tagsRetries is deliberately lower than the export's: the caller is - // waiting on this reply, so failing early beats a long retry chain. - tagsRetries = 1 -) - -// tagsFetcher returns a novel's tags. It is injected through newModule rather -// than called directly so tests can exercise the command without network -// access. -type tagsFetcher func(ctx context.Context, novelURL string) ([]string, error) - -func tagsCommand(fetch tagsFetcher) modules.Command { - return modules.Command{ - Name: tagsCommandName, - Visibility: modules.VisibilityPublic, - Description: "Show a " + AllowedHostsHint + " novel's tags as hashtags", - Parameters: tagsParameters, - Handler: tagsHandler(fetch), - } -} - -func tagsHandler(fetch tagsFetcher) modules.CommandHandler { - return func(ctx context.Context, b *bot.Bot, update *models.Update) error { - msg := update.Message - if msg == nil { - return nil - } - - arg := chathelper.ArgAfterCommand(msg.Text) - if arg == "" { - return chathelper.Reply(ctx, b, msg, tagsUsage) - } - fields := strings.Fields(arg) - if len(fields) > 1 { - return chathelper.Reply(ctx, b, msg, - fmt.Sprintf("%s.\n%s", capitalize(errTooManyArgs.Error()), tagsUsage)) - } - novelURL, err := normalizeNovelURL(fields[0]) - if err != nil { - return chathelper.Reply(ctx, b, msg, - fmt.Sprintf("%s.\n%s", capitalize(err.Error()), tagsUsage)) - } - - // Reserve part of the handler's budget for the reply itself, then cap - // the fetch so it cannot outlive the command. - fetchCtx, cancel := chathelper.FetchContext(ctx) - defer cancel() - fetchCtx, cancelTimeout := context.WithTimeout(fetchCtx, tagsFetchTimeout) - defer cancelTimeout() - - tags, err := fetch(fetchCtx, novelURL) - if err != nil { - log.Error("monkeyd tags fetch failed", - "command", tagsCommandName, "url", novelURL, "err", err) - return chathelper.Reply(ctx, b, msg, "Could not read that novel page: "+err.Error()) - } - if len(tags) == 0 { - return chathelper.Reply(ctx, b, msg, "That page lists no tags.") - } - - // Sent as a code block so the line can be copied in one tap. - return chathelper.ReplyHTML(ctx, b, msg, monospaceBlock(tagsMessage(tags, novelURL))) - } -} - -// monospaceBlock wraps text in Telegram's
 block, which renders as a
-// copyable code box. The content is escaped, so a tag containing < or & cannot
-// break the markup.
-func monospaceBlock(text string) string {
-	return "
" + html.EscapeString(text) + "
" -} - -// fetchTags reads a novel's tags from the site. -// -// It shares the export cache directory, so tags requested for a novel that was -// already exported — or an export following a tags lookup — cost no request. -func fetchTags(ctx context.Context, novelURL string) ([]string, error) { - c := &crawler.Crawler{ - Client: crawler.NewClient(export.DefaultDelay, tagsRetries), - CacheDir: filepath.Join(os.TempDir(), cacheDirName), - } - novel, err := c.NovelInfo(ctx, novelURL) - if err != nil { - return nil, err - } - return novel.Tags, nil -} diff --git a/internal/modules/monkeyd/tags_command_test.go b/internal/modules/monkeyd/tags_command_test.go deleted file mode 100644 index 4befc7a..0000000 --- a/internal/modules/monkeyd/tags_command_test.go +++ /dev/null @@ -1,151 +0,0 @@ -package monkeyd - -import ( - "context" - "errors" - "strings" - "testing" - - "github.com/tiennm99/tiennm99bot/internal/testutil" -) - -// stubTags returns a fetcher yielding fixed tags and records the URL it saw. -func stubTags(tags []string, gotURL *string) tagsFetcher { - return func(_ context.Context, novelURL string) ([]string, error) { - if gotURL != nil { - *gotURL = novelURL - } - return tags, nil - } -} - -func TestTags_RepliesWithHashtagBlock(t *testing.T) { - var gotURL string - rb, _ := installWith(t, 999, stubTags([]string{"Cổ Đại", "Gia Đình"}, &gotURL)) - - rb.Bot.ProcessUpdate(context.Background(), - testutil.NewPrivateMessage(999, "/monkeyd_tags https://monkeydd.com/truong-an-gwem.html")) - - if want := "https://monkeydd.com/truong-an-gwem.html"; gotURL != want { - t.Errorf("fetcher got %q, want %q", gotURL, want) - } - - sent := rb.LastSent() - want := "
#MonkeyD #CổĐại #GiaĐình\n\nhttps://monkeydd.com/truong-an-gwem.html
" - if got := sent.Text(); got != want { - t.Errorf("reply =\n%q\nwant\n%q", got, want) - } - // Without HTML parse mode Telegram would show the literal
 tags.
-	if got := sent.Form["parse_mode"]; got != "HTML" {
-		t.Errorf("parse_mode = %q, want HTML", got)
-	}
-}
-
-func TestTags_NoArgumentRepliesUsage(t *testing.T) {
-	rb, _ := installWith(t, 999, stubTags(nil, nil))
-	rb.Bot.ProcessUpdate(context.Background(), testutil.NewPrivateMessage(999, "/monkeyd_tags"))
-
-	if got := rb.LastSent().Text(); !strings.Contains(got, tagsUsage) {
-		t.Errorf("reply = %q, want it to contain %q", got, tagsUsage)
-	}
-}
-
-func TestTags_RejectsDisallowedHost(t *testing.T) {
-	called := false
-	rb, _ := installWith(t, 999, func(context.Context, string) ([]string, error) {
-		called = true
-		return nil, nil
-	})
-
-	rb.Bot.ProcessUpdate(context.Background(),
-		testutil.NewPrivateMessage(999, "/monkeyd_tags https://example.com/novel.html"))
-
-	if called {
-		t.Error("fetcher ran for a disallowed host")
-	}
-	if got := rb.LastSent().Text(); !strings.Contains(got, AllowedHostsHint) {
-		t.Errorf("reply = %q, want it to name %q", got, AllowedHostsHint)
-	}
-}
-
-// A bare hostname is normalised before the fetch, matching /monkeyd_crawl, and
-// the reply quotes the normalised form rather than what was typed.
-func TestTags_NormalizesURL(t *testing.T) {
-	const wantURL = "https://monkeydd.com/truong-an-gwem.html"
-
-	var gotURL string
-	rb, _ := installWith(t, 999, stubTags([]string{"Cổ Đại"}, &gotURL))
-
-	rb.Bot.ProcessUpdate(context.Background(),
-		testutil.NewPrivateMessage(999, "/monkeyd_tags monkeydd.com/truong-an-gwem.html"))
-
-	if gotURL != wantURL {
-		t.Errorf("fetcher got %q, want %q", gotURL, wantURL)
-	}
-	if got := rb.LastSent().Text(); !strings.Contains(got, wantURL) {
-		t.Errorf("reply = %q, want it to carry the normalised URL", got)
-	}
-}
-
-func TestTags_ReportsFetchFailure(t *testing.T) {
-	rb, _ := installWith(t, 999, func(context.Context, string) ([]string, error) {
-		return nil, errors.New("fetch failed: 404")
-	})
-
-	rb.Bot.ProcessUpdate(context.Background(),
-		testutil.NewPrivateMessage(999, "/monkeyd_tags "+testNovelURL))
-
-	got := rb.LastSent().Text()
-	if !strings.Contains(got, "404") {
-		t.Errorf("reply = %q, want it to carry the underlying error", got)
-	}
-	if strings.Contains(got, sourceHashtag) {
-		t.Errorf("reply = %q, want no hashtag line on failure", got)
-	}
-}
-
-// An empty tag list must say so rather than send a lone "#MonkeyD".
-func TestTags_ReportsEmptyTagList(t *testing.T) {
-	rb, _ := installWith(t, 999, stubTags(nil, nil))
-
-	rb.Bot.ProcessUpdate(context.Background(),
-		testutil.NewPrivateMessage(999, "/monkeyd_tags "+testNovelURL))
-
-	got := rb.LastSent().Text()
-	if strings.Contains(got, sourceHashtag) {
-		t.Errorf("reply = %q, want a plain explanation rather than a bare source hashtag", got)
-	}
-	if !strings.Contains(strings.ToLower(got), "no tags") {
-		t.Errorf("reply = %q, want it to say no tags were listed", got)
-	}
-}
-
-func TestTags_RejectsExtraArguments(t *testing.T) {
-	called := false
-	rb, _ := installWith(t, 999, func(context.Context, string) ([]string, error) {
-		called = true
-		return nil, nil
-	})
-
-	rb.Bot.ProcessUpdate(context.Background(),
-		testutil.NewPrivateMessage(999, "/monkeyd_tags "+testNovelURL+" extra"))
-
-	if called {
-		t.Error("fetcher ran despite extra arguments")
-	}
-	if got := rb.LastSent().Text(); !strings.Contains(got, tagsUsage) {
-		t.Errorf("reply = %q, want usage text", got)
-	}
-}
-
-// The command is public, so a non-admin in a group must get a reply.
-func TestTags_AvailableToAnySender(t *testing.T) {
-	rb, _ := installWith(t, 999, stubTags([]string{"Cổ Đại"}, nil))
-
-	rb.Bot.ProcessUpdate(context.Background(),
-		testutil.NewGroupMessage(-100, 12345, "/monkeyd_tags "+testNovelURL))
-
-	if got := rb.LastSent().Text(); !strings.Contains(got, "#CổĐại") {
-		t.Errorf("reply = %q, want the hashtag line", got)
-	}
-}
diff --git a/internal/modules/stats/startup.go b/internal/modules/stats/startup.go
index b69f893..e52af5e 100644
--- a/internal/modules/stats/startup.go
+++ b/internal/modules/stats/startup.go
@@ -36,9 +36,28 @@ var commandRenames = []commandRename{
 	{oldCmd: "noitupvp", newCmd: "noitu", markerKey: "stats:command-rename:noitupvp-to-noitu"},
 }
 
+// commandRetirement marks every usage row of the listed commands deleted
+// once, recorded under markerKey in the system collection.
+type commandRetirement struct {
+	cmds      []string
+	markerKey string
+}
+
+// commandRetirements run in order after the renames, each at most once. The
+// commands must also be in retiredCommands, which keeps any row the marker
+// missed out of the views.
+var commandRetirements = []commandRetirement{
+	{cmds: []string{"monkeyd_crawl", "monkeyd_tags"}, markerKey: "migration:stats-delete-monkeyd-v1"},
+}
+
+// deletedCommandMigrationRetries bounds the optimistic-lock retries when one
+// row is marked deleted on the document store.
+const deletedCommandMigrationRetries = 5
+
 // InitStore performs stats collection startup maintenance. It is safe to call
 // every boot: MongoDB indexes are created idempotently, and one-time command
-// rename migrations are guarded by markers in the shared system collection.
+// rename and retirement migrations are guarded by markers in the shared
+// system collection.
 func InitStore(ctx context.Context, statsColl, systemColl storage.Collection) error {
 	if mongoColl, ok := storage.MongoCollection(statsColl); ok {
 		if err := ensureUsageIndexes(ctx, mongoColl); err != nil {
@@ -50,9 +69,121 @@ func InitStore(ctx context.Context, statsColl, systemColl storage.Collection) er
 			return err
 		}
 	}
+	for _, r := range commandRetirements {
+		if err := markDeletedCommands(ctx, statsColl, systemColl, r); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
+// markDeletedCommands sets deleted: true on every usage row of r.cmds, the
+// anonymous totals and the per-user rows, and records completion with the
+// number of rows it marked. The rows themselves are kept as history. A
+// completed marker makes it a no-op.
+func markDeletedCommands(ctx context.Context, statsColl, systemColl storage.Collection, r commandRetirement) error {
+	state := systemstate.New(systemColl)
+	if rec, ok, err := state.Get(ctx, r.markerKey); err != nil {
+		return fmt.Errorf("stats command retire marker %s: %w", r.markerKey, err)
+	} else if ok && rec.Status == "complete" {
+		return nil
+	}
+
+	var marked int64
+	for _, cmd := range r.cmds {
+		var (
+			n   int64
+			err error
+		)
+		if mongoColl, ok := storage.MongoCollection(statsColl); ok {
+			n, err = markMongoUsageEntriesDeleted(ctx, mongoColl, cmd)
+		} else {
+			n, err = markDocUsageEntriesDeleted(ctx, storage.Typed[usageEntry](statsColl), cmd)
+		}
+		if err != nil {
+			return err
+		}
+		marked += n
+	}
+
+	now := time.Now().UTC().UnixMilli()
+	if err := state.Put(ctx, r.markerKey, systemstate.Record{
+		Kind:        "migration",
+		Name:        r.markerKey,
+		Status:      "complete",
+		Count:       marked,
+		CompletedAt: now,
+		UpdatedAt:   now,
+	}); err != nil {
+		return fmt.Errorf("stats command retire marker put %s: %w", r.markerKey, err)
+	}
+	return nil
+}
+
+func markMongoUsageEntriesDeleted(ctx context.Context, coll *mongo.Collection, cmd string) (int64, error) {
+	res, err := coll.UpdateMany(ctx,
+		bson.M{"cmd": cmd, "deleted": bson.M{"$ne": true}},
+		bson.M{
+			"$set":         bson.M{"deleted": true},
+			"$inc":         bson.M{"version": int64(1)},
+			"$currentDate": bson.M{"updatedAt": true},
+		},
+	)
+	if err != nil {
+		return 0, fmt.Errorf("stats command retire update %s: %w", cmd, err)
+	}
+	return res.ModifiedCount, nil
+}
+
+func markDocUsageEntriesDeleted(ctx context.Context, docs storage.DocStore[usageEntry], cmd string) (int64, error) {
+	keys, err := docs.List(ctx, cmd)
+	if err != nil {
+		return 0, fmt.Errorf("stats command retire list %s: %w", cmd, err)
+	}
+	var marked int64
+	for _, key := range keys {
+		// The key prefix also matches longer command names.
+		if key != cmd && !strings.HasPrefix(key, cmd+":") {
+			continue
+		}
+		changed, err := markUsageEntryDeleted(ctx, docs, key, cmd)
+		if err != nil {
+			return 0, err
+		}
+		if changed {
+			marked++
+		}
+	}
+	return marked, nil
+}
+
+// markUsageEntryDeleted sets the deleted flag on one row with a versioned
+// write, retrying when a concurrent increment wins the race. It reports
+// whether the row changed.
+func markUsageEntryDeleted(ctx context.Context, docs storage.DocStore[usageEntry], key, cmd string) (bool, error) {
+	for range deletedCommandMigrationRetries {
+		entry, version, err := docs.Get(ctx, key)
+		if errors.Is(err, storage.ErrNotFound) {
+			return false, nil
+		}
+		if err != nil {
+			return false, fmt.Errorf("stats command retire get %s: %w", key, err)
+		}
+		if entry.Cmd != cmd || entry.Deleted {
+			return false, nil
+		}
+		entry.Deleted = true
+		err = docs.PutVersioned(ctx, key, version, entry)
+		if err == nil {
+			return true, nil
+		}
+		if !errors.Is(err, storage.ErrConflict) {
+			return false, fmt.Errorf("stats command retire put %s: %w", key, err)
+		}
+	}
+	return false, fmt.Errorf("stats command retire put %s: %w", key, storage.ErrConflict)
+}
+
 // migrateCommandRename merges every usage row of r.oldCmd, the anonymous
 // total and the per-user rows, into the matching r.newCmd row, deletes the
 // old row, and records completion. A completed marker makes it a no-op, so a
diff --git a/internal/modules/stats/startup_mongo_test.go b/internal/modules/stats/startup_mongo_test.go
index 727dd21..0092aa4 100644
--- a/internal/modules/stats/startup_mongo_test.go
+++ b/internal/modules/stats/startup_mongo_test.go
@@ -122,3 +122,8 @@ func TestInitStore_MongoSwapsNoituCommandStats(t *testing.T) {
 	_, provider := setupMongoStatsProvider(t)
 	assertNoituCommandSwap(t, provider)
 }
+
+func TestInitStore_MongoRetiresMonkeydCommandStats(t *testing.T) {
+	_, provider := setupMongoStatsProvider(t)
+	assertMonkeydCommandsRetired(t, provider)
+}
diff --git a/internal/modules/stats/startup_test.go b/internal/modules/stats/startup_test.go
index 8a40972..c15228d 100644
--- a/internal/modules/stats/startup_test.go
+++ b/internal/modules/stats/startup_test.go
@@ -146,3 +146,93 @@ func TestMigrateCommandRename_MergesAndKeepsDeletedFlag(t *testing.T) {
 		t.Errorf("user = %+v", got)
 	}
 }
+
+func TestInitStore_RetiresMonkeydCommandStats(t *testing.T) {
+	assertMonkeydCommandsRetired(t, storage.NewMemoryProvider())
+}
+
+// assertMonkeydCommandsRetired seeds history for the removed monkeyd commands,
+// runs the startup migration twice, and checks that their rows are kept with
+// deleted: true, hidden from every view, and never counted again, while a
+// command that only shares the prefix is untouched. A row written after the
+// marker exists is left alone, which shows the marker guards the step.
+func assertMonkeydCommandsRetired(t *testing.T, provider storage.Provider) {
+	t.Helper()
+	ctx := context.Background()
+	statsColl := provider.Collection("stats")
+	systemColl := provider.Collection(systemstate.CollectionName)
+	docs := storage.Typed[usageEntry](statsColl)
+
+	seeds := map[string]usageEntry{
+		usageKey("monkeyd_crawl", 0):       {Cmd: "monkeyd_crawl", N: 4},
+		usageKey("monkeyd_crawl", 7):       {Cmd: "monkeyd_crawl", UserID: 7, Username: "alice", N: 3},
+		usageKey("monkeyd_tags", 8):        {Cmd: "monkeyd_tags", UserID: 8, Username: "bob", N: 2, Deleted: true},
+		usageKey("monkeyd_crawl_extra", 7): {Cmd: "monkeyd_crawl_extra", UserID: 7, Username: "alice", N: 1},
+		usageKey("ping", 7):                {Cmd: "ping", UserID: 7, Username: "alice", N: 5},
+	}
+	for key, entry := range seeds {
+		if err := docs.Put(ctx, key, entry); err != nil {
+			t.Fatalf("seed %s: %v", key, err)
+		}
+	}
+
+	for run := range 2 {
+		if err := InitStore(ctx, statsColl, systemColl); err != nil {
+			t.Fatalf("InitStore run %d: %v", run+1, err)
+		}
+	}
+
+	for key, seed := range seeds {
+		got, _, err := docs.Get(ctx, key)
+		if err != nil {
+			t.Fatalf("get %s: %v", key, err)
+		}
+		want := seed
+		want.Deleted = isRetiredCommand(seed.Cmd)
+		if got != want {
+			t.Errorf("%s = %+v, want %+v", key, got, want)
+		}
+	}
+	rec, ok, err := systemstate.New(systemColl).Get(ctx, "migration:stats-delete-monkeyd-v1")
+	if err != nil || !ok || rec.Status != "complete" || rec.Count != 2 {
+		t.Fatalf("marker = %+v ok=%v err=%v, want complete count 2", rec, ok, err)
+	}
+
+	store := newUsageStore(statsColl)
+	if err := store.Increment(ctx, "monkeyd_crawl", usageUser{ID: 7, Username: "alice"}, true); err != nil {
+		t.Fatalf("retired Increment: %v", err)
+	}
+	if got, _, err := docs.Get(ctx, usageKey("monkeyd_crawl", 7)); err != nil || got.N != 3 {
+		t.Fatalf("retired row after Increment = %+v err=%v, want n 3", got, err)
+	}
+	if rows, err := store.TopCommands(ctx, 10); err != nil || len(rows) != 2 || rows[0].display != "/ping" || rows[1].display != "/monkeyd_crawl_extra" {
+		t.Fatalf("top commands = %+v err=%v", rows, err)
+	}
+	if rows, err := store.TopUsers(ctx, 10); err != nil || len(rows) != 1 || rows[0].display != "@alice" || rows[0].n != 6 {
+		t.Fatalf("top users = %+v err=%v", rows, err)
+	}
+	if rows, found, err := store.CommandsByUser(ctx, "bob", 10); err != nil || found || len(rows) != 0 {
+		t.Fatalf("commands by retired-only user = %+v found=%v err=%v", rows, found, err)
+	}
+	for _, cmd := range []string{"monkeyd_crawl", "monkeyd_tags"} {
+		if rows, err := store.UsersByCommand(ctx, cmd, 10); err != nil || len(rows) != 0 {
+			t.Fatalf("users by %s = %+v err=%v", cmd, rows, err)
+		}
+	}
+
+	// A row that appears after the marker, say from the previous release
+	// still draining during a deploy, is not migrated again but stays hidden.
+	late := usageEntry{Cmd: "monkeyd_tags", UserID: 9, Username: "carol", N: 1}
+	if err := docs.Put(ctx, usageKey("monkeyd_tags", 9), late); err != nil {
+		t.Fatalf("seed late row: %v", err)
+	}
+	if err := InitStore(ctx, statsColl, systemColl); err != nil {
+		t.Fatalf("InitStore after marker: %v", err)
+	}
+	if got, _, err := docs.Get(ctx, usageKey("monkeyd_tags", 9)); err != nil || got != late {
+		t.Fatalf("late row = %+v err=%v, want untouched %+v", got, err, late)
+	}
+	if rows, err := store.TopUsers(ctx, 10); err != nil || len(rows) != 1 || rows[0].display != "@alice" {
+		t.Fatalf("top users with late retired row = %+v err=%v", rows, err)
+	}
+}
diff --git a/internal/modules/stats/usage_store.go b/internal/modules/stats/usage_store.go
index 99bf701..7619ec6 100644
--- a/internal/modules/stats/usage_store.go
+++ b/internal/modules/stats/usage_store.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"strconv"
 	"sync"
 
@@ -336,7 +337,7 @@ func (s *mongoUsageStore) Increment(ctx context.Context, cmd string, user usageU
 func (s *mongoUsageStore) TopCommands(ctx context.Context, limit int) ([]row, error) {
 	pipeline := mongo.Pipeline{
 		bson.D{bsonField("$match", bson.D{
-			bsonField("cmd", bson.D{bsonField("$type", "string"), bsonField("$ne", deletedStockDividendCommand)}),
+			bsonField("cmd", bson.D{bsonField("$type", "string"), bsonField("$nin", retiredCommands)}),
 			bsonField("deleted", bson.D{bsonField("$ne", true)}),
 		})},
 		bson.D{bsonField("$group", bson.D{bsonField("_id", "$cmd"), bsonField("n", bson.D{bsonField("$sum", "$n")})})},
@@ -370,7 +371,7 @@ func (s *mongoUsageStore) TopCommands(ctx context.Context, limit int) ([]row, er
 func (s *mongoUsageStore) TopUsers(ctx context.Context, limit int) ([]row, error) {
 	pipeline := mongo.Pipeline{
 		bson.D{bsonField("$match", bson.D{
-			bsonField("cmd", bson.D{bsonField("$ne", deletedStockDividendCommand)}),
+			bsonField("cmd", bson.D{bsonField("$nin", retiredCommands)}),
 			bsonField("uid", bson.D{bsonField("$gt", int64(0))}),
 			bsonField("user", bson.D{bsonField("$type", "string"), bsonField("$ne", "")}),
 			bsonField("deleted", bson.D{bsonField("$ne", true)}),
@@ -424,7 +425,7 @@ func (s *mongoUsageStore) CommandsByUser(ctx context.Context, username string, l
 	}
 	cur, err := s.coll.Find(ctx, bson.M{
 		"uid":     userID,
-		"cmd":     bson.M{"$type": "string", "$ne": deletedStockDividendCommand},
+		"cmd":     bson.M{"$type": "string", "$nin": retiredCommands},
 		"deleted": bson.M{"$ne": true},
 	}, opts)
 	if err != nil {
@@ -487,7 +488,7 @@ func (s *mongoUsageStore) userIDByUsername(ctx context.Context, username string)
 	}
 	err := s.coll.FindOne(ctx,
 		bson.M{
-			"cmd":     bson.M{"$ne": deletedStockDividendCommand},
+			"cmd":     bson.M{"$nin": retiredCommands},
 			"uid":     bson.M{"$gt": int64(0)},
 			"user":    username,
 			"deleted": bson.M{"$ne": true},
@@ -504,14 +505,19 @@ func (s *mongoUsageStore) userIDByUsername(ctx context.Context, username string)
 	return doc.UserID, true, nil
 }
 
-// deletedStockDividendCommand names the retired /stock_dividend command. Its
-// rows are kept with deleted: true as history.
-const deletedStockDividendCommand = "stock_dividend"
+// retiredCommands names commands removed from the bot. Their rows are kept
+// with deleted: true as history: /stock_dividend was split into the cash and
+// share dividend commands, and the monkeyd commands left with their module.
+var retiredCommands = []string{
+	"stock_dividend",
+	"monkeyd_crawl",
+	"monkeyd_tags",
+}
 
 // isRetiredCommand reports whether cmd has been removed from the bot. Its rows
 // are never incremented or shown, even ones missing the deleted flag.
 func isRetiredCommand(cmd string) bool {
-	return cmd == deletedStockDividendCommand
+	return slices.Contains(retiredCommands, cmd)
 }
 
 func withLimit(pipeline mongo.Pipeline, limit int) mongo.Pipeline {