mirror of
https://github.com/tiennm99/tiennm99bot.git
synced 2026-10-17 10:15:09 +00:00
feat(monkeyd)!: remove the monkeyd module
The novel-to-PDF export now lives in a separate bot, so /monkeyd_crawl and /monkeyd_tags are gone along with the fpdf dependency and the bundled font. Their stats rows are kept as history with deleted: true, set once at startup behind the migration:stats-delete-monkeyd-v1 marker, and a MODULES value that still lists monkeyd is skipped with a warning instead of failing startup.
This commit is contained in:
tiennm99 committed
1 parent
80e85012c7
commit
705cfda93a
41 files changed
+282
-3466
No files matched your search
+2
-2
@@ -15,8 +15,8 @@ MONGO_DATABASE=tiennm99bot
|
||||
# Comma-separated module list. Empty = load every module, including any module
|
||||
# added later — so list them explicitly when a deployment should only gain a new
|
||||
# module deliberately. wordledaily serves /wordle too; the retired name wordle
|
||||
# still loads it.
|
||||
MODULES=util,misc,random,amlich,wordledaily,loldle,noitu,lol,stock,gold,coin,stats,monkeyd,sticker,alias,blacklist,weather
|
||||
# still loads it. The removed module monkeyd is skipped with a warning.
|
||||
MODULES=util,misc,random,amlich,wordledaily,loldle,noitu,lol,stock,gold,coin,stats,sticker,alias,blacklist,weather
|
||||
# Telegram user id for owner-only commands.
|
||||
OWNER_ID=
|
||||
# Comma-separated admin Telegram user ids.
|
||||
|
||||
@@ -7,11 +7,6 @@
|
||||
in-memory otherwise, which is what tests and local no-database runs use. Read
|
||||
`README.md` before implementation work.
|
||||
|
||||
The `monkeyd` module's crawler, PDF renderer, and export flow are an in-tree
|
||||
port of `tiennm99/mttools/monkeyd-crawler` under
|
||||
`internal/modules/monkeyd/{crawler,pdf,export}`. Change them here; they no
|
||||
longer track the mttools copy.
|
||||
|
||||
`renderer/` is a separate Node 24 service (JavaScript + JSDoc, Remotion) that
|
||||
draws the `random` module's animations. It has its own `package.json`, tests,
|
||||
and `renderer/AGENTS.md`; run its npm commands from that folder. `compose.yml`
|
||||
|
||||
@@ -26,8 +26,6 @@ RUN CGO_ENABLED=0 GOOS=linux go build \
|
||||
# ffmpeg comes from Alpine's own package rather than a third-party static-ffmpeg
|
||||
# image, keeping the supply chain to bases already trusted by this build.
|
||||
FROM alpine:3.22
|
||||
# No fonts are installed here: the monkeyd module's PDF renderer falls back to a
|
||||
# font compiled into the binary when the host has none.
|
||||
RUN apk add --no-cache ffmpeg ca-certificates \
|
||||
&& addgroup -g 65532 -S nonroot \
|
||||
&& adduser -u 65532 -S -G nonroot nonroot
|
||||
|
||||
@@ -22,7 +22,6 @@ Atlas via long polling and an in-process cron scheduler.
|
||||
| `sticker` | `/addsticker` — append a replied sticker, image, video or GIF to one shared pack. See [docs/sticker-packs.md](docs/sticker-packs.md) |
|
||||
| `alias` | `/alias <name>` save a replied message under a name, then send it back with `/insert <name>`, bare `/<name>`, or inline `@botname <prefix>`; `/aliases` lists, `/unalias` deletes. See [docs/aliases.md](docs/aliases.md) |
|
||||
| `blacklist` | Per-topic text deny-list with whitelist exceptions: `/blacklist_add`, `/blacklist_del`, `/whitelist_add`, `/whitelist_del`, `/blacklist_rules` lists both, `/blacklist_check` judges a text, `/blacklist` does either, `/whitelist_rnd` picks a random exception. Passive — the bot never scans chat. See [docs/blacklist.md](docs/blacklist.md) |
|
||||
| `monkeyd` | `/monkeyd_crawl <url> [font_size]` export a monkeydd.com novel as a PDF, `/monkeyd_tags <url>` list its tags as hashtags |
|
||||
| `weather` | Weather from Open-Meteo: `/thoitiet` for the next 6 hours hour by hour, `/thoitiethomnay` for current conditions and today, `/thoitietngaymai` for tomorrow, `/thoitiettuannay` for the next 7 days. Each takes `[location... | lat,long]` (Vietnamese with or without diacritics, shorthand like `hcm`/`hn`, a foreign city, or decimal coordinates such as `10.74,106.72`); the default is Tân Thuận (formerly Q.7). Flood alerts for Tân Thuận (Q.7): `/thuyvan` shows the 5-day tide-peak forecast at Phú An and Nhà Bè (Đài KTTV Nam Bộ bulletin), the rain forecast, and live VNDMS river gauges within 30 km; `/thuyvan_subscribe` / `/thuyvan_unsubscribe` toggle a 10:30 ICT push (retried at 12:30 if that run fails) sent only when a forecast peak reaches báo động I (1.40 m) or rain reaches 50 mm |
|
||||
|
||||
Commands marked (admin) require a user ID in `ADMIN_IDS` or the owner; commands
|
||||
@@ -160,75 +159,6 @@ The stock-only `assets.<ticker>.openedAt` marker identifies the current position
|
||||
lifecycle, invalidates buttons after a full sale and later repurchase, and
|
||||
prevents a position opened after Record date from applying an older event.
|
||||
|
||||
### Novel PDF export
|
||||
|
||||
`/monkeyd_crawl <url> [font_size]` downloads every chapter of a monkeydd.com
|
||||
novel and sends it back as a single PDF document, sized for reading on a phone.
|
||||
The crawler, the PDF renderer, and the export flow live in
|
||||
`internal/modules/monkeyd/{crawler,pdf,export}`, ported from
|
||||
[mttools/monkeyd-crawler](https://github.com/tiennm99/mttools/tree/main/monkeyd-crawler)
|
||||
and trimmed to what the bot uses: a fixed phone page and the bundled DejaVu
|
||||
Sans font.
|
||||
|
||||
`font_size` is the body text size in points and accepts half points. It ranges
|
||||
from 6 to 24 and defaults to the crawler's own default of 10, which fits roughly
|
||||
43 characters per line across 26 lines on the 90×160 mm page. Larger values
|
||||
trade characters per line for legibility: 12 gives about 36. Headings and the
|
||||
title page scale with it. Omitting the argument passes no size at all, so the
|
||||
crawler's default applies rather than a second one defined here. The document
|
||||
caption reports the size that was used.
|
||||
|
||||
The command is public, so any member of a chat can run it. One invocation makes
|
||||
hundreds of outbound requests spread over several minutes, so two things bound
|
||||
the cost and are deliberate: only `monkeydd.com` URLs are accepted, and exactly
|
||||
one export runs at a time across the whole bot. A missing scheme is filled in,
|
||||
so a pasted bare hostname works.
|
||||
|
||||
The host allowlist is not only about the parser — the extractor is written
|
||||
against that site's markup and would find nothing elsewhere — it also stops the
|
||||
command being used to make the bot fetch arbitrary URLs.
|
||||
|
||||
The bot replies immediately that the export started, then sends the PDF when it
|
||||
is ready; a second request while one is in flight is told which novel is
|
||||
currently running. Requests are spaced out by the crawler, so a run is
|
||||
deliberately slow rather than aggressive.
|
||||
|
||||
Raw pages are cached under the system temp directory, so re-exporting the same
|
||||
novel costs no requests. The cache is not pruned and a container restart clears
|
||||
it. Finished PDFs are deleted after upload. Telegram caps bot uploads at 50 MB;
|
||||
a larger book is reported instead of being sent.
|
||||
|
||||
The PDF embeds a font covering Vietnamese diacritics. The crawler prefers a
|
||||
system font and falls back to one compiled into the binary, so the runtime
|
||||
image needs no fonts installed.
|
||||
|
||||
### Novel tags
|
||||
|
||||
`/monkeyd_tags <url>` reports a novel's tags as a hashtag line, sent as a code
|
||||
block so it can be copied in one tap:
|
||||
|
||||
```text
|
||||
#MonkeyD #CổĐại #GiaĐình
|
||||
|
||||
https://monkeydd.com/truong-an-gwem.html
|
||||
```
|
||||
|
||||
Each label becomes one hashtag with spaces and punctuation removed and every
|
||||
word capitalised, because Telegram ends a hashtag at the first character that is
|
||||
not a letter, digit, or underscore. Diacritics are kept. A label with no letters
|
||||
is dropped rather than emitted as a bare `#`.
|
||||
|
||||
The tags are the novel's own genres, identified by their `itemprop="genre"`
|
||||
microdata. The same page also links every genre on the site as navigation — 69
|
||||
of them against one novel's 6 in a sampled page — so matching the category URL
|
||||
shape instead would return the whole menu.
|
||||
|
||||
This command costs a single request and runs inline rather than in the
|
||||
background, bounded by a short timeout: handlers are dispatched one at a time, so
|
||||
a stalled fetch would hold up every other command. It shares the export page
|
||||
cache, so a lookup for a novel that was already exported costs no request at
|
||||
all.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
|
||||
@@ -73,8 +73,6 @@ func TestCommandDiscovery_AllPublicCommandsHaveSafeMetadata(t *testing.T) {
|
||||
"loldle": "[champion]",
|
||||
"loldle_giveup": "",
|
||||
"loldle_stats": "",
|
||||
"monkeyd_crawl": "<url> [font_size]",
|
||||
"monkeyd_tags": "<url>",
|
||||
"noitu": "",
|
||||
"noitubot": "",
|
||||
"noitutop": "",
|
||||
|
||||
+14
-3
@@ -33,7 +33,6 @@ import (
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/lol"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/loldle"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/misc"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/monkeyd"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/noitu"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/random"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/stats"
|
||||
@@ -99,7 +98,6 @@ func factories() map[string]modules.Factory {
|
||||
"misc": misc.New,
|
||||
"random": random.New,
|
||||
"amlich": amlich.New,
|
||||
"monkeyd": monkeyd.New,
|
||||
wordledaily.ShortName: wordledaily.New,
|
||||
noitu.ShortName: noitu.New,
|
||||
loldle.ShortName: loldle.New,
|
||||
@@ -422,12 +420,25 @@ var moduleAliases = map[string]string{
|
||||
wordledaily.LegacyShortName: wordledaily.ShortName,
|
||||
}
|
||||
|
||||
// removedModules names catalog keys that were deleted without a successor.
|
||||
// A MODULES value written before the removal skips them with a warning
|
||||
// instead of failing startup on an unknown module.
|
||||
var removedModules = map[string]bool{
|
||||
"monkeyd": true,
|
||||
}
|
||||
|
||||
// normalizeModules replaces retired module names in a MODULES list with
|
||||
// their successors, dropping duplicates that creates.
|
||||
// their successors, drops removed modules, and drops duplicates that creates.
|
||||
// A list naming only removed modules comes back empty and so loads every
|
||||
// module, the same as an unset MODULES.
|
||||
func normalizeModules(names []string) []string {
|
||||
out := make([]string, 0, len(names))
|
||||
seen := map[string]bool{}
|
||||
for _, name := range names {
|
||||
if removedModules[name] {
|
||||
log.Warn("MODULES names a removed module; skipping it", "module", name)
|
||||
continue
|
||||
}
|
||||
if to, ok := moduleAliases[name]; ok {
|
||||
log.Warn("MODULES names a retired module; loading its successor", "module", name, "successor", to)
|
||||
name = to
|
||||
|
||||
@@ -201,6 +201,25 @@ func TestNormalizeModules_MapsRetiredWordleToWordleDaily(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The monkeyd module was deleted: a MODULES value still naming it loads the
|
||||
// rest of the list instead of failing on an unknown module.
|
||||
func TestNormalizeModules_DropsRemovedMonkeyd(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"util,monkeyd,stats": "util stats",
|
||||
"monkeyd,misc": "misc",
|
||||
} {
|
||||
if got := strings.Join(normalizeModules(splitCSV(in)), " "); got != want {
|
||||
t.Errorf("normalizeModules(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
if _, ok := factories()["monkeyd"]; ok {
|
||||
t.Fatal("the removed monkeyd module is still in the catalog")
|
||||
}
|
||||
if _, err := modules.Build(normalizeModules(splitCSV("util,monkeyd,stats")), factories(), storage.NewMemoryProvider(), modules.BuildOptions{}); err != nil {
|
||||
t.Fatalf("Build with removed module: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An empty MODULES loads every module, so a command name that collides with an
|
||||
// existing module surfaces here as a test failure rather than as a startup
|
||||
// crash on deploy.
|
||||
|
||||
@@ -39,7 +39,7 @@ Copy [`.env.example`](../.env.example) → `.env` (gitignored) and fill in.
|
||||
| `TELEGRAM_BOT_TOKEN` | ✅ | from @BotFather; startup fails without it |
|
||||
| `MONGO_URL` | ✅ | Atlas SRV string **incl. credentials** — secret, never logged |
|
||||
| `MONGO_DATABASE` | ✅ | e.g. `tiennm99bot` |
|
||||
| `MODULES` | optional | CSV; empty = all modules, including any added later. The retired `wordle` name loads `wordledaily`, which now serves `/wordle` |
|
||||
| `MODULES` | optional | CSV; empty = all modules, including any added later. The retired `wordle` name loads `wordledaily`, which now serves `/wordle`; the removed `monkeyd` name is skipped with a warning |
|
||||
| `OWNER_ID` | optional | Telegram user id for owner-only commands, the deploy DM, and the `/addsticker` pack owner. Unset = owner-only commands are denied and `/addsticker` refuses |
|
||||
| `ADMIN_IDS` | optional | CSV of Telegram user ids for admin-only commands |
|
||||
| `BOT_USERNAME` | optional | the bot's Telegram username, without `@`; unset = asked from Telegram (`getMe`) once at startup. Used for the default sticker pack name and the lol User-Agent |
|
||||
@@ -161,9 +161,10 @@ MP4, with the same text fallback.
|
||||
- **`coin`** stores cash as `usd` and embeds positions as
|
||||
`assets.<symbol>.{quantity,base}`.
|
||||
- **`system`** holds one marker per completed one-time startup migration. Keep
|
||||
those records as audit history. No one-time migration runs at startup now;
|
||||
the completed ones were removed from the code once production data was
|
||||
verified migrated.
|
||||
those records as audit history. The stats command renames and the
|
||||
retirement of the removed `monkeyd` commands still run at startup and skip
|
||||
once their marker is complete; earlier completed migrations were removed
|
||||
from the code once production data was verified migrated.
|
||||
|
||||
## 2. Coolify
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ module github.com/tiennm99/tiennm99bot
|
||||
go 1.26.5
|
||||
|
||||
require (
|
||||
github.com/go-pdf/fpdf v0.9.0
|
||||
github.com/go-telegram/bot v1.20.0
|
||||
github.com/ledongthuc/pdf v0.0.0-20260907135840-6c8c28e0e8a0
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
@@ -11,8 +10,6 @@ require (
|
||||
github.com/testcontainers/testcontainers-go/modules/mongodb v0.43.0
|
||||
go.mongodb.org/mongo-driver/v2 v2.7.0
|
||||
golang.org/x/image v0.45.0
|
||||
golang.org/x/net v0.57.0
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/text v0.41.0
|
||||
)
|
||||
|
||||
@@ -69,6 +66,7 @@ require (
|
||||
go.opentelemetry.io/otel/metric v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.41.0 // indirect
|
||||
golang.org/x/crypto v0.54.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -41,8 +41,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-pdf/fpdf v0.9.0 h1:PPvSaUuo1iMi9KkaAn90NuKi+P4gwMedWPHhj8YlJQw=
|
||||
github.com/go-pdf/fpdf v0.9.0/go.mod h1:oO8N111TkmKb9D7VvWGLvLJlaZUQVPM+6V42pp3iV4Y=
|
||||
github.com/go-telegram/bot v1.20.0 h1:4Pea/qTidSspr4WBJw9FbHUMNhYeqszBqQUfsQEyFbc=
|
||||
github.com/go-telegram/bot v1.20.0/go.mod h1:i2TRs7fXWIeaceF3z7KzsMt/he0TwkVC680mvdTFYeM=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
@@ -145,8 +143,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
package monkeyd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Font size bounds for the optional argument. The body font drives the heading
|
||||
// and title sizes too, and the default page is only 78mm wide between margins,
|
||||
// so the useful range is narrow: at 6pt a line holds about 72 characters, at
|
||||
// 24pt about 18.
|
||||
const (
|
||||
minFontSize = 6.0
|
||||
maxFontSize = 24.0
|
||||
)
|
||||
|
||||
var errTooManyArgs = errors.New("too many arguments")
|
||||
|
||||
// crawlArgs is a validated /monkeyd_crawl invocation.
|
||||
type crawlArgs struct {
|
||||
// NovelURL is normalised and known to be on an allowed host.
|
||||
NovelURL string
|
||||
|
||||
// FontSize is the requested body font size in points, or 0 to leave the
|
||||
// crawler's default in place.
|
||||
FontSize float64
|
||||
}
|
||||
|
||||
// parseCrawlArgs validates the text after the command: a novel URL, optionally
|
||||
// followed by a body font size.
|
||||
func parseCrawlArgs(arg string) (crawlArgs, error) {
|
||||
fields := strings.Fields(arg)
|
||||
if len(fields) == 0 {
|
||||
return crawlArgs{}, errNotAURL
|
||||
}
|
||||
if len(fields) > 2 {
|
||||
return crawlArgs{}, errTooManyArgs
|
||||
}
|
||||
|
||||
novelURL, err := normalizeNovelURL(fields[0])
|
||||
if err != nil {
|
||||
return crawlArgs{}, err
|
||||
}
|
||||
parsed := crawlArgs{NovelURL: novelURL}
|
||||
|
||||
if len(fields) == 2 {
|
||||
if parsed.FontSize, err = parseFontSize(fields[1]); err != nil {
|
||||
return crawlArgs{}, err
|
||||
}
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
// parseFontSize accepts a font size in points, including half points, and
|
||||
// rejects anything outside the readable range.
|
||||
func parseFontSize(raw string) (float64, error) {
|
||||
size, err := strconv.ParseFloat(raw, 64)
|
||||
// ParseFloat accepts "NaN" and "Inf", neither of which is a font size, and
|
||||
// both would reach the renderer as a silently broken layout.
|
||||
if err != nil || math.IsNaN(size) || math.IsInf(size, 0) {
|
||||
return 0, fmt.Errorf("font size %q is not a number", raw)
|
||||
}
|
||||
if size < minFontSize || size > maxFontSize {
|
||||
return 0, fmt.Errorf("font size must be between %g and %g points", minFontSize, maxFontSize)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
package monkeyd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseCrawlArgs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
arg string
|
||||
wantURL string
|
||||
wantFontSize float64
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "url only leaves the font size unset",
|
||||
arg: testNovelURL,
|
||||
wantURL: testNovelURL,
|
||||
wantFontSize: 0,
|
||||
},
|
||||
{
|
||||
name: "url with font size",
|
||||
arg: testNovelURL + " 14",
|
||||
wantURL: testNovelURL,
|
||||
wantFontSize: 14,
|
||||
},
|
||||
{
|
||||
name: "half point font size",
|
||||
arg: testNovelURL + " 10.5",
|
||||
wantURL: testNovelURL,
|
||||
wantFontSize: 10.5,
|
||||
},
|
||||
{
|
||||
name: "extra whitespace between arguments",
|
||||
arg: " " + testNovelURL + " 12 ",
|
||||
wantURL: testNovelURL,
|
||||
wantFontSize: 12,
|
||||
},
|
||||
{
|
||||
name: "bare host gets a scheme",
|
||||
arg: "monkeydd.com/novel.html 9",
|
||||
wantURL: "https://monkeydd.com/novel.html",
|
||||
wantFontSize: 9,
|
||||
},
|
||||
{
|
||||
name: "font size at the lower bound",
|
||||
arg: testNovelURL + " 6",
|
||||
wantURL: testNovelURL,
|
||||
wantFontSize: minFontSize,
|
||||
},
|
||||
{
|
||||
name: "font size at the upper bound",
|
||||
arg: testNovelURL + " 24",
|
||||
wantURL: testNovelURL,
|
||||
wantFontSize: maxFontSize,
|
||||
},
|
||||
{
|
||||
name: "empty argument",
|
||||
arg: "",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "font size below the lower bound",
|
||||
arg: testNovelURL + " 5",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "font size above the upper bound",
|
||||
arg: testNovelURL + " 25",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "zero font size",
|
||||
arg: testNovelURL + " 0",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "negative font size",
|
||||
arg: testNovelURL + " -12",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "non-numeric font size",
|
||||
arg: testNovelURL + " big",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "NaN font size",
|
||||
arg: testNovelURL + " NaN",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "infinite font size",
|
||||
arg: testNovelURL + " Inf",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "too many arguments",
|
||||
arg: testNovelURL + " 12 a5",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "disallowed host",
|
||||
arg: "https://example.com/novel.html 12",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseCrawlArgs(tt.arg)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("parseCrawlArgs(%q) = %+v, want error", tt.arg, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("parseCrawlArgs(%q) returned error: %v", tt.arg, err)
|
||||
}
|
||||
if got.NovelURL != tt.wantURL {
|
||||
t.Errorf("NovelURL = %q, want %q", got.NovelURL, tt.wantURL)
|
||||
}
|
||||
if got.FontSize != tt.wantFontSize {
|
||||
t.Errorf("FontSize = %v, want %v", got.FontSize, tt.wantFontSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The out-of-range message must state the bounds; "invalid font size" alone
|
||||
// leaves the user guessing what to type instead.
|
||||
func TestParseFontSizeErrorNamesTheBounds(t *testing.T) {
|
||||
_, err := parseFontSize("99")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for 99")
|
||||
}
|
||||
for _, want := range []string{"6", "24"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("error %q does not mention the bound %q", err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// contentElementID is the container holding a chapter's body text.
|
||||
const contentElementID = "chapter-content-render"
|
||||
|
||||
// Chapter is one fetched chapter, reduced to plain paragraphs.
|
||||
type Chapter struct {
|
||||
Label string
|
||||
URL string
|
||||
Paragraphs []string
|
||||
}
|
||||
|
||||
// Heading is the chapter title to print. Labels are often a bare number, which
|
||||
// reads poorly as a heading, so those get the Vietnamese word for "chapter".
|
||||
func (c *Chapter) Heading() string {
|
||||
label := strings.TrimSpace(c.Label)
|
||||
if label == "" {
|
||||
return "Chương"
|
||||
}
|
||||
if _, err := strconv.Atoi(label); err == nil {
|
||||
return "Chương " + label
|
||||
}
|
||||
return label
|
||||
}
|
||||
|
||||
// WordCount is a rough word count, used to sanity-check extraction.
|
||||
func (c *Chapter) WordCount() int {
|
||||
n := 0
|
||||
for _, p := range c.Paragraphs {
|
||||
n += len(strings.Fields(p))
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// blockTags end the current paragraph when opened or closed.
|
||||
var blockTags = map[string]bool{
|
||||
"p": true, "div": true, "br": true, "hr": true, "blockquote": true,
|
||||
"h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true,
|
||||
"li": true, "ul": true, "ol": true, "tr": true,
|
||||
}
|
||||
|
||||
// skipTags never contribute prose.
|
||||
//
|
||||
// Anchors are included because inside a chapter body they are always site
|
||||
// chrome: the prev/next chapter navigation and the sponsor call-to-action are
|
||||
// both links, while novel prose never needs one. Inline emphasis tags (b, i,
|
||||
// em) are deliberately absent so italics in the prose survive; the site's icons
|
||||
// use <i> but carry no text.
|
||||
var skipTags = map[string]bool{
|
||||
"script": true, "style": true, "noscript": true, "iframe": true,
|
||||
"ins": true, "form": true, "select": true, "button": true, "textarea": true,
|
||||
"a": true, "img": true, "svg": true,
|
||||
}
|
||||
|
||||
// junkClasses marks containers the site injects into the chapter body. Their
|
||||
// whole subtree is dropped.
|
||||
//
|
||||
// These are matched on class rather than position because the blocks move: the
|
||||
// sponsor block opens the body on most chapters but is absent on others, and
|
||||
// the watermark is planted at a different paragraph in every chapter. Only
|
||||
// site-specific class names are listed; generic Bootstrap utilities such as
|
||||
// "my-4" or "text-center" are not, since prose could legitimately carry them.
|
||||
//
|
||||
// Note the sibling class "actac" is NOT junk: it wraps the real chapter text and
|
||||
// carries style="display:none", because the site gates the body behind a click
|
||||
// on the sponsor link and reveals it with JavaScript. Skipping hidden elements,
|
||||
// or skipping "act*" as a family, would therefore discard the whole chapter.
|
||||
var junkClasses = map[string]bool{
|
||||
"actcl": true, // sponsor block shown in place of the gated chapter body
|
||||
"signature": true, // "[Truyện được đăng tải duy nhất tại ...]" source watermark
|
||||
}
|
||||
|
||||
// hasJunkClass reports whether a node is an injected non-prose container.
|
||||
func hasJunkClass(n *html.Node) bool {
|
||||
for _, tok := range strings.Fields(attr(n, "class")) {
|
||||
if junkClasses[tok] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ParseChapter extracts a chapter's paragraphs, restoring the words the site
|
||||
// serves through CSS :before rules instead of markup.
|
||||
func ParseChapter(page []byte, ref ChapterRef) (*Chapter, error) {
|
||||
doc, err := html.Parse(bytes.NewReader(page))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse chapter %s: %w", ref.URL, err)
|
||||
}
|
||||
content := elementByID(doc, contentElementID)
|
||||
if content == nil {
|
||||
return nil, fmt.Errorf("chapter %s: no #%s container (page layout may have changed)",
|
||||
ref.URL, contentElementID)
|
||||
}
|
||||
|
||||
ch := &Chapter{
|
||||
Label: ref.Label,
|
||||
URL: ref.URL,
|
||||
Paragraphs: extractParagraphs(content, ParseWordClasses(page)),
|
||||
}
|
||||
if len(ch.Paragraphs) == 0 {
|
||||
return nil, fmt.Errorf("chapter %s: extracted no text", ref.URL)
|
||||
}
|
||||
ch.Paragraphs = dropRepeatedTitle(ch.Paragraphs, ref.Label)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
// extractParagraphs walks the content subtree into plain paragraphs, replacing
|
||||
// each word-carrying element with the word its CSS rule injects.
|
||||
func extractParagraphs(content *html.Node, words map[string]string) []string {
|
||||
var b strings.Builder
|
||||
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
switch n.Type {
|
||||
case html.TextNode:
|
||||
// The HTML parser has already decoded entities such as ư.
|
||||
b.WriteString(n.Data)
|
||||
return
|
||||
case html.ElementNode:
|
||||
if skipTags[n.Data] || hasJunkClass(n) {
|
||||
return
|
||||
}
|
||||
// These elements are empty in the markup; the CSS word replaces them.
|
||||
if word, ok := injectedWord(n, words); ok {
|
||||
b.WriteString(word)
|
||||
return
|
||||
}
|
||||
if blockTags[n.Data] {
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
|
||||
if n.Type == html.ElementNode && blockTags[n.Data] {
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
walk(content)
|
||||
|
||||
var paragraphs []string
|
||||
for _, line := range strings.Split(b.String(), "\n") {
|
||||
if p := collapseSpaces(line); p != "" {
|
||||
paragraphs = append(paragraphs, p)
|
||||
}
|
||||
}
|
||||
return paragraphs
|
||||
}
|
||||
|
||||
// injectedWord returns the word a node's class supplies via CSS, if any.
|
||||
func injectedWord(n *html.Node, words map[string]string) (string, bool) {
|
||||
class := attr(n, "class")
|
||||
if class == "" {
|
||||
return "", false
|
||||
}
|
||||
for _, tok := range strings.Fields(class) {
|
||||
if word, ok := words[tok]; ok {
|
||||
return word, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// dropRepeatedTitle removes a leading paragraph that only repeats the chapter
|
||||
// label, since the export prints its own heading.
|
||||
func dropRepeatedTitle(paragraphs []string, label string) []string {
|
||||
if len(paragraphs) < 2 {
|
||||
return paragraphs
|
||||
}
|
||||
first := strings.TrimSpace(paragraphs[0])
|
||||
if strings.EqualFold(first, strings.TrimSpace(label)) {
|
||||
return paragraphs[1:]
|
||||
}
|
||||
return paragraphs
|
||||
}
|
||||
@@ -1,200 +0,0 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// chapterFixture mirrors the real page shape: words split between markup and
|
||||
// CSS :before rules, HTML entities, spacer paragraphs and an ad script
|
||||
// inside the content container.
|
||||
const chapterFixture = `<!DOCTYPE html><html><head>
|
||||
<style>
|
||||
.t-aaa:before { content: "v\1ecb "; }
|
||||
.j-bbb:before { content: "tr\1ed3 "; }
|
||||
.z-ccc::before{content:"n\E0ng";}
|
||||
.unused-ddd:before { content: "khong-dung"; }
|
||||
</style></head><body>
|
||||
<h1 class="card-title">TEN TRUYEN - 1</h1>
|
||||
<div class="content-container" id="chapter-content-render">
|
||||
<p>1</p>
|
||||
<p> </p>
|
||||
<p>Nghe <span class="t-aaa"></span> trưởng tử noi.</p>
|
||||
<p> </p>
|
||||
<p>Một <span class="j-bbb"></span> và <span class="z-ccc"></span> di.</p>
|
||||
<script>ads();</script>
|
||||
</div></body></html>`
|
||||
|
||||
func TestParseWordClassesDecodesEscapes(t *testing.T) {
|
||||
words := ParseWordClasses([]byte(chapterFixture))
|
||||
|
||||
for class, want := range map[string]string{
|
||||
"t-aaa": "vị",
|
||||
"j-bbb": "trồ",
|
||||
"z-ccc": "nàng",
|
||||
} {
|
||||
if got := words[class]; got != want {
|
||||
t.Errorf("words[%q] = %q, want %q", class, got, want)
|
||||
}
|
||||
}
|
||||
if len(words) != 4 {
|
||||
t.Errorf("got %d rules, want 4", len(words))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseChapterRestoresCSSWords(t *testing.T) {
|
||||
ch, err := ParseChapter([]byte(chapterFixture), ChapterRef{Label: "1", URL: "http://x/1.html"})
|
||||
if err != nil {
|
||||
t.Fatalf("ParseChapter: %v", err)
|
||||
}
|
||||
|
||||
want := []string{
|
||||
"Nghe vị trưởng tử noi.",
|
||||
"Một trồ và nàng di.",
|
||||
}
|
||||
if len(ch.Paragraphs) != len(want) {
|
||||
t.Fatalf("got %d paragraphs %q, want %d", len(ch.Paragraphs), ch.Paragraphs, len(want))
|
||||
}
|
||||
for i, w := range want {
|
||||
if ch.Paragraphs[i] != w {
|
||||
t.Errorf("paragraph %d = %q, want %q", i, ch.Paragraphs[i], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The CSS-injected words are the difference between real text and text with
|
||||
// silent holes, so guard against a regression that drops them.
|
||||
func TestParseChapterWithoutCSSWouldLoseWords(t *testing.T) {
|
||||
withoutCSS := strings.Replace(chapterFixture, `.t-aaa:before { content: "v\1ecb "; }`, "", 1)
|
||||
|
||||
ch, err := ParseChapter([]byte(withoutCSS), ChapterRef{Label: "1", URL: "http://x/1.html"})
|
||||
if err != nil {
|
||||
t.Fatalf("ParseChapter: %v", err)
|
||||
}
|
||||
if strings.Contains(ch.Paragraphs[0], "vị") {
|
||||
t.Fatal("word appeared without its CSS rule; fixture no longer proves anything")
|
||||
}
|
||||
if want := "Nghe trưởng tử noi."; ch.Paragraphs[0] != want {
|
||||
t.Errorf("paragraph 0 = %q, want %q", ch.Paragraphs[0], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseChapterDropsScriptsAndSpacers(t *testing.T) {
|
||||
ch, err := ParseChapter([]byte(chapterFixture), ChapterRef{Label: "1", URL: "http://x/1.html"})
|
||||
if err != nil {
|
||||
t.Fatalf("ParseChapter: %v", err)
|
||||
}
|
||||
for _, p := range ch.Paragraphs {
|
||||
if strings.Contains(p, "ads()") {
|
||||
t.Errorf("script text leaked into paragraph %q", p)
|
||||
}
|
||||
if strings.TrimSpace(p) == "" {
|
||||
t.Error("empty spacer paragraph was kept")
|
||||
}
|
||||
if strings.Contains(p, " ") {
|
||||
t.Errorf("non-breaking space survived in %q", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The leading "<p>1</p>" repeats the chapter label and would print twice.
|
||||
func TestParseChapterDropsRepeatedTitle(t *testing.T) {
|
||||
ch, err := ParseChapter([]byte(chapterFixture), ChapterRef{Label: "1", URL: "http://x/1.html"})
|
||||
if err != nil {
|
||||
t.Fatalf("ParseChapter: %v", err)
|
||||
}
|
||||
if ch.Paragraphs[0] == "1" {
|
||||
t.Error("repeated chapter label was kept as a paragraph")
|
||||
}
|
||||
}
|
||||
|
||||
// gatedChapterFixture mirrors how most chapters are served: a visible sponsor
|
||||
// block (div.actcl) stands in for the body, while the real text sits in a
|
||||
// sibling div.actac hidden with display:none and revealed by the site's
|
||||
// JavaScript. A source watermark and prev/next navigation bracket the prose.
|
||||
const gatedChapterFixture = `<!DOCTYPE html><html><head>
|
||||
<style>.t-aaa:before { content: "v\1ecb"; }</style></head><body>
|
||||
<div class="content-container" id="chapter-content-render">
|
||||
<div class="actcl">
|
||||
<h4 class="text-center text-primary">Moi Quy doc gia CLICK vao lien ket</h4>
|
||||
<p class="text-center">mo ung dung Shopee, sau do quay tro lai de doc!</p>
|
||||
<a class="btn btn-primary px-3" href="https://s.shopee.vn/xxxx">
|
||||
<img src="x.jpg"><span class="text-uppercase text-danger">CLICK</span></a>
|
||||
<h4 class="text-center text-primary">MonkeyD va doi ngu Editor xin chan thanh cam on!</h4>
|
||||
</div>
|
||||
<div class="actac" style=" display:none; ">
|
||||
<p>Doan van dau tien co <span class="t-aaa"></span> tri.</p>
|
||||
<p> </p>
|
||||
<p class="signature">[Truyen duoc dang tai duy nhat tai MonkeyDD.com - https://monkeydd.com/n/10.html.]</p>
|
||||
<p>Doan van cuoi cung.</p>
|
||||
</div>
|
||||
<div class="my-4"><div class="d-flex justify-content-center">
|
||||
<a class="btn btn-primary px-3 me-2" href="/n/9.html"><i class="bx bx-chevron-left"></i>Chương trước</a>
|
||||
<a class="btn btn-primary px-3" href="/n/11.html">Chương sau<i class="bx bx-chevron-right"></i></a>
|
||||
</div></div>
|
||||
</div></body></html>`
|
||||
|
||||
// The gated body is the one thing that must survive: it is hidden with
|
||||
// display:none, so any rule that drops hidden or "act*" containers silently
|
||||
// discards the entire chapter.
|
||||
func TestParseChapterKeepsGatedBody(t *testing.T) {
|
||||
ch, err := ParseChapter([]byte(gatedChapterFixture), ChapterRef{Label: "10", URL: "http://x/10.html"})
|
||||
if err != nil {
|
||||
t.Fatalf("ParseChapter: %v", err)
|
||||
}
|
||||
|
||||
want := []string{
|
||||
"Doan van dau tien co vị tri.",
|
||||
"Doan van cuoi cung.",
|
||||
}
|
||||
if len(ch.Paragraphs) != len(want) {
|
||||
t.Fatalf("got %d paragraphs %q, want %d", len(ch.Paragraphs), ch.Paragraphs, len(want))
|
||||
}
|
||||
for i, w := range want {
|
||||
if ch.Paragraphs[i] != w {
|
||||
t.Errorf("paragraph %d = %q, want %q", i, ch.Paragraphs[i], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Everything the site injects around the prose must be gone.
|
||||
func TestParseChapterDropsInjectedBlocks(t *testing.T) {
|
||||
ch, err := ParseChapter([]byte(gatedChapterFixture), ChapterRef{Label: "10", URL: "http://x/10.html"})
|
||||
if err != nil {
|
||||
t.Fatalf("ParseChapter: %v", err)
|
||||
}
|
||||
body := strings.Join(ch.Paragraphs, "\n")
|
||||
|
||||
for _, junk := range []string{
|
||||
"Shopee", // sponsor copy
|
||||
"CLICK", // sponsor call to action
|
||||
"cam on", // sponsor sign-off
|
||||
"MonkeyDD.com", // source watermark
|
||||
"Chương trước", // navigation
|
||||
"Chương sau", // navigation
|
||||
} {
|
||||
if strings.Contains(body, junk) {
|
||||
t.Errorf("injected text %q survived extraction in %q", junk, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseChapterMissingContainer(t *testing.T) {
|
||||
if _, err := ParseChapter([]byte(`<html><body><p>hi</p></body></html>`),
|
||||
ChapterRef{URL: "http://x/1.html"}); err == nil {
|
||||
t.Fatal("want an error when the content container is absent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChapterHeading(t *testing.T) {
|
||||
for _, tc := range []struct{ label, want string }{
|
||||
{"14", "Chương 14"},
|
||||
{"Chương 12", "Chương 12"},
|
||||
{"", "Chương"},
|
||||
} {
|
||||
ch := &Chapter{Label: tc.label}
|
||||
if got := ch.Heading(); got != tc.want {
|
||||
t.Errorf("Heading(%q) = %q, want %q", tc.label, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The site rejects requests without a browser-like User-Agent.
|
||||
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||
|
||||
// maxPageSize caps how much of a response we buffer; chapter pages are ~130 KB.
|
||||
const maxPageSize = 8 << 20
|
||||
|
||||
// statusError reports an unexpected HTTP status so Get can decide whether
|
||||
// retrying is worthwhile.
|
||||
type statusError struct {
|
||||
code int
|
||||
status string
|
||||
}
|
||||
|
||||
func (e *statusError) Error() string { return "unexpected status " + e.status }
|
||||
|
||||
// retryable is true for transient failures. A 404 or 403 will not fix itself,
|
||||
// so those fail immediately instead of burning the retry budget.
|
||||
func (e *statusError) retryable() bool {
|
||||
return e.code == http.StatusTooManyRequests || e.code >= 500
|
||||
}
|
||||
|
||||
// Client fetches pages from monkeydd.com. It spaces requests out by a fixed
|
||||
// delay no matter how many goroutines call Get, so raising the worker count
|
||||
// never raises the request rate, and it retries transient failures with
|
||||
// exponential backoff.
|
||||
type Client struct {
|
||||
http *http.Client
|
||||
ua string
|
||||
delay time.Duration
|
||||
retries int
|
||||
|
||||
mu sync.Mutex
|
||||
nextSlot time.Time
|
||||
}
|
||||
|
||||
func NewClient(delay time.Duration, retries int) *Client {
|
||||
return &Client{
|
||||
http: &http.Client{Timeout: 45 * time.Second},
|
||||
ua: defaultUserAgent,
|
||||
delay: delay,
|
||||
retries: retries,
|
||||
}
|
||||
}
|
||||
|
||||
// reserve claims the next request slot and blocks until it comes due, holding
|
||||
// the global rate at one request per delay across all callers.
|
||||
func (c *Client) reserve(ctx context.Context) error {
|
||||
c.mu.Lock()
|
||||
slot := c.nextSlot
|
||||
if now := time.Now(); slot.Before(now) {
|
||||
slot = now
|
||||
}
|
||||
c.nextSlot = slot.Add(c.delay)
|
||||
c.mu.Unlock()
|
||||
|
||||
wait := time.Until(slot)
|
||||
if wait <= 0 {
|
||||
return nil
|
||||
}
|
||||
timer := time.NewTimer(wait)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Get fetches url, retrying transient failures.
|
||||
func (c *Client) Get(ctx context.Context, url string) ([]byte, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= c.retries; attempt++ {
|
||||
if attempt > 0 {
|
||||
backoff := time.Duration(1<<uint(attempt-1)) * time.Second
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
if err := c.reserve(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
body, err := c.fetch(ctx, url)
|
||||
if err == nil {
|
||||
return body, nil
|
||||
}
|
||||
lastErr = err
|
||||
|
||||
var se *statusError
|
||||
if errors.As(err, &se) && !se.retryable() {
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("fetch %s: %w", url, lastErr)
|
||||
}
|
||||
|
||||
func (c *Client) fetch(ctx context.Context, url string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", c.ua)
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "vi,en;q=0.8")
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, &statusError{code: resp.StatusCode, status: resp.Status}
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, maxPageSize))
|
||||
}
|
||||
@@ -1,198 +0,0 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// Crawler fetches a novel and its chapters.
|
||||
type Crawler struct {
|
||||
Client *Client
|
||||
|
||||
// CacheDir, when set, stores raw pages on disk and serves later runs from
|
||||
// them. Re-exporting with different font or page settings then costs no
|
||||
// requests.
|
||||
CacheDir string
|
||||
|
||||
// Workers bounds concurrent fetches. The client's delay still caps the
|
||||
// overall request rate.
|
||||
Workers int
|
||||
|
||||
// Log receives progress messages. Optional.
|
||||
Log func(format string, args ...any)
|
||||
}
|
||||
|
||||
func (c *Crawler) logf(format string, args ...any) {
|
||||
if c.Log != nil {
|
||||
c.Log(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Novel fetches a novel landing page and resolves its chapter list.
|
||||
//
|
||||
// The chapter list is taken from the landing page and cross-checked against the
|
||||
// dropdown embedded in the first chapter page, so a truncated list cannot
|
||||
// silently shorten the export.
|
||||
func (c *Crawler) Novel(ctx context.Context, novelURL string) (*Novel, error) {
|
||||
page, err := c.page(ctx, novelURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
novel, err := ParseNovelPage(page, novelURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.logf("novel: %s (%d chapters listed)", novel.Title, len(novel.Chapters))
|
||||
|
||||
base, err := url.Parse(novelURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
firstPage, err := c.page(ctx, novel.Chapters[0].URL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fromSelect, err := ChapterRefsFromSelect(firstPage, base)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
final, extra := ReconcileChapterRefs(novel.Chapters, fromSelect)
|
||||
if len(extra) > 0 {
|
||||
c.logf("warning: %d chapter(s) appear only in the chapter dropdown and were not "+
|
||||
"in the landing page list; verify the export is complete", len(extra))
|
||||
}
|
||||
if len(final) != len(novel.Chapters) {
|
||||
c.logf("chapter list reconciled to %d chapters using the in-chapter dropdown", len(final))
|
||||
}
|
||||
novel.Chapters = final
|
||||
return novel, nil
|
||||
}
|
||||
|
||||
// NovelInfo fetches only the landing page and returns what it carries: title,
|
||||
// slug, tags, and the chapter list as that page shows it.
|
||||
//
|
||||
// Unlike Novel it does not also fetch a chapter page to cross-check the chapter
|
||||
// list, so it costs a single request. Callers that only want metadata should
|
||||
// prefer it; callers about to export every chapter want Novel, whose
|
||||
// reconciliation guards against a silently truncated list.
|
||||
func (c *Crawler) NovelInfo(ctx context.Context, novelURL string) (*Novel, error) {
|
||||
page, err := c.page(ctx, novelURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ParseNovelPage(page, novelURL)
|
||||
}
|
||||
|
||||
// Chapters fetches every chapter concurrently and returns them in reading
|
||||
// order. Any chapter that cannot be fetched or parsed fails the whole run
|
||||
// rather than yielding a book with a hole in it.
|
||||
func (c *Crawler) Chapters(ctx context.Context, novel *Novel) ([]*Chapter, error) {
|
||||
chapters := make([]*Chapter, len(novel.Chapters))
|
||||
|
||||
workers := c.Workers
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
|
||||
group, groupCtx := errgroup.WithContext(ctx)
|
||||
group.SetLimit(workers)
|
||||
|
||||
var mu sync.Mutex
|
||||
done := 0
|
||||
|
||||
for i, ref := range novel.Chapters {
|
||||
i, ref := i, ref
|
||||
group.Go(func() error {
|
||||
page, err := c.page(groupCtx, ref.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chapter, err := ParseChapter(page, ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chapters[i] = chapter
|
||||
|
||||
mu.Lock()
|
||||
done++
|
||||
c.logf("fetched %d/%d: %s (%d words)", done, len(novel.Chapters),
|
||||
chapter.Heading(), chapter.WordCount())
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := group.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return chapters, nil
|
||||
}
|
||||
|
||||
// page returns a page from the cache when available, otherwise fetches and
|
||||
// caches it.
|
||||
func (c *Crawler) page(ctx context.Context, pageURL string) ([]byte, error) {
|
||||
path := c.cachePath(pageURL)
|
||||
if path != "" {
|
||||
if body, err := os.ReadFile(path); err == nil && len(body) > 0 { //nolint:gosec // G304: cachePath strips every path separator from the URL
|
||||
return body, nil
|
||||
}
|
||||
}
|
||||
|
||||
body, err := c.Client.Get(ctx, pageURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if path != "" {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o750); err == nil {
|
||||
// A failed cache write must not fail the crawl.
|
||||
_ = os.WriteFile(path, body, 0o600)
|
||||
}
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// unsafeFileChars matches everything not allowed in a cache file name.
|
||||
var unsafeFileChars = regexp.MustCompile(`[^A-Za-z0-9._-]+`)
|
||||
|
||||
// cachePath maps a page URL to a cache file, or "" when caching is disabled.
|
||||
func (c *Crawler) cachePath(pageURL string) string {
|
||||
if c.CacheDir == "" {
|
||||
return ""
|
||||
}
|
||||
u, err := url.Parse(pageURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
name := unsafeFileChars.ReplaceAllString(strings.Trim(u.Path, "/"), "_")
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
if !strings.HasSuffix(name, ".html") {
|
||||
name += ".html"
|
||||
}
|
||||
return filepath.Join(c.CacheDir, name)
|
||||
}
|
||||
|
||||
// TotalWords sums the word count across chapters.
|
||||
func TotalWords(chapters []*Chapter) int {
|
||||
n := 0
|
||||
for _, ch := range chapters {
|
||||
n += ch.WordCount()
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Describe renders a one-line summary of a crawl result.
|
||||
func Describe(novel *Novel, chapters []*Chapter) string {
|
||||
return fmt.Sprintf("%s — %d chapters, %d words", novel.Title, len(chapters), TotalWords(chapters))
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The site hides part of every chapter behind CSS rather than putting it in the
|
||||
// markup. Chapter HTML carries empty elements such as
|
||||
//
|
||||
// Nghe <span class="t-3e625e..."></span> trưởng tử
|
||||
//
|
||||
// and the page stylesheet supplies the missing word:
|
||||
//
|
||||
// .t-3e625e...:before { content: "vị"; }
|
||||
//
|
||||
// Reading DOM text alone therefore drops hundreds of words per chapter without
|
||||
// any visible error. wordRule finds those rules so the words can be put back.
|
||||
var wordRule = regexp.MustCompile(`\.([A-Za-z0-9_-]+)\s*::?before\s*\{[^}]*?content\s*:\s*"((?:[^"\\]|\\.)*)"`)
|
||||
|
||||
// cssEscape matches a CSS character escape: a hex code point, optionally
|
||||
// followed by one whitespace terminator, or an escaped literal character.
|
||||
var cssEscape = regexp.MustCompile(`\\([0-9A-Fa-f]{1,6})\s?|\\(.)`)
|
||||
|
||||
// ParseWordClasses maps CSS class name to the word its :before rule injects.
|
||||
func ParseWordClasses(page []byte) map[string]string {
|
||||
words := make(map[string]string)
|
||||
for _, m := range wordRule.FindAllSubmatch(page, -1) {
|
||||
words[string(m[1])] = decodeCSSString(string(m[2]))
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
// decodeCSSString resolves the escape sequences allowed inside a CSS string.
|
||||
func decodeCSSString(s string) string {
|
||||
if !strings.Contains(s, `\`) {
|
||||
return s
|
||||
}
|
||||
return cssEscape.ReplaceAllStringFunc(s, func(esc string) string {
|
||||
m := cssEscape.FindStringSubmatch(esc)
|
||||
if m[1] != "" {
|
||||
if cp, err := strconv.ParseInt(m[1], 16, 32); err == nil && cp > 0 {
|
||||
return string(rune(cp))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return m[2]
|
||||
})
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// attr returns the value of the named attribute, or "" when absent.
|
||||
func attr(n *html.Node, name string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == name {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hasClass reports whether the node carries the given class token.
|
||||
func hasClass(n *html.Node, class string) bool {
|
||||
for _, tok := range strings.Fields(attr(n, "class")) {
|
||||
if tok == class {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// findNode returns the first node in document order satisfying match.
|
||||
func findNode(root *html.Node, match func(*html.Node) bool) *html.Node {
|
||||
if match(root) {
|
||||
return root
|
||||
}
|
||||
for c := root.FirstChild; c != nil; c = c.NextSibling {
|
||||
if found := findNode(c, match); found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// findAllNodes returns every node satisfying match, in document order.
|
||||
func findAllNodes(root *html.Node, match func(*html.Node) bool) []*html.Node {
|
||||
var out []*html.Node
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if match(n) {
|
||||
out = append(out, n)
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(root)
|
||||
return out
|
||||
}
|
||||
|
||||
// elementByID finds an element by its id attribute.
|
||||
func elementByID(root *html.Node, id string) *html.Node {
|
||||
return findNode(root, func(n *html.Node) bool {
|
||||
return n.Type == html.ElementNode && attr(n, "id") == id
|
||||
})
|
||||
}
|
||||
|
||||
// elementByTag finds the first element with the given tag name.
|
||||
func elementByTag(root *html.Node, tag string) *html.Node {
|
||||
return findNode(root, func(n *html.Node) bool {
|
||||
return n.Type == html.ElementNode && n.Data == tag
|
||||
})
|
||||
}
|
||||
|
||||
// nodeText collects the descendant text of a node with whitespace collapsed.
|
||||
func nodeText(n *html.Node) string {
|
||||
if n == nil {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.TextNode {
|
||||
b.WriteString(n.Data)
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return collapseSpaces(b.String())
|
||||
}
|
||||
|
||||
// collapseSpaces trims the string and reduces every whitespace run, including
|
||||
// the non-breaking spaces the site emits as , to a single space.
|
||||
func collapseSpaces(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// ChapterRef points at a single chapter listed on a novel page.
|
||||
type ChapterRef struct {
|
||||
Label string // as shown on the site, e.g. "14" or "Chương 12"
|
||||
URL string
|
||||
}
|
||||
|
||||
// Novel is a novel's landing page: its title and its chapters in reading order.
|
||||
type Novel struct {
|
||||
Title string
|
||||
Slug string
|
||||
URL string
|
||||
|
||||
// Tags are the novel's own genres, as labelled on the site, in the order
|
||||
// they appear. Empty when the page lists none.
|
||||
Tags []string
|
||||
Chapters []ChapterRef
|
||||
}
|
||||
|
||||
// ParseNovelPage reads the title and chapter list from a novel landing page.
|
||||
//
|
||||
// Chapter URLs are always taken from the anchors on the page. Slugs are not
|
||||
// uniform across novels ("/14.html" on one, "/chuong-12.html" on another) and
|
||||
// numbering has gaps, so generating URLs from a chapter count would fetch 404s
|
||||
// and miss real chapters.
|
||||
func ParseNovelPage(page []byte, pageURL string) (*Novel, error) {
|
||||
doc, err := html.Parse(bytes.NewReader(page))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse novel page: %w", err)
|
||||
}
|
||||
base, err := url.Parse(pageURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse novel url: %w", err)
|
||||
}
|
||||
|
||||
novel := &Novel{
|
||||
Title: novelTitle(doc),
|
||||
Slug: slugFromNovelURL(base),
|
||||
URL: pageURL,
|
||||
Tags: tagsFromInfo(doc),
|
||||
Chapters: chapterRefsFromList(doc, base),
|
||||
}
|
||||
if novel.Title == "" {
|
||||
novel.Title = novel.Slug
|
||||
}
|
||||
if len(novel.Chapters) == 0 {
|
||||
return nil, fmt.Errorf("no chapters found on %s (page layout may have changed)", pageURL)
|
||||
}
|
||||
return novel, nil
|
||||
}
|
||||
|
||||
// novelTitle prefers the <h1> heading and falls back to the document title.
|
||||
func novelTitle(doc *html.Node) string {
|
||||
if h1 := elementByTag(doc, "h1"); h1 != nil {
|
||||
if t := nodeText(h1); t != "" {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return nodeText(elementByTag(doc, "title"))
|
||||
}
|
||||
|
||||
// tagsFromInfo reads the novel's own genres out of the info block.
|
||||
//
|
||||
// The anchors are matched on the schema.org itemprop="genre" microdata rather
|
||||
// than on their href. The page also carries a site-wide genre menu linking every
|
||||
// category — 69 distinct ones against this novel's 6 on a sampled page — so
|
||||
// matching the /the-loai/ URL shape would pull in the whole navigation.
|
||||
func tagsFromInfo(doc *html.Node) []string {
|
||||
links := findAllNodes(doc, func(n *html.Node) bool {
|
||||
return n.Type == html.ElementNode && n.Data == "a" && attr(n, "itemprop") == "genre"
|
||||
})
|
||||
|
||||
var tags []string
|
||||
seen := make(map[string]bool, len(links))
|
||||
for _, link := range links {
|
||||
// The label appears both as the link text and as its title attribute;
|
||||
// prefer the text and fall back to the attribute.
|
||||
name := nodeText(link)
|
||||
if name == "" {
|
||||
name = collapseSpaces(attr(link, "title"))
|
||||
}
|
||||
if name == "" || seen[name] {
|
||||
continue
|
||||
}
|
||||
seen[name] = true
|
||||
tags = append(tags, name)
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
// slugFromNovelURL turns https://host/tro-lai-nam-thang-cu.html into
|
||||
// "tro-lai-nam-thang-cu".
|
||||
func slugFromNovelURL(u *url.URL) string {
|
||||
seg := strings.Trim(u.Path, "/")
|
||||
if i := strings.LastIndex(seg, "/"); i >= 0 {
|
||||
seg = seg[i+1:]
|
||||
}
|
||||
return strings.TrimSuffix(seg, ".html")
|
||||
}
|
||||
|
||||
// chapterRefsFromList reads the "list-chapters" block on the landing page.
|
||||
// The site lists newest first, so the result is reversed into reading order.
|
||||
func chapterRefsFromList(doc *html.Node, base *url.URL) []ChapterRef {
|
||||
list := findNode(doc, func(n *html.Node) bool {
|
||||
return n.Type == html.ElementNode && hasClass(n, "list-chapters")
|
||||
})
|
||||
if list == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
titles := findAllNodes(list, func(n *html.Node) bool {
|
||||
return n.Type == html.ElementNode && hasClass(n, "episode-title")
|
||||
})
|
||||
|
||||
var refs []ChapterRef
|
||||
for _, title := range titles {
|
||||
link := elementByTag(title, "a")
|
||||
if link == nil {
|
||||
continue
|
||||
}
|
||||
href := strings.TrimSpace(attr(link, "href"))
|
||||
if href == "" {
|
||||
continue
|
||||
}
|
||||
abs, err := base.Parse(href)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
refs = append(refs, ChapterRef{Label: nodeText(link), URL: abs.String()})
|
||||
}
|
||||
return reverseRefs(refs)
|
||||
}
|
||||
|
||||
// ChapterRefsFromSelect reads the chapter dropdown embedded in every chapter
|
||||
// page, whose options hold "novel-slug,chapter-slug" pairs. This is a second,
|
||||
// independent view of the chapter list used to cross-check the landing page.
|
||||
func ChapterRefsFromSelect(page []byte, base *url.URL) ([]ChapterRef, error) {
|
||||
doc, err := html.Parse(bytes.NewReader(page))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse chapter page: %w", err)
|
||||
}
|
||||
sel := elementByID(doc, "selected_chapter")
|
||||
if sel == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var refs []ChapterRef
|
||||
for _, opt := range findAllNodes(sel, func(n *html.Node) bool {
|
||||
return n.Type == html.ElementNode && n.Data == "option"
|
||||
}) {
|
||||
novelSlug, chapterSlug, ok := strings.Cut(attr(opt, "value"), ",")
|
||||
if !ok || novelSlug == "" || chapterSlug == "" {
|
||||
continue
|
||||
}
|
||||
abs, err := base.Parse("/" + novelSlug + "/" + chapterSlug + ".html")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
refs = append(refs, ChapterRef{Label: nodeText(opt), URL: abs.String()})
|
||||
}
|
||||
return reverseRefs(refs), nil
|
||||
}
|
||||
|
||||
// ReconcileChapterRefs picks the chapter list to crawl from the landing page
|
||||
// list and the in-chapter dropdown.
|
||||
//
|
||||
// Both views come from the same site ordering, so when the dropdown is a
|
||||
// superset it is preferred: that keeps the run correct even if the landing page
|
||||
// ever truncates or paginates its list. Anything the dropdown alone knows about
|
||||
// while disagreeing on order is returned as extra so the caller can warn rather
|
||||
// than silently export a short book.
|
||||
func ReconcileChapterRefs(fromList, fromSelect []ChapterRef) (final, extra []ChapterRef) {
|
||||
if len(fromSelect) == 0 {
|
||||
return fromList, nil
|
||||
}
|
||||
|
||||
inSelect := refURLSet(fromSelect)
|
||||
if listIsSubsetOf(fromList, inSelect) {
|
||||
return fromSelect, nil
|
||||
}
|
||||
|
||||
inList := refURLSet(fromList)
|
||||
for _, ref := range fromSelect {
|
||||
if !inList[ref.URL] {
|
||||
extra = append(extra, ref)
|
||||
}
|
||||
}
|
||||
return fromList, extra
|
||||
}
|
||||
|
||||
func refURLSet(refs []ChapterRef) map[string]bool {
|
||||
set := make(map[string]bool, len(refs))
|
||||
for _, r := range refs {
|
||||
set[r.URL] = true
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func listIsSubsetOf(refs []ChapterRef, set map[string]bool) bool {
|
||||
for _, r := range refs {
|
||||
if !set[r.URL] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func reverseRefs(refs []ChapterRef) []ChapterRef {
|
||||
for i, j := 0, len(refs)-1; i < j; i, j = i+1, j-1 {
|
||||
refs[i], refs[j] = refs[j], refs[i]
|
||||
}
|
||||
return refs
|
||||
}
|
||||
@@ -1,170 +0,0 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novelFixture reproduces the two traits that break naive crawlers: the list is
|
||||
// newest-first, and chapter numbering has a gap (no chapter 4).
|
||||
const novelFixture = `<html><head><title>TEN TRUYEN</title></head><body>
|
||||
<h1>TEN TRUYEN</h1>
|
||||
<div class="list-chapters">
|
||||
<div class="item"><div class="episode-title"><a href="https://monkeydd.com/n/5.html">5</a></div></div>
|
||||
<div class="item"><div class="episode-title"><a href="https://monkeydd.com/n/3.html">3</a></div></div>
|
||||
<div class="item"><div class="episode-title"><a href="/n/2.html">2</a></div></div>
|
||||
<div class="item"><div class="episode-title"><a href="https://monkeydd.com/n/1.html">1</a></div></div>
|
||||
</div></body></html>`
|
||||
|
||||
func TestParseNovelPageOrdersChaptersForReading(t *testing.T) {
|
||||
novel, err := ParseNovelPage([]byte(novelFixture), "https://monkeydd.com/n.html")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseNovelPage: %v", err)
|
||||
}
|
||||
|
||||
if novel.Title != "TEN TRUYEN" {
|
||||
t.Errorf("Title = %q", novel.Title)
|
||||
}
|
||||
if novel.Slug != "n" {
|
||||
t.Errorf("Slug = %q, want %q", novel.Slug, "n")
|
||||
}
|
||||
|
||||
wantLabels := []string{"1", "2", "3", "5"}
|
||||
if len(novel.Chapters) != len(wantLabels) {
|
||||
t.Fatalf("got %d chapters, want %d", len(novel.Chapters), len(wantLabels))
|
||||
}
|
||||
for i, want := range wantLabels {
|
||||
if novel.Chapters[i].Label != want {
|
||||
t.Errorf("chapter %d label = %q, want %q", i, novel.Chapters[i].Label, want)
|
||||
}
|
||||
}
|
||||
// Relative hrefs must resolve against the novel URL.
|
||||
if got, want := novel.Chapters[1].URL, "https://monkeydd.com/n/2.html"; got != want {
|
||||
t.Errorf("chapter 2 URL = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// tagsFixture mirrors the real page: the novel's own genres carry
|
||||
// itemprop="genre", while a site-wide menu links every category without it. A
|
||||
// parser matching on the /the-loai/ href shape would swallow the whole menu.
|
||||
const tagsFixture = `<html><head><title>TEN TRUYEN</title></head><body>
|
||||
<h1>TEN TRUYEN</h1>
|
||||
<dl class="row">
|
||||
<dt class="col-sm-3">Thể loại</dt>
|
||||
<dd class="col-sm-9">
|
||||
<a class='cate-item' itemprop='genre' title='Trọng Sinh' href='https://monkeydd.com/the-loai/trong-sinh.html'>Trọng Sinh</a>
|
||||
<a class='cate-item' itemprop='genre' title='Cổ Đại' href='https://monkeydd.com/the-loai/co-dai.html'>Cổ Đại</a>
|
||||
<a class='cate-item' itemprop='genre' title='Gia Đình' href='https://monkeydd.com/the-loai/gia-dinh.html'></a>
|
||||
<a class='cate-item' itemprop='genre' title='Cổ Đại' href='https://monkeydd.com/the-loai/co-dai.html'>Cổ Đại</a>
|
||||
</dd>
|
||||
</dl>
|
||||
<nav class="site-menu">
|
||||
<a href='https://monkeydd.com/the-loai/dam-my.html'>Đam Mỹ</a>
|
||||
<a href='https://monkeydd.com/the-loai/bach-hop.html'>Bách Hợp</a>
|
||||
</nav>
|
||||
<div class="list-chapters">
|
||||
<div class="item"><div class="episode-title"><a href="/n/1.html">1</a></div></div>
|
||||
</div></body></html>`
|
||||
|
||||
func TestParseNovelPageReadsTags(t *testing.T) {
|
||||
novel, err := ParseNovelPage([]byte(tagsFixture), "https://monkeydd.com/n.html")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseNovelPage: %v", err)
|
||||
}
|
||||
|
||||
// "Cổ Đại" has its inner whitespace collapsed, the empty anchor falls back
|
||||
// to its title attribute, and the repeated genre appears once.
|
||||
want := []string{"Trọng Sinh", "Cổ Đại", "Gia Đình"}
|
||||
if len(novel.Tags) != len(want) {
|
||||
t.Fatalf("Tags = %q, want %q", novel.Tags, want)
|
||||
}
|
||||
for i := range want {
|
||||
if novel.Tags[i] != want[i] {
|
||||
t.Errorf("Tags[%d] = %q, want %q", i, novel.Tags[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNovelPageWithoutTags(t *testing.T) {
|
||||
novel, err := ParseNovelPage([]byte(novelFixture), "https://monkeydd.com/n.html")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseNovelPage: %v", err)
|
||||
}
|
||||
if len(novel.Tags) != 0 {
|
||||
t.Errorf("Tags = %q, want none", novel.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNovelPageNoChapters(t *testing.T) {
|
||||
if _, err := ParseNovelPage([]byte(`<html><title>x</title><body></body></html>`),
|
||||
"https://monkeydd.com/n.html"); err == nil {
|
||||
t.Fatal("want an error when no chapters are listed")
|
||||
}
|
||||
}
|
||||
|
||||
const selectFixture = `<html><body>
|
||||
<select name="selected_chapter" id="selected_chapter">
|
||||
<option value="n,5">5</option>
|
||||
<option value="n,chuong-3">Chương 3</option>
|
||||
<option value="n,1">1</option>
|
||||
<option value="14">malformed</option>
|
||||
</select></body></html>`
|
||||
|
||||
func TestChapterRefsFromSelect(t *testing.T) {
|
||||
base, err := url.Parse("https://monkeydd.com/n.html")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refs, err := ChapterRefsFromSelect([]byte(selectFixture), base)
|
||||
if err != nil {
|
||||
t.Fatalf("ChapterRefsFromSelect: %v", err)
|
||||
}
|
||||
|
||||
if len(refs) != 3 {
|
||||
t.Fatalf("got %d refs %+v, want 3 (malformed option skipped)", len(refs), refs)
|
||||
}
|
||||
if got, want := refs[0].URL, "https://monkeydd.com/n/1.html"; got != want {
|
||||
t.Errorf("first ref URL = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := refs[1].URL, "https://monkeydd.com/n/chuong-3.html"; got != want {
|
||||
t.Errorf("second ref URL = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileChapterRefs(t *testing.T) {
|
||||
ref := func(u string) ChapterRef { return ChapterRef{Label: u, URL: u} }
|
||||
|
||||
t.Run("dropdown superset wins", func(t *testing.T) {
|
||||
list := []ChapterRef{ref("a"), ref("b")}
|
||||
sel := []ChapterRef{ref("a"), ref("b"), ref("c")}
|
||||
|
||||
final, extra := ReconcileChapterRefs(list, sel)
|
||||
if len(final) != 3 {
|
||||
t.Errorf("got %d chapters, want the 3 from the dropdown", len(final))
|
||||
}
|
||||
if len(extra) != 0 {
|
||||
t.Errorf("got %d extra, want 0", len(extra))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("disagreement is reported not silently merged", func(t *testing.T) {
|
||||
list := []ChapterRef{ref("a"), ref("z")}
|
||||
sel := []ChapterRef{ref("a"), ref("b")}
|
||||
|
||||
final, extra := ReconcileChapterRefs(list, sel)
|
||||
if len(final) != 2 || final[1].URL != "z" {
|
||||
t.Errorf("final = %+v, want the landing page list", final)
|
||||
}
|
||||
if len(extra) != 1 || extra[0].URL != "b" {
|
||||
t.Errorf("extra = %+v, want [b] so the caller can warn", extra)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty dropdown falls back to the list", func(t *testing.T) {
|
||||
list := []ChapterRef{ref("a")}
|
||||
final, extra := ReconcileChapterRefs(list, nil)
|
||||
if len(final) != 1 || len(extra) != 0 {
|
||||
t.Errorf("final = %+v, extra = %+v", final, extra)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
// Package export turns a novel URL into a PDF file: resolve the chapter list,
|
||||
// fetch the chapters, render the PDF with the bundled font.
|
||||
package export
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/crawler"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/pdf"
|
||||
)
|
||||
|
||||
// Defaults the bot advertises to users or reuses in other crawls.
|
||||
const (
|
||||
DefaultFontSize = 10.0
|
||||
DefaultDelay = 400 * time.Millisecond
|
||||
)
|
||||
|
||||
// Fixed layout and crawl settings. The bot exposes only the font size.
|
||||
const (
|
||||
lineSpacing = 1.55 // multiple of font size
|
||||
margin = 6.0 // millimetres
|
||||
workers = 4
|
||||
retries = 3
|
||||
)
|
||||
|
||||
// Request describes one export.
|
||||
type Request struct {
|
||||
NovelURL string
|
||||
|
||||
// OutDir receives the PDF, named after the novel title.
|
||||
OutDir string
|
||||
|
||||
// FontSize is in points. 0 means DefaultFontSize.
|
||||
FontSize float64
|
||||
|
||||
// CacheDir stores raw pages so a re-export costs no requests. Empty
|
||||
// disables the cache.
|
||||
CacheDir string
|
||||
|
||||
// Log receives progress messages. Optional.
|
||||
Log func(format string, args ...any)
|
||||
}
|
||||
|
||||
// Result reports what was produced.
|
||||
type Result struct {
|
||||
Path string
|
||||
Title string
|
||||
SourceURL string
|
||||
Chapters int
|
||||
Words int
|
||||
Page pdf.PageSize
|
||||
}
|
||||
|
||||
// Summary renders a one-line description of the exported book.
|
||||
func (r *Result) Summary() string {
|
||||
return fmt.Sprintf("%s — %d chapters, %d words", r.Title, r.Chapters, r.Words)
|
||||
}
|
||||
|
||||
// Export fetches the novel at req.NovelURL and writes it as a PDF, returning
|
||||
// where it landed. The context bounds the whole crawl; cancelling it abandons
|
||||
// the run without leaving a partial PDF behind.
|
||||
func Export(ctx context.Context, req Request) (*Result, error) {
|
||||
if req.FontSize == 0 {
|
||||
req.FontSize = DefaultFontSize
|
||||
}
|
||||
if err := req.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c := &crawler.Crawler{
|
||||
Client: crawler.NewClient(DefaultDelay, retries),
|
||||
CacheDir: req.CacheDir,
|
||||
Workers: workers,
|
||||
Log: req.Log,
|
||||
}
|
||||
|
||||
novel, err := c.Novel(ctx, req.NovelURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
chapters, err := c.Chapters(ctx, novel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
outPath := filepath.Join(req.OutDir, SafeFileName(novel.Title, novel.Slug)+".pdf")
|
||||
opts := pdf.Options{
|
||||
Page: pdf.PhonePage,
|
||||
Margin: margin,
|
||||
Font: pdf.BundledFont(),
|
||||
FontSize: req.FontSize,
|
||||
LineSpacing: lineSpacing,
|
||||
Title: novel.Title,
|
||||
SourceURL: novel.URL,
|
||||
}
|
||||
if err := pdf.Write(outPath, opts, toPDFChapters(chapters)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Result{
|
||||
Path: outPath,
|
||||
Title: novel.Title,
|
||||
SourceURL: novel.URL,
|
||||
Chapters: len(chapters),
|
||||
Words: crawler.TotalWords(chapters),
|
||||
Page: pdf.PhonePage,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// validate rejects a Request before any request is made, so a typo costs no
|
||||
// fetches.
|
||||
func (r *Request) validate() error {
|
||||
if r.NovelURL == "" {
|
||||
return fmt.Errorf("novel url is required")
|
||||
}
|
||||
parsed, err := url.Parse(r.NovelURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid novel url: %w", err)
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return fmt.Errorf("invalid novel url: want an http(s) URL, got %q", r.NovelURL)
|
||||
}
|
||||
if r.FontSize <= 0 {
|
||||
return fmt.Errorf("font size must be positive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func toPDFChapters(chapters []*crawler.Chapter) []pdf.Chapter {
|
||||
out := make([]pdf.Chapter, 0, len(chapters))
|
||||
for _, ch := range chapters {
|
||||
out = append(out, pdf.Chapter{Heading: ch.Heading(), Paragraphs: ch.Paragraphs})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var unsafeNameChars = regexp.MustCompile(`[^\p{L}\p{N}]+`)
|
||||
|
||||
// SafeFileName builds a file name from the novel title, falling back to the
|
||||
// slug when the title has no usable characters. The result has no extension.
|
||||
func SafeFileName(title, fallback string) string {
|
||||
name := strings.Trim(unsafeNameChars.ReplaceAllString(title, "-"), "-")
|
||||
if name == "" {
|
||||
return fallback
|
||||
}
|
||||
return name
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
package export
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
req Request
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "url and font size are valid",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", FontSize: DefaultFontSize},
|
||||
},
|
||||
{
|
||||
name: "missing url",
|
||||
req: Request{FontSize: DefaultFontSize},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "non-http scheme",
|
||||
req: Request{NovelURL: "ftp://monkeydd.com/example.html", FontSize: DefaultFontSize},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "scheme-less url",
|
||||
req: Request{NovelURL: "monkeydd.com/example.html", FontSize: DefaultFontSize},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "negative font size",
|
||||
req: Request{NovelURL: "https://monkeydd.com/example.html", FontSize: -1},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := tt.req.validate()
|
||||
if tt.wantErr && err == nil {
|
||||
t.Error("validate() = nil, want error")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Errorf("validate() = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// An invalid request must fail before the crawler makes any request.
|
||||
func TestExportRejectsInvalidRequestWithoutFetching(t *testing.T) {
|
||||
if _, err := Export(t.Context(), Request{NovelURL: "ftp://monkeydd.com/x.html", OutDir: t.TempDir()}); err == nil {
|
||||
t.Fatal("Export = nil error, want an invalid url error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafeFileName(t *testing.T) {
|
||||
tests := []struct {
|
||||
title string
|
||||
fallback string
|
||||
want string
|
||||
}{
|
||||
{"Trở Lại Năm Tháng Cũ", "slug", "Trở-Lại-Năm-Tháng-Cũ"},
|
||||
{"Chapter: One / Two", "slug", "Chapter-One-Two"},
|
||||
{" --- ", "slug", "slug"},
|
||||
{"", "slug", "slug"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := SafeFileName(tt.title, tt.fallback); got != tt.want {
|
||||
t.Errorf("SafeFileName(%q, %q) = %q, want %q", tt.title, tt.fallback, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResultSummary(t *testing.T) {
|
||||
r := &Result{Title: "Example", Chapters: 12, Words: 3400}
|
||||
want := "Example — 12 chapters, 3400 words"
|
||||
if got := r.Summary(); got != want {
|
||||
t.Errorf("Summary() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -1,168 +0,0 @@
|
||||
package monkeyd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"github.com/go-telegram/bot"
|
||||
"github.com/go-telegram/bot/models"
|
||||
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/export"
|
||||
|
||||
"github.com/tiennm99/tiennm99bot/internal/log"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/util/chathelper"
|
||||
)
|
||||
|
||||
const (
|
||||
// crawlTimeout bounds one export. A long novel is roughly one request per
|
||||
// 400ms plus retries, so even a thousand chapters fits well inside this;
|
||||
// the ceiling exists to release the single export slot if the site stalls.
|
||||
crawlTimeout = 30 * time.Minute
|
||||
|
||||
// uploadTimeout is separate from crawlTimeout so a slow crawl cannot eat
|
||||
// the budget needed to actually deliver the finished PDF.
|
||||
uploadTimeout = 5 * time.Minute
|
||||
|
||||
// statusTimeout bounds the short progress and failure replies.
|
||||
statusTimeout = 30 * time.Second
|
||||
|
||||
// maxDocumentBytes is the Bot API's upload ceiling for sendDocument.
|
||||
// Checked before opening the upload so an oversized book fails with an
|
||||
// explanation instead of a Telegram API error.
|
||||
maxDocumentBytes = 50 << 20
|
||||
)
|
||||
|
||||
// cacheDirName is the shared page cache under the system temp directory.
|
||||
// Keeping it outside the per-run directory is what makes a repeat export of the
|
||||
// same novel cost no requests. It is not pruned; a container restart clears it.
|
||||
const cacheDirName = "tiennm99bot-monkeyd-cache"
|
||||
|
||||
// export runs one crawl to completion and delivers the PDF. It is called on its
|
||||
// own goroutine, detached from the Telegram handler context.
|
||||
func (r *runner) export(b *bot.Bot, msg *models.Message, args crawlArgs) {
|
||||
novelURL := args.NovelURL
|
||||
defer r.end()
|
||||
// A panic here would reach no handler recover and would take the whole
|
||||
// process down with it, so contain it.
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
log.Error("monkeyd export panicked",
|
||||
"command", commandName, "url", novelURL, "panic", p, "stack", string(debug.Stack()))
|
||||
r.reportFailure(b, msg, "The export crashed. Nothing was sent.")
|
||||
}
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), crawlTimeout)
|
||||
defer cancel()
|
||||
|
||||
// The PDF is a temporary artefact: it is uploaded and then dropped.
|
||||
outDir, err := os.MkdirTemp("", "monkeyd-export-*")
|
||||
if err != nil {
|
||||
log.Error("monkeyd temp dir failed", "command", commandName, "err", err)
|
||||
r.reportFailure(b, msg, "Could not create a working directory for the export.")
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(outDir); err != nil {
|
||||
log.Warn("monkeyd temp cleanup failed", "command", commandName, "dir", outDir, "err", err)
|
||||
}
|
||||
}()
|
||||
|
||||
result, err := r.exporter(ctx, export.Request{
|
||||
NovelURL: novelURL,
|
||||
OutDir: outDir,
|
||||
// Zero means the caller gave no font size, which Export reads as
|
||||
// "use the default" — exactly the intent.
|
||||
FontSize: args.FontSize,
|
||||
CacheDir: filepath.Join(os.TempDir(), cacheDirName),
|
||||
// Per-chapter progress is one line per chapter — useful when
|
||||
// diagnosing a stuck export, too noisy for the default level.
|
||||
Log: func(format string, args ...any) {
|
||||
log.Debug("monkeyd crawl: "+fmt.Sprintf(format, args...), "command", commandName, "url", novelURL)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Error("monkeyd export failed", "command", commandName, "url", novelURL, "err", err)
|
||||
r.reportFailure(b, msg, "Export failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("monkeyd export done", "command", commandName, "url", novelURL,
|
||||
"title", result.Title, "chapters", result.Chapters, "words", result.Words)
|
||||
|
||||
if err := sendPDF(b, msg, result, effectiveFontSize(args.FontSize)); err != nil {
|
||||
log.Error("monkeyd delivery failed", "command", commandName, "url", novelURL, "err", err)
|
||||
r.reportFailure(b, msg, "The novel was exported but the PDF could not be sent: "+err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// effectiveFontSize resolves what the renderer actually used, so the caption
|
||||
// can report a real number rather than "default".
|
||||
func effectiveFontSize(requested float64) float64 {
|
||||
if requested == 0 {
|
||||
return export.DefaultFontSize
|
||||
}
|
||||
return requested
|
||||
}
|
||||
|
||||
// sendPDF uploads the finished book as a Telegram document.
|
||||
func sendPDF(b *bot.Bot, msg *models.Message, result *export.Result, fontSize float64) error {
|
||||
info, err := os.Stat(result.Path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat pdf: %w", err)
|
||||
}
|
||||
if info.Size() > maxDocumentBytes {
|
||||
return fmt.Errorf("the PDF is %.1f MB, above Telegram's %d MB limit for bots",
|
||||
float64(info.Size())/(1<<20), maxDocumentBytes>>20)
|
||||
}
|
||||
|
||||
file, err := os.Open(result.Path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open pdf: %w", err)
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), uploadTimeout)
|
||||
defer cancel()
|
||||
|
||||
// MessageThreadID keeps the document in the forum topic the command came
|
||||
// from, matching chathelper.Reply's behaviour.
|
||||
_, err = b.SendDocument(ctx, &bot.SendDocumentParams{
|
||||
ChatID: msg.Chat.ID,
|
||||
MessageThreadID: msg.MessageThreadID,
|
||||
Document: &models.InputFileUpload{
|
||||
Filename: filepath.Base(result.Path),
|
||||
Data: file,
|
||||
},
|
||||
Caption: caption(result, fontSize),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// captionLimit is the Bot API's maximum caption length in characters.
|
||||
const captionLimit = 1024
|
||||
|
||||
// caption describes the book under the document. Plain text, so a title
|
||||
// containing markup characters needs no escaping.
|
||||
func caption(result *export.Result, fontSize float64) string {
|
||||
text := fmt.Sprintf("%s\n%s page (%.0f x %.0f mm), %gpt\n%s",
|
||||
result.Summary(), result.Page.Name, result.Page.W, result.Page.H, fontSize, result.SourceURL)
|
||||
if runes := []rune(text); len(runes) > captionLimit {
|
||||
text = string(runes[:captionLimit])
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
// reportFailure tells the chat the export did not produce a document. Delivery
|
||||
// failures here are only logged — there is nothing further to fall back to.
|
||||
func (r *runner) reportFailure(b *bot.Bot, msg *models.Message, text string) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), statusTimeout)
|
||||
defer cancel()
|
||||
if err := chathelper.Reply(ctx, b, msg, text); err != nil {
|
||||
log.Error("monkeyd failure reply failed", "command", commandName, "err", err)
|
||||
}
|
||||
}
|
||||
@@ -1,386 +0,0 @@
|
||||
package monkeyd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/export"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/pdf"
|
||||
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules"
|
||||
"github.com/tiennm99/tiennm99bot/internal/storage"
|
||||
"github.com/tiennm99/tiennm99bot/internal/testutil"
|
||||
)
|
||||
|
||||
const testNovelURL = "https://monkeydd.com/tro-lai-nam-thang-cu.html"
|
||||
|
||||
// install wires the module to a recording bot. The returned runner is the one
|
||||
// behind the registered command, so tests can substitute its exporter and run
|
||||
// the export synchronously instead of on a detached goroutine.
|
||||
//
|
||||
// ownerID becomes the bot owner. Both commands are public, so it grants no
|
||||
// extra access; the public-access tests below rely on a non-owner sender.
|
||||
func install(t *testing.T, ownerID int64) (*testutil.RecordingBot, *runner) {
|
||||
t.Helper()
|
||||
return installWith(t, ownerID, func(context.Context, string) ([]string, error) {
|
||||
t.Helper()
|
||||
t.Error("tags fetcher called by an export test")
|
||||
return nil, nil
|
||||
})
|
||||
}
|
||||
|
||||
// installWith is install with an explicit tag fetcher, for the tags command.
|
||||
func installWith(t *testing.T, ownerID int64, fetch tagsFetcher) (*testutil.RecordingBot, *runner) {
|
||||
t.Helper()
|
||||
rb := testutil.NewRecordingBot(t)
|
||||
r := newRunner()
|
||||
// Run the export inline so assertions do not race a detached goroutine.
|
||||
r.launch = func(job func()) { job() }
|
||||
mod := newModule(r, fetch)
|
||||
|
||||
reg := &modules.Registry{
|
||||
Modules: []modules.Module{{Name: "monkeyd", Commands: mod.Commands}},
|
||||
AllCommands: map[string]modules.Command{},
|
||||
}
|
||||
for _, c := range mod.Commands {
|
||||
reg.AllCommands[c.Name] = c
|
||||
}
|
||||
modules.Install(rb.Bot, reg, modules.Auth{BotOwnerID: ownerID})
|
||||
return rb, r
|
||||
}
|
||||
|
||||
// stubPDF creates a file of the given size standing in for a rendered book and
|
||||
// returns a Result describing it, as export.Export would. The file is sized by
|
||||
// truncation so an over-the-limit case costs no real bytes.
|
||||
func stubPDF(t *testing.T, dir, name string, size int64) *export.Result {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, name)
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatalf("create stub pdf: %v", err)
|
||||
}
|
||||
if err := file.Truncate(size); err != nil {
|
||||
_ = file.Close()
|
||||
t.Fatalf("size stub pdf: %v", err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
t.Fatalf("close stub pdf: %v", err)
|
||||
}
|
||||
return &export.Result{
|
||||
Path: path,
|
||||
Title: "Example Novel",
|
||||
SourceURL: testNovelURL,
|
||||
Chapters: 3,
|
||||
Words: 1200,
|
||||
Page: pdf.PhonePage,
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawl_NoArgumentRepliesUsage(t *testing.T) {
|
||||
rb, _ := install(t, 999)
|
||||
rb.Bot.ProcessUpdate(context.Background(), testutil.NewPrivateMessage(999, "/monkeyd_crawl"))
|
||||
|
||||
if got := rb.LastSent().Text(); !strings.Contains(got, usage) {
|
||||
t.Errorf("reply = %q, want it to contain %q", got, usage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawl_RejectsDisallowedHost(t *testing.T) {
|
||||
rb, r := install(t, 999)
|
||||
called := false
|
||||
r.exporter = func(context.Context, export.Request) (*export.Result, error) {
|
||||
called = true
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_crawl https://example.com/novel.html"))
|
||||
|
||||
if called {
|
||||
t.Error("exporter ran for a disallowed host")
|
||||
}
|
||||
if got := rb.LastSent().Text(); !strings.Contains(got, AllowedHostsHint) {
|
||||
t.Errorf("reply = %q, want it to name %q", got, AllowedHostsHint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawl_SendsPDFOnSuccess(t *testing.T) {
|
||||
rb, r := install(t, 999)
|
||||
dir := t.TempDir()
|
||||
|
||||
var gotRequest export.Request
|
||||
r.exporter = func(_ context.Context, req export.Request) (*export.Result, error) {
|
||||
gotRequest = req
|
||||
return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil
|
||||
}
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL))
|
||||
|
||||
if gotRequest.NovelURL != testNovelURL {
|
||||
t.Errorf("exporter got NovelURL %q, want %q", gotRequest.NovelURL, testNovelURL)
|
||||
}
|
||||
if gotRequest.OutDir == "" {
|
||||
t.Error("exporter got an empty OutDir; the PDF would land in the working directory")
|
||||
}
|
||||
|
||||
calls := rb.Sent()
|
||||
if len(calls) < 2 {
|
||||
t.Fatalf("expected an acknowledgement and a document, got %d calls: %+v", len(calls), calls)
|
||||
}
|
||||
if first := calls[0]; !strings.Contains(first.Text(), testNovelURL) {
|
||||
t.Errorf("first reply = %q, want it to name the novel URL", first.Text())
|
||||
}
|
||||
last := calls[len(calls)-1]
|
||||
if last.Method != "sendDocument" {
|
||||
t.Fatalf("last call = %q, want sendDocument", last.Method)
|
||||
}
|
||||
if caption := last.Form["caption"]; !strings.Contains(caption, "Example Novel") {
|
||||
t.Errorf("caption = %q, want it to name the novel", caption)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawl_PassesRequestedFontSizeToExporter(t *testing.T) {
|
||||
rb, r := install(t, 999)
|
||||
dir := t.TempDir()
|
||||
|
||||
var gotRequest export.Request
|
||||
r.exporter = func(_ context.Context, req export.Request) (*export.Result, error) {
|
||||
gotRequest = req
|
||||
return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil
|
||||
}
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL+" 14"))
|
||||
|
||||
if gotRequest.FontSize != 14 {
|
||||
t.Errorf("exporter got FontSize %v, want 14", gotRequest.FontSize)
|
||||
}
|
||||
if caption := rb.LastSent().Form["caption"]; !strings.Contains(caption, "14pt") {
|
||||
t.Errorf("caption = %q, want it to report 14pt", caption)
|
||||
}
|
||||
}
|
||||
|
||||
// Omitting the argument must leave Export to apply its own default rather than
|
||||
// the bot inventing one, so the two cannot disagree.
|
||||
func TestCrawl_OmittedFontSizeLeavesRequestZero(t *testing.T) {
|
||||
rb, r := install(t, 999)
|
||||
dir := t.TempDir()
|
||||
|
||||
var gotRequest export.Request
|
||||
r.exporter = func(_ context.Context, req export.Request) (*export.Result, error) {
|
||||
gotRequest = req
|
||||
return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil
|
||||
}
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL))
|
||||
|
||||
if gotRequest.FontSize != 0 {
|
||||
t.Errorf("exporter got FontSize %v, want 0 (defer to Export)", gotRequest.FontSize)
|
||||
}
|
||||
// The caption still has to name a real size, not "0pt".
|
||||
want := fmt.Sprintf("%gpt", export.DefaultFontSize)
|
||||
if caption := rb.LastSent().Form["caption"]; !strings.Contains(caption, want) {
|
||||
t.Errorf("caption = %q, want it to report %s", caption, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawl_RejectsBadFontSize(t *testing.T) {
|
||||
rb, r := install(t, 999)
|
||||
called := false
|
||||
r.exporter = func(context.Context, export.Request) (*export.Result, error) {
|
||||
called = true
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL+" 100"))
|
||||
|
||||
if called {
|
||||
t.Error("exporter ran despite an out-of-range font size")
|
||||
}
|
||||
if got := rb.LastSent().Text(); !strings.Contains(got, "between 6 and 24") {
|
||||
t.Errorf("reply = %q, want it to state the font size bounds", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawl_ReportsExportFailure(t *testing.T) {
|
||||
rb, r := install(t, 999)
|
||||
r.exporter = func(context.Context, export.Request) (*export.Result, error) {
|
||||
return nil, errors.New("fetch failed: 404")
|
||||
}
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL))
|
||||
|
||||
last := rb.LastSent()
|
||||
if last.Method == "sendDocument" {
|
||||
t.Fatal("a document was sent despite the export failing")
|
||||
}
|
||||
if got := last.Text(); !strings.Contains(got, "404") {
|
||||
t.Errorf("failure reply = %q, want it to carry the underlying error", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawl_RefusesOversizedPDF(t *testing.T) {
|
||||
rb, r := install(t, 999)
|
||||
dir := t.TempDir()
|
||||
r.exporter = func(context.Context, export.Request) (*export.Result, error) {
|
||||
return stubPDF(t, dir, "Huge-Novel.pdf", maxDocumentBytes+1), nil
|
||||
}
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL))
|
||||
|
||||
last := rb.LastSent()
|
||||
if last.Method == "sendDocument" {
|
||||
t.Fatal("an oversized document was uploaded instead of being refused")
|
||||
}
|
||||
if got := last.Text(); !strings.Contains(got, "limit") {
|
||||
t.Errorf("reply = %q, want it to explain the size limit", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The single export slot must be released whether the run succeeded or failed,
|
||||
// otherwise the command is dead until the process restarts.
|
||||
func TestCrawl_ReleasesSlotAfterRun(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
exporter func(t *testing.T) func(context.Context, export.Request) (*export.Result, error)
|
||||
}{
|
||||
{
|
||||
name: "after success",
|
||||
exporter: func(t *testing.T) func(context.Context, export.Request) (*export.Result, error) {
|
||||
dir := t.TempDir()
|
||||
return func(context.Context, export.Request) (*export.Result, error) {
|
||||
return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "after failure",
|
||||
exporter: func(*testing.T) func(context.Context, export.Request) (*export.Result, error) {
|
||||
return func(context.Context, export.Request) (*export.Result, error) {
|
||||
return nil, errors.New("boom")
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "after panic",
|
||||
exporter: func(*testing.T) func(context.Context, export.Request) (*export.Result, error) {
|
||||
return func(context.Context, export.Request) (*export.Result, error) {
|
||||
panic("boom")
|
||||
}
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rb, r := install(t, 999)
|
||||
r.exporter = tc.exporter(t)
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL))
|
||||
|
||||
r.mu.Lock()
|
||||
running := r.running
|
||||
r.mu.Unlock()
|
||||
if running {
|
||||
t.Error("export slot still held after the run finished")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrawl_RepliesBusyWhileAnotherExportRuns(t *testing.T) {
|
||||
rb, r := install(t, 999)
|
||||
// Simulate an in-flight export rather than racing a real one.
|
||||
if ok, _ := r.begin("https://monkeydd.com/other-novel.html"); !ok {
|
||||
t.Fatal("could not claim the export slot")
|
||||
}
|
||||
|
||||
called := false
|
||||
r.exporter = func(context.Context, export.Request) (*export.Result, error) {
|
||||
called = true
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL))
|
||||
|
||||
if called {
|
||||
t.Error("a second export started while one was already running")
|
||||
}
|
||||
got := rb.LastSent().Text()
|
||||
if !strings.Contains(got, "other-novel") {
|
||||
t.Errorf("busy reply = %q, want it to name the in-flight novel", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The command is public: a sender who is neither owner nor admin must still get
|
||||
// a reply, including in a group.
|
||||
func TestCrawl_AvailableToAnySender(t *testing.T) {
|
||||
rb, r := install(t, 999)
|
||||
dir := t.TempDir()
|
||||
r.exporter = func(context.Context, export.Request) (*export.Result, error) {
|
||||
return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil
|
||||
}
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewGroupMessage(-100, 12345, "/monkeyd_crawl "+testNovelURL))
|
||||
|
||||
calls := rb.Sent()
|
||||
if len(calls) == 0 {
|
||||
t.Fatal("a non-admin sender got no reply from a public command")
|
||||
}
|
||||
if last := calls[len(calls)-1]; last.Method != "sendDocument" {
|
||||
t.Errorf("last call = %q, want sendDocument", last.Method)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistration(t *testing.T) {
|
||||
mod := New(modules.Deps{Store: storage.NewMemoryProvider().Collection("monkeyd")})
|
||||
|
||||
wantParameters := map[string]string{
|
||||
commandName: "<url> [font_size]",
|
||||
tagsCommandName: "<url>",
|
||||
}
|
||||
if len(mod.Commands) != len(wantParameters) {
|
||||
t.Fatalf("expected %d commands, got %d", len(wantParameters), len(mod.Commands))
|
||||
}
|
||||
|
||||
for _, cmd := range mod.Commands {
|
||||
want, known := wantParameters[cmd.Name]
|
||||
if !known {
|
||||
t.Errorf("unexpected command %q", cmd.Name)
|
||||
continue
|
||||
}
|
||||
delete(wantParameters, cmd.Name)
|
||||
|
||||
if cmd.Parameters != want {
|
||||
t.Errorf("/%s parameters = %q, want %q", cmd.Name, cmd.Parameters, want)
|
||||
}
|
||||
if cmd.Visibility != modules.VisibilityPublic {
|
||||
t.Errorf("/%s visibility = %v, want Public", cmd.Name, cmd.Visibility)
|
||||
}
|
||||
if cmd.Description == "" {
|
||||
t.Errorf("/%s description is empty; command discovery requires one", cmd.Name)
|
||||
}
|
||||
if cmd.Handler == nil {
|
||||
t.Errorf("/%s handler is nil", cmd.Name)
|
||||
}
|
||||
}
|
||||
for name := range wantParameters {
|
||||
t.Errorf("command %q was not registered", name)
|
||||
}
|
||||
|
||||
if len(mod.Crons) != 0 || len(mod.Callbacks) != 0 {
|
||||
t.Errorf("expected no crons or callbacks, got %d crons and %d callbacks",
|
||||
len(mod.Crons), len(mod.Callbacks))
|
||||
}
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
package monkeyd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// sourceHashtag leads every tag line so the source is visible when the line is
|
||||
// pasted somewhere on its own.
|
||||
const sourceHashtag = "#MonkeyD"
|
||||
|
||||
// tagsMessage renders the tag line followed by the novel URL, separated by a
|
||||
// blank line. Plain text: the caller decides how to present it.
|
||||
func tagsMessage(tags []string, novelURL string) string {
|
||||
return hashtagLine(tags) + "\n\n" + novelURL
|
||||
}
|
||||
|
||||
// hashtagLine turns genre labels into a single space-separated line of
|
||||
// hashtags, led by sourceHashtag. Labels that cannot form a usable hashtag are
|
||||
// dropped, and duplicates collapse — two labels differing only in punctuation
|
||||
// or spacing produce the same hashtag.
|
||||
func hashtagLine(tags []string) string {
|
||||
parts := []string{sourceHashtag}
|
||||
seen := map[string]bool{sourceHashtag: true}
|
||||
for _, tag := range tags {
|
||||
body := hashtag(tag)
|
||||
if body == "" {
|
||||
continue
|
||||
}
|
||||
tag := "#" + body
|
||||
if seen[tag] {
|
||||
continue
|
||||
}
|
||||
seen[tag] = true
|
||||
parts = append(parts, tag)
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
// hashtag converts a genre label into a hashtag body: "Cổ Đại" -> "CổĐại".
|
||||
//
|
||||
// Telegram ends a hashtag at the first character that is not a letter, digit or
|
||||
// underscore, so spaces and punctuation are removed rather than replaced, and
|
||||
// each word is capitalised to keep the boundaries readable. Diacritics survive:
|
||||
// Telegram accepts non-ASCII letters in hashtags.
|
||||
//
|
||||
// Returns "" for a label with no letters. A digits-only hashtag is not linkified
|
||||
// by Telegram, so emitting one would only add noise.
|
||||
func hashtag(tag string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(tag))
|
||||
|
||||
hasLetter := false
|
||||
wordStart := true
|
||||
for _, r := range tag {
|
||||
switch {
|
||||
case unicode.IsLetter(r):
|
||||
hasLetter = true
|
||||
if wordStart {
|
||||
r = unicode.ToUpper(r)
|
||||
}
|
||||
b.WriteRune(r)
|
||||
wordStart = false
|
||||
case unicode.IsDigit(r):
|
||||
b.WriteRune(r)
|
||||
wordStart = false
|
||||
default:
|
||||
// Dropped, but it still ends the word, so the next letter is
|
||||
// capitalised: "sci-fi" -> "SciFi".
|
||||
wordStart = true
|
||||
}
|
||||
}
|
||||
if !hasLetter {
|
||||
return ""
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
package monkeyd
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestHashtag(t *testing.T) {
|
||||
tests := []struct {
|
||||
tag string
|
||||
want string
|
||||
}{
|
||||
{"Cổ Đại", "CổĐại"},
|
||||
{"Gia Đình", "GiaĐình"},
|
||||
{"Trọng Sinh", "TrọngSinh"},
|
||||
{"Nữ Cường", "NữCường"},
|
||||
{"Vả Mặt", "VảMặt"},
|
||||
// Single word passes through untouched.
|
||||
{"Ngôn", "Ngôn"},
|
||||
// A lower-case label is capitalised per word.
|
||||
{"ngôn tình", "NgônTình"},
|
||||
// Punctuation is removed but still ends the word.
|
||||
{"sci-fi", "SciFi"},
|
||||
{"Đông/Tây", "ĐôngTây"},
|
||||
{"Truyện (Chữ)", "TruyệnChữ"},
|
||||
// Collapsed whitespace and surrounding space.
|
||||
{" Cổ Đại ", "CổĐại"},
|
||||
// Digits are kept.
|
||||
{"3D", "3D"},
|
||||
{"18+ Tuổi", "18Tuổi"},
|
||||
// Nothing usable: Telegram will not linkify a digits-only hashtag.
|
||||
{"18+", ""},
|
||||
{"", ""},
|
||||
{"---", ""},
|
||||
{"+", ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := hashtag(tt.tag); got != tt.want {
|
||||
t.Errorf("hashtag(%q) = %q, want %q", tt.tag, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashtagLine(t *testing.T) {
|
||||
got := hashtagLine([]string{"Cổ Đại", "Gia Đình"})
|
||||
want := "#MonkeyD #CổĐại #GiaĐình"
|
||||
if got != want {
|
||||
t.Errorf("hashtagLine() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashtagLineWithoutTags(t *testing.T) {
|
||||
if got := hashtagLine(nil); got != sourceHashtag {
|
||||
t.Errorf("hashtagLine(nil) = %q, want %q", got, sourceHashtag)
|
||||
}
|
||||
}
|
||||
|
||||
// Labels differing only in punctuation collapse to one hashtag, and an unusable
|
||||
// label is dropped rather than emitted as a bare "#".
|
||||
func TestHashtagLineDropsUnusableAndDuplicates(t *testing.T) {
|
||||
got := hashtagLine([]string{"Cổ Đại", "Cổ-Đại", "18+", "Gia Đình"})
|
||||
want := "#MonkeyD #CổĐại #GiaĐình"
|
||||
if got != want {
|
||||
t.Errorf("hashtagLine() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagsMessage(t *testing.T) {
|
||||
got := tagsMessage([]string{"Cổ Đại", "Gia Đình"}, "https://monkeydd.com/truong-an-gwem.html")
|
||||
want := "#MonkeyD #CổĐại #GiaĐình\n\nhttps://monkeydd.com/truong-an-gwem.html"
|
||||
if got != want {
|
||||
t.Errorf("tagsMessage() =\n%q\nwant\n%q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The block is what makes the line copyable in one tap, and its contents must be
|
||||
// escaped so a label containing markup cannot break out of it.
|
||||
func TestMonospaceBlockEscapes(t *testing.T) {
|
||||
got := monospaceBlock("#MonkeyD #A<b>&")
|
||||
want := "<pre>#MonkeyD #A<b>&</pre>"
|
||||
if got != want {
|
||||
t.Errorf("monospaceBlock() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
// Package monkeyd exports a monkeydd.com novel as a PDF and sends it back as a
|
||||
// Telegram document (/monkeyd_crawl), and reports a novel's genre tags as a
|
||||
// copyable hashtag line (/monkeyd_tags). The crawling and rendering live in the
|
||||
// monkeyd-crawler submodule (third_party/monkeyd-crawler); this module is the
|
||||
// Telegram surface around it: argument validation, one-at-a-time scheduling,
|
||||
// and delivery.
|
||||
package monkeyd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/go-telegram/bot"
|
||||
"github.com/go-telegram/bot/models"
|
||||
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/export"
|
||||
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/util/chathelper"
|
||||
)
|
||||
|
||||
// commandName is the PDF export command; tagsCommandName is the other one.
|
||||
const commandName = "monkeyd_crawl"
|
||||
|
||||
// parameters is the display syntax shared by the command menu, /help, and the
|
||||
// usage text below, so the three cannot drift apart.
|
||||
const parameters = "<url> [font_size]"
|
||||
|
||||
// usage is shown when the command arrives without usable arguments. The bounds
|
||||
// and default are read from their definitions rather than restated, so the
|
||||
// message stays true if either changes.
|
||||
var usage = fmt.Sprintf(
|
||||
"Usage: /%s %s\nfont_size is in points, %g to %g (default %g).",
|
||||
commandName, parameters, minFontSize, maxFontSize, export.DefaultFontSize)
|
||||
|
||||
// New is the module Factory. The module keeps no persistent state — an export
|
||||
// is a one-shot job — so deps.Store is unused.
|
||||
func New(_ modules.Deps) modules.Module {
|
||||
return newModule(newRunner(), fetchTags)
|
||||
}
|
||||
|
||||
// newModule builds the module around a given runner and tag fetcher, which is
|
||||
// how tests supply a stubbed exporter, a synchronous launch, and tags without
|
||||
// network access.
|
||||
func newModule(r *runner, fetch tagsFetcher) modules.Module {
|
||||
return modules.Module{
|
||||
Commands: []modules.Command{
|
||||
{
|
||||
Name: commandName,
|
||||
Visibility: modules.VisibilityPublic,
|
||||
// Public despite being expensive: one invocation makes
|
||||
// hundreds of outbound requests over several minutes. The
|
||||
// host allowlist and the single in-flight export are what
|
||||
// bound the cost, so both are load-bearing here.
|
||||
Description: "Export a " + AllowedHostsHint + " novel as a PDF",
|
||||
Parameters: parameters,
|
||||
Handler: r.handle,
|
||||
},
|
||||
tagsCommand(fetch),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// runner serialises exports. The crawler spaces its own requests out per run,
|
||||
// so two concurrent crawls would double the request rate against the site —
|
||||
// and a novel is minutes of work, which makes queueing pointless. One at a
|
||||
// time, globally, with a clear reply to anyone who asks meanwhile.
|
||||
type runner struct {
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
current string // novel URL of the in-flight export, for the busy reply
|
||||
|
||||
// exporter is the crawl-and-render step. It is a field so tests can
|
||||
// exercise scheduling and delivery without network access.
|
||||
exporter func(context.Context, export.Request) (*export.Result, error)
|
||||
|
||||
// launch runs an export job. Production detaches it onto its own
|
||||
// goroutine; tests substitute a synchronous run for determinism.
|
||||
launch func(job func())
|
||||
}
|
||||
|
||||
func newRunner() *runner {
|
||||
return &runner{
|
||||
exporter: export.Export,
|
||||
launch: func(job func()) {
|
||||
// Detached from the handler context on purpose: handlers run one
|
||||
// at a time (the bot is built WithNotAsyncHandlers), so crawling
|
||||
// inline would block every other command for minutes. The job
|
||||
// owns its own timeout and recovers its own panics.
|
||||
go job() //nolint:gosec // G118: intentional; see runner.export
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// begin claims the single export slot, reporting the in-flight URL when it is
|
||||
// already taken.
|
||||
func (r *runner) begin(novelURL string) (ok bool, inFlight string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.running {
|
||||
return false, r.current
|
||||
}
|
||||
r.running = true
|
||||
r.current = novelURL
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func (r *runner) end() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.running = false
|
||||
r.current = ""
|
||||
}
|
||||
|
||||
func (r *runner) handle(ctx context.Context, b *bot.Bot, update *models.Update) error {
|
||||
msg := update.Message
|
||||
if msg == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
arg := chathelper.ArgAfterCommand(msg.Text)
|
||||
if arg == "" {
|
||||
return chathelper.Reply(ctx, b, msg, usage)
|
||||
}
|
||||
|
||||
args, err := parseCrawlArgs(arg)
|
||||
if err != nil {
|
||||
return chathelper.Reply(ctx, b, msg, fmt.Sprintf("%s.\n%s", capitalize(err.Error()), usage))
|
||||
}
|
||||
novelURL := args.NovelURL
|
||||
|
||||
ok, inFlight := r.begin(novelURL)
|
||||
if !ok {
|
||||
return chathelper.Reply(ctx, b, msg,
|
||||
"Already exporting "+inFlight+". Try again once it finishes.")
|
||||
}
|
||||
|
||||
// Reply before starting so the user knows the wait is expected. If the
|
||||
// reply cannot be delivered, drop the slot rather than crawling for a chat
|
||||
// that will never hear the result.
|
||||
if err := chathelper.Reply(ctx, b, msg,
|
||||
"Exporting "+novelURL+" — this takes a few minutes. I will send the PDF here when it is ready."); err != nil {
|
||||
r.end()
|
||||
return err
|
||||
}
|
||||
|
||||
r.launch(func() { r.export(b, msg, args) })
|
||||
return nil
|
||||
}
|
||||
|
||||
// capitalize upper-cases the first letter so a lower-case error string reads as
|
||||
// a sentence in a Telegram reply.
|
||||
func capitalize(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
return strings.ToUpper(s[:1]) + s[1:]
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package monkeyd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// allowedHosts are the hostnames the crawler's extractor understands. It is
|
||||
// written against monkeydd.com's specific markup — the chapter list, the
|
||||
// in-chapter dropdown, and the CSS rules that supply part of the chapter text —
|
||||
// so another host would parse to nothing useful. Refusing it up front also
|
||||
// keeps the command from being used to make the bot fetch arbitrary URLs.
|
||||
var allowedHosts = map[string]bool{
|
||||
"monkeydd.com": true,
|
||||
"www.monkeydd.com": true,
|
||||
}
|
||||
|
||||
// AllowedHostsHint lists the accepted hosts for user-facing usage text.
|
||||
const AllowedHostsHint = "monkeydd.com"
|
||||
|
||||
var (
|
||||
errNotAURL = errors.New("that does not look like a URL")
|
||||
errHostNotAllow = errors.New("only " + AllowedHostsHint + " novel URLs are supported")
|
||||
errNoPath = errors.New("that URL has no novel path — link the novel's own page")
|
||||
)
|
||||
|
||||
// normalizeNovelURL validates a user-supplied novel URL and returns the form to
|
||||
// crawl. A missing scheme is filled in with https, since people paste bare
|
||||
// hostnames; anything else that is not a plain http(s) URL on an allowed host
|
||||
// is rejected.
|
||||
func normalizeNovelURL(raw string) (string, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return "", errNotAURL
|
||||
}
|
||||
// A bare "monkeydd.com/x.html" parses as a path with no host, so give it a
|
||||
// scheme before parsing rather than trying to interpret the result.
|
||||
if !strings.Contains(raw, "://") {
|
||||
raw = "https://" + raw
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return "", errNotAURL
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return "", errNotAURL
|
||||
}
|
||||
// Credentials in the URL are never needed here and would be logged with the
|
||||
// crawl, so treat them as malformed input.
|
||||
if parsed.User != nil {
|
||||
return "", errNotAURL
|
||||
}
|
||||
// Hostname() drops any port, which the allowlist must not be fooled by.
|
||||
if !allowedHosts[strings.ToLower(parsed.Hostname())] {
|
||||
return "", errHostNotAllow
|
||||
}
|
||||
if strings.Trim(parsed.Path, "/") == "" {
|
||||
return "", errNoPath
|
||||
}
|
||||
// Rebuild from the parsed parts so the crawl uses a canonical host and no
|
||||
// fragment; query strings are kept because the site may need them.
|
||||
canonical := url.URL{
|
||||
Scheme: parsed.Scheme,
|
||||
Host: strings.ToLower(parsed.Host),
|
||||
Path: parsed.Path,
|
||||
RawQuery: parsed.RawQuery,
|
||||
}
|
||||
return canonical.String(), nil
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
package monkeyd
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizeNovelURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "https url passes through",
|
||||
raw: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
want: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
},
|
||||
{
|
||||
name: "www host is allowed",
|
||||
raw: "https://www.monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
want: "https://www.monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
},
|
||||
{
|
||||
name: "missing scheme gets https",
|
||||
raw: "monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
want: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
},
|
||||
{
|
||||
name: "http is kept",
|
||||
raw: "http://monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
want: "http://monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
},
|
||||
{
|
||||
name: "surrounding whitespace is trimmed",
|
||||
raw: " https://monkeydd.com/tro-lai-nam-thang-cu.html ",
|
||||
want: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
},
|
||||
{
|
||||
name: "uppercase host is canonicalised",
|
||||
raw: "https://MonkeyDD.com/tro-lai-nam-thang-cu.html",
|
||||
want: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
},
|
||||
{
|
||||
name: "fragment is dropped",
|
||||
raw: "https://monkeydd.com/tro-lai-nam-thang-cu.html#chuong-1",
|
||||
want: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
|
||||
},
|
||||
{
|
||||
name: "query is preserved",
|
||||
raw: "https://monkeydd.com/novel.html?page=2",
|
||||
want: "https://monkeydd.com/novel.html?page=2",
|
||||
},
|
||||
{
|
||||
name: "empty input",
|
||||
raw: "",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "other host",
|
||||
raw: "https://example.com/novel.html",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "host that merely ends with the allowed name",
|
||||
raw: "https://evilmonkeydd.com/novel.html",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "host that merely contains the allowed name",
|
||||
raw: "https://monkeydd.com.evil.example/novel.html",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "allowed host as a subdomain of another host",
|
||||
raw: "https://monkeydd.com.attacker.test/novel.html",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "non-http scheme",
|
||||
raw: "file:///etc/passwd",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "credentials in url",
|
||||
raw: "https://user:pass@monkeydd.com/novel.html",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "landing page with no novel path",
|
||||
raw: "https://monkeydd.com",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "root path only",
|
||||
raw: "https://monkeydd.com/",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := normalizeNovelURL(tt.raw)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("normalizeNovelURL(%q) = %q, want error", tt.raw, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("normalizeNovelURL(%q) returned error: %v", tt.raw, err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("normalizeNovelURL(%q) = %q, want %q", tt.raw, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A port on an allowed host must not defeat the allowlist, and must survive
|
||||
// canonicalisation so the crawl reaches the same place the user asked for.
|
||||
func TestNormalizeNovelURLKeepsPort(t *testing.T) {
|
||||
got, err := normalizeNovelURL("http://monkeydd.com:8080/novel.html")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if want := "http://monkeydd.com:8080/novel.html"; got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
package pdf
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
)
|
||||
|
||||
// bundledTTF is the only font the bot renders with. A minimal container has no
|
||||
// system fonts, and a single embedded font keeps the PDF identical on every
|
||||
// host. See fonts/NOTICE.md for provenance and licensing.
|
||||
//
|
||||
// Vietnamese text needs the Latin Extended Additional block (ư, ạ, ế, ộ …);
|
||||
// DejaVu Sans covers it, where a basic-Latin font would silently drop the
|
||||
// diacritics.
|
||||
//
|
||||
//go:embed fonts/DejaVuSans.ttf
|
||||
var bundledTTF []byte
|
||||
|
||||
// BundledFontName labels the embedded font in diagnostics. It is not a path;
|
||||
// the font is compiled into the binary.
|
||||
const BundledFontName = "DejaVu Sans (bundled)"
|
||||
|
||||
// Font is font data ready to embed in a PDF.
|
||||
//
|
||||
// The data is carried as bytes rather than as a path because fpdf joins a font
|
||||
// path onto its own font directory, which it defaults to "." — turning an
|
||||
// absolute path into a working-directory-relative one that resolves only when
|
||||
// the process happens to run from the filesystem root.
|
||||
type Font struct {
|
||||
// Name identifies the font for diagnostics.
|
||||
Name string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// BundledFont returns the font compiled into the binary.
|
||||
func BundledFont() Font {
|
||||
return Font{Name: BundledFontName, Data: bundledTTF}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package pdf
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"golang.org/x/image/font/sfnt"
|
||||
)
|
||||
|
||||
func TestBundledFontIsParseable(t *testing.T) {
|
||||
font := BundledFont()
|
||||
if font.Name != BundledFontName {
|
||||
t.Errorf("Name = %q, want %q", font.Name, BundledFontName)
|
||||
}
|
||||
if _, err := sfnt.Parse(font.Data); err != nil {
|
||||
t.Fatalf("bundled font does not parse as a TrueType font: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The bundled font exists to render Vietnamese. A replacement that lacks these
|
||||
// glyphs would silently emit blanks, so check before trusting it.
|
||||
func TestBundledFontCoversVietnamese(t *testing.T) {
|
||||
parsed, err := sfnt.Parse(BundledFont().Data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse bundled font: %v", err)
|
||||
}
|
||||
var buf sfnt.Buffer
|
||||
for _, r := range []rune{'ư', 'ơ', 'đ', 'ạ', 'ế', 'ộ', 'ữ', 'ằ', 'ỷ', 'ỹ', 'Ọ', 'Ế', 'Ư', 'Đ'} {
|
||||
index, err := parsed.GlyphIndex(&buf, r)
|
||||
if err != nil {
|
||||
t.Errorf("GlyphIndex(%q): %v", r, err)
|
||||
continue
|
||||
}
|
||||
// Glyph 0 is .notdef — the character is absent from the font.
|
||||
if index == 0 {
|
||||
t.Errorf("bundled font has no glyph for %q (U+%04X)", r, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -1,24 +0,0 @@
|
||||
# Bundled font
|
||||
|
||||
`DejaVuSans.ttf` is embedded into the binary and used when no font is supplied
|
||||
and no suitable system font is found. It covers the Latin Extended Additional
|
||||
block, which is what Vietnamese diacritics need — a font with only basic Latin
|
||||
coverage silently drops them.
|
||||
|
||||
The following is recorded in the font file's own name table:
|
||||
|
||||
- Version: `Version 2.37`
|
||||
- Copyright: `Copyright (c) 2003 by Bitstream, Inc. All Rights Reserved.`
|
||||
`Copyright (c) 2006 by Tavmjong Bah. All Rights Reserved.`
|
||||
`DejaVu changes are in public domain`
|
||||
- License information: <http://dejavu.sourceforge.net/wiki/index.php/License>
|
||||
|
||||
The DejaVu fonts are free and redistributable, which is why they ship with most
|
||||
Linux distributions. This copy came from Alpine's `font-dejavu` package, which
|
||||
does not include the license text as a separate file. For a vendored copy of the
|
||||
full license text, take `LICENSE` from the upstream DejaVu release and add it to
|
||||
this directory.
|
||||
|
||||
To swap the bundled font, replace `DejaVuSans.ttf` and update this file. Verify
|
||||
the replacement covers Vietnamese first — `TestBundledFontCoversVietnamese` in
|
||||
`../font_test.go` checks a representative set of characters.
|
||||
@@ -1,132 +0,0 @@
|
||||
package pdf
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/go-pdf/fpdf"
|
||||
)
|
||||
|
||||
// pointsToMM converts typographic points to millimetres.
|
||||
const pointsToMM = 25.4 / 72.0
|
||||
|
||||
// bodyFont is the internal family name registered with the PDF.
|
||||
const bodyFont = "body"
|
||||
|
||||
// PageSize is a page in millimetres.
|
||||
type PageSize struct {
|
||||
Name string
|
||||
W, H float64
|
||||
}
|
||||
|
||||
// PhonePage is the page every export uses.
|
||||
//
|
||||
// Phone reading depends far more on page shape than on font size. A phone
|
||||
// viewer scales a whole page to fit the screen, so a large font on an A4 page
|
||||
// still ends up tiny: the page is ~3x wider than the screen and gets shrunk to
|
||||
// match. A page cut to the phone's own aspect ratio fills the screen at 100%,
|
||||
// which is why this is a small 9:16 page rather than A4 with big type.
|
||||
var PhonePage = PageSize{Name: "phone", W: 90, H: 160}
|
||||
|
||||
// Chapter is a chapter ready to render.
|
||||
type Chapter struct {
|
||||
Heading string
|
||||
Paragraphs []string
|
||||
}
|
||||
|
||||
// Options controls the exported PDF.
|
||||
type Options struct {
|
||||
Page PageSize
|
||||
Margin float64 // mm
|
||||
Font Font // BundledFont()
|
||||
FontSize float64 // pt
|
||||
LineSpacing float64 // multiple of font size
|
||||
Title string
|
||||
SourceURL string
|
||||
}
|
||||
|
||||
// footerReserve is the vertical space kept clear for the page number.
|
||||
const footerReserve = 6.0
|
||||
|
||||
// Write renders the chapters to a PDF at path.
|
||||
func Write(path string, opts Options, chapters []Chapter) error {
|
||||
pdf := fpdf.NewCustom(&fpdf.InitType{
|
||||
UnitStr: "mm",
|
||||
Size: fpdf.SizeType{Wd: opts.Page.W, Ht: opts.Page.H},
|
||||
})
|
||||
|
||||
pdf.SetMargins(opts.Margin, opts.Margin, opts.Margin)
|
||||
pdf.SetAutoPageBreak(true, opts.Margin+footerReserve)
|
||||
|
||||
// Embeds a subset of the TrueType data, which is what makes the Vietnamese
|
||||
// diacritics render instead of falling back to "?". The bytes are passed
|
||||
// directly rather than by path: the path-taking variant joins the name onto
|
||||
// fpdf's own font directory (default "."), which mangles an absolute path
|
||||
// into a working-directory-relative one.
|
||||
pdf.AddUTF8FontFromBytes(bodyFont, "", opts.Font.Data)
|
||||
pdf.SetFont(bodyFont, "", opts.FontSize)
|
||||
pdf.SetTitle(opts.Title, true)
|
||||
|
||||
lineHeight := opts.FontSize * opts.LineSpacing * pointsToMM
|
||||
paragraphGap := lineHeight * 0.45
|
||||
headingSize := opts.FontSize * 1.35
|
||||
|
||||
addFooter(pdf, opts)
|
||||
writeTitlePage(pdf, opts, len(chapters))
|
||||
|
||||
for _, ch := range chapters {
|
||||
pdf.AddPage()
|
||||
|
||||
pdf.SetFontSize(headingSize)
|
||||
pdf.MultiCell(0, headingSize*1.3*pointsToMM, ch.Heading, "", "L", false)
|
||||
pdf.Ln(paragraphGap * 1.6)
|
||||
|
||||
pdf.SetFontSize(opts.FontSize)
|
||||
for _, p := range ch.Paragraphs {
|
||||
// "J" justifies, which keeps the short measure of a phone page tidy.
|
||||
pdf.MultiCell(0, lineHeight, p, "", "J", false)
|
||||
pdf.Ln(paragraphGap)
|
||||
}
|
||||
}
|
||||
|
||||
if err := pdf.OutputFileAndClose(path); err != nil {
|
||||
return fmt.Errorf("write pdf %s: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// addFooter prints a centred page number, restoring the body font size so the
|
||||
// footer callback cannot leak its own size into the following content.
|
||||
func addFooter(pdf *fpdf.Fpdf, opts Options) {
|
||||
pdf.SetFooterFunc(func() {
|
||||
if pdf.PageNo() <= 1 {
|
||||
return
|
||||
}
|
||||
pdf.SetY(-(opts.Margin + footerReserve*0.6))
|
||||
pdf.SetFontSize(opts.FontSize * 0.75)
|
||||
pdf.SetTextColor(120, 120, 120)
|
||||
pdf.CellFormat(0, 4, fmt.Sprintf("%d", pdf.PageNo()-1), "", 0, "C", false, 0, "")
|
||||
pdf.SetTextColor(0, 0, 0)
|
||||
pdf.SetFontSize(opts.FontSize)
|
||||
})
|
||||
}
|
||||
|
||||
func writeTitlePage(pdf *fpdf.Fpdf, opts Options, chapterCount int) {
|
||||
pdf.AddPage()
|
||||
pdf.SetY(opts.Page.H * 0.30)
|
||||
|
||||
titleSize := opts.FontSize * 1.9
|
||||
pdf.SetFontSize(titleSize)
|
||||
pdf.MultiCell(0, titleSize*1.35*pointsToMM, strings.ToUpper(opts.Title), "", "C", false)
|
||||
|
||||
pdf.Ln(opts.FontSize * pointsToMM * 2)
|
||||
pdf.SetFontSize(opts.FontSize * 0.85)
|
||||
pdf.SetTextColor(90, 90, 90)
|
||||
pdf.MultiCell(0, opts.FontSize*1.4*pointsToMM,
|
||||
fmt.Sprintf("%d chương", chapterCount), "", "C", false)
|
||||
if opts.SourceURL != "" {
|
||||
pdf.MultiCell(0, opts.FontSize*1.4*pointsToMM, opts.SourceURL, "", "C", false)
|
||||
}
|
||||
pdf.SetTextColor(0, 0, 0)
|
||||
pdf.SetFontSize(opts.FontSize)
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
package pdf
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func testOptions(t *testing.T) Options {
|
||||
t.Helper()
|
||||
return Options{
|
||||
Page: PhonePage,
|
||||
Margin: 6,
|
||||
Font: BundledFont(),
|
||||
FontSize: 12,
|
||||
LineSpacing: 1.55,
|
||||
Title: "TRỞ LẠI NĂM THÁNG CŨ",
|
||||
SourceURL: "https://example.test/n.html",
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteProducesReadablePDF(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "out.pdf")
|
||||
|
||||
chapters := []Chapter{
|
||||
{Heading: "Chương 1", Paragraphs: []string{
|
||||
"Nghe vị trưởng tử nói chuyện với nàng.",
|
||||
"Một đoạn văn khác để kiểm tra ngắt dòng tự động trên trang nhỏ.",
|
||||
}},
|
||||
{Heading: "Chương 2", Paragraphs: []string{"Đoạn cuối."}},
|
||||
}
|
||||
|
||||
if err := Write(path, testOptions(t), chapters); err != nil {
|
||||
t.Fatalf("Write: %v", err)
|
||||
}
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat output: %v", err)
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
t.Fatal("wrote an empty PDF")
|
||||
}
|
||||
|
||||
header := make([]byte, 5)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := f.Read(header); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(header) != "%PDF-" {
|
||||
t.Errorf("output does not start with a PDF header, got %q", header)
|
||||
}
|
||||
}
|
||||
|
||||
// A novel with many chapters must not overflow a page; auto page break plus the
|
||||
// footer reserve handles that, so a long chapter should span several pages.
|
||||
func TestWriteHandlesLongChapters(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "long.pdf")
|
||||
|
||||
paragraphs := make([]string, 200)
|
||||
for i := range paragraphs {
|
||||
paragraphs[i] = "Một đoạn văn dài để buộc trình kết xuất phải sang trang mới nhiều lần."
|
||||
}
|
||||
|
||||
if err := Write(path, testOptions(t), []Chapter{{Heading: "Chương 1", Paragraphs: paragraphs}}); err != nil {
|
||||
t.Fatalf("Write: %v", err)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Size() < 2000 {
|
||||
t.Errorf("output suspiciously small (%d bytes) for 200 paragraphs", info.Size())
|
||||
}
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
package monkeyd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"html"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-telegram/bot"
|
||||
"github.com/go-telegram/bot/models"
|
||||
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/crawler"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/monkeyd/export"
|
||||
|
||||
"github.com/tiennm99/tiennm99bot/internal/log"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules"
|
||||
"github.com/tiennm99/tiennm99bot/internal/modules/util/chathelper"
|
||||
)
|
||||
|
||||
// tagsCommandName is the command that reports a novel's tags.
|
||||
const tagsCommandName = "monkeyd_tags"
|
||||
|
||||
// tagsParameters is the display syntax shared by the command menu, /help, and
|
||||
// the usage text.
|
||||
const tagsParameters = "<url>"
|
||||
|
||||
const tagsUsage = "Usage: /" + tagsCommandName + " " + tagsParameters
|
||||
|
||||
const (
|
||||
// tagsFetchTimeout bounds the tag fetch. Unlike an export this is one
|
||||
// request and runs inline, and handlers are dispatched one at a time, so a
|
||||
// stalled fetch would hold up every other command until it gives up.
|
||||
tagsFetchTimeout = 30 * time.Second
|
||||
|
||||
// tagsRetries is deliberately lower than the export's: the caller is
|
||||
// waiting on this reply, so failing early beats a long retry chain.
|
||||
tagsRetries = 1
|
||||
)
|
||||
|
||||
// tagsFetcher returns a novel's tags. It is injected through newModule rather
|
||||
// than called directly so tests can exercise the command without network
|
||||
// access.
|
||||
type tagsFetcher func(ctx context.Context, novelURL string) ([]string, error)
|
||||
|
||||
func tagsCommand(fetch tagsFetcher) modules.Command {
|
||||
return modules.Command{
|
||||
Name: tagsCommandName,
|
||||
Visibility: modules.VisibilityPublic,
|
||||
Description: "Show a " + AllowedHostsHint + " novel's tags as hashtags",
|
||||
Parameters: tagsParameters,
|
||||
Handler: tagsHandler(fetch),
|
||||
}
|
||||
}
|
||||
|
||||
func tagsHandler(fetch tagsFetcher) modules.CommandHandler {
|
||||
return func(ctx context.Context, b *bot.Bot, update *models.Update) error {
|
||||
msg := update.Message
|
||||
if msg == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
arg := chathelper.ArgAfterCommand(msg.Text)
|
||||
if arg == "" {
|
||||
return chathelper.Reply(ctx, b, msg, tagsUsage)
|
||||
}
|
||||
fields := strings.Fields(arg)
|
||||
if len(fields) > 1 {
|
||||
return chathelper.Reply(ctx, b, msg,
|
||||
fmt.Sprintf("%s.\n%s", capitalize(errTooManyArgs.Error()), tagsUsage))
|
||||
}
|
||||
novelURL, err := normalizeNovelURL(fields[0])
|
||||
if err != nil {
|
||||
return chathelper.Reply(ctx, b, msg,
|
||||
fmt.Sprintf("%s.\n%s", capitalize(err.Error()), tagsUsage))
|
||||
}
|
||||
|
||||
// Reserve part of the handler's budget for the reply itself, then cap
|
||||
// the fetch so it cannot outlive the command.
|
||||
fetchCtx, cancel := chathelper.FetchContext(ctx)
|
||||
defer cancel()
|
||||
fetchCtx, cancelTimeout := context.WithTimeout(fetchCtx, tagsFetchTimeout)
|
||||
defer cancelTimeout()
|
||||
|
||||
tags, err := fetch(fetchCtx, novelURL)
|
||||
if err != nil {
|
||||
log.Error("monkeyd tags fetch failed",
|
||||
"command", tagsCommandName, "url", novelURL, "err", err)
|
||||
return chathelper.Reply(ctx, b, msg, "Could not read that novel page: "+err.Error())
|
||||
}
|
||||
if len(tags) == 0 {
|
||||
return chathelper.Reply(ctx, b, msg, "That page lists no tags.")
|
||||
}
|
||||
|
||||
// Sent as a code block so the line can be copied in one tap.
|
||||
return chathelper.ReplyHTML(ctx, b, msg, monospaceBlock(tagsMessage(tags, novelURL)))
|
||||
}
|
||||
}
|
||||
|
||||
// monospaceBlock wraps text in Telegram's <pre> block, which renders as a
|
||||
// copyable code box. The content is escaped, so a tag containing < or & cannot
|
||||
// break the markup.
|
||||
func monospaceBlock(text string) string {
|
||||
return "<pre>" + html.EscapeString(text) + "</pre>"
|
||||
}
|
||||
|
||||
// fetchTags reads a novel's tags from the site.
|
||||
//
|
||||
// It shares the export cache directory, so tags requested for a novel that was
|
||||
// already exported — or an export following a tags lookup — cost no request.
|
||||
func fetchTags(ctx context.Context, novelURL string) ([]string, error) {
|
||||
c := &crawler.Crawler{
|
||||
Client: crawler.NewClient(export.DefaultDelay, tagsRetries),
|
||||
CacheDir: filepath.Join(os.TempDir(), cacheDirName),
|
||||
}
|
||||
novel, err := c.NovelInfo(ctx, novelURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return novel.Tags, nil
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
package monkeyd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/tiennm99/tiennm99bot/internal/testutil"
|
||||
)
|
||||
|
||||
// stubTags returns a fetcher yielding fixed tags and records the URL it saw.
|
||||
func stubTags(tags []string, gotURL *string) tagsFetcher {
|
||||
return func(_ context.Context, novelURL string) ([]string, error) {
|
||||
if gotURL != nil {
|
||||
*gotURL = novelURL
|
||||
}
|
||||
return tags, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestTags_RepliesWithHashtagBlock(t *testing.T) {
|
||||
var gotURL string
|
||||
rb, _ := installWith(t, 999, stubTags([]string{"Cổ Đại", "Gia Đình"}, &gotURL))
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_tags https://monkeydd.com/truong-an-gwem.html"))
|
||||
|
||||
if want := "https://monkeydd.com/truong-an-gwem.html"; gotURL != want {
|
||||
t.Errorf("fetcher got %q, want %q", gotURL, want)
|
||||
}
|
||||
|
||||
sent := rb.LastSent()
|
||||
want := "<pre>#MonkeyD #CổĐại #GiaĐình\n\nhttps://monkeydd.com/truong-an-gwem.html</pre>"
|
||||
if got := sent.Text(); got != want {
|
||||
t.Errorf("reply =\n%q\nwant\n%q", got, want)
|
||||
}
|
||||
// Without HTML parse mode Telegram would show the literal <pre> tags.
|
||||
if got := sent.Form["parse_mode"]; got != "HTML" {
|
||||
t.Errorf("parse_mode = %q, want HTML", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTags_NoArgumentRepliesUsage(t *testing.T) {
|
||||
rb, _ := installWith(t, 999, stubTags(nil, nil))
|
||||
rb.Bot.ProcessUpdate(context.Background(), testutil.NewPrivateMessage(999, "/monkeyd_tags"))
|
||||
|
||||
if got := rb.LastSent().Text(); !strings.Contains(got, tagsUsage) {
|
||||
t.Errorf("reply = %q, want it to contain %q", got, tagsUsage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTags_RejectsDisallowedHost(t *testing.T) {
|
||||
called := false
|
||||
rb, _ := installWith(t, 999, func(context.Context, string) ([]string, error) {
|
||||
called = true
|
||||
return nil, nil
|
||||
})
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_tags https://example.com/novel.html"))
|
||||
|
||||
if called {
|
||||
t.Error("fetcher ran for a disallowed host")
|
||||
}
|
||||
if got := rb.LastSent().Text(); !strings.Contains(got, AllowedHostsHint) {
|
||||
t.Errorf("reply = %q, want it to name %q", got, AllowedHostsHint)
|
||||
}
|
||||
}
|
||||
|
||||
// A bare hostname is normalised before the fetch, matching /monkeyd_crawl, and
|
||||
// the reply quotes the normalised form rather than what was typed.
|
||||
func TestTags_NormalizesURL(t *testing.T) {
|
||||
const wantURL = "https://monkeydd.com/truong-an-gwem.html"
|
||||
|
||||
var gotURL string
|
||||
rb, _ := installWith(t, 999, stubTags([]string{"Cổ Đại"}, &gotURL))
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_tags monkeydd.com/truong-an-gwem.html"))
|
||||
|
||||
if gotURL != wantURL {
|
||||
t.Errorf("fetcher got %q, want %q", gotURL, wantURL)
|
||||
}
|
||||
if got := rb.LastSent().Text(); !strings.Contains(got, wantURL) {
|
||||
t.Errorf("reply = %q, want it to carry the normalised URL", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTags_ReportsFetchFailure(t *testing.T) {
|
||||
rb, _ := installWith(t, 999, func(context.Context, string) ([]string, error) {
|
||||
return nil, errors.New("fetch failed: 404")
|
||||
})
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_tags "+testNovelURL))
|
||||
|
||||
got := rb.LastSent().Text()
|
||||
if !strings.Contains(got, "404") {
|
||||
t.Errorf("reply = %q, want it to carry the underlying error", got)
|
||||
}
|
||||
if strings.Contains(got, sourceHashtag) {
|
||||
t.Errorf("reply = %q, want no hashtag line on failure", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An empty tag list must say so rather than send a lone "#MonkeyD".
|
||||
func TestTags_ReportsEmptyTagList(t *testing.T) {
|
||||
rb, _ := installWith(t, 999, stubTags(nil, nil))
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_tags "+testNovelURL))
|
||||
|
||||
got := rb.LastSent().Text()
|
||||
if strings.Contains(got, sourceHashtag) {
|
||||
t.Errorf("reply = %q, want a plain explanation rather than a bare source hashtag", got)
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(got), "no tags") {
|
||||
t.Errorf("reply = %q, want it to say no tags were listed", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTags_RejectsExtraArguments(t *testing.T) {
|
||||
called := false
|
||||
rb, _ := installWith(t, 999, func(context.Context, string) ([]string, error) {
|
||||
called = true
|
||||
return nil, nil
|
||||
})
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewPrivateMessage(999, "/monkeyd_tags "+testNovelURL+" extra"))
|
||||
|
||||
if called {
|
||||
t.Error("fetcher ran despite extra arguments")
|
||||
}
|
||||
if got := rb.LastSent().Text(); !strings.Contains(got, tagsUsage) {
|
||||
t.Errorf("reply = %q, want usage text", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The command is public, so a non-admin in a group must get a reply.
|
||||
func TestTags_AvailableToAnySender(t *testing.T) {
|
||||
rb, _ := installWith(t, 999, stubTags([]string{"Cổ Đại"}, nil))
|
||||
|
||||
rb.Bot.ProcessUpdate(context.Background(),
|
||||
testutil.NewGroupMessage(-100, 12345, "/monkeyd_tags "+testNovelURL))
|
||||
|
||||
if got := rb.LastSent().Text(); !strings.Contains(got, "#CổĐại") {
|
||||
t.Errorf("reply = %q, want the hashtag line", got)
|
||||
}
|
||||
}
|
||||
@@ -36,9 +36,28 @@ var commandRenames = []commandRename{
|
||||
{oldCmd: "noitupvp", newCmd: "noitu", markerKey: "stats:command-rename:noitupvp-to-noitu"},
|
||||
}
|
||||
|
||||
// commandRetirement marks every usage row of the listed commands deleted
|
||||
// once, recorded under markerKey in the system collection.
|
||||
type commandRetirement struct {
|
||||
cmds []string
|
||||
markerKey string
|
||||
}
|
||||
|
||||
// commandRetirements run in order after the renames, each at most once. The
|
||||
// commands must also be in retiredCommands, which keeps any row the marker
|
||||
// missed out of the views.
|
||||
var commandRetirements = []commandRetirement{
|
||||
{cmds: []string{"monkeyd_crawl", "monkeyd_tags"}, markerKey: "migration:stats-delete-monkeyd-v1"},
|
||||
}
|
||||
|
||||
// deletedCommandMigrationRetries bounds the optimistic-lock retries when one
|
||||
// row is marked deleted on the document store.
|
||||
const deletedCommandMigrationRetries = 5
|
||||
|
||||
// InitStore performs stats collection startup maintenance. It is safe to call
|
||||
// every boot: MongoDB indexes are created idempotently, and one-time command
|
||||
// rename migrations are guarded by markers in the shared system collection.
|
||||
// rename and retirement migrations are guarded by markers in the shared
|
||||
// system collection.
|
||||
func InitStore(ctx context.Context, statsColl, systemColl storage.Collection) error {
|
||||
if mongoColl, ok := storage.MongoCollection(statsColl); ok {
|
||||
if err := ensureUsageIndexes(ctx, mongoColl); err != nil {
|
||||
@@ -50,9 +69,121 @@ func InitStore(ctx context.Context, statsColl, systemColl storage.Collection) er
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, r := range commandRetirements {
|
||||
if err := markDeletedCommands(ctx, statsColl, systemColl, r); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// markDeletedCommands sets deleted: true on every usage row of r.cmds, the
|
||||
// anonymous totals and the per-user rows, and records completion with the
|
||||
// number of rows it marked. The rows themselves are kept as history. A
|
||||
// completed marker makes it a no-op.
|
||||
func markDeletedCommands(ctx context.Context, statsColl, systemColl storage.Collection, r commandRetirement) error {
|
||||
state := systemstate.New(systemColl)
|
||||
if rec, ok, err := state.Get(ctx, r.markerKey); err != nil {
|
||||
return fmt.Errorf("stats command retire marker %s: %w", r.markerKey, err)
|
||||
} else if ok && rec.Status == "complete" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var marked int64
|
||||
for _, cmd := range r.cmds {
|
||||
var (
|
||||
n int64
|
||||
err error
|
||||
)
|
||||
if mongoColl, ok := storage.MongoCollection(statsColl); ok {
|
||||
n, err = markMongoUsageEntriesDeleted(ctx, mongoColl, cmd)
|
||||
} else {
|
||||
n, err = markDocUsageEntriesDeleted(ctx, storage.Typed[usageEntry](statsColl), cmd)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
marked += n
|
||||
}
|
||||
|
||||
now := time.Now().UTC().UnixMilli()
|
||||
if err := state.Put(ctx, r.markerKey, systemstate.Record{
|
||||
Kind: "migration",
|
||||
Name: r.markerKey,
|
||||
Status: "complete",
|
||||
Count: marked,
|
||||
CompletedAt: now,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("stats command retire marker put %s: %w", r.markerKey, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func markMongoUsageEntriesDeleted(ctx context.Context, coll *mongo.Collection, cmd string) (int64, error) {
|
||||
res, err := coll.UpdateMany(ctx,
|
||||
bson.M{"cmd": cmd, "deleted": bson.M{"$ne": true}},
|
||||
bson.M{
|
||||
"$set": bson.M{"deleted": true},
|
||||
"$inc": bson.M{"version": int64(1)},
|
||||
"$currentDate": bson.M{"updatedAt": true},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stats command retire update %s: %w", cmd, err)
|
||||
}
|
||||
return res.ModifiedCount, nil
|
||||
}
|
||||
|
||||
func markDocUsageEntriesDeleted(ctx context.Context, docs storage.DocStore[usageEntry], cmd string) (int64, error) {
|
||||
keys, err := docs.List(ctx, cmd)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stats command retire list %s: %w", cmd, err)
|
||||
}
|
||||
var marked int64
|
||||
for _, key := range keys {
|
||||
// The key prefix also matches longer command names.
|
||||
if key != cmd && !strings.HasPrefix(key, cmd+":") {
|
||||
continue
|
||||
}
|
||||
changed, err := markUsageEntryDeleted(ctx, docs, key, cmd)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if changed {
|
||||
marked++
|
||||
}
|
||||
}
|
||||
return marked, nil
|
||||
}
|
||||
|
||||
// markUsageEntryDeleted sets the deleted flag on one row with a versioned
|
||||
// write, retrying when a concurrent increment wins the race. It reports
|
||||
// whether the row changed.
|
||||
func markUsageEntryDeleted(ctx context.Context, docs storage.DocStore[usageEntry], key, cmd string) (bool, error) {
|
||||
for range deletedCommandMigrationRetries {
|
||||
entry, version, err := docs.Get(ctx, key)
|
||||
if errors.Is(err, storage.ErrNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("stats command retire get %s: %w", key, err)
|
||||
}
|
||||
if entry.Cmd != cmd || entry.Deleted {
|
||||
return false, nil
|
||||
}
|
||||
entry.Deleted = true
|
||||
err = docs.PutVersioned(ctx, key, version, entry)
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if !errors.Is(err, storage.ErrConflict) {
|
||||
return false, fmt.Errorf("stats command retire put %s: %w", key, err)
|
||||
}
|
||||
}
|
||||
return false, fmt.Errorf("stats command retire put %s: %w", key, storage.ErrConflict)
|
||||
}
|
||||
|
||||
// migrateCommandRename merges every usage row of r.oldCmd, the anonymous
|
||||
// total and the per-user rows, into the matching r.newCmd row, deletes the
|
||||
// old row, and records completion. A completed marker makes it a no-op, so a
|
||||
|
||||
@@ -122,3 +122,8 @@ func TestInitStore_MongoSwapsNoituCommandStats(t *testing.T) {
|
||||
_, provider := setupMongoStatsProvider(t)
|
||||
assertNoituCommandSwap(t, provider)
|
||||
}
|
||||
|
||||
func TestInitStore_MongoRetiresMonkeydCommandStats(t *testing.T) {
|
||||
_, provider := setupMongoStatsProvider(t)
|
||||
assertMonkeydCommandsRetired(t, provider)
|
||||
}
|
||||
@@ -146,3 +146,93 @@ func TestMigrateCommandRename_MergesAndKeepsDeletedFlag(t *testing.T) {
|
||||
t.Errorf("user = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitStore_RetiresMonkeydCommandStats(t *testing.T) {
|
||||
assertMonkeydCommandsRetired(t, storage.NewMemoryProvider())
|
||||
}
|
||||
|
||||
// assertMonkeydCommandsRetired seeds history for the removed monkeyd commands,
|
||||
// runs the startup migration twice, and checks that their rows are kept with
|
||||
// deleted: true, hidden from every view, and never counted again, while a
|
||||
// command that only shares the prefix is untouched. A row written after the
|
||||
// marker exists is left alone, which shows the marker guards the step.
|
||||
func assertMonkeydCommandsRetired(t *testing.T, provider storage.Provider) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
statsColl := provider.Collection("stats")
|
||||
systemColl := provider.Collection(systemstate.CollectionName)
|
||||
docs := storage.Typed[usageEntry](statsColl)
|
||||
|
||||
seeds := map[string]usageEntry{
|
||||
usageKey("monkeyd_crawl", 0): {Cmd: "monkeyd_crawl", N: 4},
|
||||
usageKey("monkeyd_crawl", 7): {Cmd: "monkeyd_crawl", UserID: 7, Username: "alice", N: 3},
|
||||
usageKey("monkeyd_tags", 8): {Cmd: "monkeyd_tags", UserID: 8, Username: "bob", N: 2, Deleted: true},
|
||||
usageKey("monkeyd_crawl_extra", 7): {Cmd: "monkeyd_crawl_extra", UserID: 7, Username: "alice", N: 1},
|
||||
usageKey("ping", 7): {Cmd: "ping", UserID: 7, Username: "alice", N: 5},
|
||||
}
|
||||
for key, entry := range seeds {
|
||||
if err := docs.Put(ctx, key, entry); err != nil {
|
||||
t.Fatalf("seed %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
for run := range 2 {
|
||||
if err := InitStore(ctx, statsColl, systemColl); err != nil {
|
||||
t.Fatalf("InitStore run %d: %v", run+1, err)
|
||||
}
|
||||
}
|
||||
|
||||
for key, seed := range seeds {
|
||||
got, _, err := docs.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", key, err)
|
||||
}
|
||||
want := seed
|
||||
want.Deleted = isRetiredCommand(seed.Cmd)
|
||||
if got != want {
|
||||
t.Errorf("%s = %+v, want %+v", key, got, want)
|
||||
}
|
||||
}
|
||||
rec, ok, err := systemstate.New(systemColl).Get(ctx, "migration:stats-delete-monkeyd-v1")
|
||||
if err != nil || !ok || rec.Status != "complete" || rec.Count != 2 {
|
||||
t.Fatalf("marker = %+v ok=%v err=%v, want complete count 2", rec, ok, err)
|
||||
}
|
||||
|
||||
store := newUsageStore(statsColl)
|
||||
if err := store.Increment(ctx, "monkeyd_crawl", usageUser{ID: 7, Username: "alice"}, true); err != nil {
|
||||
t.Fatalf("retired Increment: %v", err)
|
||||
}
|
||||
if got, _, err := docs.Get(ctx, usageKey("monkeyd_crawl", 7)); err != nil || got.N != 3 {
|
||||
t.Fatalf("retired row after Increment = %+v err=%v, want n 3", got, err)
|
||||
}
|
||||
if rows, err := store.TopCommands(ctx, 10); err != nil || len(rows) != 2 || rows[0].display != "/ping" || rows[1].display != "/monkeyd_crawl_extra" {
|
||||
t.Fatalf("top commands = %+v err=%v", rows, err)
|
||||
}
|
||||
if rows, err := store.TopUsers(ctx, 10); err != nil || len(rows) != 1 || rows[0].display != "@alice" || rows[0].n != 6 {
|
||||
t.Fatalf("top users = %+v err=%v", rows, err)
|
||||
}
|
||||
if rows, found, err := store.CommandsByUser(ctx, "bob", 10); err != nil || found || len(rows) != 0 {
|
||||
t.Fatalf("commands by retired-only user = %+v found=%v err=%v", rows, found, err)
|
||||
}
|
||||
for _, cmd := range []string{"monkeyd_crawl", "monkeyd_tags"} {
|
||||
if rows, err := store.UsersByCommand(ctx, cmd, 10); err != nil || len(rows) != 0 {
|
||||
t.Fatalf("users by %s = %+v err=%v", cmd, rows, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A row that appears after the marker, say from the previous release
|
||||
// still draining during a deploy, is not migrated again but stays hidden.
|
||||
late := usageEntry{Cmd: "monkeyd_tags", UserID: 9, Username: "carol", N: 1}
|
||||
if err := docs.Put(ctx, usageKey("monkeyd_tags", 9), late); err != nil {
|
||||
t.Fatalf("seed late row: %v", err)
|
||||
}
|
||||
if err := InitStore(ctx, statsColl, systemColl); err != nil {
|
||||
t.Fatalf("InitStore after marker: %v", err)
|
||||
}
|
||||
if got, _, err := docs.Get(ctx, usageKey("monkeyd_tags", 9)); err != nil || got != late {
|
||||
t.Fatalf("late row = %+v err=%v, want untouched %+v", got, err, late)
|
||||
}
|
||||
if rows, err := store.TopUsers(ctx, 10); err != nil || len(rows) != 1 || rows[0].display != "@alice" {
|
||||
t.Fatalf("top users with late retired row = %+v err=%v", rows, err)
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
@@ -336,7 +337,7 @@ func (s *mongoUsageStore) Increment(ctx context.Context, cmd string, user usageU
|
||||
func (s *mongoUsageStore) TopCommands(ctx context.Context, limit int) ([]row, error) {
|
||||
pipeline := mongo.Pipeline{
|
||||
bson.D{bsonField("$match", bson.D{
|
||||
bsonField("cmd", bson.D{bsonField("$type", "string"), bsonField("$ne", deletedStockDividendCommand)}),
|
||||
bsonField("cmd", bson.D{bsonField("$type", "string"), bsonField("$nin", retiredCommands)}),
|
||||
bsonField("deleted", bson.D{bsonField("$ne", true)}),
|
||||
})},
|
||||
bson.D{bsonField("$group", bson.D{bsonField("_id", "$cmd"), bsonField("n", bson.D{bsonField("$sum", "$n")})})},
|
||||
@@ -370,7 +371,7 @@ func (s *mongoUsageStore) TopCommands(ctx context.Context, limit int) ([]row, er
|
||||
func (s *mongoUsageStore) TopUsers(ctx context.Context, limit int) ([]row, error) {
|
||||
pipeline := mongo.Pipeline{
|
||||
bson.D{bsonField("$match", bson.D{
|
||||
bsonField("cmd", bson.D{bsonField("$ne", deletedStockDividendCommand)}),
|
||||
bsonField("cmd", bson.D{bsonField("$nin", retiredCommands)}),
|
||||
bsonField("uid", bson.D{bsonField("$gt", int64(0))}),
|
||||
bsonField("user", bson.D{bsonField("$type", "string"), bsonField("$ne", "")}),
|
||||
bsonField("deleted", bson.D{bsonField("$ne", true)}),
|
||||
@@ -424,7 +425,7 @@ func (s *mongoUsageStore) CommandsByUser(ctx context.Context, username string, l
|
||||
}
|
||||
cur, err := s.coll.Find(ctx, bson.M{
|
||||
"uid": userID,
|
||||
"cmd": bson.M{"$type": "string", "$ne": deletedStockDividendCommand},
|
||||
"cmd": bson.M{"$type": "string", "$nin": retiredCommands},
|
||||
"deleted": bson.M{"$ne": true},
|
||||
}, opts)
|
||||
if err != nil {
|
||||
@@ -487,7 +488,7 @@ func (s *mongoUsageStore) userIDByUsername(ctx context.Context, username string)
|
||||
}
|
||||
err := s.coll.FindOne(ctx,
|
||||
bson.M{
|
||||
"cmd": bson.M{"$ne": deletedStockDividendCommand},
|
||||
"cmd": bson.M{"$nin": retiredCommands},
|
||||
"uid": bson.M{"$gt": int64(0)},
|
||||
"user": username,
|
||||
"deleted": bson.M{"$ne": true},
|
||||
@@ -504,14 +505,19 @@ func (s *mongoUsageStore) userIDByUsername(ctx context.Context, username string)
|
||||
return doc.UserID, true, nil
|
||||
}
|
||||
|
||||
// deletedStockDividendCommand names the retired /stock_dividend command. Its
|
||||
// rows are kept with deleted: true as history.
|
||||
const deletedStockDividendCommand = "stock_dividend"
|
||||
// retiredCommands names commands removed from the bot. Their rows are kept
|
||||
// with deleted: true as history: /stock_dividend was split into the cash and
|
||||
// share dividend commands, and the monkeyd commands left with their module.
|
||||
var retiredCommands = []string{
|
||||
"stock_dividend",
|
||||
"monkeyd_crawl",
|
||||
"monkeyd_tags",
|
||||
}
|
||||
|
||||
// isRetiredCommand reports whether cmd has been removed from the bot. Its rows
|
||||
// are never incremented or shown, even ones missing the deleted flag.
|
||||
func isRetiredCommand(cmd string) bool {
|
||||
return cmd == deletedStockDividendCommand
|
||||
return slices.Contains(retiredCommands, cmd)
|
||||
}
|
||||
|
||||
func withLimit(pipeline mongo.Pipeline, limit int) mongo.Pipeline {
|
||||
|
||||
Reference in new issue
Block a user