feat(monkeyd): export monkeydd.com novels as PDF via /monkeyd_crawl

Add the monkeyd module, which crawls a novel and sends the rendered PDF back
as a Telegram document. Crawling and rendering come from the monkeyd-crawler
submodule, resolved through a go.mod replace directive.

The command is admin-only and restricted to monkeydd.com: one run makes
hundreds of outbound requests over minutes, and the extractor only understands
that site. Exports run one at a time and on a detached goroutine, because
handlers are dispatched synchronously and an inline crawl would block every
other command.

The runtime image gains DejaVuSans; font discovery probes system paths and the
distroless base ships none, so PDF rendering would otherwise fail in
production. CI checks out submodules and the builder copies the submodule
go.mod before go mod download, which needs it to resolve the build list.

internal/modules/monkeyd/export_job.go

@@ -0,0 +1,155 @@
+package monkeyd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"time"
+
+	"github.com/go-telegram/bot"
+	"github.com/go-telegram/bot/models"
+
+	"github.com/tiennm99/monkeyd-crawler/export"
+
+	"github.com/tiennm99/miti99bot/internal/log"
+	"github.com/tiennm99/miti99bot/internal/modules/util/chathelper"
+)
+
+const (
+	// crawlTimeout bounds one export. A long novel is roughly one request per
+	// 400ms plus retries, so even a thousand chapters fits well inside this;
+	// the ceiling exists to release the single export slot if the site stalls.
+	crawlTimeout = 30 * time.Minute
+
+	// uploadTimeout is separate from crawlTimeout so a slow crawl cannot eat
+	// the budget needed to actually deliver the finished PDF.
+	uploadTimeout = 5 * time.Minute
+
+	// statusTimeout bounds the short progress and failure replies.
+	statusTimeout = 30 * time.Second
+
+	// maxDocumentBytes is the Bot API's upload ceiling for sendDocument.
+	// Checked before opening the upload so an oversized book fails with an
+	// explanation instead of a Telegram API error.
+	maxDocumentBytes = 50 << 20
+)
+
+// cacheDirName is the shared page cache under the system temp directory.
+// Keeping it outside the per-run directory is what makes a repeat export of the
+// same novel cost no requests. It is not pruned; a container restart clears it.
+const cacheDirName = "miti99bot-monkeyd-cache"
+
+// export runs one crawl to completion and delivers the PDF. It is called on its
+// own goroutine, detached from the Telegram handler context.
+func (r *runner) export(b *bot.Bot, msg *models.Message, novelURL string) {
+	defer r.end()
+	// A panic here would reach no handler recover and would take the whole
+	// process down with it, so contain it.
+	defer func() {
+		if p := recover(); p != nil {
+			log.Error("monkeyd export panicked",
+				"command", commandName, "url", novelURL, "panic", p, "stack", string(debug.Stack()))
+			r.reportFailure(b, msg, "The export crashed. Nothing was sent.")
+		}
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), crawlTimeout)
+	defer cancel()
+
+	// The PDF is a temporary artefact: it is uploaded and then dropped.
+	outDir, err := os.MkdirTemp("", "monkeyd-export-*")
+	if err != nil {
+		log.Error("monkeyd temp dir failed", "command", commandName, "err", err)
+		r.reportFailure(b, msg, "Could not create a working directory for the export.")
+		return
+	}
+	defer func() {
+		if err := os.RemoveAll(outDir); err != nil {
+			log.Warn("monkeyd temp cleanup failed", "command", commandName, "dir", outDir, "err", err)
+		}
+	}()
+
+	result, err := r.exporter(ctx, export.Request{
+		NovelURL: novelURL,
+		OutDir:   outDir,
+		CacheDir: filepath.Join(os.TempDir(), cacheDirName),
+		// Per-chapter progress is one line per chapter — useful when
+		// diagnosing a stuck export, too noisy for the default level.
+		Log: func(format string, args ...any) {
+			log.Debug("monkeyd crawl: "+fmt.Sprintf(format, args...), "command", commandName, "url", novelURL)
+		},
+	})
+	if err != nil {
+		log.Error("monkeyd export failed", "command", commandName, "url", novelURL, "err", err)
+		r.reportFailure(b, msg, "Export failed: "+err.Error())
+		return
+	}
+
+	log.Info("monkeyd export done", "command", commandName, "url", novelURL,
+		"title", result.Title, "chapters", result.Chapters, "words", result.Words)
+
+	if err := sendPDF(b, msg, result); err != nil {
+		log.Error("monkeyd delivery failed", "command", commandName, "url", novelURL, "err", err)
+		r.reportFailure(b, msg, "The novel was exported but the PDF could not be sent: "+err.Error())
+	}
+}
+
+// sendPDF uploads the finished book as a Telegram document.
+func sendPDF(b *bot.Bot, msg *models.Message, result *export.Result) error {
+	info, err := os.Stat(result.Path)
+	if err != nil {
+		return fmt.Errorf("stat pdf: %w", err)
+	}
+	if info.Size() > maxDocumentBytes {
+		return fmt.Errorf("the PDF is %.1f MB, above Telegram's %d MB limit for bots",
+			float64(info.Size())/(1<<20), maxDocumentBytes>>20)
+	}
+
+	file, err := os.Open(result.Path)
+	if err != nil {
+		return fmt.Errorf("open pdf: %w", err)
+	}
+	defer func() { _ = file.Close() }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), uploadTimeout)
+	defer cancel()
+
+	// MessageThreadID keeps the document in the forum topic the command came
+	// from, matching chathelper.Reply's behaviour.
+	_, err = b.SendDocument(ctx, &bot.SendDocumentParams{
+		ChatID:          msg.Chat.ID,
+		MessageThreadID: msg.MessageThreadID,
+		Document: &models.InputFileUpload{
+			Filename: filepath.Base(result.Path),
+			Data:     file,
+		},
+		Caption: caption(result),
+	})
+	return err
+}
+
+// captionLimit is the Bot API's maximum caption length in characters.
+const captionLimit = 1024
+
+// caption describes the book under the document. Plain text, so a title
+// containing markup characters needs no escaping.
+func caption(result *export.Result) string {
+	text := fmt.Sprintf("%s\n%s page (%.0f x %.0f mm)\n%s",
+		result.Summary(), result.Page.Name, result.Page.W, result.Page.H, result.SourceURL)
+	if runes := []rune(text); len(runes) > captionLimit {
+		text = string(runes[:captionLimit])
+	}
+	return text
+}
+
+// reportFailure tells the chat the export did not produce a document. Delivery
+// failures here are only logged — there is nothing further to fall back to.
+func (r *runner) reportFailure(b *bot.Bot, msg *models.Message, text string) {
+	ctx, cancel := context.WithTimeout(context.Background(), statusTimeout)
+	defer cancel()
+	if err := chathelper.Reply(ctx, b, msg, text); err != nil {
+		log.Error("monkeyd failure reply failed", "command", commandName, "err", err)
+	}
+}

internal/modules/monkeyd/handlers_test.go

@@ -0,0 +1,286 @@
+package monkeyd
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/tiennm99/monkeyd-crawler/export"
+	"github.com/tiennm99/monkeyd-crawler/pdfout"
+
+	"github.com/tiennm99/miti99bot/internal/modules"
+	"github.com/tiennm99/miti99bot/internal/storage"
+	"github.com/tiennm99/miti99bot/internal/testutil"
+)
+
+const testNovelURL = "https://monkeydd.com/tro-lai-nam-thang-cu.html"
+
+// install wires the module to a recording bot. The returned runner is the one
+// behind the registered command, so tests can substitute its exporter and run
+// the export synchronously instead of on a detached goroutine.
+//
+// ownerID is permitted, which /monkeyd_crawl requires: the command is
+// Protected, and the dispatcher drops unauthorized calls silently.
+func install(t *testing.T, ownerID int64) (*testutil.RecordingBot, *runner) {
+	t.Helper()
+	rb := testutil.NewRecordingBot(t)
+	r := newRunner()
+	// Run the export inline so assertions do not race a detached goroutine.
+	r.launch = func(job func()) { job() }
+	mod := newModule(r)
+
+	reg := &modules.Registry{
+		Modules:     []modules.Module{{Name: "monkeyd", Commands: mod.Commands}},
+		AllCommands: map[string]modules.Command{},
+	}
+	for _, c := range mod.Commands {
+		reg.AllCommands[c.Name] = c
+	}
+	modules.Install(rb.Bot, reg, modules.Auth{BotOwnerID: ownerID})
+	return rb, r
+}
+
+// stubPDF creates a file of the given size standing in for a rendered book and
+// returns a Result describing it, as export.Export would. The file is sized by
+// truncation so an over-the-limit case costs no real bytes.
+func stubPDF(t *testing.T, dir, name string, size int64) *export.Result {
+	t.Helper()
+	path := filepath.Join(dir, name)
+	file, err := os.Create(path)
+	if err != nil {
+		t.Fatalf("create stub pdf: %v", err)
+	}
+	if err := file.Truncate(size); err != nil {
+		_ = file.Close()
+		t.Fatalf("size stub pdf: %v", err)
+	}
+	if err := file.Close(); err != nil {
+		t.Fatalf("close stub pdf: %v", err)
+	}
+	return &export.Result{
+		Path:      path,
+		Title:     "Example Novel",
+		SourceURL: testNovelURL,
+		Chapters:  3,
+		Words:     1200,
+		Page:      pdfout.Presets["phone"],
+	}
+}
+
+func TestCrawl_NoArgumentRepliesUsage(t *testing.T) {
+	rb, _ := install(t, 999)
+	rb.Bot.ProcessUpdate(context.Background(), testutil.NewPrivateMessage(999, "/monkeyd_crawl"))
+
+	if got := rb.LastSent().Text(); !strings.Contains(got, usage) {
+		t.Errorf("reply = %q, want it to contain %q", got, usage)
+	}
+}
+
+func TestCrawl_RejectsDisallowedHost(t *testing.T) {
+	rb, r := install(t, 999)
+	called := false
+	r.exporter = func(context.Context, export.Request) (*export.Result, error) {
+		called = true
+		return nil, nil
+	}
+
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewPrivateMessage(999, "/monkeyd_crawl https://example.com/novel.html"))
+
+	if called {
+		t.Error("exporter ran for a disallowed host")
+	}
+	if got := rb.LastSent().Text(); !strings.Contains(got, AllowedHostsHint) {
+		t.Errorf("reply = %q, want it to name %q", got, AllowedHostsHint)
+	}
+}
+
+func TestCrawl_SendsPDFOnSuccess(t *testing.T) {
+	rb, r := install(t, 999)
+	dir := t.TempDir()
+
+	var gotRequest export.Request
+	r.exporter = func(_ context.Context, req export.Request) (*export.Result, error) {
+		gotRequest = req
+		return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil
+	}
+
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL))
+
+	if gotRequest.NovelURL != testNovelURL {
+		t.Errorf("exporter got NovelURL %q, want %q", gotRequest.NovelURL, testNovelURL)
+	}
+	if gotRequest.OutDir == "" {
+		t.Error("exporter got an empty OutDir; the PDF would land in the working directory")
+	}
+
+	calls := rb.Sent()
+	if len(calls) < 2 {
+		t.Fatalf("expected an acknowledgement and a document, got %d calls: %+v", len(calls), calls)
+	}
+	if first := calls[0]; !strings.Contains(first.Text(), testNovelURL) {
+		t.Errorf("first reply = %q, want it to name the novel URL", first.Text())
+	}
+	last := calls[len(calls)-1]
+	if last.Method != "sendDocument" {
+		t.Fatalf("last call = %q, want sendDocument", last.Method)
+	}
+	if caption := last.Form["caption"]; !strings.Contains(caption, "Example Novel") {
+		t.Errorf("caption = %q, want it to name the novel", caption)
+	}
+}
+
+func TestCrawl_ReportsExportFailure(t *testing.T) {
+	rb, r := install(t, 999)
+	r.exporter = func(context.Context, export.Request) (*export.Result, error) {
+		return nil, errors.New("fetch failed: 404")
+	}
+
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL))
+
+	last := rb.LastSent()
+	if last.Method == "sendDocument" {
+		t.Fatal("a document was sent despite the export failing")
+	}
+	if got := last.Text(); !strings.Contains(got, "404") {
+		t.Errorf("failure reply = %q, want it to carry the underlying error", got)
+	}
+}
+
+func TestCrawl_RefusesOversizedPDF(t *testing.T) {
+	rb, r := install(t, 999)
+	dir := t.TempDir()
+	r.exporter = func(context.Context, export.Request) (*export.Result, error) {
+		return stubPDF(t, dir, "Huge-Novel.pdf", maxDocumentBytes+1), nil
+	}
+
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL))
+
+	last := rb.LastSent()
+	if last.Method == "sendDocument" {
+		t.Fatal("an oversized document was uploaded instead of being refused")
+	}
+	if got := last.Text(); !strings.Contains(got, "limit") {
+		t.Errorf("reply = %q, want it to explain the size limit", got)
+	}
+}
+
+// The single export slot must be released whether the run succeeded or failed,
+// otherwise the command is dead until the process restarts.
+func TestCrawl_ReleasesSlotAfterRun(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		exporter func(t *testing.T) func(context.Context, export.Request) (*export.Result, error)
+	}{
+		{
+			name: "after success",
+			exporter: func(t *testing.T) func(context.Context, export.Request) (*export.Result, error) {
+				dir := t.TempDir()
+				return func(context.Context, export.Request) (*export.Result, error) {
+					return stubPDF(t, dir, "Example-Novel.pdf", 2048), nil
+				}
+			},
+		},
+		{
+			name: "after failure",
+			exporter: func(*testing.T) func(context.Context, export.Request) (*export.Result, error) {
+				return func(context.Context, export.Request) (*export.Result, error) {
+					return nil, errors.New("boom")
+				}
+			},
+		},
+		{
+			name: "after panic",
+			exporter: func(*testing.T) func(context.Context, export.Request) (*export.Result, error) {
+				return func(context.Context, export.Request) (*export.Result, error) {
+					panic("boom")
+				}
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			rb, r := install(t, 999)
+			r.exporter = tc.exporter(t)
+
+			rb.Bot.ProcessUpdate(context.Background(),
+				testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL))
+
+			r.mu.Lock()
+			running := r.running
+			r.mu.Unlock()
+			if running {
+				t.Error("export slot still held after the run finished")
+			}
+		})
+	}
+}
+
+func TestCrawl_RepliesBusyWhileAnotherExportRuns(t *testing.T) {
+	rb, r := install(t, 999)
+	// Simulate an in-flight export rather than racing a real one.
+	if ok, _ := r.begin("https://monkeydd.com/other-novel.html"); !ok {
+		t.Fatal("could not claim the export slot")
+	}
+
+	called := false
+	r.exporter = func(context.Context, export.Request) (*export.Result, error) {
+		called = true
+		return nil, nil
+	}
+
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewPrivateMessage(999, "/monkeyd_crawl "+testNovelURL))
+
+	if called {
+		t.Error("a second export started while one was already running")
+	}
+	got := rb.LastSent().Text()
+	if !strings.Contains(got, "other-novel") {
+		t.Errorf("busy reply = %q, want it to name the in-flight novel", got)
+	}
+}
+
+// A Protected command must not respond at all to an unauthorized sender, or its
+// existence leaks.
+func TestCrawl_SilentForUnauthorizedSender(t *testing.T) {
+	rb, _ := install(t, 999)
+	rb.Bot.ProcessUpdate(context.Background(),
+		testutil.NewPrivateMessage(12345, "/monkeyd_crawl "+testNovelURL))
+
+	if calls := rb.Sent(); len(calls) != 0 {
+		t.Errorf("expected no reply to an unauthorized sender, got %+v", calls)
+	}
+}
+
+func TestRegistration(t *testing.T) {
+	mod := New(modules.Deps{Store: storage.NewMemoryProvider().Collection("monkeyd")})
+	if len(mod.Commands) != 1 {
+		t.Fatalf("expected 1 command, got %d", len(mod.Commands))
+	}
+	cmd := mod.Commands[0]
+	if cmd.Name != commandName {
+		t.Errorf("Name = %q, want %q", cmd.Name, commandName)
+	}
+	if cmd.Visibility != modules.VisibilityProtected {
+		t.Errorf("Visibility = %v, want Protected", cmd.Visibility)
+	}
+	if cmd.Parameters != "<url>" {
+		t.Errorf("Parameters = %q, want %q", cmd.Parameters, "<url>")
+	}
+	if cmd.Description == "" {
+		t.Error("Description is empty; command discovery requires one")
+	}
+	if cmd.Handler == nil {
+		t.Error("Handler is nil")
+	}
+	if len(mod.Crons) != 0 || len(mod.Callbacks) != 0 {
+		t.Errorf("expected no crons or callbacks, got %d crons and %d callbacks",
+			len(mod.Crons), len(mod.Callbacks))
+	}
+}

internal/modules/monkeyd/monkeyd.go

@@ -0,0 +1,151 @@
+// Package monkeyd exports a monkeydd.com novel as a PDF and sends it back as a
+// Telegram document. The crawling and rendering live in the monkeyd-crawler
+// submodule (third_party/monkeyd-crawler); this module is the Telegram surface
+// around it: argument validation, one-at-a-time scheduling, and delivery.
+package monkeyd
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/go-telegram/bot"
+	"github.com/go-telegram/bot/models"
+
+	"github.com/tiennm99/monkeyd-crawler/export"
+
+	"github.com/tiennm99/miti99bot/internal/modules"
+	"github.com/tiennm99/miti99bot/internal/modules/util/chathelper"
+)
+
+// commandName is the single command this module exposes.
+const commandName = "monkeyd_crawl"
+
+// usage is shown when the command arrives without a usable URL. It repeats the
+// Parameters syntax so the error and the command menu agree.
+const usage = "Usage: /" + commandName + " <url>"
+
+// New is the module Factory. The module keeps no persistent state — an export
+// is a one-shot job — so deps.Store is unused.
+func New(_ modules.Deps) modules.Module {
+	return newModule(newRunner())
+}
+
+// newModule builds the module around a given runner, which is how tests supply
+// one with a stubbed exporter and a synchronous launch.
+func newModule(r *runner) modules.Module {
+	return modules.Module{
+		Commands: []modules.Command{
+			{
+				Name:       commandName,
+				Visibility: modules.VisibilityProtected,
+				// One invocation makes hundreds of outbound requests over
+				// several minutes, so it stays off the public surface.
+				Description: "Export a " + AllowedHostsHint + " novel as a PDF",
+				Parameters:  "<url>",
+				Handler:     r.handle,
+			},
+		},
+	}
+}
+
+// runner serialises exports. The crawler spaces its own requests out per run,
+// so two concurrent crawls would double the request rate against the site —
+// and a novel is minutes of work, which makes queueing pointless. One at a
+// time, globally, with a clear reply to anyone who asks meanwhile.
+type runner struct {
+	mu      sync.Mutex
+	running bool
+	current string // novel URL of the in-flight export, for the busy reply
+
+	// exporter is the crawl-and-render step. It is a field so tests can
+	// exercise scheduling and delivery without network access.
+	exporter func(context.Context, export.Request) (*export.Result, error)
+
+	// launch runs an export job. Production detaches it onto its own
+	// goroutine; tests substitute a synchronous run for determinism.
+	launch func(job func())
+}
+
+func newRunner() *runner {
+	return &runner{
+		exporter: export.Export,
+		launch: func(job func()) {
+			// Detached from the handler context on purpose: handlers run one
+			// at a time (the bot is built WithNotAsyncHandlers), so crawling
+			// inline would block every other command for minutes. The job
+			// owns its own timeout and recovers its own panics.
+			go job() //nolint:gosec // G118: intentional; see runner.export
+		},
+	}
+}
+
+// begin claims the single export slot, reporting the in-flight URL when it is
+// already taken.
+func (r *runner) begin(novelURL string) (ok bool, inFlight string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.running {
+		return false, r.current
+	}
+	r.running = true
+	r.current = novelURL
+	return true, ""
+}
+
+func (r *runner) end() {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.running = false
+	r.current = ""
+}
+
+func (r *runner) handle(ctx context.Context, b *bot.Bot, update *models.Update) error {
+	msg := update.Message
+	if msg == nil {
+		return nil
+	}
+
+	arg := chathelper.ArgAfterCommand(msg.Text)
+	if arg == "" {
+		return chathelper.Reply(ctx, b, msg, usage)
+	}
+	// Telegram may hand the URL over with trailing punctuation or a stray
+	// second word; only the first token can be the URL.
+	if fields := strings.Fields(arg); len(fields) > 0 {
+		arg = fields[0]
+	}
+
+	novelURL, err := normalizeNovelURL(arg)
+	if err != nil {
+		return chathelper.Reply(ctx, b, msg, fmt.Sprintf("%s.\n%s", capitalize(err.Error()), usage))
+	}
+
+	ok, inFlight := r.begin(novelURL)
+	if !ok {
+		return chathelper.Reply(ctx, b, msg,
+			"Already exporting "+inFlight+". Try again once it finishes.")
+	}
+
+	// Reply before starting so the user knows the wait is expected. If the
+	// reply cannot be delivered, drop the slot rather than crawling for a chat
+	// that will never hear the result.
+	if err := chathelper.Reply(ctx, b, msg,
+		"Exporting "+novelURL+" — this takes a few minutes. I will send the PDF here when it is ready."); err != nil {
+		r.end()
+		return err
+	}
+
+	r.launch(func() { r.export(b, msg, novelURL) })
+	return nil
+}
+
+// capitalize upper-cases the first letter so a lower-case error string reads as
+// a sentence in a Telegram reply.
+func capitalize(s string) string {
+	if s == "" {
+		return s
+	}
+	return strings.ToUpper(s[:1]) + s[1:]
+}

internal/modules/monkeyd/novel_url.go

@@ -0,0 +1,71 @@
+package monkeyd
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+)
+
+// allowedHosts are the hostnames the crawler's extractor understands. It is
+// written against monkeydd.com's specific markup — the chapter list, the
+// in-chapter dropdown, and the CSS rules that supply part of the chapter text —
+// so another host would parse to nothing useful. Refusing it up front also
+// keeps the command from being used to make the bot fetch arbitrary URLs.
+var allowedHosts = map[string]bool{
+	"monkeydd.com":     true,
+	"www.monkeydd.com": true,
+}
+
+// AllowedHostsHint lists the accepted hosts for user-facing usage text.
+const AllowedHostsHint = "monkeydd.com"
+
+var (
+	errNotAURL      = errors.New("that does not look like a URL")
+	errHostNotAllow = errors.New("only " + AllowedHostsHint + " novel URLs are supported")
+	errNoPath       = errors.New("that URL has no novel path — link the novel's own page")
+)
+
+// normalizeNovelURL validates a user-supplied novel URL and returns the form to
+// crawl. A missing scheme is filled in with https, since people paste bare
+// hostnames; anything else that is not a plain http(s) URL on an allowed host
+// is rejected.
+func normalizeNovelURL(raw string) (string, error) {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return "", errNotAURL
+	}
+	// A bare "monkeydd.com/x.html" parses as a path with no host, so give it a
+	// scheme before parsing rather than trying to interpret the result.
+	if !strings.Contains(raw, "://") {
+		raw = "https://" + raw
+	}
+
+	parsed, err := url.Parse(raw)
+	if err != nil {
+		return "", errNotAURL
+	}
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		return "", errNotAURL
+	}
+	// Credentials in the URL are never needed here and would be logged with the
+	// crawl, so treat them as malformed input.
+	if parsed.User != nil {
+		return "", errNotAURL
+	}
+	// Hostname() drops any port, which the allowlist must not be fooled by.
+	if !allowedHosts[strings.ToLower(parsed.Hostname())] {
+		return "", errHostNotAllow
+	}
+	if strings.Trim(parsed.Path, "/") == "" {
+		return "", errNoPath
+	}
+	// Rebuild from the parsed parts so the crawl uses a canonical host and no
+	// fragment; query strings are kept because the site may need them.
+	canonical := url.URL{
+		Scheme:   parsed.Scheme,
+		Host:     strings.ToLower(parsed.Host),
+		Path:     parsed.Path,
+		RawQuery: parsed.RawQuery,
+	}
+	return canonical.String(), nil
+}

internal/modules/monkeyd/novel_url_test.go

@@ -0,0 +1,128 @@
+package monkeyd
+
+import "testing"
+
+func TestNormalizeNovelURL(t *testing.T) {
+	tests := []struct {
+		name    string
+		raw     string
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "https url passes through",
+			raw:  "https://monkeydd.com/tro-lai-nam-thang-cu.html",
+			want: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
+		},
+		{
+			name: "www host is allowed",
+			raw:  "https://www.monkeydd.com/tro-lai-nam-thang-cu.html",
+			want: "https://www.monkeydd.com/tro-lai-nam-thang-cu.html",
+		},
+		{
+			name: "missing scheme gets https",
+			raw:  "monkeydd.com/tro-lai-nam-thang-cu.html",
+			want: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
+		},
+		{
+			name: "http is kept",
+			raw:  "http://monkeydd.com/tro-lai-nam-thang-cu.html",
+			want: "http://monkeydd.com/tro-lai-nam-thang-cu.html",
+		},
+		{
+			name: "surrounding whitespace is trimmed",
+			raw:  "  https://monkeydd.com/tro-lai-nam-thang-cu.html  ",
+			want: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
+		},
+		{
+			name: "uppercase host is canonicalised",
+			raw:  "https://MonkeyDD.com/tro-lai-nam-thang-cu.html",
+			want: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
+		},
+		{
+			name: "fragment is dropped",
+			raw:  "https://monkeydd.com/tro-lai-nam-thang-cu.html#chuong-1",
+			want: "https://monkeydd.com/tro-lai-nam-thang-cu.html",
+		},
+		{
+			name: "query is preserved",
+			raw:  "https://monkeydd.com/novel.html?page=2",
+			want: "https://monkeydd.com/novel.html?page=2",
+		},
+		{
+			name:    "empty input",
+			raw:     "",
+			wantErr: true,
+		},
+		{
+			name:    "other host",
+			raw:     "https://example.com/novel.html",
+			wantErr: true,
+		},
+		{
+			name:    "host that merely ends with the allowed name",
+			raw:     "https://evilmonkeydd.com/novel.html",
+			wantErr: true,
+		},
+		{
+			name:    "host that merely contains the allowed name",
+			raw:     "https://monkeydd.com.evil.example/novel.html",
+			wantErr: true,
+		},
+		{
+			name:    "allowed host as a subdomain of another host",
+			raw:     "https://monkeydd.com.attacker.test/novel.html",
+			wantErr: true,
+		},
+		{
+			name:    "non-http scheme",
+			raw:     "file:///etc/passwd",
+			wantErr: true,
+		},
+		{
+			name:    "credentials in url",
+			raw:     "https://user:pass@monkeydd.com/novel.html",
+			wantErr: true,
+		},
+		{
+			name:    "landing page with no novel path",
+			raw:     "https://monkeydd.com",
+			wantErr: true,
+		},
+		{
+			name:    "root path only",
+			raw:     "https://monkeydd.com/",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := normalizeNovelURL(tt.raw)
+			if tt.wantErr {
+				if err == nil {
+					t.Fatalf("normalizeNovelURL(%q) = %q, want error", tt.raw, got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("normalizeNovelURL(%q) returned error: %v", tt.raw, err)
+			}
+			if got != tt.want {
+				t.Errorf("normalizeNovelURL(%q) = %q, want %q", tt.raw, got, tt.want)
+			}
+		})
+	}
+}
+
+// A port on an allowed host must not defeat the allowlist, and must survive
+// canonicalisation so the crawl reaches the same place the user asked for.
+func TestNormalizeNovelURLKeepsPort(t *testing.T) {
+	got, err := normalizeNovelURL("http://monkeydd.com:8080/novel.html")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if want := "http://monkeydd.com:8080/novel.html"; got != want {
+		t.Errorf("got %q, want %q", got, want)
+	}
+}