refactor: make crawler importable and add export pipeline

Move monkeyd and pdfout out of internal/ so other modules can import them,
and add export.Export, which holds the crawl-to-PDF sequence the CLI used to
inline. The CLI now parses flags and delegates, so an embedding program gets
the same defaults and validation.
This commit is contained in:
tiennm99 committed 2026-07-29 22:49:30 +07:00
1 parent 2855ce5e53
commit 3633e96cf5
15 files changed
+490 -128

No files matched your search

+119
View File
@@ -0,0 +1,119 @@
package monkeyd
import (
"net/url"
"testing"
)
// novelFixture reproduces the two traits that break naive crawlers: the list is
// newest-first, and chapter numbering has a gap (no chapter 4).
const novelFixture = `<html><head><title>TEN TRUYEN</title></head><body>
<h1>TEN TRUYEN</h1>
<div class="list-chapters">
<div class="item"><div class="episode-title"><a href="https://monkeydd.com/n/5.html">5</a></div></div>
<div class="item"><div class="episode-title"><a href="https://monkeydd.com/n/3.html">3</a></div></div>
<div class="item"><div class="episode-title"><a href="/n/2.html">2</a></div></div>
<div class="item"><div class="episode-title"><a href="https://monkeydd.com/n/1.html">1</a></div></div>
</div></body></html>`
func TestParseNovelPageOrdersChaptersForReading(t *testing.T) {
novel, err := ParseNovelPage([]byte(novelFixture), "https://monkeydd.com/n.html")
if err != nil {
t.Fatalf("ParseNovelPage: %v", err)
}
if novel.Title != "TEN TRUYEN" {
t.Errorf("Title = %q", novel.Title)
}
if novel.Slug != "n" {
t.Errorf("Slug = %q, want %q", novel.Slug, "n")
}
wantLabels := []string{"1", "2", "3", "5"}
if len(novel.Chapters) != len(wantLabels) {
t.Fatalf("got %d chapters, want %d", len(novel.Chapters), len(wantLabels))
}
for i, want := range wantLabels {
if novel.Chapters[i].Label != want {
t.Errorf("chapter %d label = %q, want %q", i, novel.Chapters[i].Label, want)
}
}
// Relative hrefs must resolve against the novel URL.
if got, want := novel.Chapters[1].URL, "https://monkeydd.com/n/2.html"; got != want {
t.Errorf("chapter 2 URL = %q, want %q", got, want)
}
}
func TestParseNovelPageNoChapters(t *testing.T) {
if _, err := ParseNovelPage([]byte(`<html><title>x</title><body></body></html>`),
"https://monkeydd.com/n.html"); err == nil {
t.Fatal("want an error when no chapters are listed")
}
}
const selectFixture = `<html><body>
<select name="selected_chapter" id="selected_chapter">
<option value="n,5">5</option>
<option value="n,chuong-3">Chương 3</option>
<option value="n,1">1</option>
<option value="14">malformed</option>
</select></body></html>`
func TestChapterRefsFromSelect(t *testing.T) {
base, err := url.Parse("https://monkeydd.com/n.html")
if err != nil {
t.Fatal(err)
}
refs, err := ChapterRefsFromSelect([]byte(selectFixture), base)
if err != nil {
t.Fatalf("ChapterRefsFromSelect: %v", err)
}
if len(refs) != 3 {
t.Fatalf("got %d refs %+v, want 3 (malformed option skipped)", len(refs), refs)
}
if got, want := refs[0].URL, "https://monkeydd.com/n/1.html"; got != want {
t.Errorf("first ref URL = %q, want %q", got, want)
}
if got, want := refs[1].URL, "https://monkeydd.com/n/chuong-3.html"; got != want {
t.Errorf("second ref URL = %q, want %q", got, want)
}
}
func TestReconcileChapterRefs(t *testing.T) {
ref := func(u string) ChapterRef { return ChapterRef{Label: u, URL: u} }
t.Run("dropdown superset wins", func(t *testing.T) {
list := []ChapterRef{ref("a"), ref("b")}
sel := []ChapterRef{ref("a"), ref("b"), ref("c")}
final, extra := ReconcileChapterRefs(list, sel)
if len(final) != 3 {
t.Errorf("got %d chapters, want the 3 from the dropdown", len(final))
}
if len(extra) != 0 {
t.Errorf("got %d extra, want 0", len(extra))
}
})
t.Run("disagreement is reported not silently merged", func(t *testing.T) {
list := []ChapterRef{ref("a"), ref("z")}
sel := []ChapterRef{ref("a"), ref("b")}
final, extra := ReconcileChapterRefs(list, sel)
if len(final) != 2 || final[1].URL != "z" {
t.Errorf("final = %+v, want the landing page list", final)
}
if len(extra) != 1 || extra[0].URL != "b" {
t.Errorf("extra = %+v, want [b] so the caller can warn", extra)
}
})
t.Run("empty dropdown falls back to the list", func(t *testing.T) {
list := []ChapterRef{ref("a")}
final, extra := ReconcileChapterRefs(list, nil)
if len(final) != 1 || len(extra) != 0 {
t.Errorf("final = %+v, extra = %+v", final, extra)
}
})
}