mirror of
https://github.com/tiennm99/MTTools.git
synced 2026-10-19 20:15:27 +00:00
refactor: make crawler importable and add export pipeline
Move monkeyd and pdfout out of internal/ so other modules can import them, and add export.Export, which holds the crawl-to-PDF sequence the CLI used to inline. The CLI now parses flags and delegates, so an embedding program gets the same defaults and validation.
This commit is contained in:
1 parent
2855ce5e53
commit
3633e96cf5
15 files changed
+490
-128
No files matched your search
@@ -0,0 +1,119 @@
|
||||
package monkeyd
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novelFixture reproduces the two traits that break naive crawlers: the list is
|
||||
// newest-first, and chapter numbering has a gap (no chapter 4).
|
||||
const novelFixture = `<html><head><title>TEN TRUYEN</title></head><body>
|
||||
<h1>TEN TRUYEN</h1>
|
||||
<div class="list-chapters">
|
||||
<div class="item"><div class="episode-title"><a href="https://monkeydd.com/n/5.html">5</a></div></div>
|
||||
<div class="item"><div class="episode-title"><a href="https://monkeydd.com/n/3.html">3</a></div></div>
|
||||
<div class="item"><div class="episode-title"><a href="/n/2.html">2</a></div></div>
|
||||
<div class="item"><div class="episode-title"><a href="https://monkeydd.com/n/1.html">1</a></div></div>
|
||||
</div></body></html>`
|
||||
|
||||
func TestParseNovelPageOrdersChaptersForReading(t *testing.T) {
|
||||
novel, err := ParseNovelPage([]byte(novelFixture), "https://monkeydd.com/n.html")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseNovelPage: %v", err)
|
||||
}
|
||||
|
||||
if novel.Title != "TEN TRUYEN" {
|
||||
t.Errorf("Title = %q", novel.Title)
|
||||
}
|
||||
if novel.Slug != "n" {
|
||||
t.Errorf("Slug = %q, want %q", novel.Slug, "n")
|
||||
}
|
||||
|
||||
wantLabels := []string{"1", "2", "3", "5"}
|
||||
if len(novel.Chapters) != len(wantLabels) {
|
||||
t.Fatalf("got %d chapters, want %d", len(novel.Chapters), len(wantLabels))
|
||||
}
|
||||
for i, want := range wantLabels {
|
||||
if novel.Chapters[i].Label != want {
|
||||
t.Errorf("chapter %d label = %q, want %q", i, novel.Chapters[i].Label, want)
|
||||
}
|
||||
}
|
||||
// Relative hrefs must resolve against the novel URL.
|
||||
if got, want := novel.Chapters[1].URL, "https://monkeydd.com/n/2.html"; got != want {
|
||||
t.Errorf("chapter 2 URL = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNovelPageNoChapters(t *testing.T) {
|
||||
if _, err := ParseNovelPage([]byte(`<html><title>x</title><body></body></html>`),
|
||||
"https://monkeydd.com/n.html"); err == nil {
|
||||
t.Fatal("want an error when no chapters are listed")
|
||||
}
|
||||
}
|
||||
|
||||
const selectFixture = `<html><body>
|
||||
<select name="selected_chapter" id="selected_chapter">
|
||||
<option value="n,5">5</option>
|
||||
<option value="n,chuong-3">Chương 3</option>
|
||||
<option value="n,1">1</option>
|
||||
<option value="14">malformed</option>
|
||||
</select></body></html>`
|
||||
|
||||
func TestChapterRefsFromSelect(t *testing.T) {
|
||||
base, err := url.Parse("https://monkeydd.com/n.html")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refs, err := ChapterRefsFromSelect([]byte(selectFixture), base)
|
||||
if err != nil {
|
||||
t.Fatalf("ChapterRefsFromSelect: %v", err)
|
||||
}
|
||||
|
||||
if len(refs) != 3 {
|
||||
t.Fatalf("got %d refs %+v, want 3 (malformed option skipped)", len(refs), refs)
|
||||
}
|
||||
if got, want := refs[0].URL, "https://monkeydd.com/n/1.html"; got != want {
|
||||
t.Errorf("first ref URL = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := refs[1].URL, "https://monkeydd.com/n/chuong-3.html"; got != want {
|
||||
t.Errorf("second ref URL = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileChapterRefs(t *testing.T) {
|
||||
ref := func(u string) ChapterRef { return ChapterRef{Label: u, URL: u} }
|
||||
|
||||
t.Run("dropdown superset wins", func(t *testing.T) {
|
||||
list := []ChapterRef{ref("a"), ref("b")}
|
||||
sel := []ChapterRef{ref("a"), ref("b"), ref("c")}
|
||||
|
||||
final, extra := ReconcileChapterRefs(list, sel)
|
||||
if len(final) != 3 {
|
||||
t.Errorf("got %d chapters, want the 3 from the dropdown", len(final))
|
||||
}
|
||||
if len(extra) != 0 {
|
||||
t.Errorf("got %d extra, want 0", len(extra))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("disagreement is reported not silently merged", func(t *testing.T) {
|
||||
list := []ChapterRef{ref("a"), ref("z")}
|
||||
sel := []ChapterRef{ref("a"), ref("b")}
|
||||
|
||||
final, extra := ReconcileChapterRefs(list, sel)
|
||||
if len(final) != 2 || final[1].URL != "z" {
|
||||
t.Errorf("final = %+v, want the landing page list", final)
|
||||
}
|
||||
if len(extra) != 1 || extra[0].URL != "b" {
|
||||
t.Errorf("extra = %+v, want [b] so the caller can warn", extra)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty dropdown falls back to the list", func(t *testing.T) {
|
||||
list := []ChapterRef{ref("a")}
|
||||
final, extra := ReconcileChapterRefs(list, nil)
|
||||
if len(final) != 1 || len(extra) != 0 {
|
||||
t.Errorf("final = %+v, extra = %+v", final, extra)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user