diff --git a/monkeyd-crawler/internal/monkeyd/chapter.go b/monkeyd-crawler/internal/monkeyd/chapter.go index 40d0548..b9466a2 100644 --- a/monkeyd-crawler/internal/monkeyd/chapter.go +++ b/monkeyd-crawler/internal/monkeyd/chapter.go @@ -48,11 +48,45 @@ var blockTags = map[string]bool{ "li": true, "ul": true, "ol": true, "tr": true, } -// skipTags never contribute text: scripts, styles and the ad/report widgets -// the site injects inside the content container. +// skipTags never contribute prose. +// +// Anchors are included because inside a chapter body they are always site +// chrome: the prev/next chapter navigation and the sponsor call-to-action are +// both links, while novel prose never needs one. Inline emphasis tags (b, i, +// em) are deliberately absent so italics in the prose survive; the site's icons +// use but carry no text. var skipTags = map[string]bool{ "script": true, "style": true, "noscript": true, "iframe": true, "ins": true, "form": true, "select": true, "button": true, "textarea": true, + "a": true, "img": true, "svg": true, +} + +// junkClasses marks containers the site injects into the chapter body. Their +// whole subtree is dropped. +// +// These are matched on class rather than position because the blocks move: the +// sponsor block opens the body on most chapters but is absent on others, and +// the watermark is planted at a different paragraph in every chapter. Only +// site-specific class names are listed; generic Bootstrap utilities such as +// "my-4" or "text-center" are not, since prose could legitimately carry them. +// +// Note the sibling class "actac" is NOT junk: it wraps the real chapter text and +// carries style="display:none", because the site gates the body behind a click +// on the sponsor link and reveals it with JavaScript. Skipping hidden elements, +// or skipping "act*" as a family, would therefore discard the whole chapter. +var junkClasses = map[string]bool{ + "actcl": true, // sponsor block shown in place of the gated chapter body + "signature": true, // "[Truyện được đăng tải duy nhất tại ...]" source watermark +} + +// hasJunkClass reports whether a node is an injected non-prose container. +func hasJunkClass(n *html.Node) bool { + for _, tok := range strings.Fields(attr(n, "class")) { + if junkClasses[tok] { + return true + } + } + return false } // ParseChapter extracts a chapter's paragraphs, restoring the words the site @@ -93,7 +127,7 @@ func extractParagraphs(content *html.Node, words map[string]string) []string { b.WriteString(n.Data) return case html.ElementNode: - if skipTags[n.Data] { + if skipTags[n.Data] || hasJunkClass(n) { return } // These elements are empty in the markup; the CSS word replaces them. diff --git a/monkeyd-crawler/internal/monkeyd/chapter_test.go b/monkeyd-crawler/internal/monkeyd/chapter_test.go index 94e64a2..2cd877e 100644 --- a/monkeyd-crawler/internal/monkeyd/chapter_test.go +++ b/monkeyd-crawler/internal/monkeyd/chapter_test.go @@ -108,6 +108,77 @@ func TestParseChapterDropsRepeatedTitle(t *testing.T) { } } +// gatedChapterFixture mirrors how most chapters are served: a visible sponsor +// block (div.actcl) stands in for the body, while the real text sits in a +// sibling div.actac hidden with display:none and revealed by the site's +// JavaScript. A source watermark and prev/next navigation bracket the prose. +const gatedChapterFixture = ` + +
+
+

Moi Quy doc gia CLICK vao lien ket

+

mo ung dung Shopee, sau do quay tro lai de doc!

+ + CLICK +

MonkeyD va doi ngu Editor xin chan thanh cam on!

+
+
+

Doan van dau tien co tri.

+

 

+

[Truyen duoc dang tai duy nhat tai MonkeyDD.com - https://monkeydd.com/n/10.html.]

+

Doan van cuoi cung.

+
+ +
` + +// The gated body is the one thing that must survive: it is hidden with +// display:none, so any rule that drops hidden or "act*" containers silently +// discards the entire chapter. +func TestParseChapterKeepsGatedBody(t *testing.T) { + ch, err := ParseChapter([]byte(gatedChapterFixture), ChapterRef{Label: "10", URL: "http://x/10.html"}) + if err != nil { + t.Fatalf("ParseChapter: %v", err) + } + + want := []string{ + "Doan van dau tien co vị tri.", + "Doan van cuoi cung.", + } + if len(ch.Paragraphs) != len(want) { + t.Fatalf("got %d paragraphs %q, want %d", len(ch.Paragraphs), ch.Paragraphs, len(want)) + } + for i, w := range want { + if ch.Paragraphs[i] != w { + t.Errorf("paragraph %d = %q, want %q", i, ch.Paragraphs[i], w) + } + } +} + +// Everything the site injects around the prose must be gone. +func TestParseChapterDropsInjectedBlocks(t *testing.T) { + ch, err := ParseChapter([]byte(gatedChapterFixture), ChapterRef{Label: "10", URL: "http://x/10.html"}) + if err != nil { + t.Fatalf("ParseChapter: %v", err) + } + body := strings.Join(ch.Paragraphs, "\n") + + for _, junk := range []string{ + "Shopee", // sponsor copy + "CLICK", // sponsor call to action + "cam on", // sponsor sign-off + "MonkeyDD.com", // source watermark + "Chương trước", // navigation + "Chương sau", // navigation + } { + if strings.Contains(body, junk) { + t.Errorf("injected text %q survived extraction in %q", junk, body) + } + } +} + func TestParseChapterMissingContainer(t *testing.T) { if _, err := ParseChapter([]byte(`

hi

`), ChapterRef{URL: "http://x/1.html"}); err == nil {