From 2855ce5e533f0284c5b8517b6c78787c08473964 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Wed, 29 Jul 2026 22:36:15 +0700 Subject: [PATCH] fix: drop sponsor, watermark and navigation blocks from chapter text Three blocks the site injects inside the chapter body were being extracted as prose: the sponsor block, a source watermark planted mid-chapter, and the prev/next chapter links. Anchors and images are now skipped, since inside a chapter body they are always site chrome, and the sponsor wrapper and watermark are skipped by class. Inline emphasis tags are left alone so italics in the prose survive. The sponsor wrapper has a sibling that must NOT be skipped: on most chapters the real text sits in a container hidden with display:none, because the body is gated behind a click on the sponsor link and revealed with JavaScript. Skipping hidden elements, or treating the two sibling classes as one family, discards the entire chapter. A test pins the gated body as kept while its sibling is dropped. Removes 662 words of injected text across the sample novel and lets the repeated chapter-number paragraph be recognised again, which the sponsor block had been masking by displacing it from the top of the body. --- monkeyd-crawler/internal/monkeyd/chapter.go | 40 ++++++++++- .../internal/monkeyd/chapter_test.go | 71 +++++++++++++++++++ 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/monkeyd-crawler/internal/monkeyd/chapter.go b/monkeyd-crawler/internal/monkeyd/chapter.go index 40d0548..b9466a2 100644 --- a/monkeyd-crawler/internal/monkeyd/chapter.go +++ b/monkeyd-crawler/internal/monkeyd/chapter.go @@ -48,11 +48,45 @@ var blockTags = map[string]bool{ "li": true, "ul": true, "ol": true, "tr": true, } -// skipTags never contribute text: scripts, styles and the ad/report widgets -// the site injects inside the content container. +// skipTags never contribute prose. +// +// Anchors are included because inside a chapter body they are always site +// chrome: the prev/next chapter navigation and the sponsor call-to-action are +// both links, while novel prose never needs one. Inline emphasis tags (b, i, +// em) are deliberately absent so italics in the prose survive; the site's icons +// use but carry no text. var skipTags = map[string]bool{ "script": true, "style": true, "noscript": true, "iframe": true, "ins": true, "form": true, "select": true, "button": true, "textarea": true, + "a": true, "img": true, "svg": true, +} + +// junkClasses marks containers the site injects into the chapter body. Their +// whole subtree is dropped. +// +// These are matched on class rather than position because the blocks move: the +// sponsor block opens the body on most chapters but is absent on others, and +// the watermark is planted at a different paragraph in every chapter. Only +// site-specific class names are listed; generic Bootstrap utilities such as +// "my-4" or "text-center" are not, since prose could legitimately carry them. +// +// Note the sibling class "actac" is NOT junk: it wraps the real chapter text and +// carries style="display:none", because the site gates the body behind a click +// on the sponsor link and reveals it with JavaScript. Skipping hidden elements, +// or skipping "act*" as a family, would therefore discard the whole chapter. +var junkClasses = map[string]bool{ + "actcl": true, // sponsor block shown in place of the gated chapter body + "signature": true, // "[Truyện được đăng tải duy nhất tại ...]" source watermark +} + +// hasJunkClass reports whether a node is an injected non-prose container. +func hasJunkClass(n *html.Node) bool { + for _, tok := range strings.Fields(attr(n, "class")) { + if junkClasses[tok] { + return true + } + } + return false } // ParseChapter extracts a chapter's paragraphs, restoring the words the site @@ -93,7 +127,7 @@ func extractParagraphs(content *html.Node, words map[string]string) []string { b.WriteString(n.Data) return case html.ElementNode: - if skipTags[n.Data] { + if skipTags[n.Data] || hasJunkClass(n) { return } // These elements are empty in the markup; the CSS word replaces them. diff --git a/monkeyd-crawler/internal/monkeyd/chapter_test.go b/monkeyd-crawler/internal/monkeyd/chapter_test.go index 94e64a2..2cd877e 100644 --- a/monkeyd-crawler/internal/monkeyd/chapter_test.go +++ b/monkeyd-crawler/internal/monkeyd/chapter_test.go @@ -108,6 +108,77 @@ func TestParseChapterDropsRepeatedTitle(t *testing.T) { } } +// gatedChapterFixture mirrors how most chapters are served: a visible sponsor +// block (div.actcl) stands in for the body, while the real text sits in a +// sibling div.actac hidden with display:none and revealed by the site's +// JavaScript. A source watermark and prev/next navigation bracket the prose. +const gatedChapterFixture = ` + +
+
+

Moi Quy doc gia CLICK vao lien ket

+

mo ung dung Shopee, sau do quay tro lai de doc!

+ + CLICK +

MonkeyD va doi ngu Editor xin chan thanh cam on!

+
+
+

Doan van dau tien co tri.

+

 

+

[Truyen duoc dang tai duy nhat tai MonkeyDD.com - https://monkeydd.com/n/10.html.]

+

Doan van cuoi cung.

+
+
+ Chương trước + Chương sau +
+
` + +// The gated body is the one thing that must survive: it is hidden with +// display:none, so any rule that drops hidden or "act*" containers silently +// discards the entire chapter. +func TestParseChapterKeepsGatedBody(t *testing.T) { + ch, err := ParseChapter([]byte(gatedChapterFixture), ChapterRef{Label: "10", URL: "http://x/10.html"}) + if err != nil { + t.Fatalf("ParseChapter: %v", err) + } + + want := []string{ + "Doan van dau tien co vị tri.", + "Doan van cuoi cung.", + } + if len(ch.Paragraphs) != len(want) { + t.Fatalf("got %d paragraphs %q, want %d", len(ch.Paragraphs), ch.Paragraphs, len(want)) + } + for i, w := range want { + if ch.Paragraphs[i] != w { + t.Errorf("paragraph %d = %q, want %q", i, ch.Paragraphs[i], w) + } + } +} + +// Everything the site injects around the prose must be gone. +func TestParseChapterDropsInjectedBlocks(t *testing.T) { + ch, err := ParseChapter([]byte(gatedChapterFixture), ChapterRef{Label: "10", URL: "http://x/10.html"}) + if err != nil { + t.Fatalf("ParseChapter: %v", err) + } + body := strings.Join(ch.Paragraphs, "\n") + + for _, junk := range []string{ + "Shopee", // sponsor copy + "CLICK", // sponsor call to action + "cam on", // sponsor sign-off + "MonkeyDD.com", // source watermark + "Chương trước", // navigation + "Chương sau", // navigation + } { + if strings.Contains(body, junk) { + t.Errorf("injected text %q survived extraction in %q", junk, body) + } + } +} + func TestParseChapterMissingContainer(t *testing.T) { if _, err := ParseChapter([]byte(`

hi

`), ChapterRef{URL: "http://x/1.html"}); err == nil {