+ 
CLICK
+
+
+ Moi Quy doc gia CLICK vao lien ket
+mo ung dung Shopee, sau do quay tro lai de doc!
+ +CLICK
+ MonkeyD va doi ngu Editor xin chan thanh cam on!
+
+ Chương trước
+ Chương sau
+
1
++
Nghe trưởng tử noi.
++
Một và di.
+ +1
" repeats the chapter label and would print twice. +func TestParseChapterDropsRepeatedTitle(t *testing.T) { + ch, err := ParseChapter([]byte(chapterFixture), ChapterRef{Label: "1", URL: "http://x/1.html"}) + if err != nil { + t.Fatalf("ParseChapter: %v", err) + } + if ch.Paragraphs[0] == "1" { + t.Error("repeated chapter label was kept as a paragraph") + } +} + +// gatedChapterFixture mirrors how most chapters are served: a visible sponsor +// block (div.actcl) stands in for the body, while the real text sits in a +// sibling div.actac hidden with display:none and revealed by the site's +// JavaScript. A source watermark and prev/next navigation bracket the prose. +const gatedChapterFixture = ` + +mo ung dung Shopee, sau do quay tro lai de doc!
+ +CLICK
+ hi
`), + ChapterRef{URL: "http://x/1.html"}); err == nil { + t.Fatal("want an error when the content container is absent") + } +} + +func TestChapterHeading(t *testing.T) { + for _, tc := range []struct{ label, want string }{ + {"14", "Chương 14"}, + {"Chương 12", "Chương 12"}, + {"", "Chương"}, + } { + ch := &Chapter{Label: tc.label} + if got := ch.Heading(); got != tc.want { + t.Errorf("Heading(%q) = %q, want %q", tc.label, got, tc.want) + } + } +} diff --git a/internal/modules/monkeyd/crawler/client.go b/internal/modules/monkeyd/crawler/client.go new file mode 100644 index 0000000..a0cdb16 --- /dev/null +++ b/internal/modules/monkeyd/crawler/client.go @@ -0,0 +1,135 @@ +package crawler + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "sync" + "time" +) + +// The site rejects requests without a browser-like User-Agent. +const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" + +// maxPageSize caps how much of a response we buffer; chapter pages are ~130 KB. +const maxPageSize = 8 << 20 + +// statusError reports an unexpected HTTP status so Get can decide whether +// retrying is worthwhile. +type statusError struct { + code int + status string +} + +func (e *statusError) Error() string { return "unexpected status " + e.status } + +// retryable is true for transient failures. A 404 or 403 will not fix itself, +// so those fail immediately instead of burning the retry budget. +func (e *statusError) retryable() bool { + return e.code == http.StatusTooManyRequests || e.code >= 500 +} + +// Client fetches pages from monkeydd.com. It spaces requests out by a fixed +// delay no matter how many goroutines call Get, so raising the worker count +// never raises the request rate, and it retries transient failures with +// exponential backoff. +type Client struct { + http *http.Client + ua string + delay time.Duration + retries int + + mu sync.Mutex + nextSlot time.Time +} + +func NewClient(delay time.Duration, retries int) *Client { + return &Client{ + http: &http.Client{Timeout: 45 * time.Second}, + ua: defaultUserAgent, + delay: delay, + retries: retries, + } +} + +// reserve claims the next request slot and blocks until it comes due, holding +// the global rate at one request per delay across all callers. +func (c *Client) reserve(ctx context.Context) error { + c.mu.Lock() + slot := c.nextSlot + if now := time.Now(); slot.Before(now) { + slot = now + } + c.nextSlot = slot.Add(c.delay) + c.mu.Unlock() + + wait := time.Until(slot) + if wait <= 0 { + return nil + } + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// Get fetches url, retrying transient failures. +func (c *Client) Get(ctx context.Context, url string) ([]byte, error) { + var lastErr error + for attempt := 0; attempt <= c.retries; attempt++ { + if attempt > 0 { + backoff := time.Duration(1<