-
+
- + Minh Họa 01 +01/01/2024
+
Chương 01
` + + `Trời hôm nay đẹp lắm.
` + + `Người anh trai mỉm cười.[note1]
` + + `` + + ch, err := ParseChapter([]byte(chapterPage(protectedBody(t, body, testKey))), testRef) + if err != nil { + t.Fatalf("parse: %v", err) + } + + // The leading paragraph only repeats the label, the blank one carries no + // text, and the footnote marker is page furniture rather than prose. + want := []string{"Trời hôm nay đẹp lắm.", "Người anh trai mỉm cười."} + if len(ch.Paragraphs) != len(want) { + t.Fatalf("paragraphs = %q, want %q", ch.Paragraphs, want) + } + for i, w := range want { + if ch.Paragraphs[i] != w { + t.Errorf("paragraph %d = %q, want %q", i, ch.Paragraphs[i], w) + } + } + if ch.Volume != testRef.Volume { + t.Errorf("volume = %q, want %q", ch.Volume, testRef.Volume) + } + if ch.WordCount() != 10 { + t.Errorf("word count = %d, want 10", ch.WordCount()) + } +} + +// A chapter still served as plain markup must keep working, and the hidden +// title beside the body must not become a paragraph. +func TestParseChapterPlainMarkup(t *testing.T) { + ch, err := ParseChapter([]byte(chapterPage(`
Một buổi sáng yên tĩnh.
`)), testRef) + if err != nil { + t.Fatalf("parse: %v", err) + } + if len(ch.Paragraphs) != 1 || ch.Paragraphs[0] != "Một buổi sáng yên tĩnh." { + t.Fatalf("paragraphs = %q", ch.Paragraphs) + } +} + +// Illustration chapters are real chapters that carry pictures and little or no +// text. They must not fail the run. +func TestParseChapterWithoutProse(t *testing.T) { + ch, err := ParseChapter([]byte(chapterPage(protectedBody(t, `
nope
`), testRef); err == nil { + t.Fatal("expected an error when the content container is absent, got none") + } +} + +// A rotated scheme must surface as an error naming the chapter, not as an empty +// or garbled one. +func TestParseChapterUnknownScheme(t *testing.T) { + page := chapterPage(``) + _, err := ParseChapter([]byte(page), testRef) + if err == nil { + t.Fatal("expected an error for an unknown scheme, got none") + } + if !strings.Contains(err.Error(), testRef.URL) { + t.Errorf("error should name the chapter URL, got: %v", err) + } +} + +func TestHeading(t *testing.T) { + for _, tc := range []struct{ label, want string }{ + {"Chương 01", "Chương 01"}, + {"12", "Chương 12"}, + {"", "Chương"}, + {"Minh Họa 01", "Minh Họa 01"}, + } { + if got := (&Chapter{Label: tc.label}).Heading(); got != tc.want { + t.Errorf("Heading(%q) = %q, want %q", tc.label, got, tc.want) + } + } +} diff --git a/hako-crawler/hako/client.go b/hako-crawler/hako/client.go new file mode 100644 index 0000000..2dce6ae --- /dev/null +++ b/hako-crawler/hako/client.go @@ -0,0 +1,141 @@ +package hako + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "sync" + "time" +) + +// The site serves a trimmed page to clients without a browser-like User-Agent. +const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" + +// maxPageSize caps how much of a response is buffered. Chapter pages carry the +// body twice over — once as the encoded blob — and run to ~170 KB. +const maxPageSize = 8 << 20 + +// statusError carries an unexpected HTTP status so Get can decide whether a +// retry is worth spending. +type statusError struct { + code int + status string +} + +func (e *statusError) Error() string { return "unexpected status " + e.status } + +// retryable is true only for failures that a later attempt might survive. +// Hako answers a burst with 429, which is the case worth waiting out; a 404 or +// 403 will not fix itself and must not burn the retry budget. +func (e *statusError) retryable() bool { + return e.code == http.StatusTooManyRequests || e.code >= 500 +} + +// Client fetches pages from ln.hako.vn. +// +// Requests are spaced by a fixed delay across every caller, so raising the +// worker count raises concurrency without raising the request rate — which is +// what keeps a whole-novel crawl from tripping the 429 the old single-threaded +// script kept hitting. +type Client struct { + http *http.Client + ua string + delay time.Duration + retries int + + mu sync.Mutex + nextSlot time.Time +} + +// NewClient returns a client that spaces requests by delay and retries a +// transient failure up to retries times. +func NewClient(delay time.Duration, retries int) *Client { + return &Client{ + http: &http.Client{Timeout: 45 * time.Second}, + ua: defaultUserAgent, + delay: delay, + retries: retries, + } +} + +// reserve claims the next request slot and waits for it, holding the global +// rate at one request per delay no matter how many goroutines are fetching. +func (c *Client) reserve(ctx context.Context) error { + c.mu.Lock() + slot := c.nextSlot + if now := time.Now(); slot.Before(now) { + slot = now + } + c.nextSlot = slot.Add(c.delay) + c.mu.Unlock() + + wait := time.Until(slot) + if wait <= 0 { + return nil + } + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// Get fetches url, retrying transient failures with exponential backoff. +func (c *Client) Get(ctx context.Context, url string) ([]byte, error) { + var lastErr error + for attempt := 0; attempt <= c.retries; attempt++ { + if attempt > 0 { + backoff := time.Duration(1<Trời hôm nay đẹp lắm.
Người anh trai mỉm cười.
` +) + +func TestDecodeXORShuffle(t *testing.T) { + for _, chunkLen := range []int{7, 16, 31, 4096} { + load, err := parsePayload(schemeXORShuffle, testKey, + jsonArray(encodeXORShuffle(t, testBody, testKey, chunkLen))) + if err != nil { + t.Fatalf("chunk length %d: parse payload: %v", chunkLen, err) + } + got, err := load.decode() + if err != nil { + t.Fatalf("chunk length %d: decode: %v", chunkLen, err) + } + if got != testBody { + t.Errorf("chunk length %d: body round-trip failed\n got: %q\nwant: %q", chunkLen, got, testBody) + } + } +} + +func TestDecodeRejectsWrongKey(t *testing.T) { + load, err := parsePayload(schemeXORShuffle, "0000000000000000", + jsonArray(encodeXORShuffle(t, testBody, testKey, 16))) + if err != nil { + t.Fatalf("parse payload: %v", err) + } + // A wrong key yields bytes that are not valid UTF-8; the decoder must say + // so rather than hand back a chapter of mojibake. + if _, err := load.decode(); err == nil { + t.Fatal("expected an error for a wrong key, got none") + } +} + +func TestDecodeRejectsUnknownScheme(t *testing.T) { + // When the site rotates its encoding, failing loudly here is what tells the + // user why the export stopped working. + load, err := parsePayload("aes_gcm_2027", testKey, `["0000AAAA"]`) + if err != nil { + t.Fatalf("parse payload: %v", err) + } + if _, err := load.decode(); err == nil { + t.Fatal("expected an error for an unknown scheme, got none") + } +} + +func TestDecodeRejectsMissingKey(t *testing.T) { + load, err := parsePayload(schemeXORShuffle, "", `["0000AAAA"]`) + if err != nil { + t.Fatalf("parse payload: %v", err) + } + if _, err := load.decode(); err == nil { + t.Fatal("expected an error when the key is absent, got none") + } +} + +func TestDecodeRejectsEmptyPayload(t *testing.T) { + load, err := parsePayload(schemeXORShuffle, testKey, `[]`) + if err != nil { + t.Fatalf("parse payload: %v", err) + } + if _, err := load.decode(); err == nil { + t.Fatal("expected an error for an empty chunk list, got none") + } +} + +// base64 and base64_reverse are the site's simpler schemes; the decoder still +// carries them so a chapter served either way keeps working. +func TestDecodePlainBase64Schemes(t *testing.T) { + encoded := base64.StdEncoding.EncodeToString([]byte(testBody)) + + for _, tc := range []struct { + scheme string + chunk string + }{ + {schemeBase64, encoded}, + {schemeBase64Reverse, reverseASCII(encoded)}, + } { + load, err := parsePayload(tc.scheme, "", jsonArray([]string{"0000" + tc.chunk})) + if err != nil { + t.Fatalf("%s: parse payload: %v", tc.scheme, err) + } + got, err := load.decode() + if err != nil { + t.Fatalf("%s: decode: %v", tc.scheme, err) + } + if got != testBody { + t.Errorf("%s: got %q, want %q", tc.scheme, got, testBody) + } + } +} + +func TestSortChunksRejectsBadPrefix(t *testing.T) { + if _, err := sortChunks([]string{"abcdAAAA"}); err == nil { + t.Fatal("expected an error for a non-numeric position prefix, got none") + } + if _, err := sortChunks([]string{"01"}); err == nil { + t.Fatal("expected an error for a chunk shorter than its prefix, got none") + } +} diff --git a/hako-crawler/main.go b/hako-crawler/main.go deleted file mode 100644 index 8f8173c..0000000 --- a/hako-crawler/main.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -import ( - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/PuerkitoBio/goquery" -) - -const DEBUG = false - -var sanitizeRe = regexp.MustCompile(`[\\\/:*?"<>|]`) - -func isDigit(id string) bool { - if id == "" { - return false - } - for _, r := range id { - if r < '0' || r > '9' { - return false - } - } - return true -} - -func getFromURL(url string) (string, error) { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return "", err - } - req.Header.Set("User-Agent", "Mozilla/5.0") - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("HTTP error: %s", resp.Status) - } - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - return string(body), nil -} - -func writeTextToFile(text, filename string) error { - filename = sanitizeRe.ReplaceAllString(filename, "_") - dir := "data" - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - path := filepath.Join(dir, filename) - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - _, err = f.WriteString(text) - return err -} - -func readTextFromFile(filename string) (string, error) { - path := filepath.Join("data", filename) - data, err := os.ReadFile(path) - if err != nil { - return "", err - } - return string(data), nil -} - -func main() { - var html string - var err error - if DEBUG { - html, err = readTextFromFile("_.txt") - } else { - html, err = getFromURL("https://ln.hako.vn/sang-tac/8476-kiep-nay-la-anh-trai-cua-nhan-vat-chinh") - // writeTextToFile(html, "_.txt") - } - if err != nil { - fmt.Printf("Error: %v\n", err) - return - } - - doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) - if err != nil { - fmt.Printf("Parse error: %v\n", err) - return - } - - chapters := doc.Find("div.chapter-name") - /* - you may get `HTTP Error 429: Too Many Requests` - you can try again later, and skip downloaded chapters - example skip 50 first chapters like: chapters = chapters.Slice(50, chapters.Length()) - */ - // chapters = chapters.Slice(100, chapters.Length()) - for i := 0; i < chapters.Length(); i++ { - chapterSel := chapters.Eq(i) - children := chapterSel.ChildrenFiltered("a") - if children.Length() == 0 { - continue - } - linkSel := children.First() - title, hasTitle := linkSel.Attr("title") - if !hasTitle { - continue - } - href, hasHref := linkSel.Attr("href") - if !hasHref { - continue - } - chapterURL := "https://ln.hako.vn" + href - chapterHTML, err := getFromURL(chapterURL) - if err != nil { - fmt.Printf("Error fetching %s: %v\n", chapterURL, err) - continue - } - chapterDoc, err := goquery.NewDocumentFromReader(strings.NewReader(chapterHTML)) - if err != nil { - fmt.Printf("Parse chapter %s: %v\n", title, err) - continue - } - content := chapterDoc.Find("#chapter-content") - var chapterData strings.Builder - content.Find("p").Each(func(_ int, s *goquery.Selection) { - id, exists := s.Attr("id") - if exists && isDigit(id) { - chapterData.WriteString(s.Text()) - chapterData.WriteString("\n") - } - }) - filename := title + ".txt" - if err := writeTextToFile(chapterData.String(), filename); err != nil { - fmt.Printf("Error writing %s: %v\n", filename, err) - } - } -}