mirror of
https://github.com/tiennm99/MTTools.git
synced 2026-10-13 02:14:49 +00:00
refactor!: generalise single-novel script into a hako library
The crawler took one hardcoded novel URL, wrote per-chapter text files into ./data, and fetched serially until the site answered 429. Rename the module to hako-crawler and split fetching, page parsing and crawl orchestration into a hako package that takes any novel URL. The chapter body is no longer served as markup: #chapter-content now carries a shuffled, XOR-obfuscated payload that the site's own script expands in the browser, so the previous p[id=<digits>] extractor returned nothing at all. Decode it, keeping the plain-markup path as a fallback. The scheme is read from the page and an unrecognised one is a hard error, because the alternative is exporting a book of mojibake. Space requests globally instead of per-goroutine, so concurrency no longer raises the request rate, and retry only what a retry can fix. Cache raw pages so re-runs cost no requests. BREAKING CHANGE: module path is now github.com/tiennm99/hako-crawler, and the novel URL is a required argument rather than a constant.
This commit is contained in:
tiennm99 committed
1 parent
c5a0487cf3
commit
ad8d2649c1
13 files changed
+1158
-167
No files matched your search
+8
-10
@@ -1,10 +1,12 @@
|
||||
.idea
|
||||
# Build output
|
||||
/hako-crawler
|
||||
/hako-crawler.exe
|
||||
|
||||
# Cached pages and exported books
|
||||
/.cache/
|
||||
/data/
|
||||
*.pdf
|
||||
|
||||
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
#
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
@@ -21,9 +23,6 @@ coverage.*
|
||||
*.coverprofile
|
||||
profile.cov
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
go.work.sum
|
||||
@@ -32,5 +31,4 @@ go.work.sum
|
||||
.env
|
||||
|
||||
# Editor/IDE
|
||||
# .idea/
|
||||
# .vscode/
|
||||
.idea
|
||||
@@ -1,2 +0,0 @@
|
||||
*
|
||||
!.gitignore
|
||||
+8
-6
@@ -1,12 +1,14 @@
|
||||
module github.com/tiennm99/atnvc-crawler
|
||||
module github.com/tiennm99/hako-crawler
|
||||
|
||||
go 1.24.0
|
||||
go 1.25.0
|
||||
|
||||
toolchain go1.24.10
|
||||
|
||||
require github.com/PuerkitoBio/goquery v1.11.0
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.12.0
|
||||
github.com/go-pdf/fpdf v0.9.0
|
||||
golang.org/x/sync v0.22.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
)
|
||||
+8
-4
@@ -1,7 +1,9 @@
|
||||
github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw=
|
||||
github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ=
|
||||
github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo=
|
||||
github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/go-pdf/fpdf v0.9.0 h1:PPvSaUuo1iMi9KkaAn90NuKi+P4gwMedWPHhj8YlJQw=
|
||||
github.com/go-pdf/fpdf v0.9.0/go.mod h1:oO8N111TkmKb9D7VvWGLvLJlaZUQVPM+6V42pp3iV4Y=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
@@ -24,8 +26,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -33,6 +35,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package hako
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
// contentID is the container holding a chapter's body.
|
||||
const contentID = "chapter-content"
|
||||
|
||||
// Chapter is one fetched chapter reduced to plain paragraphs.
|
||||
type Chapter struct {
|
||||
Label string
|
||||
Volume string
|
||||
URL string
|
||||
Paragraphs []string
|
||||
}
|
||||
|
||||
// Heading is the chapter title to print. A bare number reads poorly as a
|
||||
// heading, so those get the Vietnamese word for "chapter".
|
||||
func (c *Chapter) Heading() string {
|
||||
label := strings.TrimSpace(c.Label)
|
||||
if label == "" {
|
||||
return "Chương"
|
||||
}
|
||||
if isDigits(label) {
|
||||
return "Chương " + label
|
||||
}
|
||||
return label
|
||||
}
|
||||
|
||||
// WordCount is a rough word count, used to sanity-check extraction.
|
||||
func (c *Chapter) WordCount() int {
|
||||
n := 0
|
||||
for _, p := range c.Paragraphs {
|
||||
n += len(strings.Fields(p))
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func isDigits(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range s {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// noteMarker matches the footnote placeholders left in the body text. The site
|
||||
// swaps each one for a tooltip icon whose text lives elsewhere on the page, so
|
||||
// in a plain-text export the marker itself is noise.
|
||||
var noteMarker = regexp.MustCompile(`(?i)\[note\d+\]`)
|
||||
|
||||
// ParseChapter extracts a chapter's paragraphs.
|
||||
//
|
||||
// The body is normally delivered as an encoded payload rather than as markup
|
||||
// (see protected.go); when it is, the payload is decoded first and the
|
||||
// paragraphs are read out of the result. A chapter still served as plain markup
|
||||
// is read directly, so both layouts work.
|
||||
func ParseChapter(page []byte, ref ChapterRef) (*Chapter, error) {
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(page))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse chapter %s: %w", ref.URL, err)
|
||||
}
|
||||
content := doc.Find("#" + contentID).First()
|
||||
if content.Length() == 0 {
|
||||
return nil, fmt.Errorf("chapter %s: no #%s container (page layout may have changed)",
|
||||
ref.URL, contentID)
|
||||
}
|
||||
|
||||
body, err := chapterBody(content, ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Chapter{
|
||||
Label: ref.Label,
|
||||
Volume: ref.Volume,
|
||||
URL: ref.URL,
|
||||
Paragraphs: dropRepeatedTitle(extractParagraphs(body), ref.Label),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// chapterBody returns the selection actually holding the prose: the decoded
|
||||
// payload when the body is encoded, otherwise the container itself.
|
||||
func chapterBody(content *goquery.Selection, ref ChapterRef) (*goquery.Selection, error) {
|
||||
protected := content.Find("#" + protectedID).First()
|
||||
if protected.Length() == 0 {
|
||||
return content, nil
|
||||
}
|
||||
|
||||
load, err := parsePayload(
|
||||
protected.AttrOr("data-s", ""),
|
||||
protected.AttrOr("data-k", ""),
|
||||
protected.AttrOr("data-c", "[]"),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chapter %s: %w", ref.URL, err)
|
||||
}
|
||||
html, err := load.decode()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chapter %s: %w", ref.URL, err)
|
||||
}
|
||||
|
||||
// The payload is a fragment of the chapter body only, with none of the
|
||||
// page's own furniture around it.
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chapter %s: parse decoded body: %w", ref.URL, err)
|
||||
}
|
||||
return doc.Selection, nil
|
||||
}
|
||||
|
||||
// extractParagraphs reads the paragraph text out of a body selection.
|
||||
func extractParagraphs(body *goquery.Selection) []string {
|
||||
var paragraphs []string
|
||||
body.Find("p").Each(func(_ int, p *goquery.Selection) {
|
||||
// The page plants a hidden copy of the chapter title next to the body.
|
||||
if strings.Contains(strings.ReplaceAll(p.AttrOr("style", ""), " ", ""), "display:none") {
|
||||
return
|
||||
}
|
||||
text := collapseSpaces(noteMarker.ReplaceAllString(p.Text(), ""))
|
||||
if text == "" {
|
||||
return
|
||||
}
|
||||
paragraphs = append(paragraphs, text)
|
||||
})
|
||||
return paragraphs
|
||||
}
|
||||
|
||||
// dropRepeatedTitle removes a leading paragraph that only repeats the chapter
|
||||
// label, since the export prints its own heading.
|
||||
func dropRepeatedTitle(paragraphs []string, label string) []string {
|
||||
if len(paragraphs) < 2 {
|
||||
return paragraphs
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(paragraphs[0]), strings.TrimSpace(label)) {
|
||||
return paragraphs[1:]
|
||||
}
|
||||
return paragraphs
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package hako
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// chapterPage wraps a body in the page furniture the parser has to see past:
|
||||
// the container, and the hidden copy of the chapter title the site plants
|
||||
// beside it.
|
||||
func chapterPage(body string) string {
|
||||
return `<html><body><div id="chapter-content" class="long-text text-justify">
|
||||
<p style="display: none">Chương 01</p>` + body + `</div></body></html>`
|
||||
}
|
||||
|
||||
// protectedBody renders the encoded container the site now serves.
|
||||
func protectedBody(t *testing.T, html, key string) string {
|
||||
t.Helper()
|
||||
chunks := encodeXORShuffle(t, html, key, 24)
|
||||
quoted := make([]string, len(chunks))
|
||||
for i, c := range chunks {
|
||||
// The attribute is HTML-escaped on the page; the parser decodes it.
|
||||
quoted[i] = """ + c + """
|
||||
}
|
||||
return `<div id="` + protectedID + `" data-s="` + schemeXORShuffle +
|
||||
`" data-k="` + key + `" data-c="[` + strings.Join(quoted, ",") + `]" aria-hidden="true"></div>`
|
||||
}
|
||||
|
||||
var testRef = ChapterRef{Label: "Chương 01", Volume: "Tập 01", URL: "https://ln.hako.vn/sang-tac/1-x/c1-chuong-01"}
|
||||
|
||||
func TestParseChapterProtected(t *testing.T) {
|
||||
body := `<p id="1">Chương 01</p>` +
|
||||
`<p id="2">Trời hôm nay đẹp lắm.</p>` +
|
||||
`<p id="3">Người anh trai <em>mỉm cười</em>.[note1]</p>` +
|
||||
`<p id="4"> </p>`
|
||||
|
||||
ch, err := ParseChapter([]byte(chapterPage(protectedBody(t, body, testKey))), testRef)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
|
||||
// The leading paragraph only repeats the label, the blank one carries no
|
||||
// text, and the footnote marker is page furniture rather than prose.
|
||||
want := []string{"Trời hôm nay đẹp lắm.", "Người anh trai mỉm cười."}
|
||||
if len(ch.Paragraphs) != len(want) {
|
||||
t.Fatalf("paragraphs = %q, want %q", ch.Paragraphs, want)
|
||||
}
|
||||
for i, w := range want {
|
||||
if ch.Paragraphs[i] != w {
|
||||
t.Errorf("paragraph %d = %q, want %q", i, ch.Paragraphs[i], w)
|
||||
}
|
||||
}
|
||||
if ch.Volume != testRef.Volume {
|
||||
t.Errorf("volume = %q, want %q", ch.Volume, testRef.Volume)
|
||||
}
|
||||
if ch.WordCount() != 10 {
|
||||
t.Errorf("word count = %d, want 10", ch.WordCount())
|
||||
}
|
||||
}
|
||||
|
||||
// A chapter still served as plain markup must keep working, and the hidden
|
||||
// title beside the body must not become a paragraph.
|
||||
func TestParseChapterPlainMarkup(t *testing.T) {
|
||||
ch, err := ParseChapter([]byte(chapterPage(`<p id="1">Một buổi sáng yên tĩnh.</p>`)), testRef)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(ch.Paragraphs) != 1 || ch.Paragraphs[0] != "Một buổi sáng yên tĩnh." {
|
||||
t.Fatalf("paragraphs = %q", ch.Paragraphs)
|
||||
}
|
||||
}
|
||||
|
||||
// Illustration chapters are real chapters that carry pictures and little or no
|
||||
// text. They must not fail the run.
|
||||
func TestParseChapterWithoutProse(t *testing.T) {
|
||||
ch, err := ParseChapter([]byte(chapterPage(protectedBody(t, `<p id="1"><img src="/a.png"></p>`, testKey))),
|
||||
ChapterRef{Label: "Minh Họa 01", URL: testRef.URL})
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(ch.Paragraphs) != 0 {
|
||||
t.Errorf("paragraphs = %q, want none", ch.Paragraphs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseChapterMissingContainer(t *testing.T) {
|
||||
if _, err := ParseChapter([]byte(`<html><body><p>nope</p></body></html>`), testRef); err == nil {
|
||||
t.Fatal("expected an error when the content container is absent, got none")
|
||||
}
|
||||
}
|
||||
|
||||
// A rotated scheme must surface as an error naming the chapter, not as an empty
|
||||
// or garbled one.
|
||||
func TestParseChapterUnknownScheme(t *testing.T) {
|
||||
page := chapterPage(`<div id="` + protectedID + `" data-s="aes_gcm_2027" data-k="k" data-c="["0000AAAA"]"></div>`)
|
||||
_, err := ParseChapter([]byte(page), testRef)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for an unknown scheme, got none")
|
||||
}
|
||||
if !strings.Contains(err.Error(), testRef.URL) {
|
||||
t.Errorf("error should name the chapter URL, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeading(t *testing.T) {
|
||||
for _, tc := range []struct{ label, want string }{
|
||||
{"Chương 01", "Chương 01"},
|
||||
{"12", "Chương 12"},
|
||||
{"", "Chương"},
|
||||
{"Minh Họa 01", "Minh Họa 01"},
|
||||
} {
|
||||
if got := (&Chapter{Label: tc.label}).Heading(); got != tc.want {
|
||||
t.Errorf("Heading(%q) = %q, want %q", tc.label, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package hako
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The site serves a trimmed page to clients without a browser-like User-Agent.
|
||||
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||
|
||||
// maxPageSize caps how much of a response is buffered. Chapter pages carry the
|
||||
// body twice over — once as the encoded blob — and run to ~170 KB.
|
||||
const maxPageSize = 8 << 20
|
||||
|
||||
// statusError carries an unexpected HTTP status so Get can decide whether a
|
||||
// retry is worth spending.
|
||||
type statusError struct {
|
||||
code int
|
||||
status string
|
||||
}
|
||||
|
||||
func (e *statusError) Error() string { return "unexpected status " + e.status }
|
||||
|
||||
// retryable is true only for failures that a later attempt might survive.
|
||||
// Hako answers a burst with 429, which is the case worth waiting out; a 404 or
|
||||
// 403 will not fix itself and must not burn the retry budget.
|
||||
func (e *statusError) retryable() bool {
|
||||
return e.code == http.StatusTooManyRequests || e.code >= 500
|
||||
}
|
||||
|
||||
// Client fetches pages from ln.hako.vn.
|
||||
//
|
||||
// Requests are spaced by a fixed delay across every caller, so raising the
|
||||
// worker count raises concurrency without raising the request rate — which is
|
||||
// what keeps a whole-novel crawl from tripping the 429 the old single-threaded
|
||||
// script kept hitting.
|
||||
type Client struct {
|
||||
http *http.Client
|
||||
ua string
|
||||
delay time.Duration
|
||||
retries int
|
||||
|
||||
mu sync.Mutex
|
||||
nextSlot time.Time
|
||||
}
|
||||
|
||||
// NewClient returns a client that spaces requests by delay and retries a
|
||||
// transient failure up to retries times.
|
||||
func NewClient(delay time.Duration, retries int) *Client {
|
||||
return &Client{
|
||||
http: &http.Client{Timeout: 45 * time.Second},
|
||||
ua: defaultUserAgent,
|
||||
delay: delay,
|
||||
retries: retries,
|
||||
}
|
||||
}
|
||||
|
||||
// reserve claims the next request slot and waits for it, holding the global
|
||||
// rate at one request per delay no matter how many goroutines are fetching.
|
||||
func (c *Client) reserve(ctx context.Context) error {
|
||||
c.mu.Lock()
|
||||
slot := c.nextSlot
|
||||
if now := time.Now(); slot.Before(now) {
|
||||
slot = now
|
||||
}
|
||||
c.nextSlot = slot.Add(c.delay)
|
||||
c.mu.Unlock()
|
||||
|
||||
wait := time.Until(slot)
|
||||
if wait <= 0 {
|
||||
return nil
|
||||
}
|
||||
timer := time.NewTimer(wait)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Get fetches url, retrying transient failures with exponential backoff.
|
||||
func (c *Client) Get(ctx context.Context, url string) ([]byte, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= c.retries; attempt++ {
|
||||
if attempt > 0 {
|
||||
backoff := time.Duration(1<<uint(attempt-1)) * time.Second
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
}
|
||||
if err := c.reserve(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
body, err := c.fetch(ctx, url)
|
||||
if err == nil {
|
||||
return body, nil
|
||||
}
|
||||
lastErr = err
|
||||
|
||||
var se *statusError
|
||||
if errors.As(err, &se) && !se.retryable() {
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("fetch %s: %w", url, lastErr)
|
||||
}
|
||||
|
||||
func (c *Client) fetch(ctx context.Context, url string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", c.ua)
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "vi,en;q=0.8")
|
||||
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, &statusError{code: resp.StatusCode, status: resp.Status}
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, maxPageSize))
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package hako
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// Crawler fetches a novel and its chapters.
|
||||
type Crawler struct {
|
||||
Client *Client
|
||||
|
||||
// CacheDir, when set, stores raw pages on disk and serves later runs from
|
||||
// them, so re-rendering a book with different page or font settings costs
|
||||
// no requests.
|
||||
CacheDir string
|
||||
|
||||
// Workers bounds concurrent fetches. The client's delay still caps the
|
||||
// overall request rate, so this trades latency, not politeness.
|
||||
Workers int
|
||||
|
||||
// Log receives progress messages. Optional.
|
||||
Log func(format string, args ...any)
|
||||
}
|
||||
|
||||
func (c *Crawler) logf(format string, args ...any) {
|
||||
if c.Log != nil {
|
||||
c.Log(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Novel fetches a novel landing page and reads its chapter list.
|
||||
func (c *Crawler) Novel(ctx context.Context, novelURL string) (*Novel, error) {
|
||||
page, err := c.page(ctx, novelURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
novel, err := ParseNovelPage(page, novelURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.logf("novel: %s (%d chapters)", novel.Title, len(novel.Chapters))
|
||||
return novel, nil
|
||||
}
|
||||
|
||||
// Chapters fetches every chapter concurrently and returns them in reading
|
||||
// order. A chapter that cannot be fetched or decoded fails the whole run
|
||||
// rather than yielding a book with a silent hole in it.
|
||||
func (c *Crawler) Chapters(ctx context.Context, novel *Novel) ([]*Chapter, error) {
|
||||
chapters := make([]*Chapter, len(novel.Chapters))
|
||||
|
||||
workers := c.Workers
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
|
||||
group, groupCtx := errgroup.WithContext(ctx)
|
||||
group.SetLimit(workers)
|
||||
|
||||
var mu sync.Mutex
|
||||
done := 0
|
||||
|
||||
for i, ref := range novel.Chapters {
|
||||
group.Go(func() error {
|
||||
page, err := c.page(groupCtx, ref.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chapter, err := ParseChapter(page, ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chapters[i] = chapter
|
||||
|
||||
mu.Lock()
|
||||
done++
|
||||
c.logf("fetched %d/%d: %s (%d words)", done, len(novel.Chapters),
|
||||
chapter.Heading(), chapter.WordCount())
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := group.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return chapters, nil
|
||||
}
|
||||
|
||||
// page returns a page from the cache when it is there, otherwise fetches and
|
||||
// caches it.
|
||||
func (c *Crawler) page(ctx context.Context, pageURL string) ([]byte, error) {
|
||||
path := c.cachePath(pageURL)
|
||||
if path != "" {
|
||||
if body, err := os.ReadFile(path); err == nil && len(body) > 0 {
|
||||
return body, nil
|
||||
}
|
||||
}
|
||||
|
||||
body, err := c.Client.Get(ctx, pageURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if path != "" {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err == nil {
|
||||
// A cache that cannot be written must not fail the crawl.
|
||||
_ = os.WriteFile(path, body, 0o644)
|
||||
}
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// unsafeFileChars matches everything not allowed in a cache file name.
|
||||
var unsafeFileChars = regexp.MustCompile(`[^A-Za-z0-9._-]+`)
|
||||
|
||||
// cachePath maps a page URL to a cache file, or "" when caching is off.
|
||||
func (c *Crawler) cachePath(pageURL string) string {
|
||||
if c.CacheDir == "" {
|
||||
return ""
|
||||
}
|
||||
u, err := url.Parse(pageURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
name := unsafeFileChars.ReplaceAllString(strings.Trim(u.Path, "/"), "_")
|
||||
if name == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Join(c.CacheDir, name+".html")
|
||||
}
|
||||
|
||||
// TotalWords sums the word count across chapters.
|
||||
func TotalWords(chapters []*Chapter) int {
|
||||
n := 0
|
||||
for _, ch := range chapters {
|
||||
n += ch.WordCount()
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Describe renders a one-line summary of a crawl result.
|
||||
func Describe(novel *Novel, chapters []*Chapter) string {
|
||||
return fmt.Sprintf("%s — %d chapters, %d words", novel.Title, len(chapters), TotalWords(chapters))
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package hako
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
// ChapterRef points at one chapter listed on a novel page.
|
||||
type ChapterRef struct {
|
||||
Label string // as shown on the site, e.g. "Chương 01"
|
||||
Volume string // the volume section it sits under; empty if ungrouped
|
||||
URL string
|
||||
}
|
||||
|
||||
// Novel is a novel's landing page: what it is called and its chapters in
|
||||
// reading order.
|
||||
type Novel struct {
|
||||
Title string
|
||||
Slug string
|
||||
URL string
|
||||
|
||||
// Tags are the novel's genres as labelled on the site, in page order.
|
||||
Tags []string
|
||||
Chapters []ChapterRef
|
||||
}
|
||||
|
||||
// ParseNovelPage reads the title, genres and chapter list off a novel page.
|
||||
//
|
||||
// Chapter URLs always come from the anchors on the page. Hako's chapter slugs
|
||||
// carry an opaque id ("c75189-chuong-01") that cannot be derived from a chapter
|
||||
// number, so a generated URL would simply 404.
|
||||
func ParseNovelPage(page []byte, pageURL string) (*Novel, error) {
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(page))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse novel page: %w", err)
|
||||
}
|
||||
base, err := url.Parse(pageURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse novel url: %w", err)
|
||||
}
|
||||
|
||||
novel := &Novel{
|
||||
Title: novelTitle(doc),
|
||||
Slug: slugFromURL(base),
|
||||
URL: pageURL,
|
||||
Tags: novelTags(doc),
|
||||
Chapters: chapterRefs(doc, base),
|
||||
}
|
||||
if novel.Title == "" {
|
||||
novel.Title = novel.Slug
|
||||
}
|
||||
if len(novel.Chapters) == 0 {
|
||||
return nil, fmt.Errorf("no chapters found on %s (page layout may have changed)", pageURL)
|
||||
}
|
||||
return novel, nil
|
||||
}
|
||||
|
||||
// novelTitle prefers the series heading and falls back to the document title,
|
||||
// which carries the site's name as a suffix and so is only a last resort.
|
||||
func novelTitle(doc *goquery.Document) string {
|
||||
if t := collapseSpaces(doc.Find(".series-name a").First().Text()); t != "" {
|
||||
return t
|
||||
}
|
||||
if t := collapseSpaces(doc.Find(".series-name").First().Text()); t != "" {
|
||||
return t
|
||||
}
|
||||
title := collapseSpaces(doc.Find("title").First().Text())
|
||||
if head, _, found := strings.Cut(title, " - "); found {
|
||||
return head
|
||||
}
|
||||
return title
|
||||
}
|
||||
|
||||
// novelTags reads the genre links out of the series info block.
|
||||
//
|
||||
// The class spelling is the site's own ("gerne"). Some entries are rendered
|
||||
// hidden behind a "show more" toggle; they are still this novel's genres, so
|
||||
// they are kept.
|
||||
func novelTags(doc *goquery.Document) []string {
|
||||
var tags []string
|
||||
seen := make(map[string]bool)
|
||||
doc.Find("a.series-gerne-item").Each(func(_ int, s *goquery.Selection) {
|
||||
name := collapseSpaces(s.Text())
|
||||
if name == "" || seen[name] {
|
||||
return
|
||||
}
|
||||
seen[name] = true
|
||||
tags = append(tags, name)
|
||||
})
|
||||
return tags
|
||||
}
|
||||
|
||||
// slugFromURL turns /sang-tac/8476-kiep-nay-la-anh-trai-cua-nhan-vat-chinh
|
||||
// into "8476-kiep-nay-la-anh-trai-cua-nhan-vat-chinh".
|
||||
func slugFromURL(u *url.URL) string {
|
||||
seg := strings.Trim(u.Path, "/")
|
||||
if i := strings.LastIndex(seg, "/"); i >= 0 {
|
||||
seg = seg[i+1:]
|
||||
}
|
||||
return seg
|
||||
}
|
||||
|
||||
// chapterRefs collects every chapter link, tagged with the volume it sits
|
||||
// under. The site lists volumes oldest first and chapters ascending within a
|
||||
// volume, so document order is already reading order.
|
||||
func chapterRefs(doc *goquery.Document, base *url.URL) []ChapterRef {
|
||||
var refs []ChapterRef
|
||||
seen := make(map[string]bool)
|
||||
|
||||
add := func(volume string, sel *goquery.Selection) {
|
||||
sel.Find("div.chapter-name a").Each(func(_ int, a *goquery.Selection) {
|
||||
href, ok := a.Attr("href")
|
||||
if !ok || strings.TrimSpace(href) == "" {
|
||||
return
|
||||
}
|
||||
abs, err := base.Parse(strings.TrimSpace(href))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
link := abs.String()
|
||||
if seen[link] {
|
||||
return
|
||||
}
|
||||
seen[link] = true
|
||||
|
||||
// The visible text is the chapter name; the title attribute
|
||||
// repeats it and survives when the text is decorated with icons.
|
||||
label := collapseSpaces(a.Text())
|
||||
if label == "" {
|
||||
label = collapseSpaces(a.AttrOr("title", ""))
|
||||
}
|
||||
refs = append(refs, ChapterRef{Label: label, Volume: volume, URL: link})
|
||||
})
|
||||
}
|
||||
|
||||
volumes := doc.Find(".volume-list")
|
||||
volumes.Each(func(_ int, vol *goquery.Selection) {
|
||||
add(collapseSpaces(vol.Find("header .sect-title").First().Text()), vol)
|
||||
})
|
||||
// A layout without volume sections still has chapter links; picking them up
|
||||
// from the document keeps the crawl working rather than reporting an empty
|
||||
// novel. Links already taken from a volume are skipped by seen.
|
||||
add("", doc.Selection)
|
||||
|
||||
return refs
|
||||
}
|
||||
|
||||
// collapseSpaces trims and reduces every whitespace run — including the
|
||||
// non-breaking spaces the site emits as — to a single space.
|
||||
func collapseSpaces(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package hako
|
||||
|
||||
import "testing"
|
||||
|
||||
// A landing page reduced to the structure the parser depends on: a series name,
|
||||
// genre links (including one hidden behind the site's "show more" toggle), and
|
||||
// two volume sections whose chapters are listed in reading order.
|
||||
const novelFixture = `<!doctype html><html><head>
|
||||
<title>Truyện Thử - Cổng Light Novel - Đọc Light Novel</title></head><body>
|
||||
<div class="series-name-group"><span class="series-name">
|
||||
<a href="/sang-tac/1234-truyen-thu">Truyện Thử</a></span></div>
|
||||
<div class="series-gernes">
|
||||
<a class="series-gerne-item" href="/the-loai/comedy">Comedy</a>
|
||||
<a class="series-gerne-item" href="/the-loai/fantasy">Fantasy</a>
|
||||
<a class="series-gerne-item" style="display: none;" href="/the-loai/magic">Magic</a>
|
||||
<a class="series-gerne-item" href="/the-loai/comedy">Comedy</a>
|
||||
</div>
|
||||
<section class="volume-list at-series basic-section">
|
||||
<header id="volume_1" class="sect-header"><span class="sect-title"> Minh Họa </span></header>
|
||||
<main><ul>
|
||||
<li><div class="chapter-name"><i class="fas fa-image"></i>
|
||||
<a href="/sang-tac/1234-truyen-thu/c11-minh-hoa-01" title="Minh Họa 01">Minh Họa 01</a>
|
||||
</div><div class="chapter-time">01/01/2024</div></li>
|
||||
</ul></main>
|
||||
</section>
|
||||
<section class="volume-list at-series basic-section">
|
||||
<header id="volume_2" class="sect-header"><span class="sect-title">Tập 01</span></header>
|
||||
<main><ul>
|
||||
<li><div class="chapter-name">
|
||||
<a href="/sang-tac/1234-truyen-thu/c21-chuong-01" title="Chương 01">Chương 01</a></div></li>
|
||||
<li><div class="chapter-name">
|
||||
<a href="https://ln.hako.vn/sang-tac/1234-truyen-thu/c22-chuong-02" title="Chương 02">Chương 02</a></div></li>
|
||||
<li><div class="chapter-name">
|
||||
<a href="/sang-tac/1234-truyen-thu/c21-chuong-01" title="Chương 01">Chương 01</a></div></li>
|
||||
</ul></main>
|
||||
</section>
|
||||
</body></html>`
|
||||
|
||||
const novelURL = "https://ln.hako.vn/sang-tac/1234-truyen-thu"
|
||||
|
||||
func TestParseNovelPage(t *testing.T) {
|
||||
novel, err := ParseNovelPage([]byte(novelFixture), novelURL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
|
||||
if novel.Title != "Truyện Thử" {
|
||||
t.Errorf("title = %q, want %q", novel.Title, "Truyện Thử")
|
||||
}
|
||||
if novel.Slug != "1234-truyen-thu" {
|
||||
t.Errorf("slug = %q, want %q", novel.Slug, "1234-truyen-thu")
|
||||
}
|
||||
|
||||
// The hidden genre is still this novel's genre; the repeated one is not a
|
||||
// second genre.
|
||||
wantTags := []string{"Comedy", "Fantasy", "Magic"}
|
||||
if len(novel.Tags) != len(wantTags) {
|
||||
t.Fatalf("tags = %v, want %v", novel.Tags, wantTags)
|
||||
}
|
||||
for i, want := range wantTags {
|
||||
if novel.Tags[i] != want {
|
||||
t.Errorf("tag %d = %q, want %q", i, novel.Tags[i], want)
|
||||
}
|
||||
}
|
||||
|
||||
// Document order is reading order, volume by volume, and the duplicated
|
||||
// link must not produce a duplicated chapter.
|
||||
want := []ChapterRef{
|
||||
{Label: "Minh Họa 01", Volume: "Minh Họa", URL: novelURL + "/c11-minh-hoa-01"},
|
||||
{Label: "Chương 01", Volume: "Tập 01", URL: novelURL + "/c21-chuong-01"},
|
||||
{Label: "Chương 02", Volume: "Tập 01", URL: novelURL + "/c22-chuong-02"},
|
||||
}
|
||||
if len(novel.Chapters) != len(want) {
|
||||
t.Fatalf("got %d chapters, want %d: %+v", len(novel.Chapters), len(want), novel.Chapters)
|
||||
}
|
||||
for i, w := range want {
|
||||
if novel.Chapters[i] != w {
|
||||
t.Errorf("chapter %d = %+v, want %+v", i, novel.Chapters[i], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A layout with no volume sections must still yield its chapters rather than
|
||||
// reporting an empty novel.
|
||||
func TestParseNovelPageWithoutVolumes(t *testing.T) {
|
||||
page := `<html><body><span class="series-name"><a href="/x">Truyện Thử</a></span>
|
||||
<div class="chapter-name"><a href="/sang-tac/1234-truyen-thu/c1-chuong-01">Chương 01</a></div>
|
||||
</body></html>`
|
||||
|
||||
novel, err := ParseNovelPage([]byte(page), novelURL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if len(novel.Chapters) != 1 {
|
||||
t.Fatalf("got %d chapters, want 1", len(novel.Chapters))
|
||||
}
|
||||
if novel.Chapters[0].Volume != "" {
|
||||
t.Errorf("volume = %q, want empty", novel.Chapters[0].Volume)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNovelPageWithNoChapters(t *testing.T) {
|
||||
page := `<html><body><span class="series-name"><a href="/x">Truyện Thử</a></span></body></html>`
|
||||
if _, err := ParseNovelPage([]byte(page), novelURL); err == nil {
|
||||
t.Fatal("expected an error when the page lists no chapters, got none")
|
||||
}
|
||||
}
|
||||
|
||||
// Without a series heading the document title is the only source, and it
|
||||
// carries the site's name as a suffix that must not end up in the file name.
|
||||
func TestParseNovelPageTitleFallback(t *testing.T) {
|
||||
page := `<html><head><title>Truyện Thử - Cổng Light Novel - Đọc Light Novel</title></head>
|
||||
<body><div class="chapter-name"><a href="/c1">Chương 01</a></div></body></html>`
|
||||
|
||||
novel, err := ParseNovelPage([]byte(page), novelURL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
if novel.Title != "Truyện Thử" {
|
||||
t.Errorf("title = %q, want %q", novel.Title, "Truyện Thử")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package hako
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// The site no longer ships chapter text as markup. The body arrives as a
|
||||
// container carrying an encoded payload, which its own JavaScript turns back
|
||||
// into the original HTML in the browser:
|
||||
//
|
||||
// <div id="chapter-c-protected" data-s="xor_shuffle"
|
||||
// data-k="6b9dd83fad5a3169" data-c="["0001BVsF…", …]">
|
||||
//
|
||||
// data-c is a JSON array of chunks in shuffled order, each prefixed with its
|
||||
// own 4-digit position. Reading DOM text alone yields an empty chapter, which
|
||||
// is why the plain-paragraph extractor this project started with stopped
|
||||
// returning anything.
|
||||
const (
|
||||
protectedID = "chapter-c-protected"
|
||||
|
||||
// chunkIndexLen is the width of the decimal position prefix on each chunk.
|
||||
chunkIndexLen = 4
|
||||
|
||||
schemeXORShuffle = "xor_shuffle"
|
||||
schemeBase64 = "base64"
|
||||
schemeBase64Reverse = "base64_reverse"
|
||||
)
|
||||
|
||||
// payload is the encoded chapter body read off the container's attributes.
|
||||
type payload struct {
|
||||
Scheme string // data-s
|
||||
Key string // data-k
|
||||
Chunks []string
|
||||
}
|
||||
|
||||
// parsePayload reads the attribute trio into a payload. The document parser has
|
||||
// already turned " back into a quote, so data-c is plain JSON here.
|
||||
func parsePayload(scheme, key, chunksJSON string) (*payload, error) {
|
||||
var chunks []string
|
||||
if err := json.Unmarshal([]byte(chunksJSON), &chunks); err != nil {
|
||||
return nil, fmt.Errorf("parse protected chunk list: %w", err)
|
||||
}
|
||||
if scheme == "" {
|
||||
scheme = schemeBase64
|
||||
}
|
||||
return &payload{Scheme: scheme, Key: key, Chunks: chunks}, nil
|
||||
}
|
||||
|
||||
// decode reassembles the chapter HTML.
|
||||
//
|
||||
// An unrecognised scheme is an error rather than a best-effort guess: the
|
||||
// alternative is emitting a chapter of mojibake that looks like a successful
|
||||
// export. When the site rotates its scheme this is the message that says so.
|
||||
func (p *payload) decode() (string, error) {
|
||||
if len(p.Chunks) == 0 {
|
||||
return "", fmt.Errorf("protected body carries no chunks")
|
||||
}
|
||||
switch p.Scheme {
|
||||
case schemeXORShuffle, schemeBase64, schemeBase64Reverse:
|
||||
default:
|
||||
return "", fmt.Errorf("unknown content scheme %q (the site's encoding changed)", p.Scheme)
|
||||
}
|
||||
if p.Scheme == schemeXORShuffle && p.Key == "" {
|
||||
return "", fmt.Errorf("scheme %s needs a key but the page carried none", p.Scheme)
|
||||
}
|
||||
|
||||
ordered, err := sortChunks(p.Chunks)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var out []byte
|
||||
for i, chunk := range ordered {
|
||||
part, err := p.decodeChunk(chunk)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("chunk %d of %d: %w", i+1, len(ordered), err)
|
||||
}
|
||||
out = append(out, part...)
|
||||
}
|
||||
|
||||
// Each chunk is decoded on its own, so a multi-byte rune split across a
|
||||
// chunk boundary would only show up once they are joined.
|
||||
if !utf8.Valid(out) {
|
||||
return "", fmt.Errorf("decoded body is not valid UTF-8 (wrong key or changed scheme)")
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// sortChunks puts the shuffled chunks back in order and strips the position
|
||||
// prefix that carried the ordering.
|
||||
func sortChunks(chunks []string) ([]string, error) {
|
||||
type indexed struct {
|
||||
pos int
|
||||
data string
|
||||
}
|
||||
|
||||
items := make([]indexed, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
if len(chunk) < chunkIndexLen {
|
||||
return nil, fmt.Errorf("chunk shorter than its %d-char position prefix", chunkIndexLen)
|
||||
}
|
||||
pos, err := strconv.Atoi(chunk[:chunkIndexLen])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chunk position prefix %q is not a number", chunk[:chunkIndexLen])
|
||||
}
|
||||
items = append(items, indexed{pos: pos, data: chunk[chunkIndexLen:]})
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool { return items[i].pos < items[j].pos })
|
||||
|
||||
ordered := make([]string, len(items))
|
||||
for i, item := range items {
|
||||
ordered[i] = item.data
|
||||
}
|
||||
return ordered, nil
|
||||
}
|
||||
|
||||
// decodeChunk turns one chunk back into its slice of the chapter HTML.
|
||||
func (p *payload) decodeChunk(chunk string) ([]byte, error) {
|
||||
if p.Scheme == schemeBase64Reverse {
|
||||
chunk = reverseASCII(chunk)
|
||||
}
|
||||
data, err := base64.StdEncoding.DecodeString(chunk)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("base64: %w", err)
|
||||
}
|
||||
if p.Scheme == schemeXORShuffle {
|
||||
// The key cycles from the start of every chunk, not across the joined
|
||||
// stream — the site decodes each chunk independently.
|
||||
for i := range data {
|
||||
data[i] ^= p.Key[i%len(p.Key)]
|
||||
}
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// reverseASCII reverses a string by byte. base64 is ASCII-only, so this cannot
|
||||
// split a rune; the site reverses the same way (a JS split/reverse/join).
|
||||
func reverseASCII(s string) string {
|
||||
b := []byte(s)
|
||||
for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 {
|
||||
b[i], b[j] = b[j], b[i]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package hako
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// encodeXORShuffle builds a payload the way the site does: split the body into
|
||||
// chunks, XOR each one from the start of the key, base64 it, and prefix its
|
||||
// position. Chunks are emitted out of order on purpose — the shuffle is the
|
||||
// part the decoder has to undo.
|
||||
func encodeXORShuffle(t *testing.T, body, key string, chunkLen int) []string {
|
||||
t.Helper()
|
||||
|
||||
var chunks []string
|
||||
raw := []byte(body)
|
||||
for pos := 0; pos*chunkLen < len(raw); pos++ {
|
||||
start := pos * chunkLen
|
||||
end := min(start+chunkLen, len(raw))
|
||||
|
||||
part := make([]byte, end-start)
|
||||
copy(part, raw[start:end])
|
||||
for i := range part {
|
||||
part[i] ^= key[i%len(key)]
|
||||
}
|
||||
chunks = append(chunks, fmt.Sprintf("%04d%s", pos, base64.StdEncoding.EncodeToString(part)))
|
||||
}
|
||||
|
||||
// Reverse so the decoder cannot pass by accident on input order alone.
|
||||
for i, j := 0, len(chunks)-1; i < j; i, j = i+1, j-1 {
|
||||
chunks[i], chunks[j] = chunks[j], chunks[i]
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
func jsonArray(items []string) string {
|
||||
quoted := make([]string, len(items))
|
||||
for i, s := range items {
|
||||
quoted[i] = `"` + s + `"`
|
||||
}
|
||||
return "[" + strings.Join(quoted, ",") + "]"
|
||||
}
|
||||
|
||||
// The body is deliberately split so multi-byte runes straddle chunk
|
||||
// boundaries, which is where a decoder that XORs across the joined stream, or
|
||||
// that decodes each chunk as its own UTF-8 string, falls apart.
|
||||
const (
|
||||
testKey = "6b9dd83fad5a3169"
|
||||
testBody = `<p id="1">Trời hôm nay đẹp lắm.</p><p id="2">Người anh trai mỉm cười.</p>`
|
||||
)
|
||||
|
||||
func TestDecodeXORShuffle(t *testing.T) {
|
||||
for _, chunkLen := range []int{7, 16, 31, 4096} {
|
||||
load, err := parsePayload(schemeXORShuffle, testKey,
|
||||
jsonArray(encodeXORShuffle(t, testBody, testKey, chunkLen)))
|
||||
if err != nil {
|
||||
t.Fatalf("chunk length %d: parse payload: %v", chunkLen, err)
|
||||
}
|
||||
got, err := load.decode()
|
||||
if err != nil {
|
||||
t.Fatalf("chunk length %d: decode: %v", chunkLen, err)
|
||||
}
|
||||
if got != testBody {
|
||||
t.Errorf("chunk length %d: body round-trip failed\n got: %q\nwant: %q", chunkLen, got, testBody)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeRejectsWrongKey(t *testing.T) {
|
||||
load, err := parsePayload(schemeXORShuffle, "0000000000000000",
|
||||
jsonArray(encodeXORShuffle(t, testBody, testKey, 16)))
|
||||
if err != nil {
|
||||
t.Fatalf("parse payload: %v", err)
|
||||
}
|
||||
// A wrong key yields bytes that are not valid UTF-8; the decoder must say
|
||||
// so rather than hand back a chapter of mojibake.
|
||||
if _, err := load.decode(); err == nil {
|
||||
t.Fatal("expected an error for a wrong key, got none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeRejectsUnknownScheme(t *testing.T) {
|
||||
// When the site rotates its encoding, failing loudly here is what tells the
|
||||
// user why the export stopped working.
|
||||
load, err := parsePayload("aes_gcm_2027", testKey, `["0000AAAA"]`)
|
||||
if err != nil {
|
||||
t.Fatalf("parse payload: %v", err)
|
||||
}
|
||||
if _, err := load.decode(); err == nil {
|
||||
t.Fatal("expected an error for an unknown scheme, got none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeRejectsMissingKey(t *testing.T) {
|
||||
load, err := parsePayload(schemeXORShuffle, "", `["0000AAAA"]`)
|
||||
if err != nil {
|
||||
t.Fatalf("parse payload: %v", err)
|
||||
}
|
||||
if _, err := load.decode(); err == nil {
|
||||
t.Fatal("expected an error when the key is absent, got none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeRejectsEmptyPayload(t *testing.T) {
|
||||
load, err := parsePayload(schemeXORShuffle, testKey, `[]`)
|
||||
if err != nil {
|
||||
t.Fatalf("parse payload: %v", err)
|
||||
}
|
||||
if _, err := load.decode(); err == nil {
|
||||
t.Fatal("expected an error for an empty chunk list, got none")
|
||||
}
|
||||
}
|
||||
|
||||
// base64 and base64_reverse are the site's simpler schemes; the decoder still
|
||||
// carries them so a chapter served either way keeps working.
|
||||
func TestDecodePlainBase64Schemes(t *testing.T) {
|
||||
encoded := base64.StdEncoding.EncodeToString([]byte(testBody))
|
||||
|
||||
for _, tc := range []struct {
|
||||
scheme string
|
||||
chunk string
|
||||
}{
|
||||
{schemeBase64, encoded},
|
||||
{schemeBase64Reverse, reverseASCII(encoded)},
|
||||
} {
|
||||
load, err := parsePayload(tc.scheme, "", jsonArray([]string{"0000" + tc.chunk}))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: parse payload: %v", tc.scheme, err)
|
||||
}
|
||||
got, err := load.decode()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: decode: %v", tc.scheme, err)
|
||||
}
|
||||
if got != testBody {
|
||||
t.Errorf("%s: got %q, want %q", tc.scheme, got, testBody)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortChunksRejectsBadPrefix(t *testing.T) {
|
||||
if _, err := sortChunks([]string{"abcdAAAA"}); err == nil {
|
||||
t.Fatal("expected an error for a non-numeric position prefix, got none")
|
||||
}
|
||||
if _, err := sortChunks([]string{"01"}); err == nil {
|
||||
t.Fatal("expected an error for a chunk shorter than its prefix, got none")
|
||||
}
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
const DEBUG = false
|
||||
|
||||
var sanitizeRe = regexp.MustCompile(`[\\\/:*?"<>|]`)
|
||||
|
||||
func isDigit(id string) bool {
|
||||
if id == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range id {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func getFromURL(url string) (string, error) {
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0")
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP error: %s", resp.Status)
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
func writeTextToFile(text, filename string) error {
|
||||
filename = sanitizeRe.ReplaceAllString(filename, "_")
|
||||
dir := "data"
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
path := filepath.Join(dir, filename)
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
_, err = f.WriteString(text)
|
||||
return err
|
||||
}
|
||||
|
||||
func readTextFromFile(filename string) (string, error) {
|
||||
path := filepath.Join("data", filename)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
var html string
|
||||
var err error
|
||||
if DEBUG {
|
||||
html, err = readTextFromFile("_.txt")
|
||||
} else {
|
||||
html, err = getFromURL("https://ln.hako.vn/sang-tac/8476-kiep-nay-la-anh-trai-cua-nhan-vat-chinh")
|
||||
// writeTextToFile(html, "_.txt")
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Printf("Error: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
|
||||
if err != nil {
|
||||
fmt.Printf("Parse error: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
chapters := doc.Find("div.chapter-name")
|
||||
/*
|
||||
you may get `HTTP Error 429: Too Many Requests`
|
||||
you can try again later, and skip downloaded chapters
|
||||
example skip 50 first chapters like: chapters = chapters.Slice(50, chapters.Length())
|
||||
*/
|
||||
// chapters = chapters.Slice(100, chapters.Length())
|
||||
for i := 0; i < chapters.Length(); i++ {
|
||||
chapterSel := chapters.Eq(i)
|
||||
children := chapterSel.ChildrenFiltered("a")
|
||||
if children.Length() == 0 {
|
||||
continue
|
||||
}
|
||||
linkSel := children.First()
|
||||
title, hasTitle := linkSel.Attr("title")
|
||||
if !hasTitle {
|
||||
continue
|
||||
}
|
||||
href, hasHref := linkSel.Attr("href")
|
||||
if !hasHref {
|
||||
continue
|
||||
}
|
||||
chapterURL := "https://ln.hako.vn" + href
|
||||
chapterHTML, err := getFromURL(chapterURL)
|
||||
if err != nil {
|
||||
fmt.Printf("Error fetching %s: %v\n", chapterURL, err)
|
||||
continue
|
||||
}
|
||||
chapterDoc, err := goquery.NewDocumentFromReader(strings.NewReader(chapterHTML))
|
||||
if err != nil {
|
||||
fmt.Printf("Parse chapter %s: %v\n", title, err)
|
||||
continue
|
||||
}
|
||||
content := chapterDoc.Find("#chapter-content")
|
||||
var chapterData strings.Builder
|
||||
content.Find("p").Each(func(_ int, s *goquery.Selection) {
|
||||
id, exists := s.Attr("id")
|
||||
if exists && isDigit(id) {
|
||||
chapterData.WriteString(s.Text())
|
||||
chapterData.WriteString("\n")
|
||||
}
|
||||
})
|
||||
filename := title + ".txt"
|
||||
if err := writeTextToFile(chapterData.String(), filename); err != nil {
|
||||
fmt.Printf("Error writing %s: %v\n", filename, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user