mirror of
https://github.com/tiennm99/MTTools.git
synced 2026-10-15 16:15:02 +00:00
feat: parse novel tags and add a single-request metadata fetch
Novel gains Tags, read from the anchors carrying schema.org itemprop="genre". Matching that microdata rather than the /the-loai/ href shape is what keeps the site-wide genre navigation out: a sampled page links 69 categories against the novel's own 6. NovelInfo fetches just the landing page and parses it, so a caller that only wants metadata pays one request instead of the two Novel needs for its chapter list cross-check.
This commit is contained in:
1 parent
5b213da6be
commit
e56338e5e3
3 files changed
+103
-3
No files matched your search
@@ -77,6 +77,21 @@ func (c *Crawler) Novel(ctx context.Context, novelURL string) (*Novel, error) {
|
|||||||
return novel, nil
|
return novel, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NovelInfo fetches only the landing page and returns what it carries: title,
|
||||||
|
// slug, tags, and the chapter list as that page shows it.
|
||||||
|
//
|
||||||
|
// Unlike Novel it does not also fetch a chapter page to cross-check the chapter
|
||||||
|
// list, so it costs a single request. Callers that only want metadata should
|
||||||
|
// prefer it; callers about to export every chapter want Novel, whose
|
||||||
|
// reconciliation guards against a silently truncated list.
|
||||||
|
func (c *Crawler) NovelInfo(ctx context.Context, novelURL string) (*Novel, error) {
|
||||||
|
page, err := c.page(ctx, novelURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return ParseNovelPage(page, novelURL)
|
||||||
|
}
|
||||||
|
|
||||||
// Chapters fetches every chapter concurrently and returns them in reading
|
// Chapters fetches every chapter concurrently and returns them in reading
|
||||||
// order. Any chapter that cannot be fetched or parsed fails the whole run
|
// order. Any chapter that cannot be fetched or parsed fails the whole run
|
||||||
// rather than yielding a book with a hole in it.
|
// rather than yielding a book with a hole in it.
|
||||||
|
|||||||
@@ -17,9 +17,13 @@ type ChapterRef struct {
|
|||||||
|
|
||||||
// Novel is a novel's landing page: its title and its chapters in reading order.
|
// Novel is a novel's landing page: its title and its chapters in reading order.
|
||||||
type Novel struct {
|
type Novel struct {
|
||||||
Title string
|
Title string
|
||||||
Slug string
|
Slug string
|
||||||
URL string
|
URL string
|
||||||
|
|
||||||
|
// Tags are the novel's own genres, as labelled on the site, in the order
|
||||||
|
// they appear. Empty when the page lists none.
|
||||||
|
Tags []string
|
||||||
Chapters []ChapterRef
|
Chapters []ChapterRef
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -43,6 +47,7 @@ func ParseNovelPage(page []byte, pageURL string) (*Novel, error) {
|
|||||||
Title: novelTitle(doc),
|
Title: novelTitle(doc),
|
||||||
Slug: slugFromNovelURL(base),
|
Slug: slugFromNovelURL(base),
|
||||||
URL: pageURL,
|
URL: pageURL,
|
||||||
|
Tags: tagsFromInfo(doc),
|
||||||
Chapters: chapterRefsFromList(doc, base),
|
Chapters: chapterRefsFromList(doc, base),
|
||||||
}
|
}
|
||||||
if novel.Title == "" {
|
if novel.Title == "" {
|
||||||
@@ -64,6 +69,35 @@ func novelTitle(doc *html.Node) string {
|
|||||||
return nodeText(elementByTag(doc, "title"))
|
return nodeText(elementByTag(doc, "title"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// tagsFromInfo reads the novel's own genres out of the info block.
|
||||||
|
//
|
||||||
|
// The anchors are matched on the schema.org itemprop="genre" microdata rather
|
||||||
|
// than on their href. The page also carries a site-wide genre menu linking every
|
||||||
|
// category — 69 distinct ones against this novel's 6 on a sampled page — so
|
||||||
|
// matching the /the-loai/ URL shape would pull in the whole navigation.
|
||||||
|
func tagsFromInfo(doc *html.Node) []string {
|
||||||
|
links := findAllNodes(doc, func(n *html.Node) bool {
|
||||||
|
return n.Type == html.ElementNode && n.Data == "a" && attr(n, "itemprop") == "genre"
|
||||||
|
})
|
||||||
|
|
||||||
|
var tags []string
|
||||||
|
seen := make(map[string]bool, len(links))
|
||||||
|
for _, link := range links {
|
||||||
|
// The label appears both as the link text and as its title attribute;
|
||||||
|
// prefer the text and fall back to the attribute.
|
||||||
|
name := nodeText(link)
|
||||||
|
if name == "" {
|
||||||
|
name = collapseSpaces(attr(link, "title"))
|
||||||
|
}
|
||||||
|
if name == "" || seen[name] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[name] = true
|
||||||
|
tags = append(tags, name)
|
||||||
|
}
|
||||||
|
return tags
|
||||||
|
}
|
||||||
|
|
||||||
// slugFromNovelURL turns https://host/tro-lai-nam-thang-cu.html into
|
// slugFromNovelURL turns https://host/tro-lai-nam-thang-cu.html into
|
||||||
// "tro-lai-nam-thang-cu".
|
// "tro-lai-nam-thang-cu".
|
||||||
func slugFromNovelURL(u *url.URL) string {
|
func slugFromNovelURL(u *url.URL) string {
|
||||||
|
|||||||
@@ -44,6 +44,57 @@ func TestParseNovelPageOrdersChaptersForReading(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// tagsFixture mirrors the real page: the novel's own genres carry
|
||||||
|
// itemprop="genre", while a site-wide menu links every category without it. A
|
||||||
|
// parser matching on the /the-loai/ href shape would swallow the whole menu.
|
||||||
|
const tagsFixture = `<html><head><title>TEN TRUYEN</title></head><body>
|
||||||
|
<h1>TEN TRUYEN</h1>
|
||||||
|
<dl class="row">
|
||||||
|
<dt class="col-sm-3">Thể loại</dt>
|
||||||
|
<dd class="col-sm-9">
|
||||||
|
<a class='cate-item' itemprop='genre' title='Trọng Sinh' href='https://monkeydd.com/the-loai/trong-sinh.html'>Trọng Sinh</a>
|
||||||
|
<a class='cate-item' itemprop='genre' title='Cổ Đại' href='https://monkeydd.com/the-loai/co-dai.html'>Cổ Đại</a>
|
||||||
|
<a class='cate-item' itemprop='genre' title='Gia Đình' href='https://monkeydd.com/the-loai/gia-dinh.html'></a>
|
||||||
|
<a class='cate-item' itemprop='genre' title='Cổ Đại' href='https://monkeydd.com/the-loai/co-dai.html'>Cổ Đại</a>
|
||||||
|
</dd>
|
||||||
|
</dl>
|
||||||
|
<nav class="site-menu">
|
||||||
|
<a href='https://monkeydd.com/the-loai/dam-my.html'>Đam Mỹ</a>
|
||||||
|
<a href='https://monkeydd.com/the-loai/bach-hop.html'>Bách Hợp</a>
|
||||||
|
</nav>
|
||||||
|
<div class="list-chapters">
|
||||||
|
<div class="item"><div class="episode-title"><a href="/n/1.html">1</a></div></div>
|
||||||
|
</div></body></html>`
|
||||||
|
|
||||||
|
func TestParseNovelPageReadsTags(t *testing.T) {
|
||||||
|
novel, err := ParseNovelPage([]byte(tagsFixture), "https://monkeydd.com/n.html")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseNovelPage: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// "Cổ Đại" has its inner whitespace collapsed, the empty anchor falls back
|
||||||
|
// to its title attribute, and the repeated genre appears once.
|
||||||
|
want := []string{"Trọng Sinh", "Cổ Đại", "Gia Đình"}
|
||||||
|
if len(novel.Tags) != len(want) {
|
||||||
|
t.Fatalf("Tags = %q, want %q", novel.Tags, want)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if novel.Tags[i] != want[i] {
|
||||||
|
t.Errorf("Tags[%d] = %q, want %q", i, novel.Tags[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNovelPageWithoutTags(t *testing.T) {
|
||||||
|
novel, err := ParseNovelPage([]byte(novelFixture), "https://monkeydd.com/n.html")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ParseNovelPage: %v", err)
|
||||||
|
}
|
||||||
|
if len(novel.Tags) != 0 {
|
||||||
|
t.Errorf("Tags = %q, want none", novel.Tags)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseNovelPageNoChapters(t *testing.T) {
|
func TestParseNovelPageNoChapters(t *testing.T) {
|
||||||
if _, err := ParseNovelPage([]byte(`<html><title>x</title><body></body></html>`),
|
if _, err := ParseNovelPage([]byte(`<html><title>x</title><body></body></html>`),
|
||||||
"https://monkeydd.com/n.html"); err == nil {
|
"https://monkeydd.com/n.html"); err == nil {
|
||||||
|
|||||||
Reference in new issue
Block a user