mirror of
https://github.com/tiennm99/MTTools.git
synced 2026-10-14 12:16:55 +00:00
refactor: make crawler importable and add export pipeline
Move monkeyd and pdfout out of internal/ so other modules can import them, and add export.Export, which holds the crawl-to-PDF sequence the CLI used to inline. The CLI now parses flags and delegates, so an embedding program gets the same defaults and validation.
This commit is contained in:
1 parent
2855ce5e53
commit
3633e96cf5
15 files changed
+490
-128
No files matched your search
@@ -0,0 +1,95 @@
|
||||
package monkeyd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// attr returns the value of the named attribute, or "" when absent.
|
||||
func attr(n *html.Node, name string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == name {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hasClass reports whether the node carries the given class token.
|
||||
func hasClass(n *html.Node, class string) bool {
|
||||
for _, tok := range strings.Fields(attr(n, "class")) {
|
||||
if tok == class {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// findNode returns the first node in document order satisfying match.
|
||||
func findNode(root *html.Node, match func(*html.Node) bool) *html.Node {
|
||||
if match(root) {
|
||||
return root
|
||||
}
|
||||
for c := root.FirstChild; c != nil; c = c.NextSibling {
|
||||
if found := findNode(c, match); found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// findAllNodes returns every node satisfying match, in document order.
|
||||
func findAllNodes(root *html.Node, match func(*html.Node) bool) []*html.Node {
|
||||
var out []*html.Node
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if match(n) {
|
||||
out = append(out, n)
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(root)
|
||||
return out
|
||||
}
|
||||
|
||||
// elementByID finds an element by its id attribute.
|
||||
func elementByID(root *html.Node, id string) *html.Node {
|
||||
return findNode(root, func(n *html.Node) bool {
|
||||
return n.Type == html.ElementNode && attr(n, "id") == id
|
||||
})
|
||||
}
|
||||
|
||||
// elementByTag finds the first element with the given tag name.
|
||||
func elementByTag(root *html.Node, tag string) *html.Node {
|
||||
return findNode(root, func(n *html.Node) bool {
|
||||
return n.Type == html.ElementNode && n.Data == tag
|
||||
})
|
||||
}
|
||||
|
||||
// nodeText collects the descendant text of a node with whitespace collapsed.
|
||||
func nodeText(n *html.Node) string {
|
||||
if n == nil {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.TextNode {
|
||||
b.WriteString(n.Data)
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return collapseSpaces(b.String())
|
||||
}
|
||||
|
||||
// collapseSpaces trims the string and reduces every whitespace run, including
|
||||
// the non-breaking spaces the site emits as , to a single space.
|
||||
func collapseSpaces(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
Reference in new issue
Block a user