mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-13 22:15:47 +00:00
fix(web_fetch): replace regex HTML parsing with DOM-based extraction
Regex-based htmlToMarkdown/htmlToText leaked CSS, JS, and non-content elements. Replaced with golang.org/x/net/html DOM parser that extracts <body> only and skips 16 non-content element types (script, style, noscript, svg, template, iframe, form, nav, footer, etc.). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
viettranxandClaude Opus 4.6 committed
1 parent
e8c634f611
commit
37158af231
2 files changed
+884
-131
No files matched your search
+515
-131
@@ -2,8 +2,12 @@ package tools
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
"golang.org/x/net/html/atom"
|
||||
)
|
||||
|
||||
// extractJSON pretty-prints JSON content.
|
||||
@@ -16,173 +20,553 @@ func extractJSON(body []byte) (string, string) {
|
||||
return string(body), "raw"
|
||||
}
|
||||
|
||||
// --- HTML extraction utilities ---
|
||||
// --- DOM-based HTML extraction ---
|
||||
|
||||
var (
|
||||
reScript = regexp.MustCompile(`(?is)<script[\s\S]*?</script>`)
|
||||
reStyle = regexp.MustCompile(`(?is)<style[\s\S]*?</style>`)
|
||||
reComment = regexp.MustCompile(`<!--[\s\S]*?-->`)
|
||||
reNav = regexp.MustCompile(`(?is)<nav[\s\S]*?</nav>`)
|
||||
reFooter = regexp.MustCompile(`(?is)<footer[\s\S]*?</footer>`)
|
||||
reHeader = regexp.MustCompile(`(?is)<header[\s\S]*?</header>`)
|
||||
reTag = regexp.MustCompile(`<[^>]+>`)
|
||||
reMultiNL = regexp.MustCompile(`\n{3,}`)
|
||||
reMultiSP = regexp.MustCompile(`[ \t]{2,}`)
|
||||
reH1 = regexp.MustCompile(`(?i)<h1[^>]*>([\s\S]*?)</h1>`)
|
||||
reH2 = regexp.MustCompile(`(?i)<h2[^>]*>([\s\S]*?)</h2>`)
|
||||
reH3 = regexp.MustCompile(`(?i)<h3[^>]*>([\s\S]*?)</h3>`)
|
||||
reH4 = regexp.MustCompile(`(?i)<h4[^>]*>([\s\S]*?)</h4>`)
|
||||
reH5 = regexp.MustCompile(`(?i)<h5[^>]*>([\s\S]*?)</h5>`)
|
||||
reH6 = regexp.MustCompile(`(?i)<h6[^>]*>([\s\S]*?)</h6>`)
|
||||
reParagraph = regexp.MustCompile(`(?i)<p[^>]*>([\s\S]*?)</p>`)
|
||||
reBreak = regexp.MustCompile(`(?i)<br\s*/?>`)
|
||||
reListItem = regexp.MustCompile(`(?i)<li[^>]*>([\s\S]*?)</li>`)
|
||||
reAnchor = regexp.MustCompile(`(?i)<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)</a>`)
|
||||
rePre = regexp.MustCompile(`(?is)<pre[^>]*>([\s\S]*?)</pre>`)
|
||||
reCode = regexp.MustCompile(`(?i)<code[^>]*>([\s\S]*?)</code>`)
|
||||
reStrong = regexp.MustCompile(`(?i)<(?:strong|b)[^>]*>([\s\S]*?)</(?:strong|b)>`)
|
||||
reEm = regexp.MustCompile(`(?i)<(?:em|i)[^>]*>([\s\S]*?)</(?:em|i)>`)
|
||||
reBlockq = regexp.MustCompile(`(?is)<blockquote[^>]*>([\s\S]*?)</blockquote>`)
|
||||
reImg = regexp.MustCompile(`(?i)<img[^>]*alt="([^"]*)"[^>]*/?>`)
|
||||
type convertMode int
|
||||
|
||||
const (
|
||||
modeMarkdown convertMode = iota
|
||||
modeText
|
||||
)
|
||||
|
||||
// htmlToMarkdown converts HTML to a markdown-like format.
|
||||
// Not a full Readability implementation but covers common patterns.
|
||||
func htmlToMarkdown(html string) string {
|
||||
// Remove non-content elements
|
||||
s := reScript.ReplaceAllString(html, "")
|
||||
s = reStyle.ReplaceAllString(s, "")
|
||||
s = reComment.ReplaceAllString(s, "")
|
||||
s = reNav.ReplaceAllString(s, "")
|
||||
s = reFooter.ReplaceAllString(s, "")
|
||||
// converter walks a parsed HTML DOM tree and emits markdown or plain text.
|
||||
type converter struct {
|
||||
buf strings.Builder
|
||||
mode convertMode
|
||||
inPre bool
|
||||
listDepth int
|
||||
listType []atom.Atom // stack: atom.Ul / atom.Ol
|
||||
listIndex []int // ordered list counters
|
||||
inLink bool
|
||||
}
|
||||
|
||||
// Convert headings
|
||||
s = reH1.ReplaceAllString(s, "\n# $1\n")
|
||||
s = reH2.ReplaceAllString(s, "\n## $1\n")
|
||||
s = reH3.ReplaceAllString(s, "\n### $1\n")
|
||||
s = reH4.ReplaceAllString(s, "\n#### $1\n")
|
||||
s = reH5.ReplaceAllString(s, "\n##### $1\n")
|
||||
s = reH6.ReplaceAllString(s, "\n###### $1\n")
|
||||
// Elements to skip entirely (element + all descendants).
|
||||
var skipElements = map[atom.Atom]bool{
|
||||
atom.Head: true,
|
||||
atom.Script: true,
|
||||
atom.Style: true,
|
||||
atom.Noscript: true,
|
||||
atom.Svg: true,
|
||||
atom.Template: true,
|
||||
atom.Iframe: true,
|
||||
atom.Select: true,
|
||||
atom.Option: true,
|
||||
atom.Button: true,
|
||||
atom.Input: true,
|
||||
atom.Form: true,
|
||||
atom.Nav: true,
|
||||
atom.Footer: true,
|
||||
atom.Picture: true,
|
||||
atom.Source: true,
|
||||
}
|
||||
|
||||
// Pre/code blocks (before stripping other tags)
|
||||
s = rePre.ReplaceAllString(s, "\n```\n$1\n```\n")
|
||||
s = reCode.ReplaceAllString(s, "`$1`")
|
||||
// Additional elements to skip in text mode only.
|
||||
var skipInTextMode = map[atom.Atom]bool{
|
||||
atom.Header: true,
|
||||
atom.Aside: true,
|
||||
}
|
||||
|
||||
// Blockquotes
|
||||
s = reBlockq.ReplaceAllStringFunc(s, func(match string) string {
|
||||
inner := reBlockq.FindStringSubmatch(match)
|
||||
if len(inner) < 2 {
|
||||
return match
|
||||
// Block elements that need surrounding newlines.
|
||||
var blockElements = map[atom.Atom]bool{
|
||||
atom.P: true, atom.Div: true, atom.Section: true, atom.Article: true,
|
||||
atom.Main: true, atom.H1: true, atom.H2: true, atom.H3: true,
|
||||
atom.H4: true, atom.H5: true, atom.H6: true, atom.Blockquote: true,
|
||||
atom.Pre: true, atom.Ul: true, atom.Ol: true, atom.Li: true,
|
||||
atom.Table: true, atom.Tr: true, atom.Hr: true, atom.Dl: true,
|
||||
atom.Dt: true, atom.Dd: true, atom.Figure: true, atom.Figcaption: true,
|
||||
atom.Details: true, atom.Summary: true, atom.Address: true,
|
||||
}
|
||||
|
||||
// htmlToMarkdown converts HTML to a markdown-like format using DOM parsing.
|
||||
func htmlToMarkdown(rawHTML string) string {
|
||||
doc, err := html.Parse(strings.NewReader(rawHTML))
|
||||
if err != nil {
|
||||
return stripTagsFallback(rawHTML)
|
||||
}
|
||||
body := findBody(doc)
|
||||
c := &converter{mode: modeMarkdown}
|
||||
c.walkChildren(body)
|
||||
return cleanOutput(c.buf.String())
|
||||
}
|
||||
|
||||
// htmlToText extracts plain text from HTML content using DOM parsing.
|
||||
func htmlToText(rawHTML string) string {
|
||||
doc, err := html.Parse(strings.NewReader(rawHTML))
|
||||
if err != nil {
|
||||
return stripTagsFallback(rawHTML)
|
||||
}
|
||||
body := findBody(doc)
|
||||
c := &converter{mode: modeText}
|
||||
c.walkChildren(body)
|
||||
return cleanTextOutput(c.buf.String())
|
||||
}
|
||||
|
||||
func (c *converter) walk(n *html.Node) {
|
||||
switch n.Type {
|
||||
case html.TextNode:
|
||||
c.handleText(n)
|
||||
return
|
||||
case html.ElementNode:
|
||||
// handled below
|
||||
case html.DocumentNode:
|
||||
c.walkChildren(n)
|
||||
return
|
||||
default:
|
||||
return
|
||||
}
|
||||
|
||||
tag := n.DataAtom
|
||||
|
||||
if skipElements[tag] {
|
||||
return
|
||||
}
|
||||
if c.mode == modeText && skipInTextMode[tag] {
|
||||
return
|
||||
}
|
||||
|
||||
switch tag {
|
||||
case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6:
|
||||
c.handleHeading(n)
|
||||
case atom.P:
|
||||
c.handleParagraph(n)
|
||||
case atom.A:
|
||||
c.handleLink(n)
|
||||
case atom.Img:
|
||||
c.handleImage(n)
|
||||
case atom.Pre:
|
||||
c.handlePre(n)
|
||||
case atom.Code:
|
||||
c.handleCode(n)
|
||||
case atom.Blockquote:
|
||||
c.handleBlockquote(n)
|
||||
case atom.Strong, atom.B:
|
||||
c.handleStrong(n)
|
||||
case atom.Em, atom.I:
|
||||
c.handleEmphasis(n)
|
||||
case atom.Br:
|
||||
c.buf.WriteByte('\n')
|
||||
case atom.Hr:
|
||||
c.ensureNewline()
|
||||
if c.mode == modeMarkdown {
|
||||
c.buf.WriteString("---\n")
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(inner[1]), "\n")
|
||||
var quoted []string
|
||||
for _, l := range lines {
|
||||
quoted = append(quoted, "> "+strings.TrimSpace(l))
|
||||
case atom.Ul, atom.Ol:
|
||||
c.handleList(n)
|
||||
case atom.Li:
|
||||
c.handleListItem(n)
|
||||
case atom.Table:
|
||||
c.handleTable(n)
|
||||
case atom.Dt:
|
||||
c.handleDefinitionTerm(n)
|
||||
case atom.Dd:
|
||||
c.handleDefinitionDesc(n)
|
||||
default:
|
||||
if blockElements[tag] {
|
||||
c.ensureNewline()
|
||||
c.walkChildren(n)
|
||||
c.ensureNewline()
|
||||
} else {
|
||||
c.walkChildren(n)
|
||||
}
|
||||
return "\n" + strings.Join(quoted, "\n") + "\n"
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Links: <a href="url">text</a> → [text](url)
|
||||
s = reAnchor.ReplaceAllString(s, "[$2]($1)")
|
||||
func (c *converter) walkChildren(n *html.Node) {
|
||||
for child := n.FirstChild; child != nil; child = child.NextSibling {
|
||||
c.walk(child)
|
||||
}
|
||||
}
|
||||
|
||||
// Images: <img alt="text" ... /> → ![text]
|
||||
s = reImg.ReplaceAllString(s, "![$1]")
|
||||
func (c *converter) handleText(n *html.Node) {
|
||||
text := n.Data
|
||||
if c.inPre {
|
||||
c.buf.WriteString(text)
|
||||
return
|
||||
}
|
||||
text = collapseWhitespace(text)
|
||||
if text == "" {
|
||||
return
|
||||
}
|
||||
if text == " " && c.buf.Len() > 0 {
|
||||
c.buf.WriteByte(' ')
|
||||
return
|
||||
}
|
||||
c.buf.WriteString(text)
|
||||
}
|
||||
|
||||
// Bold/italic
|
||||
s = reStrong.ReplaceAllString(s, "**$1**")
|
||||
s = reEm.ReplaceAllString(s, "*$1*")
|
||||
func (c *converter) handleHeading(n *html.Node) {
|
||||
c.ensureDoubleNewline()
|
||||
if c.mode == modeMarkdown && len(n.Data) == 2 && n.Data[0] == 'h' {
|
||||
level := int(n.Data[1] - '0')
|
||||
for i := 0; i < level; i++ {
|
||||
c.buf.WriteByte('#')
|
||||
}
|
||||
c.buf.WriteByte(' ')
|
||||
}
|
||||
c.walkChildren(n)
|
||||
c.buf.WriteByte('\n')
|
||||
}
|
||||
|
||||
// Paragraphs and breaks
|
||||
s = reParagraph.ReplaceAllString(s, "\n$1\n")
|
||||
s = reBreak.ReplaceAllString(s, "\n")
|
||||
func (c *converter) handleParagraph(n *html.Node) {
|
||||
c.ensureDoubleNewline()
|
||||
c.walkChildren(n)
|
||||
c.buf.WriteByte('\n')
|
||||
}
|
||||
|
||||
// List items
|
||||
s = reListItem.ReplaceAllString(s, "\n- $1")
|
||||
func (c *converter) handleLink(n *html.Node) {
|
||||
href := getAttr(n, "href")
|
||||
if c.mode == modeText || c.inLink || href == "" || strings.HasPrefix(href, "javascript:") {
|
||||
c.walkChildren(n)
|
||||
return
|
||||
}
|
||||
c.inLink = true
|
||||
c.buf.WriteByte('[')
|
||||
c.walkChildren(n)
|
||||
c.buf.WriteString("](")
|
||||
c.buf.WriteString(href)
|
||||
c.buf.WriteByte(')')
|
||||
c.inLink = false
|
||||
}
|
||||
|
||||
// Strip remaining tags
|
||||
s = reTag.ReplaceAllString(s, "")
|
||||
func (c *converter) handleImage(n *html.Node) {
|
||||
alt := getAttr(n, "alt")
|
||||
src := getAttr(n, "src")
|
||||
if c.mode == modeMarkdown {
|
||||
c.buf.WriteString("![")
|
||||
c.buf.WriteString(alt)
|
||||
c.buf.WriteByte(']')
|
||||
if src != "" {
|
||||
c.buf.WriteByte('(')
|
||||
c.buf.WriteString(src)
|
||||
c.buf.WriteByte(')')
|
||||
}
|
||||
} else if alt != "" {
|
||||
c.buf.WriteString(alt)
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up
|
||||
s = decodeHTMLEntities(s)
|
||||
func (c *converter) handlePre(n *html.Node) {
|
||||
c.ensureDoubleNewline()
|
||||
if c.mode == modeMarkdown {
|
||||
lang := ""
|
||||
if code := findChild(n, atom.Code); code != nil {
|
||||
cls := getAttr(code, "class")
|
||||
for _, part := range strings.Fields(cls) {
|
||||
if rest, ok := strings.CutPrefix(part, "language-"); ok {
|
||||
lang = rest
|
||||
break
|
||||
}
|
||||
if rest, ok := strings.CutPrefix(part, "lang-"); ok {
|
||||
lang = rest
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
c.buf.WriteString("```")
|
||||
c.buf.WriteString(lang)
|
||||
c.buf.WriteByte('\n')
|
||||
}
|
||||
c.inPre = true
|
||||
c.walkChildren(n)
|
||||
c.inPre = false
|
||||
if c.mode == modeMarkdown {
|
||||
c.ensureNewline()
|
||||
c.buf.WriteString("```\n")
|
||||
} else {
|
||||
c.buf.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
|
||||
func (c *converter) handleCode(n *html.Node) {
|
||||
if c.inPre {
|
||||
c.walkChildren(n)
|
||||
return
|
||||
}
|
||||
if c.mode == modeMarkdown {
|
||||
c.buf.WriteByte('`')
|
||||
c.walkChildren(n)
|
||||
c.buf.WriteByte('`')
|
||||
} else {
|
||||
c.walkChildren(n)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *converter) handleBlockquote(n *html.Node) {
|
||||
c.ensureDoubleNewline()
|
||||
if c.mode == modeMarkdown {
|
||||
sub := &converter{mode: c.mode, inPre: c.inPre}
|
||||
sub.walkChildren(n)
|
||||
for i, line := range strings.Split(strings.TrimSpace(sub.buf.String()), "\n") {
|
||||
if i > 0 {
|
||||
c.buf.WriteByte('\n')
|
||||
}
|
||||
c.buf.WriteString("> ")
|
||||
c.buf.WriteString(line)
|
||||
}
|
||||
c.buf.WriteByte('\n')
|
||||
} else {
|
||||
c.walkChildren(n)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *converter) handleStrong(n *html.Node) {
|
||||
if c.mode == modeMarkdown {
|
||||
c.buf.WriteString("**")
|
||||
c.walkChildren(n)
|
||||
c.buf.WriteString("**")
|
||||
} else {
|
||||
c.walkChildren(n)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *converter) handleEmphasis(n *html.Node) {
|
||||
if c.mode == modeMarkdown {
|
||||
c.buf.WriteByte('*')
|
||||
c.walkChildren(n)
|
||||
c.buf.WriteByte('*')
|
||||
} else {
|
||||
c.walkChildren(n)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *converter) handleList(n *html.Node) {
|
||||
c.ensureNewline()
|
||||
c.listDepth++
|
||||
c.listType = append(c.listType, n.DataAtom)
|
||||
c.listIndex = append(c.listIndex, 0)
|
||||
c.walkChildren(n)
|
||||
c.listDepth--
|
||||
c.listType = c.listType[:len(c.listType)-1]
|
||||
c.listIndex = c.listIndex[:len(c.listIndex)-1]
|
||||
c.ensureNewline()
|
||||
}
|
||||
|
||||
func (c *converter) handleListItem(n *html.Node) {
|
||||
c.ensureNewline()
|
||||
indent := strings.Repeat(" ", max(0, c.listDepth-1))
|
||||
c.buf.WriteString(indent)
|
||||
|
||||
if len(c.listType) > 0 && c.listType[len(c.listType)-1] == atom.Ol {
|
||||
idx := len(c.listIndex) - 1
|
||||
c.listIndex[idx]++
|
||||
fmt.Fprintf(&c.buf, "%d. ", c.listIndex[idx])
|
||||
} else {
|
||||
c.buf.WriteString("- ")
|
||||
}
|
||||
c.walkChildren(n)
|
||||
}
|
||||
|
||||
func (c *converter) handleTable(n *html.Node) {
|
||||
c.ensureDoubleNewline()
|
||||
rows := collectTableRows(n, c.mode)
|
||||
if len(rows) == 0 {
|
||||
return
|
||||
}
|
||||
colCount := 0
|
||||
for _, row := range rows {
|
||||
if len(row) > colCount {
|
||||
colCount = len(row)
|
||||
}
|
||||
}
|
||||
if c.mode == modeMarkdown {
|
||||
for i, row := range rows {
|
||||
c.buf.WriteByte('|')
|
||||
for j := 0; j < colCount; j++ {
|
||||
cell := ""
|
||||
if j < len(row) {
|
||||
cell = row[j]
|
||||
}
|
||||
c.buf.WriteByte(' ')
|
||||
c.buf.WriteString(cell)
|
||||
c.buf.WriteString(" |")
|
||||
}
|
||||
c.buf.WriteByte('\n')
|
||||
if i == 0 {
|
||||
c.buf.WriteByte('|')
|
||||
for j := 0; j < colCount; j++ {
|
||||
c.buf.WriteString(" --- |")
|
||||
}
|
||||
c.buf.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for _, row := range rows {
|
||||
c.buf.WriteString(strings.Join(row, " | "))
|
||||
c.buf.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
c.buf.WriteByte('\n')
|
||||
}
|
||||
|
||||
func (c *converter) handleDefinitionTerm(n *html.Node) {
|
||||
c.ensureDoubleNewline()
|
||||
if c.mode == modeMarkdown {
|
||||
c.buf.WriteString("**")
|
||||
c.walkChildren(n)
|
||||
c.buf.WriteString("**")
|
||||
} else {
|
||||
c.walkChildren(n)
|
||||
}
|
||||
c.buf.WriteByte('\n')
|
||||
}
|
||||
|
||||
func (c *converter) handleDefinitionDesc(n *html.Node) {
|
||||
c.ensureNewline()
|
||||
if c.mode == modeMarkdown {
|
||||
c.buf.WriteString(": ")
|
||||
}
|
||||
c.walkChildren(n)
|
||||
c.buf.WriteByte('\n')
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
func getAttr(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func findChild(n *html.Node, tag atom.Atom) *html.Node {
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type == html.ElementNode && c.DataAtom == tag {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func findBody(doc *html.Node) *html.Node {
|
||||
var find func(*html.Node) *html.Node
|
||||
find = func(n *html.Node) *html.Node {
|
||||
if n.Type == html.ElementNode && n.DataAtom == atom.Body {
|
||||
return n
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if found := find(c); found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if body := find(doc); body != nil {
|
||||
return body
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
func collapseWhitespace(s string) string {
|
||||
var buf strings.Builder
|
||||
buf.Grow(len(s))
|
||||
inSpace := false
|
||||
for _, r := range s {
|
||||
if r == ' ' || r == '\t' || r == '\n' || r == '\r' || r == '\f' {
|
||||
if !inSpace {
|
||||
buf.WriteByte(' ')
|
||||
inSpace = true
|
||||
}
|
||||
} else {
|
||||
buf.WriteRune(r)
|
||||
inSpace = false
|
||||
}
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func (c *converter) ensureNewline() {
|
||||
if c.buf.Len() == 0 {
|
||||
return
|
||||
}
|
||||
s := c.buf.String()
|
||||
if s[len(s)-1] != '\n' {
|
||||
c.buf.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
|
||||
func (c *converter) ensureDoubleNewline() {
|
||||
if c.buf.Len() == 0 {
|
||||
return
|
||||
}
|
||||
s := c.buf.String()
|
||||
if len(s) >= 2 && s[len(s)-1] == '\n' && s[len(s)-2] == '\n' {
|
||||
return
|
||||
}
|
||||
if s[len(s)-1] == '\n' {
|
||||
c.buf.WriteByte('\n')
|
||||
} else {
|
||||
c.buf.WriteString("\n\n")
|
||||
}
|
||||
}
|
||||
|
||||
// collectTableRows extracts rows from a table node. Each row is a slice of cell strings.
|
||||
func collectTableRows(table *html.Node, mode convertMode) [][]string {
|
||||
var rows [][]string
|
||||
var findRows func(*html.Node)
|
||||
findRows = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode && n.DataAtom == atom.Tr {
|
||||
var cells []string
|
||||
for td := n.FirstChild; td != nil; td = td.NextSibling {
|
||||
if td.Type == html.ElementNode && (td.DataAtom == atom.Td || td.DataAtom == atom.Th) {
|
||||
sub := &converter{mode: mode}
|
||||
sub.walkChildren(td)
|
||||
cells = append(cells, strings.TrimSpace(sub.buf.String()))
|
||||
}
|
||||
}
|
||||
if len(cells) > 0 {
|
||||
rows = append(rows, cells)
|
||||
}
|
||||
return
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
findRows(c)
|
||||
}
|
||||
}
|
||||
findRows(table)
|
||||
return rows
|
||||
}
|
||||
|
||||
// --- output cleanup ---
|
||||
|
||||
var reMultiNL = regexp.MustCompile(`\n{3,}`)
|
||||
|
||||
func cleanOutput(s string) string {
|
||||
s = reMultiNL.ReplaceAllString(s, "\n\n")
|
||||
s = reMultiSP.ReplaceAllString(s, " ")
|
||||
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// htmlToText extracts plain text from HTML content.
|
||||
func htmlToText(html string) string {
|
||||
s := reScript.ReplaceAllString(html, "")
|
||||
s = reStyle.ReplaceAllString(s, "")
|
||||
s = reComment.ReplaceAllString(s, "")
|
||||
s = reNav.ReplaceAllString(s, "")
|
||||
s = reFooter.ReplaceAllString(s, "")
|
||||
s = reHeader.ReplaceAllString(s, "")
|
||||
|
||||
// Structural breaks
|
||||
s = reParagraph.ReplaceAllString(s, "\n$1\n")
|
||||
s = reBreak.ReplaceAllString(s, "\n")
|
||||
s = reListItem.ReplaceAllString(s, "\n- $1")
|
||||
|
||||
// Strip all tags
|
||||
s = reTag.ReplaceAllString(s, "")
|
||||
|
||||
s = decodeHTMLEntities(s)
|
||||
s = reMultiSP.ReplaceAllString(s, " ")
|
||||
s = reMultiNL.ReplaceAllString(s, "\n\n")
|
||||
|
||||
// Clean lines
|
||||
func cleanTextOutput(s string) string {
|
||||
lines := strings.Split(s, "\n")
|
||||
var clean []string
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" {
|
||||
clean = append(clean, line)
|
||||
}
|
||||
line = strings.TrimRight(line, " \t")
|
||||
clean = append(clean, line)
|
||||
}
|
||||
return strings.Join(clean, "\n")
|
||||
s = strings.Join(clean, "\n")
|
||||
s = reMultiNL.ReplaceAllString(s, "\n\n")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// stripTagsFallback is a last-resort fallback if the HTML parser fails.
|
||||
var reStripTags = regexp.MustCompile(`<[^>]+>`)
|
||||
|
||||
func stripTagsFallback(s string) string {
|
||||
return strings.TrimSpace(reStripTags.ReplaceAllString(s, ""))
|
||||
}
|
||||
|
||||
// markdownToText strips markdown formatting for text mode.
|
||||
func markdownToText(md string) string {
|
||||
s := md
|
||||
// Remove headers markers
|
||||
s = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(s, "")
|
||||
// Remove bold/italic markers
|
||||
s = strings.ReplaceAll(s, "**", "")
|
||||
s = strings.ReplaceAll(s, "__", "")
|
||||
// Remove inline code
|
||||
s = regexp.MustCompile("`[^`]+`").ReplaceAllStringFunc(s, func(m string) string {
|
||||
return strings.Trim(m, "`")
|
||||
})
|
||||
// Remove links: [text](url) → text
|
||||
s = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(s, "$1")
|
||||
// Remove images
|
||||
s = regexp.MustCompile(`!\[([^\]]*)\]\([^)]+\)`).ReplaceAllString(s, "$1")
|
||||
// Clean whitespace
|
||||
s = reMultiNL.ReplaceAllString(s, "\n\n")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// decodeHTMLEntities handles common HTML entities.
|
||||
func decodeHTMLEntities(s string) string {
|
||||
replacer := strings.NewReplacer(
|
||||
"&", "&",
|
||||
"<", "<",
|
||||
">", ">",
|
||||
""", `"`,
|
||||
"'", "'",
|
||||
"'", "'",
|
||||
" ", " ",
|
||||
"—", "\u2014",
|
||||
"–", "\u2013",
|
||||
"«", "\u00ab",
|
||||
"»", "\u00bb",
|
||||
"•", "\u2022",
|
||||
"…", "...",
|
||||
"©", "(c)",
|
||||
"®", "(R)",
|
||||
"™", "(TM)",
|
||||
)
|
||||
return replacer.Replace(s)
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHtmlToMarkdown_Headings(t *testing.T) {
|
||||
html := `<html><body><h1>Title</h1><h2>Subtitle</h2><h3>Section</h3></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
for _, want := range []string{"# Title", "## Subtitle", "### Section"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q in:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_Paragraphs(t *testing.T) {
|
||||
html := `<html><body><p>First paragraph.</p><p>Second paragraph.</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "First paragraph.") || !strings.Contains(got, "Second paragraph.") {
|
||||
t.Errorf("unexpected output:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_Links(t *testing.T) {
|
||||
html := `<html><body><p>Visit <a href="https://example.com">Example</a> site.</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "[Example](https://example.com)") {
|
||||
t.Errorf("missing link in:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_Images(t *testing.T) {
|
||||
html := `<html><body><img alt="logo" src="logo.png"></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "") {
|
||||
t.Errorf("missing image in:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_BoldItalic(t *testing.T) {
|
||||
html := `<html><body><p><strong>bold</strong> and <em>italic</em></p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "**bold**") {
|
||||
t.Errorf("missing bold in:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "*italic*") {
|
||||
t.Errorf("missing italic in:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_PreCode(t *testing.T) {
|
||||
html := `<html><body><pre><code class="language-go">func main() {}</code></pre></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "```go") {
|
||||
t.Errorf("missing fenced code block with language in:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "func main() {}") {
|
||||
t.Errorf("missing code content in:\n%s", got)
|
||||
}
|
||||
if strings.Count(got, "```") < 2 {
|
||||
t.Errorf("missing closing fence in:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_InlineCode(t *testing.T) {
|
||||
html := `<html><body><p>Use <code>fmt.Println</code> to print.</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "`fmt.Println`") {
|
||||
t.Errorf("missing inline code in:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_Blockquote(t *testing.T) {
|
||||
html := `<html><body><blockquote><p>A wise quote.</p></blockquote></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "> ") {
|
||||
t.Errorf("missing blockquote prefix in:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "A wise quote.") {
|
||||
t.Errorf("missing quote content in:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_UnorderedList(t *testing.T) {
|
||||
html := `<html><body><ul><li>One</li><li>Two</li><li>Three</li></ul></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "- One") || !strings.Contains(got, "- Two") || !strings.Contains(got, "- Three") {
|
||||
t.Errorf("missing list items in:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_OrderedList(t *testing.T) {
|
||||
html := `<html><body><ol><li>First</li><li>Second</li></ol></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "1. First") || !strings.Contains(got, "2. Second") {
|
||||
t.Errorf("missing ordered list items in:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_NestedList(t *testing.T) {
|
||||
html := `<html><body><ul><li>A<ul><li>A1</li><li>A2</li></ul></li><li>B</li></ul></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "- A") || !strings.Contains(got, " - A1") || !strings.Contains(got, " - A2") {
|
||||
t.Errorf("missing nested list in:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_Table(t *testing.T) {
|
||||
html := `<html><body><table><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr></table></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "| Name") || !strings.Contains(got, "| Age") {
|
||||
t.Errorf("missing table header in:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "| ---") {
|
||||
t.Errorf("missing table separator in:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "Alice") || !strings.Contains(got, "30") {
|
||||
t.Errorf("missing table data in:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_HorizontalRule(t *testing.T) {
|
||||
html := `<html><body><p>Above</p><hr><p>Below</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "---") {
|
||||
t.Errorf("missing horizontal rule in:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Stripping non-content elements ---
|
||||
|
||||
func TestHtmlToMarkdown_StripsHead(t *testing.T) {
|
||||
html := `<html><head><title>Page Title</title><meta name="desc" content="description"><link rel="stylesheet" href="style.css"><style>.foo{color:red}</style><script>var x=1;</script></head><body><p>Content</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if strings.Contains(got, "Page Title") {
|
||||
t.Errorf("head title should not appear in output:\n%s", got)
|
||||
}
|
||||
if strings.Contains(got, "color:red") || strings.Contains(got, "var x=1") {
|
||||
t.Errorf("head CSS/JS leaked into output:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "Content") {
|
||||
t.Errorf("body content missing:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_StripsScript(t *testing.T) {
|
||||
html := `<html><body><p>Hello</p><script>alert('xss')</script><p>World</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if strings.Contains(got, "alert") || strings.Contains(got, "xss") {
|
||||
t.Errorf("script content leaked:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_StripsStyle(t *testing.T) {
|
||||
html := `<html><body><style>.foo { display: none; }</style><p>Text</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if strings.Contains(got, "display") || strings.Contains(got, ".foo") {
|
||||
t.Errorf("style content leaked:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_StripsNoscript(t *testing.T) {
|
||||
html := `<html><body><noscript><style>.ns{color:blue}</style><p>Enable JavaScript</p></noscript><p>Real content</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if strings.Contains(got, "Enable JavaScript") || strings.Contains(got, "color:blue") {
|
||||
t.Errorf("noscript content leaked:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "Real content") {
|
||||
t.Errorf("real content missing:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_StripsSvg(t *testing.T) {
|
||||
html := `<html><body><svg xmlns="http://www.w3.org/2000/svg"><path d="M10 20 L30 40"/><text>icon</text></svg><p>Content</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if strings.Contains(got, "M10") || strings.Contains(got, "icon") {
|
||||
t.Errorf("SVG content leaked:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_StripsNav(t *testing.T) {
|
||||
html := `<html><body><nav><a href="/">Home</a><a href="/about">About</a></nav><p>Article</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if strings.Contains(got, "Home") || strings.Contains(got, "About") {
|
||||
t.Errorf("nav content leaked:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_StripsFooter(t *testing.T) {
|
||||
html := `<html><body><p>Article</p><footer><p>Copyright 2024</p></footer></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if strings.Contains(got, "Copyright") {
|
||||
t.Errorf("footer content leaked:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_StripsForm(t *testing.T) {
|
||||
html := `<html><body><form><input type="text" value="name"><select><option>A</option><option>B</option></select><button>Submit</button></form><p>Content</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if strings.Contains(got, "Submit") || strings.Contains(got, "name") {
|
||||
t.Errorf("form content leaked:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_StripsIframe(t *testing.T) {
|
||||
html := `<html><body><iframe src="https://ads.example.com">Fallback</iframe><p>Content</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if strings.Contains(got, "Fallback") || strings.Contains(got, "ads.example") {
|
||||
t.Errorf("iframe content leaked:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_StripsTemplate(t *testing.T) {
|
||||
html := `<html><body><template><div>Template content</div></template><p>Visible</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if strings.Contains(got, "Template content") {
|
||||
t.Errorf("template content leaked:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Entity handling ---
|
||||
|
||||
func TestHtmlToMarkdown_Entities(t *testing.T) {
|
||||
html := `<html><body><p>A & B < C > D "E" 'F'</p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, `A & B < C > D "E" 'F'`) {
|
||||
t.Errorf("entities not decoded properly:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Whitespace handling ---
|
||||
|
||||
func TestHtmlToMarkdown_WhitespaceCollapse(t *testing.T) {
|
||||
html := `<html><body><p> lots of spaces </p></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if strings.Contains(got, " ") {
|
||||
t.Errorf("whitespace not collapsed:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToMarkdown_PreservesPreWhitespace(t *testing.T) {
|
||||
html := `<html><body><pre> line1
|
||||
line2
|
||||
line3</pre></body></html>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, " line1\n line2\n line3") {
|
||||
t.Errorf("pre whitespace not preserved:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Malformed HTML ---
|
||||
|
||||
func TestHtmlToMarkdown_MalformedHTML(t *testing.T) {
|
||||
html := `<p>Unclosed paragraph<div>Nested <b>bold</div></p><p>OK</p>`
|
||||
got := htmlToMarkdown(html)
|
||||
if !strings.Contains(got, "Unclosed paragraph") || !strings.Contains(got, "bold") || !strings.Contains(got, "OK") {
|
||||
t.Errorf("malformed HTML not handled:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Text mode ---
|
||||
|
||||
func TestHtmlToText_NoMarkdownFormatting(t *testing.T) {
|
||||
html := `<html><body><h1>Title</h1><p>Text with <strong>bold</strong> and <a href="https://example.com">link</a>.</p></body></html>`
|
||||
got := htmlToText(html)
|
||||
if strings.Contains(got, "#") || strings.Contains(got, "**") || strings.Contains(got, "[link]") {
|
||||
t.Errorf("markdown formatting in text mode:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "Title") || !strings.Contains(got, "bold") || !strings.Contains(got, "link") {
|
||||
t.Errorf("content missing in text mode:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToText_StripsHeader(t *testing.T) {
|
||||
html := `<html><body><header><h1>Site Name</h1></header><p>Article</p></body></html>`
|
||||
got := htmlToText(html)
|
||||
if strings.Contains(got, "Site Name") {
|
||||
t.Errorf("header content should be stripped in text mode:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "Article") {
|
||||
t.Errorf("body content missing:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHtmlToText_StripsAside(t *testing.T) {
|
||||
html := `<html><body><aside><p>Sidebar</p></aside><p>Main content</p></body></html>`
|
||||
got := htmlToText(html)
|
||||
if strings.Contains(got, "Sidebar") {
|
||||
t.Errorf("aside content should be stripped in text mode:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Realistic SPA page ---
|
||||
|
||||
func TestHtmlToMarkdown_SPAPage(t *testing.T) {
|
||||
html := `<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>My App</title>
|
||||
<link rel="stylesheet" href="/static/css/main.abc123.css">
|
||||
<style>
|
||||
body { margin: 0; font-family: sans-serif; }
|
||||
.header { background: #333; color: white; }
|
||||
.container { max-width: 800px; margin: auto; }
|
||||
@media (max-width: 768px) { .container { padding: 10px; } }
|
||||
</style>
|
||||
<script>
|
||||
window.__INITIAL_STATE__ = {"user":null,"theme":"dark"};
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
<nav>
|
||||
<a href="/">Home</a>
|
||||
<a href="/about">About</a>
|
||||
<a href="/contact">Contact</a>
|
||||
</nav>
|
||||
<div id="root">
|
||||
<h1>Welcome to My App</h1>
|
||||
<p>This is the main content of the page.</p>
|
||||
<ul>
|
||||
<li>Feature 1</li>
|
||||
<li>Feature 2</li>
|
||||
</ul>
|
||||
</div>
|
||||
<footer>
|
||||
<p>© 2024 My Company. All rights reserved.</p>
|
||||
</footer>
|
||||
<script src="/static/js/main.def456.js"></script>
|
||||
<script>
|
||||
(function() { console.log("analytics loaded"); })();
|
||||
</script>
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
got := htmlToMarkdown(html)
|
||||
|
||||
// Should NOT contain CSS/JS artifacts
|
||||
for _, bad := range []string{
|
||||
"margin: 0", "sans-serif", "background: #333",
|
||||
"__INITIAL_STATE__", "theme", "dark",
|
||||
"analytics loaded", "console.log",
|
||||
"main.abc123.css", "main.def456.js",
|
||||
"enable JavaScript",
|
||||
"viewport", // meta from head
|
||||
"All rights", // footer
|
||||
"Home", "About", "Contact", // nav
|
||||
} {
|
||||
if strings.Contains(got, bad) {
|
||||
t.Errorf("non-content %q leaked into output:\n%s", bad, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Should contain actual content
|
||||
for _, good := range []string{
|
||||
"# Welcome to My App",
|
||||
"main content of the page",
|
||||
"- Feature 1",
|
||||
"- Feature 2",
|
||||
} {
|
||||
if !strings.Contains(got, good) {
|
||||
t.Errorf("expected content %q missing from output:\n%s", good, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user