From 37158af2318958c8d8d727bff3cb667ecb7db196 Mon Sep 17 00:00:00 2001 From: viettranx Date: Tue, 3 Mar 2026 17:10:55 +0700 Subject: [PATCH] fix(web_fetch): replace regex HTML parsing with DOM-based extraction Regex-based htmlToMarkdown/htmlToText leaked CSS, JS, and non-content elements. Replaced with golang.org/x/net/html DOM parser that extracts only and skips 16 non-content element types (script, style, noscript, svg, template, iframe, form, nav, footer, etc.). Co-Authored-By: Claude Opus 4.6 --- internal/tools/web_fetch_convert.go | 646 ++++++++++++++++++----- internal/tools/web_fetch_convert_test.go | 369 +++++++++++++ 2 files changed, 884 insertions(+), 131 deletions(-) create mode 100644 internal/tools/web_fetch_convert_test.go diff --git a/internal/tools/web_fetch_convert.go b/internal/tools/web_fetch_convert.go index 04ecddcd..c1eb3dda 100644 --- a/internal/tools/web_fetch_convert.go +++ b/internal/tools/web_fetch_convert.go @@ -2,8 +2,12 @@ package tools import ( "encoding/json" + "fmt" "regexp" "strings" + + "golang.org/x/net/html" + "golang.org/x/net/html/atom" ) // extractJSON pretty-prints JSON content. @@ -16,173 +20,553 @@ func extractJSON(body []byte) (string, string) { return string(body), "raw" } -// --- HTML extraction utilities --- +// --- DOM-based HTML extraction --- -var ( - reScript = regexp.MustCompile(`(?is)`) - reStyle = regexp.MustCompile(`(?is)`) - reComment = regexp.MustCompile(``) - reNav = regexp.MustCompile(`(?is)`) - reFooter = regexp.MustCompile(`(?is)`) - reHeader = regexp.MustCompile(`(?is)`) - reTag = regexp.MustCompile(`<[^>]+>`) - reMultiNL = regexp.MustCompile(`\n{3,}`) - reMultiSP = regexp.MustCompile(`[ \t]{2,}`) - reH1 = regexp.MustCompile(`(?i)]*>([\s\S]*?)`) - reH2 = regexp.MustCompile(`(?i)]*>([\s\S]*?)`) - reH3 = regexp.MustCompile(`(?i)]*>([\s\S]*?)`) - reH4 = regexp.MustCompile(`(?i)]*>([\s\S]*?)`) - reH5 = regexp.MustCompile(`(?i)]*>([\s\S]*?)`) - reH6 = regexp.MustCompile(`(?i)]*>([\s\S]*?)`) - reParagraph = regexp.MustCompile(`(?i)]*>([\s\S]*?)

`) - reBreak = regexp.MustCompile(`(?i)`) - reListItem = regexp.MustCompile(`(?i)]*>([\s\S]*?)`) - reAnchor = regexp.MustCompile(`(?i)]*href="([^"]*)"[^>]*>([\s\S]*?)`) - rePre = regexp.MustCompile(`(?is)]*>([\s\S]*?)`) - reCode = regexp.MustCompile(`(?i)]*>([\s\S]*?)`) - reStrong = regexp.MustCompile(`(?i)<(?:strong|b)[^>]*>([\s\S]*?)`) - reEm = regexp.MustCompile(`(?i)<(?:em|i)[^>]*>([\s\S]*?)`) - reBlockq = regexp.MustCompile(`(?is)]*>([\s\S]*?)`) - reImg = regexp.MustCompile(`(?i)]*alt="([^"]*)"[^>]*/?>`) +type convertMode int + +const ( + modeMarkdown convertMode = iota + modeText ) -// htmlToMarkdown converts HTML to a markdown-like format. -// Not a full Readability implementation but covers common patterns. -func htmlToMarkdown(html string) string { - // Remove non-content elements - s := reScript.ReplaceAllString(html, "") - s = reStyle.ReplaceAllString(s, "") - s = reComment.ReplaceAllString(s, "") - s = reNav.ReplaceAllString(s, "") - s = reFooter.ReplaceAllString(s, "") +// converter walks a parsed HTML DOM tree and emits markdown or plain text. +type converter struct { + buf strings.Builder + mode convertMode + inPre bool + listDepth int + listType []atom.Atom // stack: atom.Ul / atom.Ol + listIndex []int // ordered list counters + inLink bool +} - // Convert headings - s = reH1.ReplaceAllString(s, "\n# $1\n") - s = reH2.ReplaceAllString(s, "\n## $1\n") - s = reH3.ReplaceAllString(s, "\n### $1\n") - s = reH4.ReplaceAllString(s, "\n#### $1\n") - s = reH5.ReplaceAllString(s, "\n##### $1\n") - s = reH6.ReplaceAllString(s, "\n###### $1\n") +// Elements to skip entirely (element + all descendants). +var skipElements = map[atom.Atom]bool{ + atom.Head: true, + atom.Script: true, + atom.Style: true, + atom.Noscript: true, + atom.Svg: true, + atom.Template: true, + atom.Iframe: true, + atom.Select: true, + atom.Option: true, + atom.Button: true, + atom.Input: true, + atom.Form: true, + atom.Nav: true, + atom.Footer: true, + atom.Picture: true, + atom.Source: true, +} - // Pre/code blocks (before stripping other tags) - s = rePre.ReplaceAllString(s, "\n```\n$1\n```\n") - s = reCode.ReplaceAllString(s, "`$1`") +// Additional elements to skip in text mode only. +var skipInTextMode = map[atom.Atom]bool{ + atom.Header: true, + atom.Aside: true, +} - // Blockquotes - s = reBlockq.ReplaceAllStringFunc(s, func(match string) string { - inner := reBlockq.FindStringSubmatch(match) - if len(inner) < 2 { - return match +// Block elements that need surrounding newlines. +var blockElements = map[atom.Atom]bool{ + atom.P: true, atom.Div: true, atom.Section: true, atom.Article: true, + atom.Main: true, atom.H1: true, atom.H2: true, atom.H3: true, + atom.H4: true, atom.H5: true, atom.H6: true, atom.Blockquote: true, + atom.Pre: true, atom.Ul: true, atom.Ol: true, atom.Li: true, + atom.Table: true, atom.Tr: true, atom.Hr: true, atom.Dl: true, + atom.Dt: true, atom.Dd: true, atom.Figure: true, atom.Figcaption: true, + atom.Details: true, atom.Summary: true, atom.Address: true, +} + +// htmlToMarkdown converts HTML to a markdown-like format using DOM parsing. +func htmlToMarkdown(rawHTML string) string { + doc, err := html.Parse(strings.NewReader(rawHTML)) + if err != nil { + return stripTagsFallback(rawHTML) + } + body := findBody(doc) + c := &converter{mode: modeMarkdown} + c.walkChildren(body) + return cleanOutput(c.buf.String()) +} + +// htmlToText extracts plain text from HTML content using DOM parsing. +func htmlToText(rawHTML string) string { + doc, err := html.Parse(strings.NewReader(rawHTML)) + if err != nil { + return stripTagsFallback(rawHTML) + } + body := findBody(doc) + c := &converter{mode: modeText} + c.walkChildren(body) + return cleanTextOutput(c.buf.String()) +} + +func (c *converter) walk(n *html.Node) { + switch n.Type { + case html.TextNode: + c.handleText(n) + return + case html.ElementNode: + // handled below + case html.DocumentNode: + c.walkChildren(n) + return + default: + return + } + + tag := n.DataAtom + + if skipElements[tag] { + return + } + if c.mode == modeText && skipInTextMode[tag] { + return + } + + switch tag { + case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6: + c.handleHeading(n) + case atom.P: + c.handleParagraph(n) + case atom.A: + c.handleLink(n) + case atom.Img: + c.handleImage(n) + case atom.Pre: + c.handlePre(n) + case atom.Code: + c.handleCode(n) + case atom.Blockquote: + c.handleBlockquote(n) + case atom.Strong, atom.B: + c.handleStrong(n) + case atom.Em, atom.I: + c.handleEmphasis(n) + case atom.Br: + c.buf.WriteByte('\n') + case atom.Hr: + c.ensureNewline() + if c.mode == modeMarkdown { + c.buf.WriteString("---\n") } - lines := strings.Split(strings.TrimSpace(inner[1]), "\n") - var quoted []string - for _, l := range lines { - quoted = append(quoted, "> "+strings.TrimSpace(l)) + case atom.Ul, atom.Ol: + c.handleList(n) + case atom.Li: + c.handleListItem(n) + case atom.Table: + c.handleTable(n) + case atom.Dt: + c.handleDefinitionTerm(n) + case atom.Dd: + c.handleDefinitionDesc(n) + default: + if blockElements[tag] { + c.ensureNewline() + c.walkChildren(n) + c.ensureNewline() + } else { + c.walkChildren(n) } - return "\n" + strings.Join(quoted, "\n") + "\n" - }) + } +} - // Links: text → [text](url) - s = reAnchor.ReplaceAllString(s, "[$2]($1)") +func (c *converter) walkChildren(n *html.Node) { + for child := n.FirstChild; child != nil; child = child.NextSibling { + c.walk(child) + } +} - // Images: text → ![text] - s = reImg.ReplaceAllString(s, "![$1]") +func (c *converter) handleText(n *html.Node) { + text := n.Data + if c.inPre { + c.buf.WriteString(text) + return + } + text = collapseWhitespace(text) + if text == "" { + return + } + if text == " " && c.buf.Len() > 0 { + c.buf.WriteByte(' ') + return + } + c.buf.WriteString(text) +} - // Bold/italic - s = reStrong.ReplaceAllString(s, "**$1**") - s = reEm.ReplaceAllString(s, "*$1*") +func (c *converter) handleHeading(n *html.Node) { + c.ensureDoubleNewline() + if c.mode == modeMarkdown && len(n.Data) == 2 && n.Data[0] == 'h' { + level := int(n.Data[1] - '0') + for i := 0; i < level; i++ { + c.buf.WriteByte('#') + } + c.buf.WriteByte(' ') + } + c.walkChildren(n) + c.buf.WriteByte('\n') +} - // Paragraphs and breaks - s = reParagraph.ReplaceAllString(s, "\n$1\n") - s = reBreak.ReplaceAllString(s, "\n") +func (c *converter) handleParagraph(n *html.Node) { + c.ensureDoubleNewline() + c.walkChildren(n) + c.buf.WriteByte('\n') +} - // List items - s = reListItem.ReplaceAllString(s, "\n- $1") +func (c *converter) handleLink(n *html.Node) { + href := getAttr(n, "href") + if c.mode == modeText || c.inLink || href == "" || strings.HasPrefix(href, "javascript:") { + c.walkChildren(n) + return + } + c.inLink = true + c.buf.WriteByte('[') + c.walkChildren(n) + c.buf.WriteString("](") + c.buf.WriteString(href) + c.buf.WriteByte(')') + c.inLink = false +} - // Strip remaining tags - s = reTag.ReplaceAllString(s, "") +func (c *converter) handleImage(n *html.Node) { + alt := getAttr(n, "alt") + src := getAttr(n, "src") + if c.mode == modeMarkdown { + c.buf.WriteString("![") + c.buf.WriteString(alt) + c.buf.WriteByte(']') + if src != "" { + c.buf.WriteByte('(') + c.buf.WriteString(src) + c.buf.WriteByte(')') + } + } else if alt != "" { + c.buf.WriteString(alt) + } +} - // Clean up - s = decodeHTMLEntities(s) +func (c *converter) handlePre(n *html.Node) { + c.ensureDoubleNewline() + if c.mode == modeMarkdown { + lang := "" + if code := findChild(n, atom.Code); code != nil { + cls := getAttr(code, "class") + for _, part := range strings.Fields(cls) { + if rest, ok := strings.CutPrefix(part, "language-"); ok { + lang = rest + break + } + if rest, ok := strings.CutPrefix(part, "lang-"); ok { + lang = rest + break + } + } + } + c.buf.WriteString("```") + c.buf.WriteString(lang) + c.buf.WriteByte('\n') + } + c.inPre = true + c.walkChildren(n) + c.inPre = false + if c.mode == modeMarkdown { + c.ensureNewline() + c.buf.WriteString("```\n") + } else { + c.buf.WriteByte('\n') + } +} + +func (c *converter) handleCode(n *html.Node) { + if c.inPre { + c.walkChildren(n) + return + } + if c.mode == modeMarkdown { + c.buf.WriteByte('`') + c.walkChildren(n) + c.buf.WriteByte('`') + } else { + c.walkChildren(n) + } +} + +func (c *converter) handleBlockquote(n *html.Node) { + c.ensureDoubleNewline() + if c.mode == modeMarkdown { + sub := &converter{mode: c.mode, inPre: c.inPre} + sub.walkChildren(n) + for i, line := range strings.Split(strings.TrimSpace(sub.buf.String()), "\n") { + if i > 0 { + c.buf.WriteByte('\n') + } + c.buf.WriteString("> ") + c.buf.WriteString(line) + } + c.buf.WriteByte('\n') + } else { + c.walkChildren(n) + } +} + +func (c *converter) handleStrong(n *html.Node) { + if c.mode == modeMarkdown { + c.buf.WriteString("**") + c.walkChildren(n) + c.buf.WriteString("**") + } else { + c.walkChildren(n) + } +} + +func (c *converter) handleEmphasis(n *html.Node) { + if c.mode == modeMarkdown { + c.buf.WriteByte('*') + c.walkChildren(n) + c.buf.WriteByte('*') + } else { + c.walkChildren(n) + } +} + +func (c *converter) handleList(n *html.Node) { + c.ensureNewline() + c.listDepth++ + c.listType = append(c.listType, n.DataAtom) + c.listIndex = append(c.listIndex, 0) + c.walkChildren(n) + c.listDepth-- + c.listType = c.listType[:len(c.listType)-1] + c.listIndex = c.listIndex[:len(c.listIndex)-1] + c.ensureNewline() +} + +func (c *converter) handleListItem(n *html.Node) { + c.ensureNewline() + indent := strings.Repeat(" ", max(0, c.listDepth-1)) + c.buf.WriteString(indent) + + if len(c.listType) > 0 && c.listType[len(c.listType)-1] == atom.Ol { + idx := len(c.listIndex) - 1 + c.listIndex[idx]++ + fmt.Fprintf(&c.buf, "%d. ", c.listIndex[idx]) + } else { + c.buf.WriteString("- ") + } + c.walkChildren(n) +} + +func (c *converter) handleTable(n *html.Node) { + c.ensureDoubleNewline() + rows := collectTableRows(n, c.mode) + if len(rows) == 0 { + return + } + colCount := 0 + for _, row := range rows { + if len(row) > colCount { + colCount = len(row) + } + } + if c.mode == modeMarkdown { + for i, row := range rows { + c.buf.WriteByte('|') + for j := 0; j < colCount; j++ { + cell := "" + if j < len(row) { + cell = row[j] + } + c.buf.WriteByte(' ') + c.buf.WriteString(cell) + c.buf.WriteString(" |") + } + c.buf.WriteByte('\n') + if i == 0 { + c.buf.WriteByte('|') + for j := 0; j < colCount; j++ { + c.buf.WriteString(" --- |") + } + c.buf.WriteByte('\n') + } + } + } else { + for _, row := range rows { + c.buf.WriteString(strings.Join(row, " | ")) + c.buf.WriteByte('\n') + } + } + c.buf.WriteByte('\n') +} + +func (c *converter) handleDefinitionTerm(n *html.Node) { + c.ensureDoubleNewline() + if c.mode == modeMarkdown { + c.buf.WriteString("**") + c.walkChildren(n) + c.buf.WriteString("**") + } else { + c.walkChildren(n) + } + c.buf.WriteByte('\n') +} + +func (c *converter) handleDefinitionDesc(n *html.Node) { + c.ensureNewline() + if c.mode == modeMarkdown { + c.buf.WriteString(": ") + } + c.walkChildren(n) + c.buf.WriteByte('\n') +} + +// --- helpers --- + +func getAttr(n *html.Node, key string) string { + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + return "" +} + +func findChild(n *html.Node, tag atom.Atom) *html.Node { + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.ElementNode && c.DataAtom == tag { + return c + } + } + return nil +} + +func findBody(doc *html.Node) *html.Node { + var find func(*html.Node) *html.Node + find = func(n *html.Node) *html.Node { + if n.Type == html.ElementNode && n.DataAtom == atom.Body { + return n + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if found := find(c); found != nil { + return found + } + } + return nil + } + if body := find(doc); body != nil { + return body + } + return doc +} + +func collapseWhitespace(s string) string { + var buf strings.Builder + buf.Grow(len(s)) + inSpace := false + for _, r := range s { + if r == ' ' || r == '\t' || r == '\n' || r == '\r' || r == '\f' { + if !inSpace { + buf.WriteByte(' ') + inSpace = true + } + } else { + buf.WriteRune(r) + inSpace = false + } + } + return buf.String() +} + +func (c *converter) ensureNewline() { + if c.buf.Len() == 0 { + return + } + s := c.buf.String() + if s[len(s)-1] != '\n' { + c.buf.WriteByte('\n') + } +} + +func (c *converter) ensureDoubleNewline() { + if c.buf.Len() == 0 { + return + } + s := c.buf.String() + if len(s) >= 2 && s[len(s)-1] == '\n' && s[len(s)-2] == '\n' { + return + } + if s[len(s)-1] == '\n' { + c.buf.WriteByte('\n') + } else { + c.buf.WriteString("\n\n") + } +} + +// collectTableRows extracts rows from a table node. Each row is a slice of cell strings. +func collectTableRows(table *html.Node, mode convertMode) [][]string { + var rows [][]string + var findRows func(*html.Node) + findRows = func(n *html.Node) { + if n.Type == html.ElementNode && n.DataAtom == atom.Tr { + var cells []string + for td := n.FirstChild; td != nil; td = td.NextSibling { + if td.Type == html.ElementNode && (td.DataAtom == atom.Td || td.DataAtom == atom.Th) { + sub := &converter{mode: mode} + sub.walkChildren(td) + cells = append(cells, strings.TrimSpace(sub.buf.String())) + } + } + if len(cells) > 0 { + rows = append(rows, cells) + } + return + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + findRows(c) + } + } + findRows(table) + return rows +} + +// --- output cleanup --- + +var reMultiNL = regexp.MustCompile(`\n{3,}`) + +func cleanOutput(s string) string { s = reMultiNL.ReplaceAllString(s, "\n\n") - s = reMultiSP.ReplaceAllString(s, " ") - return strings.TrimSpace(s) } -// htmlToText extracts plain text from HTML content. -func htmlToText(html string) string { - s := reScript.ReplaceAllString(html, "") - s = reStyle.ReplaceAllString(s, "") - s = reComment.ReplaceAllString(s, "") - s = reNav.ReplaceAllString(s, "") - s = reFooter.ReplaceAllString(s, "") - s = reHeader.ReplaceAllString(s, "") - - // Structural breaks - s = reParagraph.ReplaceAllString(s, "\n$1\n") - s = reBreak.ReplaceAllString(s, "\n") - s = reListItem.ReplaceAllString(s, "\n- $1") - - // Strip all tags - s = reTag.ReplaceAllString(s, "") - - s = decodeHTMLEntities(s) - s = reMultiSP.ReplaceAllString(s, " ") - s = reMultiNL.ReplaceAllString(s, "\n\n") - - // Clean lines +func cleanTextOutput(s string) string { lines := strings.Split(s, "\n") var clean []string for _, line := range lines { - line = strings.TrimSpace(line) - if line != "" { - clean = append(clean, line) - } + line = strings.TrimRight(line, " \t") + clean = append(clean, line) } - return strings.Join(clean, "\n") + s = strings.Join(clean, "\n") + s = reMultiNL.ReplaceAllString(s, "\n\n") + return strings.TrimSpace(s) +} + +// stripTagsFallback is a last-resort fallback if the HTML parser fails. +var reStripTags = regexp.MustCompile(`<[^>]+>`) + +func stripTagsFallback(s string) string { + return strings.TrimSpace(reStripTags.ReplaceAllString(s, "")) } // markdownToText strips markdown formatting for text mode. func markdownToText(md string) string { s := md - // Remove headers markers s = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(s, "") - // Remove bold/italic markers s = strings.ReplaceAll(s, "**", "") s = strings.ReplaceAll(s, "__", "") - // Remove inline code s = regexp.MustCompile("`[^`]+`").ReplaceAllStringFunc(s, func(m string) string { return strings.Trim(m, "`") }) - // Remove links: [text](url) → text s = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(s, "$1") - // Remove images s = regexp.MustCompile(`!\[([^\]]*)\]\([^)]+\)`).ReplaceAllString(s, "$1") - // Clean whitespace s = reMultiNL.ReplaceAllString(s, "\n\n") return strings.TrimSpace(s) } - -// decodeHTMLEntities handles common HTML entities. -func decodeHTMLEntities(s string) string { - replacer := strings.NewReplacer( - "&", "&", - "<", "<", - ">", ">", - """, `"`, - "'", "'", - "'", "'", - " ", " ", - "—", "\u2014", - "–", "\u2013", - "«", "\u00ab", - "»", "\u00bb", - "•", "\u2022", - "…", "...", - "©", "(c)", - "®", "(R)", - "™", "(TM)", - ) - return replacer.Replace(s) -} diff --git a/internal/tools/web_fetch_convert_test.go b/internal/tools/web_fetch_convert_test.go new file mode 100644 index 00000000..42da43af --- /dev/null +++ b/internal/tools/web_fetch_convert_test.go @@ -0,0 +1,369 @@ +package tools + +import ( + "strings" + "testing" +) + +func TestHtmlToMarkdown_Headings(t *testing.T) { + html := `

Title

Subtitle

Section

` + got := htmlToMarkdown(html) + for _, want := range []string{"# Title", "## Subtitle", "### Section"} { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } +} + +func TestHtmlToMarkdown_Paragraphs(t *testing.T) { + html := `

First paragraph.

Second paragraph.

` + got := htmlToMarkdown(html) + if !strings.Contains(got, "First paragraph.") || !strings.Contains(got, "Second paragraph.") { + t.Errorf("unexpected output:\n%s", got) + } +} + +func TestHtmlToMarkdown_Links(t *testing.T) { + html := `

Visit Example site.

` + got := htmlToMarkdown(html) + if !strings.Contains(got, "[Example](https://example.com)") { + t.Errorf("missing link in:\n%s", got) + } +} + +func TestHtmlToMarkdown_Images(t *testing.T) { + html := `logo` + got := htmlToMarkdown(html) + if !strings.Contains(got, "![logo](logo.png)") { + t.Errorf("missing image in:\n%s", got) + } +} + +func TestHtmlToMarkdown_BoldItalic(t *testing.T) { + html := `

bold and italic

` + got := htmlToMarkdown(html) + if !strings.Contains(got, "**bold**") { + t.Errorf("missing bold in:\n%s", got) + } + if !strings.Contains(got, "*italic*") { + t.Errorf("missing italic in:\n%s", got) + } +} + +func TestHtmlToMarkdown_PreCode(t *testing.T) { + html := `
func main() {}
` + got := htmlToMarkdown(html) + if !strings.Contains(got, "```go") { + t.Errorf("missing fenced code block with language in:\n%s", got) + } + if !strings.Contains(got, "func main() {}") { + t.Errorf("missing code content in:\n%s", got) + } + if strings.Count(got, "```") < 2 { + t.Errorf("missing closing fence in:\n%s", got) + } +} + +func TestHtmlToMarkdown_InlineCode(t *testing.T) { + html := `

Use fmt.Println to print.

` + got := htmlToMarkdown(html) + if !strings.Contains(got, "`fmt.Println`") { + t.Errorf("missing inline code in:\n%s", got) + } +} + +func TestHtmlToMarkdown_Blockquote(t *testing.T) { + html := `

A wise quote.

` + got := htmlToMarkdown(html) + if !strings.Contains(got, "> ") { + t.Errorf("missing blockquote prefix in:\n%s", got) + } + if !strings.Contains(got, "A wise quote.") { + t.Errorf("missing quote content in:\n%s", got) + } +} + +func TestHtmlToMarkdown_UnorderedList(t *testing.T) { + html := `
  • One
  • Two
  • Three
` + got := htmlToMarkdown(html) + if !strings.Contains(got, "- One") || !strings.Contains(got, "- Two") || !strings.Contains(got, "- Three") { + t.Errorf("missing list items in:\n%s", got) + } +} + +func TestHtmlToMarkdown_OrderedList(t *testing.T) { + html := `
  1. First
  2. Second
` + got := htmlToMarkdown(html) + if !strings.Contains(got, "1. First") || !strings.Contains(got, "2. Second") { + t.Errorf("missing ordered list items in:\n%s", got) + } +} + +func TestHtmlToMarkdown_NestedList(t *testing.T) { + html := `
  • A
    • A1
    • A2
  • B
` + got := htmlToMarkdown(html) + if !strings.Contains(got, "- A") || !strings.Contains(got, " - A1") || !strings.Contains(got, " - A2") { + t.Errorf("missing nested list in:\n%s", got) + } +} + +func TestHtmlToMarkdown_Table(t *testing.T) { + html := `
NameAge
Alice30
` + got := htmlToMarkdown(html) + if !strings.Contains(got, "| Name") || !strings.Contains(got, "| Age") { + t.Errorf("missing table header in:\n%s", got) + } + if !strings.Contains(got, "| ---") { + t.Errorf("missing table separator in:\n%s", got) + } + if !strings.Contains(got, "Alice") || !strings.Contains(got, "30") { + t.Errorf("missing table data in:\n%s", got) + } +} + +func TestHtmlToMarkdown_HorizontalRule(t *testing.T) { + html := `

Above

Below

` + got := htmlToMarkdown(html) + if !strings.Contains(got, "---") { + t.Errorf("missing horizontal rule in:\n%s", got) + } +} + +// --- Stripping non-content elements --- + +func TestHtmlToMarkdown_StripsHead(t *testing.T) { + html := `Page Title

Content

` + got := htmlToMarkdown(html) + if strings.Contains(got, "Page Title") { + t.Errorf("head title should not appear in output:\n%s", got) + } + if strings.Contains(got, "color:red") || strings.Contains(got, "var x=1") { + t.Errorf("head CSS/JS leaked into output:\n%s", got) + } + if !strings.Contains(got, "Content") { + t.Errorf("body content missing:\n%s", got) + } +} + +func TestHtmlToMarkdown_StripsScript(t *testing.T) { + html := `

Hello

World

` + got := htmlToMarkdown(html) + if strings.Contains(got, "alert") || strings.Contains(got, "xss") { + t.Errorf("script content leaked:\n%s", got) + } +} + +func TestHtmlToMarkdown_StripsStyle(t *testing.T) { + html := `

Text

` + got := htmlToMarkdown(html) + if strings.Contains(got, "display") || strings.Contains(got, ".foo") { + t.Errorf("style content leaked:\n%s", got) + } +} + +func TestHtmlToMarkdown_StripsNoscript(t *testing.T) { + html := `

Real content

` + got := htmlToMarkdown(html) + if strings.Contains(got, "Enable JavaScript") || strings.Contains(got, "color:blue") { + t.Errorf("noscript content leaked:\n%s", got) + } + if !strings.Contains(got, "Real content") { + t.Errorf("real content missing:\n%s", got) + } +} + +func TestHtmlToMarkdown_StripsSvg(t *testing.T) { + html := `icon

Content

` + got := htmlToMarkdown(html) + if strings.Contains(got, "M10") || strings.Contains(got, "icon") { + t.Errorf("SVG content leaked:\n%s", got) + } +} + +func TestHtmlToMarkdown_StripsNav(t *testing.T) { + html := `

Article

` + got := htmlToMarkdown(html) + if strings.Contains(got, "Home") || strings.Contains(got, "About") { + t.Errorf("nav content leaked:\n%s", got) + } +} + +func TestHtmlToMarkdown_StripsFooter(t *testing.T) { + html := `

Article

Copyright 2024

` + got := htmlToMarkdown(html) + if strings.Contains(got, "Copyright") { + t.Errorf("footer content leaked:\n%s", got) + } +} + +func TestHtmlToMarkdown_StripsForm(t *testing.T) { + html := `

Content

` + got := htmlToMarkdown(html) + if strings.Contains(got, "Submit") || strings.Contains(got, "name") { + t.Errorf("form content leaked:\n%s", got) + } +} + +func TestHtmlToMarkdown_StripsIframe(t *testing.T) { + html := `

Content

` + got := htmlToMarkdown(html) + if strings.Contains(got, "Fallback") || strings.Contains(got, "ads.example") { + t.Errorf("iframe content leaked:\n%s", got) + } +} + +func TestHtmlToMarkdown_StripsTemplate(t *testing.T) { + html := `

Visible

` + got := htmlToMarkdown(html) + if strings.Contains(got, "Template content") { + t.Errorf("template content leaked:\n%s", got) + } +} + +// --- Entity handling --- + +func TestHtmlToMarkdown_Entities(t *testing.T) { + html := `

A & B < C > D "E" 'F'

` + got := htmlToMarkdown(html) + if !strings.Contains(got, `A & B < C > D "E" 'F'`) { + t.Errorf("entities not decoded properly:\n%s", got) + } +} + +// --- Whitespace handling --- + +func TestHtmlToMarkdown_WhitespaceCollapse(t *testing.T) { + html := `

lots of spaces

` + got := htmlToMarkdown(html) + if strings.Contains(got, " ") { + t.Errorf("whitespace not collapsed:\n%s", got) + } +} + +func TestHtmlToMarkdown_PreservesPreWhitespace(t *testing.T) { + html := `
  line1
+  line2
+  line3
` + got := htmlToMarkdown(html) + if !strings.Contains(got, " line1\n line2\n line3") { + t.Errorf("pre whitespace not preserved:\n%s", got) + } +} + +// --- Malformed HTML --- + +func TestHtmlToMarkdown_MalformedHTML(t *testing.T) { + html := `

Unclosed paragraph

Nested bold

OK

` + got := htmlToMarkdown(html) + if !strings.Contains(got, "Unclosed paragraph") || !strings.Contains(got, "bold") || !strings.Contains(got, "OK") { + t.Errorf("malformed HTML not handled:\n%s", got) + } +} + +// --- Text mode --- + +func TestHtmlToText_NoMarkdownFormatting(t *testing.T) { + html := `

Title

Text with bold and link.

` + got := htmlToText(html) + if strings.Contains(got, "#") || strings.Contains(got, "**") || strings.Contains(got, "[link]") { + t.Errorf("markdown formatting in text mode:\n%s", got) + } + if !strings.Contains(got, "Title") || !strings.Contains(got, "bold") || !strings.Contains(got, "link") { + t.Errorf("content missing in text mode:\n%s", got) + } +} + +func TestHtmlToText_StripsHeader(t *testing.T) { + html := `

Site Name

Article

` + got := htmlToText(html) + if strings.Contains(got, "Site Name") { + t.Errorf("header content should be stripped in text mode:\n%s", got) + } + if !strings.Contains(got, "Article") { + t.Errorf("body content missing:\n%s", got) + } +} + +func TestHtmlToText_StripsAside(t *testing.T) { + html := `

Main content

` + got := htmlToText(html) + if strings.Contains(got, "Sidebar") { + t.Errorf("aside content should be stripped in text mode:\n%s", got) + } +} + +// --- Realistic SPA page --- + +func TestHtmlToMarkdown_SPAPage(t *testing.T) { + html := ` + + + + +My App + + + + + + + +
+

Welcome to My App

+

This is the main content of the page.

+
    +
  • Feature 1
    • +
  • Feature 2
    • +
+
+
+

© 2024 My Company. All rights reserved.

+
+ + + +` + + got := htmlToMarkdown(html) + + // Should NOT contain CSS/JS artifacts + for _, bad := range []string{ + "margin: 0", "sans-serif", "background: #333", + "__INITIAL_STATE__", "theme", "dark", + "analytics loaded", "console.log", + "main.abc123.css", "main.def456.js", + "enable JavaScript", + "viewport", // meta from head + "All rights", // footer + "Home", "About", "Contact", // nav + } { + if strings.Contains(got, bad) { + t.Errorf("non-content %q leaked into output:\n%s", bad, got) + } + } + + // Should contain actual content + for _, good := range []string{ + "# Welcome to My App", + "main content of the page", + "- Feature 1", + "- Feature 2", + } { + if !strings.Contains(got, good) { + t.Errorf("expected content %q missing from output:\n%s", good, got) + } + } +}