diff --git a/internal/tools/web_fetch_convert.go b/internal/tools/web_fetch_convert.go index 04ecddcd..c1eb3dda 100644 --- a/internal/tools/web_fetch_convert.go +++ b/internal/tools/web_fetch_convert.go @@ -2,8 +2,12 @@ package tools import ( "encoding/json" + "fmt" "regexp" "strings" + + "golang.org/x/net/html" + "golang.org/x/net/html/atom" ) // extractJSON pretty-prints JSON content. @@ -16,173 +20,553 @@ func extractJSON(body []byte) (string, string) { return string(body), "raw" } -// --- HTML extraction utilities --- +// --- DOM-based HTML extraction --- -var ( - reScript = regexp.MustCompile(`(?is)
Content