feat(tools): add Defuddle extractor chain for web_fetch (#296)

Add Cloudflare Worker (fetch.goclaw.sh) as primary markdown extractor
with waterfall fallback to built-in HTML→Markdown converter.

Architecture:
- ExtractorChain pattern with quality gate (min 100 chars, 10 words)
- Settings stored in builtin_tools DB table (not config.json5)
- ResolveExtractorChain reads chain from context per-request
- InProcessExtractor delegates to fetchRawContent (full SSRF + domain
  policy checks on redirects)
- DefuddleExtractor with configurable base_url + timeout
- Seed default chain [defuddle, html-to-markdown] for new deployments
- Backfill migration for existing deployments

Web UI:
- Dedicated DnD extractor chain form on builtin tools page
- Drag-and-drop ordering, enable/disable per extractor
- Timeout + base URL config for Defuddle
- i18n support (en/vi/zh)
This commit is contained in:
viettranx committed 2026-03-20 14:21:46 +07:00
1 parent b2330086ad
commit 38dfcf8bb0
12 files changed
+1072 -22

No files matched your search

+1
View File
@@ -402,6 +402,7 @@ func runGateway() {
if pgStores.BuiltinTools != nil {
seedBuiltinTools(context.Background(), pgStores.BuiltinTools)
migrateBuiltinToolSettings(context.Background(), pgStores.BuiltinTools)
backfillWebFetchSettings(context.Background(), pgStores.BuiltinTools)
applyBuiltinToolDisables(context.Background(), pgStores.BuiltinTools, toolsReg)
}
+22 -1
View File
@@ -28,7 +28,9 @@ func builtinToolSeedData() []store.BuiltinToolDef {
{Name: "web_search", DisplayName: "Web Search", Description: "Search the web for information using a search engine (Brave or DuckDuckGo)", Category: "web", Enabled: true,
Metadata: json.RawMessage(`{"config_hint":"Config → Tools → Web Search"}`),
},
{Name: "web_fetch", DisplayName: "Web Fetch", Description: "Fetch a web page or API endpoint and extract its text content", Category: "web", Enabled: true},
{Name: "web_fetch", DisplayName: "Web Fetch", Description: "Fetch a web page or API endpoint and extract its text content", Category: "web", Enabled: true,
Settings: json.RawMessage(`{"extractors":[{"name":"defuddle","enabled":true,"base_url":"https://fetch.goclaw.sh/"},{"name":"html-to-markdown","enabled":true}]}`),
},
// memory
{Name: "memory_search", DisplayName: "Memory Search", Description: "Search through the agent's long-term memory using semantic similarity", Category: "memory", Enabled: true,
@@ -200,6 +202,25 @@ func migrateBuiltinToolSettings(ctx context.Context, bts store.BuiltinToolStore)
}
}
// backfillWebFetchSettings ensures the web_fetch tool has extractor chain settings.
// Existing deployments may have a web_fetch row with null/empty settings from a prior seed.
// This backfills the default chain so Defuddle is available out of the box.
func backfillWebFetchSettings(ctx context.Context, bts store.BuiltinToolStore) {
t, err := bts.Get(ctx, "web_fetch")
if err != nil || t == nil {
return // not seeded yet, will be populated by next seed
}
if len(t.Settings) > 0 && string(t.Settings) != "{}" && string(t.Settings) != "null" {
return // already has settings, don't overwrite
}
defaultSettings := json.RawMessage(`{"extractors":[{"name":"defuddle","enabled":true,"base_url":"https://fetch.goclaw.sh/"},{"name":"html-to-markdown","enabled":true}]}`)
if err := bts.Update(ctx, "web_fetch", map[string]any{"settings": defaultSettings}); err != nil {
slog.Warn("builtin_tools: failed to backfill web_fetch settings", "error", err)
return
}
slog.Info("builtin_tools: backfilled web_fetch extractor chain settings")
}
// applyBuiltinToolDisables unregisters disabled builtin tools from the registry.
// Called at startup and on cache invalidation.
func applyBuiltinToolDisables(ctx context.Context, bts store.BuiltinToolStore, toolsReg *tools.Registry) {
+71 -19
View File
@@ -213,9 +213,40 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result
}
func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) {
// For markdown mode, use the extractor chain (Defuddle → InProcess waterfall)
// resolved from builtin_tools settings stored in context.
// InProcessExtractor delegates to fetchRawContent (same path as doDirectFetch),
// so no fallthrough is needed — it would just retry the same request.
if extractMode == "markdown" {
chain := ResolveExtractorChain(ctx, t)
if chain != nil {
result, err := chain.Extract(ctx, rawURL)
if err == nil {
return formatFetchResult(result.Content, result.Extractor, rawURL, maxChars, ctx), nil
}
return "", fmt.Errorf("all extractors failed: %w", err)
}
}
// Text mode or no chain available — use direct HTTP fetch.
return t.doDirectFetch(ctx, rawURL, extractMode, maxChars, policy)
}
// fetchRawResult holds the output from fetchRawContent.
type fetchRawResult struct {
content string
extractor string
finalURL string
statusCode int
}
// fetchRawContent performs HTTP GET with full security checks (SSRF, domain policy on
// redirects) and routes content by type. Returns raw extracted content without formatting.
// Used by both doDirectFetch (text mode) and InProcessExtractor (chain fallback).
func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (fetchRawResult, error) {
req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
if err != nil {
return "", fmt.Errorf("create request: %w", err)
return fetchRawResult{}, fmt.Errorf("create request: %w", err)
}
req.Header.Set("User-Agent", fetchUserAgent)
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
@@ -234,16 +265,13 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
if redirectCount > defaultFetchMaxRedirect {
return fmt.Errorf("stopped after %d redirects", defaultFetchMaxRedirect)
}
// Check SSRF on redirect target
if err := CheckSSRF(req.URL.String()); err != nil {
return fmt.Errorf("redirect SSRF protection: %w", err)
}
// Check domain blocklist on redirect target
redirectHost := req.URL.Hostname()
if t.isDomainBlocked(redirectHost) {
return fmt.Errorf("redirect to %q blocked: domain is in blocklist", redirectHost)
}
// Check domain allowlist on redirect target
if policy == "allowlist" && !t.isDomainAllowed(redirectHost) {
return fmt.Errorf("redirect to %q blocked: domain not in allowlist", redirectHost)
}
@@ -253,16 +281,14 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
resp, err := client.Do(req)
if err != nil {
return "", err
return fetchRawResult{}, err
}
defer resp.Body.Close()
// Read enough HTML to reach <body> content — pages often have 30-50KB+ <head> sections.
readLimit := int64(max(maxChars*10, 512*1024))
limitReader := io.LimitReader(resp.Body, readLimit)
body, err := io.ReadAll(limitReader)
body, err := io.ReadAll(io.LimitReader(resp.Body, readLimit))
if err != nil {
return "", fmt.Errorf("read body: %w", err)
return fetchRawResult{}, fmt.Errorf("read body: %w", err)
}
contentType := resp.Header.Get("Content-Type")
@@ -301,21 +327,49 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
extractor = "raw"
}
// Format response metadata
return fetchRawResult{
content: text,
extractor: extractor,
finalURL: finalURL,
statusCode: resp.StatusCode,
}, nil
}
// doDirectFetch wraps fetchRawContent with full HTTP metadata formatting.
// Used for text mode extraction and as ultimate fallback.
func (t *WebFetchTool) doDirectFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) {
raw, err := t.fetchRawContent(ctx, rawURL, extractMode, maxChars, policy)
if err != nil {
return "", err
}
var sb strings.Builder
sb.WriteString(fmt.Sprintf("URL: %s\n", finalURL))
if finalURL != rawURL {
sb.WriteString(fmt.Sprintf("URL: %s\n", raw.finalURL))
if raw.finalURL != rawURL {
sb.WriteString(fmt.Sprintf("Redirected from: %s\n", rawURL))
}
sb.WriteString(fmt.Sprintf("Status: %d\n", resp.StatusCode))
sb.WriteString(fmt.Sprintf("Extractor: %s\n", extractor))
sb.WriteString(fmt.Sprintf("Status: %d\n", raw.statusCode))
sb.WriteString(fmt.Sprintf("Extractor: %s\n", raw.extractor))
appendContent(&sb, raw.content, maxChars, raw.finalURL, ctx)
// If content exceeds maxChars, save full content to a file in workspace
return sb.String(), nil
}
// formatFetchResult builds the metadata-prefixed response for chain-extracted content.
func formatFetchResult(content, extractorName, rawURL string, maxChars int, ctx context.Context) string {
var sb strings.Builder
sb.WriteString(fmt.Sprintf("URL: %s\n", rawURL))
sb.WriteString(fmt.Sprintf("Extractor: %s\n", extractorName))
appendContent(&sb, content, maxChars, rawURL, ctx)
return sb.String()
}
// appendContent writes content to the builder, handling truncation and temp file overflow.
func appendContent(sb *strings.Builder, text string, maxChars int, sourceURL string, ctx context.Context) {
if len(text) > maxChars {
workspace := ToolWorkspaceFromCtx(ctx)
tmpPath, writeErr := writeWebFetchTempFile(workspace, text, finalURL)
tmpPath, writeErr := writeWebFetchTempFile(workspace, text, sourceURL)
if writeErr != nil {
// Fallback: truncate inline if temp file write fails
slog.Warn("web_fetch: failed to write temp file, falling back to truncation", "error", writeErr)
text = text[:maxChars]
sb.WriteString(fmt.Sprintf("Truncated: true (limit: %d chars)\n", maxChars))
@@ -336,8 +390,6 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
sb.WriteString("\n")
sb.WriteString(text)
}
return sb.String(), nil
}
// writeWebFetchTempFile saves fetched content to a file with security sanitization.
+124
View File
@@ -0,0 +1,124 @@
package tools
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"strings"
)
// ContentExtractor extracts readable content from a URL.
type ContentExtractor interface {
Extract(ctx context.Context, rawURL string) (string, error)
Name() string
}
// ExtractResult holds the output from a successful extraction.
type ExtractResult struct {
Content string
Extractor string // name of the extractor that succeeded
}
// ExtractorChain tries extractors in order until one returns quality content.
type ExtractorChain struct {
extractors []ContentExtractor
}
// NewExtractorChain creates a chain from ordered extractors.
func NewExtractorChain(extractors ...ContentExtractor) *ExtractorChain {
return &ExtractorChain{extractors: extractors}
}
// Extract runs each extractor in order, returning the first quality result.
func (c *ExtractorChain) Extract(ctx context.Context, rawURL string) (ExtractResult, error) {
var lastErr error
for _, ext := range c.extractors {
content, err := ext.Extract(ctx, rawURL)
if err != nil {
slog.Debug("extractor failed", "extractor", ext.Name(), "url", rawURL, "error", err)
lastErr = err
continue
}
if !isQualityContent(content) {
slog.Debug("extractor returned low quality content", "extractor", ext.Name(), "url", rawURL, "chars", len(content))
lastErr = fmt.Errorf("%s: content below quality threshold (%d chars)", ext.Name(), len(content))
continue
}
return ExtractResult{Content: content, Extractor: ext.Name()}, nil
}
if lastErr != nil {
return ExtractResult{}, fmt.Errorf("all extractors failed for %s: %w", rawURL, lastErr)
}
return ExtractResult{}, fmt.Errorf("no extractors configured")
}
// isQualityContent checks if extracted content meets minimum quality thresholds.
// Returns false for empty, very short (<100 chars), or low word count (<10 words) content.
func isQualityContent(content string) bool {
trimmed := strings.TrimSpace(content)
if len(trimmed) < 100 {
return false
}
return len(strings.Fields(trimmed)) >= 10
}
// ---------------------------------------------------------------------------
// Extractor chain settings — stored in builtin_tools.settings for web_fetch
// ---------------------------------------------------------------------------
// ExtractorEntry represents a single extractor in the chain settings JSON.
type ExtractorEntry struct {
Name string `json:"name"`
Enabled bool `json:"enabled"`
Timeout int `json:"timeout,omitempty"` // seconds, 0 = use extractor default
BaseURL string `json:"base_url,omitempty"` // for defuddle: CF Worker URL
}
// extractorChainSettings is the JSON schema for web_fetch builtin_tools.settings.
type extractorChainSettings struct {
Extractors []ExtractorEntry `json:"extractors,omitempty"`
}
// ResolveExtractorChain parses builtin_tools.settings from context and builds
// an ordered ExtractorChain. Returns nil if no extractors are enabled.
func ResolveExtractorChain(ctx context.Context, tool *WebFetchTool) *ExtractorChain {
if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil {
if raw, ok := settings["web_fetch"]; ok && len(raw) > 0 {
chain := parseExtractorChainSettings(raw, tool)
if chain != nil {
return chain
}
}
}
// Default fallback: InProcess only (no external extractors).
return NewExtractorChain(&InProcessExtractor{tool: tool})
}
// parseExtractorChainSettings parses the settings JSON and builds a chain.
func parseExtractorChainSettings(raw []byte, tool *WebFetchTool) *ExtractorChain {
var settings extractorChainSettings
if err := json.Unmarshal(raw, &settings); err != nil {
slog.Warn("web_fetch: failed to parse extractor chain settings", "error", err)
return nil
}
var extractors []ContentExtractor
for _, entry := range settings.Extractors {
if !entry.Enabled || entry.Name == "" {
continue
}
switch entry.Name {
case "defuddle":
extractors = append(extractors, NewDefuddleExtractorFromEntry(entry))
case "html-to-markdown":
extractors = append(extractors, &InProcessExtractor{tool: tool})
default:
slog.Warn("web_fetch: unknown extractor in chain, skipping", "name", entry.Name)
}
}
if len(extractors) == 0 {
return nil
}
return NewExtractorChain(extractors...)
}
@@ -0,0 +1,93 @@
package tools
import (
"context"
"fmt"
"io"
"net/http"
"strings"
"time"
)
const (
defuddleBaseURL = "https://fetch.goclaw.sh/"
defuddleTimeout = 10 * time.Second
defuddleMaxBody = 1 << 20 // 1MB
)
// DefuddleExtractor calls the fetch.goclaw.sh Cloudflare Worker to extract
// clean markdown via Defuddle. The CF Worker handles HTTP fetch + content extraction.
type DefuddleExtractor struct {
baseURL string
client *http.Client
}
// NewDefuddleExtractorFromEntry creates a DefuddleExtractor from chain settings.
func NewDefuddleExtractorFromEntry(entry ExtractorEntry) *DefuddleExtractor {
baseURL := entry.BaseURL
if baseURL == "" {
baseURL = defuddleBaseURL
}
// Ensure trailing slash for URL construction.
if !strings.HasSuffix(baseURL, "/") {
baseURL += "/"
}
timeout := time.Duration(entry.Timeout) * time.Second
if timeout <= 0 {
timeout = defuddleTimeout
}
return newDefuddleExtractor(baseURL, timeout)
}
// newDefuddleExtractor creates a DefuddleExtractor with custom base URL and timeout (for testing).
func newDefuddleExtractor(baseURL string, timeout time.Duration) *DefuddleExtractor {
return &DefuddleExtractor{
baseURL: baseURL,
client: &http.Client{
Timeout: timeout,
Transport: &http.Transport{
ForceAttemptHTTP2: true,
MaxIdleConns: 5,
IdleConnTimeout: 30 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
},
},
}
}
func (d *DefuddleExtractor) Name() string { return "defuddle" }
// Extract sends a GET request to fetch.goclaw.sh/<domain>/<path> (no scheme)
// and returns the plain markdown response.
func (d *DefuddleExtractor) Extract(ctx context.Context, rawURL string) (string, error) {
// Strip scheme: https://example.com/path → example.com/path
target := strings.TrimPrefix(strings.TrimPrefix(rawURL, "https://"), "http://")
fetchURL := d.baseURL + target
// Context timeout ensures cancellation propagates even if transport ignores client.Timeout.
ctx, cancel := context.WithTimeout(ctx, d.client.Timeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", fetchURL, nil)
if err != nil {
return "", fmt.Errorf("create defuddle request: %w", err)
}
req.Header.Set("User-Agent", fetchUserAgent)
resp, err := d.client.Do(req)
if err != nil {
return "", fmt.Errorf("defuddle fetch: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("defuddle returned status %d", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, defuddleMaxBody))
if err != nil {
return "", fmt.Errorf("read defuddle response: %w", err)
}
return string(body), nil
}
@@ -0,0 +1,28 @@
package tools
import (
"context"
)
// InProcessExtractor delegates to WebFetchTool.fetchRawContent for HTML→markdown
// extraction with full security checks (SSRF, domain policy on redirects).
// This is the fallback when external extractors (Defuddle) are unavailable.
type InProcessExtractor struct {
tool *WebFetchTool
}
func (e *InProcessExtractor) Name() string { return "html-to-markdown" }
// Extract fetches the URL via the tool's fetchRawContent (full security checks)
// and returns the raw extracted markdown content.
func (e *InProcessExtractor) Extract(ctx context.Context, rawURL string) (string, error) {
e.tool.mu.RLock()
policy := e.tool.policy
e.tool.mu.RUnlock()
raw, err := e.tool.fetchRawContent(ctx, rawURL, "markdown", defaultFetchMaxChars, policy)
if err != nil {
return "", err
}
return raw.content, nil
}
+438
View File
@@ -0,0 +1,438 @@
package tools
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
// --- Quality Gate Tests ---
func TestIsQualityContent(t *testing.T) {
tests := []struct {
name string
content string
want bool
}{
{"empty", "", false},
{"short", "hello world", false},
{"under_100_chars", strings.Repeat("x", 99), false},
{"100_chars_few_words", strings.Repeat("x", 100), false}, // 1 word
{"whitespace_heavy", strings.Repeat(" a ", 30), true}, // 30 words, >100 chars
{"valid_paragraph", "The quick brown fox jumps over the lazy dog. This is a sample paragraph with enough words and characters to pass the quality threshold for content extraction.", true},
{"exactly_10_words_short", "one two three four five six seven eight nine ten", false}, // <100 chars
{"10_words_long_enough", "word1_padding word2_padding word3_padding word4_padding word5_padding word6_padding word7_padding word8_padding word9_padding word10_padding", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := isQualityContent(tt.content)
if got != tt.want {
t.Errorf("isQualityContent(%d chars, %d words) = %v, want %v",
len(tt.content), len(strings.Fields(tt.content)), got, tt.want)
}
})
}
}
// --- Mock Extractor ---
type mockExtractor struct {
name string
content string
err error
}
func (m *mockExtractor) Name() string { return m.name }
func (m *mockExtractor) Extract(_ context.Context, _ string) (string, error) {
if m.err != nil {
return "", m.err
}
return m.content, nil
}
// qualityContent returns a string that passes isQualityContent.
func qualityContent() string {
return "This is a sufficiently long paragraph with more than ten words and well over one hundred characters to satisfy the quality content threshold check in the extractor chain."
}
// --- ExtractorChain Tests ---
func TestExtractorChain_FirstSuccess(t *testing.T) {
chain := NewExtractorChain(
&mockExtractor{name: "first", content: qualityContent()},
&mockExtractor{name: "second", content: "should not reach"},
)
result, err := chain.Extract(context.Background(), "https://example.com")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Extractor != "first" {
t.Errorf("expected extractor 'first', got %q", result.Extractor)
}
}
func TestExtractorChain_FirstFailsSecondSucceeds(t *testing.T) {
chain := NewExtractorChain(
&mockExtractor{name: "first", err: fmt.Errorf("network error")},
&mockExtractor{name: "second", content: qualityContent()},
)
result, err := chain.Extract(context.Background(), "https://example.com")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Extractor != "second" {
t.Errorf("expected extractor 'second', got %q", result.Extractor)
}
}
func TestExtractorChain_FirstLowQualitySecondSucceeds(t *testing.T) {
chain := NewExtractorChain(
&mockExtractor{name: "first", content: "too short"},
&mockExtractor{name: "second", content: qualityContent()},
)
result, err := chain.Extract(context.Background(), "https://example.com")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Extractor != "second" {
t.Errorf("expected extractor 'second', got %q", result.Extractor)
}
}
func TestExtractorChain_AllFail(t *testing.T) {
chain := NewExtractorChain(
&mockExtractor{name: "first", err: fmt.Errorf("fail1")},
&mockExtractor{name: "second", err: fmt.Errorf("fail2")},
)
_, err := chain.Extract(context.Background(), "https://example.com")
if err == nil {
t.Fatal("expected error when all extractors fail")
}
if !strings.Contains(err.Error(), "all extractors failed") {
t.Errorf("unexpected error message: %v", err)
}
}
func TestExtractorChain_AllLowQuality(t *testing.T) {
chain := NewExtractorChain(
&mockExtractor{name: "first", content: "short"},
&mockExtractor{name: "second", content: "also short"},
)
_, err := chain.Extract(context.Background(), "https://example.com")
if err == nil {
t.Fatal("expected error when all extractors return low quality")
}
}
func TestExtractorChain_Empty(t *testing.T) {
chain := NewExtractorChain()
_, err := chain.Extract(context.Background(), "https://example.com")
if err == nil {
t.Fatal("expected error with empty chain")
}
if !strings.Contains(err.Error(), "no extractors configured") {
t.Errorf("unexpected error: %v", err)
}
}
func TestExtractorChain_SingleSuccess(t *testing.T) {
chain := NewExtractorChain(
&mockExtractor{name: "only", content: qualityContent()},
)
result, err := chain.Extract(context.Background(), "https://example.com")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Extractor != "only" {
t.Errorf("expected 'only', got %q", result.Extractor)
}
}
// --- DefuddleExtractor Tests ---
// newTestDefuddleExtractor creates a DefuddleExtractor pointing at a test server.
func newTestDefuddleExtractor(serverURL string) *DefuddleExtractor {
return newDefuddleExtractor(serverURL+"/", defuddleTimeout)
}
func TestDefuddleExtractor_Success(t *testing.T) {
expected := qualityContent()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/example.com/page" {
t.Errorf("unexpected path: %s", r.URL.Path)
}
w.Header().Set("Content-Type", "text/markdown")
w.Write([]byte(expected))
}))
defer server.Close()
ext := newTestDefuddleExtractor(server.URL)
result, err := ext.Extract(context.Background(), "https://example.com/page")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result != expected {
t.Errorf("content mismatch: got %d chars, want %d chars", len(result), len(expected))
}
}
func TestDefuddleExtractor_URLConstruction(t *testing.T) {
tests := []struct {
input string
expectedPath string
}{
{"https://example.com/page", "/example.com/page"},
{"http://example.com/page", "/example.com/page"},
{"https://x.com/user/status/123", "/x.com/user/status/123"},
}
for _, tt := range tests {
var gotPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
w.Write([]byte(qualityContent()))
}))
ext := newTestDefuddleExtractor(server.URL)
_, _ = ext.Extract(context.Background(), tt.input)
server.Close()
if gotPath != tt.expectedPath {
t.Errorf("input=%q: got path %q, want %q", tt.input, gotPath, tt.expectedPath)
}
}
}
func TestDefuddleExtractor_HTTP404(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
ext := newTestDefuddleExtractor(server.URL)
_, err := ext.Extract(context.Background(), "https://example.com/missing")
if err == nil {
t.Fatal("expected error for 404 response")
}
if !strings.Contains(err.Error(), "status 404") {
t.Errorf("unexpected error: %v", err)
}
}
func TestDefuddleExtractor_HTTP500(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer server.Close()
ext := newTestDefuddleExtractor(server.URL)
_, err := ext.Extract(context.Background(), "https://example.com/error")
if err == nil {
t.Fatal("expected error for 500 response")
}
if !strings.Contains(err.Error(), "status 500") {
t.Errorf("unexpected error: %v", err)
}
}
func TestDefuddleExtractor_EmptyBody(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
ext := newTestDefuddleExtractor(server.URL)
result, err := ext.Extract(context.Background(), "https://example.com/empty")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result != "" {
t.Errorf("expected empty result, got %q", result)
}
}
func TestDefuddleExtractorFromEntry(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Write([]byte(qualityContent()))
}))
defer server.Close()
ext := NewDefuddleExtractorFromEntry(ExtractorEntry{
Name: "defuddle",
Enabled: true,
Timeout: 5,
BaseURL: server.URL + "/",
})
result, err := ext.Extract(context.Background(), "https://example.com/page")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result != qualityContent() {
t.Errorf("content mismatch")
}
// Verify timeout was applied
if ext.client.Timeout != 5*time.Second {
t.Errorf("expected 5s timeout, got %v", ext.client.Timeout)
}
}
func TestDefuddleExtractorFromEntry_Defaults(t *testing.T) {
ext := NewDefuddleExtractorFromEntry(ExtractorEntry{Name: "defuddle", Enabled: true})
if ext.baseURL != defuddleBaseURL {
t.Errorf("expected default base URL %q, got %q", defuddleBaseURL, ext.baseURL)
}
if ext.client.Timeout != defuddleTimeout {
t.Errorf("expected default timeout %v, got %v", defuddleTimeout, ext.client.Timeout)
}
}
// --- InProcessExtractor Tests (delegate to fetchRawContent) ---
func TestInProcessExtractor_HTML(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/html")
w.Write([]byte(`<html><body><h1>Hello World</h1><p>This is a test paragraph with enough content to be meaningful and pass quality checks.</p></body></html>`))
}))
defer server.Close()
tool := NewWebFetchTool(WebFetchConfig{})
ext := &InProcessExtractor{tool: tool}
result, err := ext.Extract(context.Background(), server.URL)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !strings.Contains(result, "Hello World") {
t.Errorf("expected heading content, got: %s", result)
}
if !strings.Contains(result, "test paragraph") {
t.Errorf("expected paragraph content, got: %s", result)
}
}
func TestInProcessExtractor_JSON(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(`{"key":"value","nested":{"a":1}}`))
}))
defer server.Close()
tool := NewWebFetchTool(WebFetchConfig{})
ext := &InProcessExtractor{tool: tool}
result, err := ext.Extract(context.Background(), server.URL)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !strings.Contains(result, `"key": "value"`) {
t.Errorf("expected pretty-printed JSON, got: %s", result)
}
}
// --- ResolveExtractorChain Tests ---
func TestResolveExtractorChain_FromSettings(t *testing.T) {
settings := `{"extractors":[{"name":"defuddle","enabled":true,"timeout":5,"base_url":"https://test.example.com/"},{"name":"html-to-markdown","enabled":true}]}`
ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{
"web_fetch": json.RawMessage(settings),
})
tool := NewWebFetchTool(WebFetchConfig{})
chain := ResolveExtractorChain(ctx, tool)
if chain == nil {
t.Fatal("expected non-nil chain")
}
if len(chain.extractors) != 2 {
t.Fatalf("expected 2 extractors, got %d", len(chain.extractors))
}
if chain.extractors[0].Name() != "defuddle" {
t.Errorf("first extractor should be 'defuddle', got %q", chain.extractors[0].Name())
}
if chain.extractors[1].Name() != "html-to-markdown" {
t.Errorf("second extractor should be 'html-to-markdown', got %q", chain.extractors[1].Name())
}
}
func TestResolveExtractorChain_DisabledExtractor(t *testing.T) {
settings := `{"extractors":[{"name":"defuddle","enabled":false},{"name":"html-to-markdown","enabled":true}]}`
ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{
"web_fetch": json.RawMessage(settings),
})
tool := NewWebFetchTool(WebFetchConfig{})
chain := ResolveExtractorChain(ctx, tool)
if chain == nil {
t.Fatal("expected non-nil chain")
}
if len(chain.extractors) != 1 {
t.Fatalf("expected 1 extractor (defuddle disabled), got %d", len(chain.extractors))
}
if chain.extractors[0].Name() != "html-to-markdown" {
t.Errorf("expected 'html-to-markdown', got %q", chain.extractors[0].Name())
}
}
func TestResolveExtractorChain_NoSettings(t *testing.T) {
tool := NewWebFetchTool(WebFetchConfig{})
chain := ResolveExtractorChain(context.Background(), tool)
if chain == nil {
t.Fatal("expected default chain")
}
if len(chain.extractors) != 1 {
t.Fatalf("expected 1 default extractor, got %d", len(chain.extractors))
}
if chain.extractors[0].Name() != "html-to-markdown" {
t.Errorf("expected 'html-to-markdown' default, got %q", chain.extractors[0].Name())
}
}
func TestResolveExtractorChain_UnknownExtractor(t *testing.T) {
settings := `{"extractors":[{"name":"unknown-thing","enabled":true},{"name":"html-to-markdown","enabled":true}]}`
ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{
"web_fetch": json.RawMessage(settings),
})
tool := NewWebFetchTool(WebFetchConfig{})
chain := ResolveExtractorChain(ctx, tool)
if len(chain.extractors) != 1 {
t.Fatalf("expected 1 extractor (unknown skipped), got %d", len(chain.extractors))
}
}
func TestResolveExtractorChain_MalformedJSON(t *testing.T) {
ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{
"web_fetch": []byte(`not valid json`),
})
tool := NewWebFetchTool(WebFetchConfig{})
chain := ResolveExtractorChain(ctx, tool)
// Malformed JSON falls through to default InProcess chain.
if chain == nil {
t.Fatal("expected default chain on malformed JSON")
}
if len(chain.extractors) != 1 || chain.extractors[0].Name() != "html-to-markdown" {
t.Errorf("expected single html-to-markdown fallback, got %d extractors", len(chain.extractors))
}
}
// --- formatFetchResult Tests ---
func TestFormatFetchResult_Basic(t *testing.T) {
result := formatFetchResult("hello world", "defuddle", "https://example.com", 60000, context.Background())
if !strings.Contains(result, "URL: https://example.com") {
t.Error("missing URL in result")
}
if !strings.Contains(result, "Extractor: defuddle") {
t.Error("missing extractor name in result")
}
if !strings.Contains(result, "hello world") {
t.Error("missing content in result")
}
}
func TestFormatFetchResult_Truncation(t *testing.T) {
longContent := strings.Repeat("x", 200)
result := formatFetchResult(longContent, "test", "https://example.com", 100, context.Background())
if !strings.Contains(result, "Truncated: true") && !strings.Contains(result, "Content-Length:") {
if !strings.Contains(result, "Content truncated") {
t.Error("expected truncation indicator in result")
}
}
}
+15
View File
@@ -132,6 +132,21 @@
"save": "Save",
"saving": "Saving..."
},
"extractorChain": {
"title": "Web Fetch — Extractor Chain",
"description": "Ordered list of content extractors. Each URL is tried top-to-bottom until one returns quality content.",
"timeout": "Timeout",
"baseUrl": "Base URL",
"save": "Save",
"saving": "Saving…",
"cancel": "Cancel",
"extractors": {
"defuddle": "Defuddle (Cloudflare Worker)",
"defuddleDesc": "Extracts clean markdown via fetch.goclaw.sh using the Defuddle library.",
"html-to-markdown": "Built-in HTML→Markdown",
"html-to-markdownDesc": "In-process DOM-based converter with hidden element detection."
}
},
"mediaChain": {
"providerChainSuffix": "— Provider Chain",
"description": "Configure and order provider fallbacks. Drag to reorder — the first enabled provider is tried first.",
+15
View File
@@ -132,6 +132,21 @@
"save": "Lưu",
"saving": "Đang lưu..."
},
"extractorChain": {
"title": "Web Fetch — Chuỗi bộ trích xuất",
"description": "Danh sách bộ trích xuất nội dung theo thứ tự. Mỗi URL được thử từ trên xuống cho đến khi một bộ trả về nội dung chất lượng.",
"timeout": "Thời gian chờ",
"baseUrl": "URL cơ sở",
"save": "Lưu",
"saving": "Đang lưu…",
"cancel": "Hủy",
"extractors": {
"defuddle": "Defuddle (Cloudflare Worker)",
"defuddleDesc": "Trích xuất markdown sạch qua fetch.goclaw.sh sử dụng thư viện Defuddle.",
"html-to-markdown": "HTML→Markdown tích hợp",
"html-to-markdownDesc": "Bộ chuyển đổi dựa trên DOM trong tiến trình với khả năng phát hiện phần tử ẩn."
}
},
"mediaChain": {
"providerChainSuffix": "— Chuỗi provider",
"description": "Cấu hình và sắp xếp các provider dự phòng. Kéo để sắp xếp lại — provider đầu tiên được thử trước.",
+15
View File
@@ -46,6 +46,21 @@
"saving": "保存中...",
"title": "知识图谱设置"
},
"extractorChain": {
"title": "Web Fetch — 提取器链",
"description": "有序内容提取器列表。每个 URL 从上到下依次尝试,直到某个提取器返回高质量内容。",
"timeout": "超时",
"baseUrl": "基础 URL",
"save": "保存",
"saving": "保存中…",
"cancel": "取消",
"extractors": {
"defuddle": "Defuddle(Cloudflare Worker)",
"defuddleDesc": "通过 fetch.goclaw.sh 使用 Defuddle 库提取干净的 Markdown。",
"html-to-markdown": "内置 HTML→Markdown",
"html-to-markdownDesc": "进程内基于 DOM 的转换器,具备隐藏元素检测功能。"
}
},
"mediaChain": {
"addProvider": "添加Provider",
"cancel": "取消",
@@ -14,8 +14,10 @@ import type { BuiltinToolData } from "./hooks/use-builtin-tools";
import { MEDIA_TOOLS } from "./media-provider-params-schema";
import { MediaProviderChainForm } from "./media-provider-chain-form";
import { KGSettingsForm } from "./kg-settings-form";
import { WebFetchExtractorChainForm } from "./web-fetch-extractor-chain-form";
const KG_TOOL = "knowledge_graph_search";
const WEB_FETCH_TOOL = "web_fetch";
interface Props {
tool: BuiltinToolData | null;
@@ -27,12 +29,19 @@ interface Props {
export function BuiltinToolSettingsDialog({ tool, open, onOpenChange, onSave }: Props) {
const isMedia = tool ? MEDIA_TOOLS.has(tool.name) : false;
const isKG = tool?.name === KG_TOOL;
const wide = isMedia || isKG;
const isWebFetch = tool?.name === WEB_FETCH_TOOL;
const wide = isMedia || isKG || isWebFetch;
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className={wide ? "sm:max-w-2xl" : "sm:max-w-md"}>
{isMedia && tool ? (
{isWebFetch && tool ? (
<WebFetchExtractorChainForm
initialSettings={tool.settings ?? {}}
onSave={(settings) => onSave(tool.name, settings).then(() => onOpenChange(false))}
onCancel={() => onOpenChange(false)}
/>
) : isMedia && tool ? (
<MediaProviderChainForm
toolName={tool.name}
initialSettings={tool.settings ?? {}}
@@ -0,0 +1,239 @@
import { useState, useCallback } from "react";
import { useTranslation } from "react-i18next";
import {
DndContext,
closestCenter,
KeyboardSensor,
PointerSensor,
useSensor,
useSensors,
type DragEndEvent,
} from "@dnd-kit/core";
import {
SortableContext,
verticalListSortingStrategy,
useSortable,
arrayMove,
sortableKeyboardCoordinates,
} from "@dnd-kit/sortable";
import { CSS } from "@dnd-kit/utilities";
import { GripVertical } from "lucide-react";
import { uniqueId } from "@/lib/utils";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Switch } from "@/components/ui/switch";
import { DialogHeader, DialogTitle, DialogDescription, DialogFooter } from "@/components/ui/dialog";
interface ExtractorEntry {
id: string;
name: string;
enabled: boolean;
timeout: number;
base_url: string;
}
interface Props {
initialSettings: Record<string, unknown>;
onSave: (settings: Record<string, unknown>) => Promise<void>;
onCancel: () => void;
}
const FIXED_EXTRACTORS = ["defuddle", "html-to-markdown"] as const;
function parseInitialEntries(settings: Record<string, unknown>): ExtractorEntry[] {
const raw = Array.isArray(settings.extractors)
? (settings.extractors as Record<string, unknown>[])
: [];
// Build a map from name → raw entry for fast lookup
const byName = new Map(raw.map((e) => [String(e.name ?? ""), e]));
return FIXED_EXTRACTORS.map((name) => {
const e = byName.get(name) ?? {};
return {
id: uniqueId(),
name,
enabled: Boolean(e.enabled ?? true),
timeout: Number(e.timeout ?? 0),
base_url: String(e.base_url ?? ""),
};
});
}
interface SortableCardProps {
entry: ExtractorEntry;
index: number;
onUpdate: (id: string, patch: Partial<ExtractorEntry>) => void;
}
function SortableExtractorCard({ entry, index, onUpdate }: SortableCardProps) {
const { t } = useTranslation("tools");
const { attributes, listeners, setNodeRef, transform, transition, isDragging } = useSortable({
id: entry.id,
});
const style = {
transform: CSS.Transform.toString(transform),
transition,
opacity: isDragging ? 0.5 : 1,
};
const displayName = t(`builtin.extractorChain.extractors.${entry.name}`, {
defaultValue: entry.name,
});
const desc = t(`builtin.extractorChain.extractors.${entry.name}Desc`, {
defaultValue: "",
});
const showTimeout = entry.timeout > 0 || entry.name === "defuddle";
const showBaseUrl = entry.name === "defuddle";
return (
<div
ref={setNodeRef}
style={style}
className={`border rounded-lg bg-card ${!entry.enabled ? "opacity-60" : ""}`}
>
{/* Row 1: drag handle, index, toggle, name */}
<div className="flex items-center gap-2 px-3 pt-3 pb-1">
<button
type="button"
className="cursor-grab text-muted-foreground hover:text-foreground shrink-0"
{...attributes}
{...listeners}
>
<GripVertical className="size-4" />
</button>
<span className="text-xs text-muted-foreground font-mono shrink-0">
#{index + 1}
</span>
<Switch
size="sm"
checked={entry.enabled}
onCheckedChange={(v) => onUpdate(entry.id, { enabled: v })}
/>
<div className="flex-1 min-w-0">
<span className="text-sm font-medium">{displayName}</span>
{desc && (
<p className="text-xs text-muted-foreground mt-0.5 truncate">{desc}</p>
)}
</div>
</div>
{/* Optional fields */}
{(showTimeout || showBaseUrl) && (
<div className="flex flex-wrap items-center gap-3 px-3 pb-3 pt-1">
{showTimeout && (
<div className="flex items-center gap-1.5">
<Label className="text-xs text-muted-foreground whitespace-nowrap">
{t("builtin.extractorChain.timeout")}
</Label>
<div className="relative">
<Input
type="number"
min={0}
max={600}
value={entry.timeout}
onChange={(e) => onUpdate(entry.id, { timeout: Number(e.target.value) })}
className="h-7 w-20 text-base md:text-sm pr-5"
/>
<span className="absolute right-2 top-1/2 -translate-y-1/2 text-xs text-muted-foreground pointer-events-none">
s
</span>
</div>
</div>
)}
{showBaseUrl && (
<div className="flex items-center gap-1.5 flex-1 min-w-[200px]">
<Label className="text-xs text-muted-foreground whitespace-nowrap">
{t("builtin.extractorChain.baseUrl")}
</Label>
<Input
value={entry.base_url}
onChange={(e) => onUpdate(entry.id, { base_url: e.target.value })}
placeholder="https://fetch.goclaw.sh/"
className="h-7 text-base md:text-sm"
/>
</div>
)}
</div>
)}
</div>
);
}
export function WebFetchExtractorChainForm({ initialSettings, onSave, onCancel }: Props) {
const { t } = useTranslation("tools");
const [entries, setEntries] = useState<ExtractorEntry[]>(() =>
parseInitialEntries(initialSettings),
);
const [saving, setSaving] = useState(false);
const sensors = useSensors(
useSensor(PointerSensor),
useSensor(KeyboardSensor, { coordinateGetter: sortableKeyboardCoordinates }),
);
const handleDragEnd = useCallback((event: DragEndEvent) => {
const { active, over } = event;
if (over && active.id !== over.id) {
setEntries((prev) => {
const oldIndex = prev.findIndex((e) => e.id === active.id);
const newIndex = prev.findIndex((e) => e.id === over.id);
return arrayMove(prev, oldIndex, newIndex);
});
}
}, []);
const handleUpdate = useCallback((id: string, patch: Partial<ExtractorEntry>) => {
setEntries((prev) => prev.map((e) => (e.id === id ? { ...e, ...patch } : e)));
}, []);
const handleSave = async () => {
setSaving(true);
try {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const serialized = entries.map(({ id: _id, ...rest }) => rest);
await onSave({ extractors: serialized });
} finally {
setSaving(false);
}
};
return (
<div>
<DialogHeader>
<DialogTitle>{t("builtin.extractorChain.title")}</DialogTitle>
<DialogDescription>{t("builtin.extractorChain.description")}</DialogDescription>
</DialogHeader>
<div className="space-y-2 max-h-[60vh] overflow-y-auto pr-1 my-4">
<DndContext sensors={sensors} collisionDetection={closestCenter} onDragEnd={handleDragEnd}>
<SortableContext items={entries.map((e) => e.id)} strategy={verticalListSortingStrategy}>
{entries.map((entry, index) => (
<SortableExtractorCard
key={entry.id}
entry={entry}
index={index}
onUpdate={handleUpdate}
/>
))}
</SortableContext>
</DndContext>
</div>
<DialogFooter>
<Button variant="outline" onClick={onCancel}>
{t("builtin.extractorChain.cancel")}
</Button>
<Button onClick={handleSave} disabled={saving}>
{saving ? t("builtin.extractorChain.saving") : t("builtin.extractorChain.save")}
</Button>
</DialogFooter>
</div>
);
}