From 38dfcf8bb06f6c085f37ba189882a4ed16af2442 Mon Sep 17 00:00:00 2001 From: viettranx Date: Fri, 20 Mar 2026 14:21:46 +0700 Subject: [PATCH] feat(tools): add Defuddle extractor chain for web_fetch (#296) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Cloudflare Worker (fetch.goclaw.sh) as primary markdown extractor with waterfall fallback to built-in HTML→Markdown converter. Architecture: - ExtractorChain pattern with quality gate (min 100 chars, 10 words) - Settings stored in builtin_tools DB table (not config.json5) - ResolveExtractorChain reads chain from context per-request - InProcessExtractor delegates to fetchRawContent (full SSRF + domain policy checks on redirects) - DefuddleExtractor with configurable base_url + timeout - Seed default chain [defuddle, html-to-markdown] for new deployments - Backfill migration for existing deployments Web UI: - Dedicated DnD extractor chain form on builtin tools page - Drag-and-drop ordering, enable/disable per extractor - Timeout + base URL config for Defuddle - i18n support (en/vi/zh) --- cmd/gateway.go | 1 + cmd/gateway_builtin_tools.go | 23 +- internal/tools/web_fetch.go | 90 +++- internal/tools/web_fetch_extractor.go | 124 +++++ .../tools/web_fetch_extractor_defuddle.go | 93 ++++ .../tools/web_fetch_extractor_inprocess.go | 28 ++ internal/tools/web_fetch_extractor_test.go | 438 ++++++++++++++++++ ui/web/src/i18n/locales/en/tools.json | 15 + ui/web/src/i18n/locales/vi/tools.json | 15 + ui/web/src/i18n/locales/zh/tools.json | 15 + .../builtin-tool-settings-dialog.tsx | 13 +- .../web-fetch-extractor-chain-form.tsx | 239 ++++++++++ 12 files changed, 1072 insertions(+), 22 deletions(-) create mode 100644 internal/tools/web_fetch_extractor.go create mode 100644 internal/tools/web_fetch_extractor_defuddle.go create mode 100644 internal/tools/web_fetch_extractor_inprocess.go create mode 100644 internal/tools/web_fetch_extractor_test.go create mode 100644 ui/web/src/pages/builtin-tools/web-fetch-extractor-chain-form.tsx diff --git a/cmd/gateway.go b/cmd/gateway.go index f4a6c6b8..7a46bf47 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -402,6 +402,7 @@ func runGateway() { if pgStores.BuiltinTools != nil { seedBuiltinTools(context.Background(), pgStores.BuiltinTools) migrateBuiltinToolSettings(context.Background(), pgStores.BuiltinTools) + backfillWebFetchSettings(context.Background(), pgStores.BuiltinTools) applyBuiltinToolDisables(context.Background(), pgStores.BuiltinTools, toolsReg) } diff --git a/cmd/gateway_builtin_tools.go b/cmd/gateway_builtin_tools.go index 68305ede..354d58f8 100644 --- a/cmd/gateway_builtin_tools.go +++ b/cmd/gateway_builtin_tools.go @@ -28,7 +28,9 @@ func builtinToolSeedData() []store.BuiltinToolDef { {Name: "web_search", DisplayName: "Web Search", Description: "Search the web for information using a search engine (Brave or DuckDuckGo)", Category: "web", Enabled: true, Metadata: json.RawMessage(`{"config_hint":"Config → Tools → Web Search"}`), }, - {Name: "web_fetch", DisplayName: "Web Fetch", Description: "Fetch a web page or API endpoint and extract its text content", Category: "web", Enabled: true}, + {Name: "web_fetch", DisplayName: "Web Fetch", Description: "Fetch a web page or API endpoint and extract its text content", Category: "web", Enabled: true, + Settings: json.RawMessage(`{"extractors":[{"name":"defuddle","enabled":true,"base_url":"https://fetch.goclaw.sh/"},{"name":"html-to-markdown","enabled":true}]}`), + }, // memory {Name: "memory_search", DisplayName: "Memory Search", Description: "Search through the agent's long-term memory using semantic similarity", Category: "memory", Enabled: true, @@ -200,6 +202,25 @@ func migrateBuiltinToolSettings(ctx context.Context, bts store.BuiltinToolStore) } } +// backfillWebFetchSettings ensures the web_fetch tool has extractor chain settings. +// Existing deployments may have a web_fetch row with null/empty settings from a prior seed. +// This backfills the default chain so Defuddle is available out of the box. +func backfillWebFetchSettings(ctx context.Context, bts store.BuiltinToolStore) { + t, err := bts.Get(ctx, "web_fetch") + if err != nil || t == nil { + return // not seeded yet, will be populated by next seed + } + if len(t.Settings) > 0 && string(t.Settings) != "{}" && string(t.Settings) != "null" { + return // already has settings, don't overwrite + } + defaultSettings := json.RawMessage(`{"extractors":[{"name":"defuddle","enabled":true,"base_url":"https://fetch.goclaw.sh/"},{"name":"html-to-markdown","enabled":true}]}`) + if err := bts.Update(ctx, "web_fetch", map[string]any{"settings": defaultSettings}); err != nil { + slog.Warn("builtin_tools: failed to backfill web_fetch settings", "error", err) + return + } + slog.Info("builtin_tools: backfilled web_fetch extractor chain settings") +} + // applyBuiltinToolDisables unregisters disabled builtin tools from the registry. // Called at startup and on cache invalidation. func applyBuiltinToolDisables(ctx context.Context, bts store.BuiltinToolStore, toolsReg *tools.Registry) { diff --git a/internal/tools/web_fetch.go b/internal/tools/web_fetch.go index 85e82a47..52044e2c 100644 --- a/internal/tools/web_fetch.go +++ b/internal/tools/web_fetch.go @@ -213,9 +213,40 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result } func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) { + // For markdown mode, use the extractor chain (Defuddle → InProcess waterfall) + // resolved from builtin_tools settings stored in context. + // InProcessExtractor delegates to fetchRawContent (same path as doDirectFetch), + // so no fallthrough is needed — it would just retry the same request. + if extractMode == "markdown" { + chain := ResolveExtractorChain(ctx, t) + if chain != nil { + result, err := chain.Extract(ctx, rawURL) + if err == nil { + return formatFetchResult(result.Content, result.Extractor, rawURL, maxChars, ctx), nil + } + return "", fmt.Errorf("all extractors failed: %w", err) + } + } + + // Text mode or no chain available — use direct HTTP fetch. + return t.doDirectFetch(ctx, rawURL, extractMode, maxChars, policy) +} + +// fetchRawResult holds the output from fetchRawContent. +type fetchRawResult struct { + content string + extractor string + finalURL string + statusCode int +} + +// fetchRawContent performs HTTP GET with full security checks (SSRF, domain policy on +// redirects) and routes content by type. Returns raw extracted content without formatting. +// Used by both doDirectFetch (text mode) and InProcessExtractor (chain fallback). +func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (fetchRawResult, error) { req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil) if err != nil { - return "", fmt.Errorf("create request: %w", err) + return fetchRawResult{}, fmt.Errorf("create request: %w", err) } req.Header.Set("User-Agent", fetchUserAgent) req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") @@ -234,16 +265,13 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, if redirectCount > defaultFetchMaxRedirect { return fmt.Errorf("stopped after %d redirects", defaultFetchMaxRedirect) } - // Check SSRF on redirect target if err := CheckSSRF(req.URL.String()); err != nil { return fmt.Errorf("redirect SSRF protection: %w", err) } - // Check domain blocklist on redirect target redirectHost := req.URL.Hostname() if t.isDomainBlocked(redirectHost) { return fmt.Errorf("redirect to %q blocked: domain is in blocklist", redirectHost) } - // Check domain allowlist on redirect target if policy == "allowlist" && !t.isDomainAllowed(redirectHost) { return fmt.Errorf("redirect to %q blocked: domain not in allowlist", redirectHost) } @@ -253,16 +281,14 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, resp, err := client.Do(req) if err != nil { - return "", err + return fetchRawResult{}, err } defer resp.Body.Close() - // Read enough HTML to reach content — pages often have 30-50KB+ sections. readLimit := int64(max(maxChars*10, 512*1024)) - limitReader := io.LimitReader(resp.Body, readLimit) - body, err := io.ReadAll(limitReader) + body, err := io.ReadAll(io.LimitReader(resp.Body, readLimit)) if err != nil { - return "", fmt.Errorf("read body: %w", err) + return fetchRawResult{}, fmt.Errorf("read body: %w", err) } contentType := resp.Header.Get("Content-Type") @@ -301,21 +327,49 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, extractor = "raw" } - // Format response metadata + return fetchRawResult{ + content: text, + extractor: extractor, + finalURL: finalURL, + statusCode: resp.StatusCode, + }, nil +} + +// doDirectFetch wraps fetchRawContent with full HTTP metadata formatting. +// Used for text mode extraction and as ultimate fallback. +func (t *WebFetchTool) doDirectFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) { + raw, err := t.fetchRawContent(ctx, rawURL, extractMode, maxChars, policy) + if err != nil { + return "", err + } + var sb strings.Builder - sb.WriteString(fmt.Sprintf("URL: %s\n", finalURL)) - if finalURL != rawURL { + sb.WriteString(fmt.Sprintf("URL: %s\n", raw.finalURL)) + if raw.finalURL != rawURL { sb.WriteString(fmt.Sprintf("Redirected from: %s\n", rawURL)) } - sb.WriteString(fmt.Sprintf("Status: %d\n", resp.StatusCode)) - sb.WriteString(fmt.Sprintf("Extractor: %s\n", extractor)) + sb.WriteString(fmt.Sprintf("Status: %d\n", raw.statusCode)) + sb.WriteString(fmt.Sprintf("Extractor: %s\n", raw.extractor)) + appendContent(&sb, raw.content, maxChars, raw.finalURL, ctx) - // If content exceeds maxChars, save full content to a file in workspace + return sb.String(), nil +} + +// formatFetchResult builds the metadata-prefixed response for chain-extracted content. +func formatFetchResult(content, extractorName, rawURL string, maxChars int, ctx context.Context) string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("URL: %s\n", rawURL)) + sb.WriteString(fmt.Sprintf("Extractor: %s\n", extractorName)) + appendContent(&sb, content, maxChars, rawURL, ctx) + return sb.String() +} + +// appendContent writes content to the builder, handling truncation and temp file overflow. +func appendContent(sb *strings.Builder, text string, maxChars int, sourceURL string, ctx context.Context) { if len(text) > maxChars { workspace := ToolWorkspaceFromCtx(ctx) - tmpPath, writeErr := writeWebFetchTempFile(workspace, text, finalURL) + tmpPath, writeErr := writeWebFetchTempFile(workspace, text, sourceURL) if writeErr != nil { - // Fallback: truncate inline if temp file write fails slog.Warn("web_fetch: failed to write temp file, falling back to truncation", "error", writeErr) text = text[:maxChars] sb.WriteString(fmt.Sprintf("Truncated: true (limit: %d chars)\n", maxChars)) @@ -336,8 +390,6 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, sb.WriteString("\n") sb.WriteString(text) } - - return sb.String(), nil } // writeWebFetchTempFile saves fetched content to a file with security sanitization. diff --git a/internal/tools/web_fetch_extractor.go b/internal/tools/web_fetch_extractor.go new file mode 100644 index 00000000..041b8f79 --- /dev/null +++ b/internal/tools/web_fetch_extractor.go @@ -0,0 +1,124 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "strings" +) + +// ContentExtractor extracts readable content from a URL. +type ContentExtractor interface { + Extract(ctx context.Context, rawURL string) (string, error) + Name() string +} + +// ExtractResult holds the output from a successful extraction. +type ExtractResult struct { + Content string + Extractor string // name of the extractor that succeeded +} + +// ExtractorChain tries extractors in order until one returns quality content. +type ExtractorChain struct { + extractors []ContentExtractor +} + +// NewExtractorChain creates a chain from ordered extractors. +func NewExtractorChain(extractors ...ContentExtractor) *ExtractorChain { + return &ExtractorChain{extractors: extractors} +} + +// Extract runs each extractor in order, returning the first quality result. +func (c *ExtractorChain) Extract(ctx context.Context, rawURL string) (ExtractResult, error) { + var lastErr error + for _, ext := range c.extractors { + content, err := ext.Extract(ctx, rawURL) + if err != nil { + slog.Debug("extractor failed", "extractor", ext.Name(), "url", rawURL, "error", err) + lastErr = err + continue + } + if !isQualityContent(content) { + slog.Debug("extractor returned low quality content", "extractor", ext.Name(), "url", rawURL, "chars", len(content)) + lastErr = fmt.Errorf("%s: content below quality threshold (%d chars)", ext.Name(), len(content)) + continue + } + return ExtractResult{Content: content, Extractor: ext.Name()}, nil + } + if lastErr != nil { + return ExtractResult{}, fmt.Errorf("all extractors failed for %s: %w", rawURL, lastErr) + } + return ExtractResult{}, fmt.Errorf("no extractors configured") +} + +// isQualityContent checks if extracted content meets minimum quality thresholds. +// Returns false for empty, very short (<100 chars), or low word count (<10 words) content. +func isQualityContent(content string) bool { + trimmed := strings.TrimSpace(content) + if len(trimmed) < 100 { + return false + } + return len(strings.Fields(trimmed)) >= 10 +} + +// --------------------------------------------------------------------------- +// Extractor chain settings — stored in builtin_tools.settings for web_fetch +// --------------------------------------------------------------------------- + +// ExtractorEntry represents a single extractor in the chain settings JSON. +type ExtractorEntry struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Timeout int `json:"timeout,omitempty"` // seconds, 0 = use extractor default + BaseURL string `json:"base_url,omitempty"` // for defuddle: CF Worker URL +} + +// extractorChainSettings is the JSON schema for web_fetch builtin_tools.settings. +type extractorChainSettings struct { + Extractors []ExtractorEntry `json:"extractors,omitempty"` +} + +// ResolveExtractorChain parses builtin_tools.settings from context and builds +// an ordered ExtractorChain. Returns nil if no extractors are enabled. +func ResolveExtractorChain(ctx context.Context, tool *WebFetchTool) *ExtractorChain { + if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil { + if raw, ok := settings["web_fetch"]; ok && len(raw) > 0 { + chain := parseExtractorChainSettings(raw, tool) + if chain != nil { + return chain + } + } + } + // Default fallback: InProcess only (no external extractors). + return NewExtractorChain(&InProcessExtractor{tool: tool}) +} + +// parseExtractorChainSettings parses the settings JSON and builds a chain. +func parseExtractorChainSettings(raw []byte, tool *WebFetchTool) *ExtractorChain { + var settings extractorChainSettings + if err := json.Unmarshal(raw, &settings); err != nil { + slog.Warn("web_fetch: failed to parse extractor chain settings", "error", err) + return nil + } + + var extractors []ContentExtractor + for _, entry := range settings.Extractors { + if !entry.Enabled || entry.Name == "" { + continue + } + switch entry.Name { + case "defuddle": + extractors = append(extractors, NewDefuddleExtractorFromEntry(entry)) + case "html-to-markdown": + extractors = append(extractors, &InProcessExtractor{tool: tool}) + default: + slog.Warn("web_fetch: unknown extractor in chain, skipping", "name", entry.Name) + } + } + if len(extractors) == 0 { + return nil + } + return NewExtractorChain(extractors...) +} diff --git a/internal/tools/web_fetch_extractor_defuddle.go b/internal/tools/web_fetch_extractor_defuddle.go new file mode 100644 index 00000000..ace1e35c --- /dev/null +++ b/internal/tools/web_fetch_extractor_defuddle.go @@ -0,0 +1,93 @@ +package tools + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +const ( + defuddleBaseURL = "https://fetch.goclaw.sh/" + defuddleTimeout = 10 * time.Second + defuddleMaxBody = 1 << 20 // 1MB +) + +// DefuddleExtractor calls the fetch.goclaw.sh Cloudflare Worker to extract +// clean markdown via Defuddle. The CF Worker handles HTTP fetch + content extraction. +type DefuddleExtractor struct { + baseURL string + client *http.Client +} + +// NewDefuddleExtractorFromEntry creates a DefuddleExtractor from chain settings. +func NewDefuddleExtractorFromEntry(entry ExtractorEntry) *DefuddleExtractor { + baseURL := entry.BaseURL + if baseURL == "" { + baseURL = defuddleBaseURL + } + // Ensure trailing slash for URL construction. + if !strings.HasSuffix(baseURL, "/") { + baseURL += "/" + } + timeout := time.Duration(entry.Timeout) * time.Second + if timeout <= 0 { + timeout = defuddleTimeout + } + return newDefuddleExtractor(baseURL, timeout) +} + +// newDefuddleExtractor creates a DefuddleExtractor with custom base URL and timeout (for testing). +func newDefuddleExtractor(baseURL string, timeout time.Duration) *DefuddleExtractor { + return &DefuddleExtractor{ + baseURL: baseURL, + client: &http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + ForceAttemptHTTP2: true, + MaxIdleConns: 5, + IdleConnTimeout: 30 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + }, + }, + } +} + +func (d *DefuddleExtractor) Name() string { return "defuddle" } + +// Extract sends a GET request to fetch.goclaw.sh// (no scheme) +// and returns the plain markdown response. +func (d *DefuddleExtractor) Extract(ctx context.Context, rawURL string) (string, error) { + // Strip scheme: https://example.com/path → example.com/path + target := strings.TrimPrefix(strings.TrimPrefix(rawURL, "https://"), "http://") + fetchURL := d.baseURL + target + + // Context timeout ensures cancellation propagates even if transport ignores client.Timeout. + ctx, cancel := context.WithTimeout(ctx, d.client.Timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", fetchURL, nil) + if err != nil { + return "", fmt.Errorf("create defuddle request: %w", err) + } + req.Header.Set("User-Agent", fetchUserAgent) + + resp, err := d.client.Do(req) + if err != nil { + return "", fmt.Errorf("defuddle fetch: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("defuddle returned status %d", resp.StatusCode) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, defuddleMaxBody)) + if err != nil { + return "", fmt.Errorf("read defuddle response: %w", err) + } + + return string(body), nil +} diff --git a/internal/tools/web_fetch_extractor_inprocess.go b/internal/tools/web_fetch_extractor_inprocess.go new file mode 100644 index 00000000..77c24687 --- /dev/null +++ b/internal/tools/web_fetch_extractor_inprocess.go @@ -0,0 +1,28 @@ +package tools + +import ( + "context" +) + +// InProcessExtractor delegates to WebFetchTool.fetchRawContent for HTML→markdown +// extraction with full security checks (SSRF, domain policy on redirects). +// This is the fallback when external extractors (Defuddle) are unavailable. +type InProcessExtractor struct { + tool *WebFetchTool +} + +func (e *InProcessExtractor) Name() string { return "html-to-markdown" } + +// Extract fetches the URL via the tool's fetchRawContent (full security checks) +// and returns the raw extracted markdown content. +func (e *InProcessExtractor) Extract(ctx context.Context, rawURL string) (string, error) { + e.tool.mu.RLock() + policy := e.tool.policy + e.tool.mu.RUnlock() + + raw, err := e.tool.fetchRawContent(ctx, rawURL, "markdown", defaultFetchMaxChars, policy) + if err != nil { + return "", err + } + return raw.content, nil +} diff --git a/internal/tools/web_fetch_extractor_test.go b/internal/tools/web_fetch_extractor_test.go new file mode 100644 index 00000000..46b5e873 --- /dev/null +++ b/internal/tools/web_fetch_extractor_test.go @@ -0,0 +1,438 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// --- Quality Gate Tests --- + +func TestIsQualityContent(t *testing.T) { + tests := []struct { + name string + content string + want bool + }{ + {"empty", "", false}, + {"short", "hello world", false}, + {"under_100_chars", strings.Repeat("x", 99), false}, + {"100_chars_few_words", strings.Repeat("x", 100), false}, // 1 word + {"whitespace_heavy", strings.Repeat(" a ", 30), true}, // 30 words, >100 chars + {"valid_paragraph", "The quick brown fox jumps over the lazy dog. This is a sample paragraph with enough words and characters to pass the quality threshold for content extraction.", true}, + {"exactly_10_words_short", "one two three four five six seven eight nine ten", false}, // <100 chars + {"10_words_long_enough", "word1_padding word2_padding word3_padding word4_padding word5_padding word6_padding word7_padding word8_padding word9_padding word10_padding", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isQualityContent(tt.content) + if got != tt.want { + t.Errorf("isQualityContent(%d chars, %d words) = %v, want %v", + len(tt.content), len(strings.Fields(tt.content)), got, tt.want) + } + }) + } +} + +// --- Mock Extractor --- + +type mockExtractor struct { + name string + content string + err error +} + +func (m *mockExtractor) Name() string { return m.name } +func (m *mockExtractor) Extract(_ context.Context, _ string) (string, error) { + if m.err != nil { + return "", m.err + } + return m.content, nil +} + +// qualityContent returns a string that passes isQualityContent. +func qualityContent() string { + return "This is a sufficiently long paragraph with more than ten words and well over one hundred characters to satisfy the quality content threshold check in the extractor chain." +} + +// --- ExtractorChain Tests --- + +func TestExtractorChain_FirstSuccess(t *testing.T) { + chain := NewExtractorChain( + &mockExtractor{name: "first", content: qualityContent()}, + &mockExtractor{name: "second", content: "should not reach"}, + ) + result, err := chain.Extract(context.Background(), "https://example.com") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Extractor != "first" { + t.Errorf("expected extractor 'first', got %q", result.Extractor) + } +} + +func TestExtractorChain_FirstFailsSecondSucceeds(t *testing.T) { + chain := NewExtractorChain( + &mockExtractor{name: "first", err: fmt.Errorf("network error")}, + &mockExtractor{name: "second", content: qualityContent()}, + ) + result, err := chain.Extract(context.Background(), "https://example.com") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Extractor != "second" { + t.Errorf("expected extractor 'second', got %q", result.Extractor) + } +} + +func TestExtractorChain_FirstLowQualitySecondSucceeds(t *testing.T) { + chain := NewExtractorChain( + &mockExtractor{name: "first", content: "too short"}, + &mockExtractor{name: "second", content: qualityContent()}, + ) + result, err := chain.Extract(context.Background(), "https://example.com") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Extractor != "second" { + t.Errorf("expected extractor 'second', got %q", result.Extractor) + } +} + +func TestExtractorChain_AllFail(t *testing.T) { + chain := NewExtractorChain( + &mockExtractor{name: "first", err: fmt.Errorf("fail1")}, + &mockExtractor{name: "second", err: fmt.Errorf("fail2")}, + ) + _, err := chain.Extract(context.Background(), "https://example.com") + if err == nil { + t.Fatal("expected error when all extractors fail") + } + if !strings.Contains(err.Error(), "all extractors failed") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestExtractorChain_AllLowQuality(t *testing.T) { + chain := NewExtractorChain( + &mockExtractor{name: "first", content: "short"}, + &mockExtractor{name: "second", content: "also short"}, + ) + _, err := chain.Extract(context.Background(), "https://example.com") + if err == nil { + t.Fatal("expected error when all extractors return low quality") + } +} + +func TestExtractorChain_Empty(t *testing.T) { + chain := NewExtractorChain() + _, err := chain.Extract(context.Background(), "https://example.com") + if err == nil { + t.Fatal("expected error with empty chain") + } + if !strings.Contains(err.Error(), "no extractors configured") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestExtractorChain_SingleSuccess(t *testing.T) { + chain := NewExtractorChain( + &mockExtractor{name: "only", content: qualityContent()}, + ) + result, err := chain.Extract(context.Background(), "https://example.com") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.Extractor != "only" { + t.Errorf("expected 'only', got %q", result.Extractor) + } +} + +// --- DefuddleExtractor Tests --- + +// newTestDefuddleExtractor creates a DefuddleExtractor pointing at a test server. +func newTestDefuddleExtractor(serverURL string) *DefuddleExtractor { + return newDefuddleExtractor(serverURL+"/", defuddleTimeout) +} + +func TestDefuddleExtractor_Success(t *testing.T) { + expected := qualityContent() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/example.com/page" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "text/markdown") + w.Write([]byte(expected)) + })) + defer server.Close() + + ext := newTestDefuddleExtractor(server.URL) + result, err := ext.Extract(context.Background(), "https://example.com/page") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result != expected { + t.Errorf("content mismatch: got %d chars, want %d chars", len(result), len(expected)) + } +} + +func TestDefuddleExtractor_URLConstruction(t *testing.T) { + tests := []struct { + input string + expectedPath string + }{ + {"https://example.com/page", "/example.com/page"}, + {"http://example.com/page", "/example.com/page"}, + {"https://x.com/user/status/123", "/x.com/user/status/123"}, + } + for _, tt := range tests { + var gotPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Write([]byte(qualityContent())) + })) + ext := newTestDefuddleExtractor(server.URL) + _, _ = ext.Extract(context.Background(), tt.input) + server.Close() + + if gotPath != tt.expectedPath { + t.Errorf("input=%q: got path %q, want %q", tt.input, gotPath, tt.expectedPath) + } + } +} + +func TestDefuddleExtractor_HTTP404(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + ext := newTestDefuddleExtractor(server.URL) + _, err := ext.Extract(context.Background(), "https://example.com/missing") + if err == nil { + t.Fatal("expected error for 404 response") + } + if !strings.Contains(err.Error(), "status 404") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestDefuddleExtractor_HTTP500(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + ext := newTestDefuddleExtractor(server.URL) + _, err := ext.Extract(context.Background(), "https://example.com/error") + if err == nil { + t.Fatal("expected error for 500 response") + } + if !strings.Contains(err.Error(), "status 500") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestDefuddleExtractor_EmptyBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + ext := newTestDefuddleExtractor(server.URL) + result, err := ext.Extract(context.Background(), "https://example.com/empty") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result != "" { + t.Errorf("expected empty result, got %q", result) + } +} + +func TestDefuddleExtractorFromEntry(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte(qualityContent())) + })) + defer server.Close() + + ext := NewDefuddleExtractorFromEntry(ExtractorEntry{ + Name: "defuddle", + Enabled: true, + Timeout: 5, + BaseURL: server.URL + "/", + }) + result, err := ext.Extract(context.Background(), "https://example.com/page") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result != qualityContent() { + t.Errorf("content mismatch") + } + // Verify timeout was applied + if ext.client.Timeout != 5*time.Second { + t.Errorf("expected 5s timeout, got %v", ext.client.Timeout) + } +} + +func TestDefuddleExtractorFromEntry_Defaults(t *testing.T) { + ext := NewDefuddleExtractorFromEntry(ExtractorEntry{Name: "defuddle", Enabled: true}) + if ext.baseURL != defuddleBaseURL { + t.Errorf("expected default base URL %q, got %q", defuddleBaseURL, ext.baseURL) + } + if ext.client.Timeout != defuddleTimeout { + t.Errorf("expected default timeout %v, got %v", defuddleTimeout, ext.client.Timeout) + } +} + +// --- InProcessExtractor Tests (delegate to fetchRawContent) --- + +func TestInProcessExtractor_HTML(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.Write([]byte(`

Hello World

This is a test paragraph with enough content to be meaningful and pass quality checks.

`)) + })) + defer server.Close() + + tool := NewWebFetchTool(WebFetchConfig{}) + ext := &InProcessExtractor{tool: tool} + result, err := ext.Extract(context.Background(), server.URL) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(result, "Hello World") { + t.Errorf("expected heading content, got: %s", result) + } + if !strings.Contains(result, "test paragraph") { + t.Errorf("expected paragraph content, got: %s", result) + } +} + +func TestInProcessExtractor_JSON(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"key":"value","nested":{"a":1}}`)) + })) + defer server.Close() + + tool := NewWebFetchTool(WebFetchConfig{}) + ext := &InProcessExtractor{tool: tool} + result, err := ext.Extract(context.Background(), server.URL) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.Contains(result, `"key": "value"`) { + t.Errorf("expected pretty-printed JSON, got: %s", result) + } +} + +// --- ResolveExtractorChain Tests --- + +func TestResolveExtractorChain_FromSettings(t *testing.T) { + settings := `{"extractors":[{"name":"defuddle","enabled":true,"timeout":5,"base_url":"https://test.example.com/"},{"name":"html-to-markdown","enabled":true}]}` + ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{ + "web_fetch": json.RawMessage(settings), + }) + tool := NewWebFetchTool(WebFetchConfig{}) + chain := ResolveExtractorChain(ctx, tool) + if chain == nil { + t.Fatal("expected non-nil chain") + } + if len(chain.extractors) != 2 { + t.Fatalf("expected 2 extractors, got %d", len(chain.extractors)) + } + if chain.extractors[0].Name() != "defuddle" { + t.Errorf("first extractor should be 'defuddle', got %q", chain.extractors[0].Name()) + } + if chain.extractors[1].Name() != "html-to-markdown" { + t.Errorf("second extractor should be 'html-to-markdown', got %q", chain.extractors[1].Name()) + } +} + +func TestResolveExtractorChain_DisabledExtractor(t *testing.T) { + settings := `{"extractors":[{"name":"defuddle","enabled":false},{"name":"html-to-markdown","enabled":true}]}` + ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{ + "web_fetch": json.RawMessage(settings), + }) + tool := NewWebFetchTool(WebFetchConfig{}) + chain := ResolveExtractorChain(ctx, tool) + if chain == nil { + t.Fatal("expected non-nil chain") + } + if len(chain.extractors) != 1 { + t.Fatalf("expected 1 extractor (defuddle disabled), got %d", len(chain.extractors)) + } + if chain.extractors[0].Name() != "html-to-markdown" { + t.Errorf("expected 'html-to-markdown', got %q", chain.extractors[0].Name()) + } +} + +func TestResolveExtractorChain_NoSettings(t *testing.T) { + tool := NewWebFetchTool(WebFetchConfig{}) + chain := ResolveExtractorChain(context.Background(), tool) + if chain == nil { + t.Fatal("expected default chain") + } + if len(chain.extractors) != 1 { + t.Fatalf("expected 1 default extractor, got %d", len(chain.extractors)) + } + if chain.extractors[0].Name() != "html-to-markdown" { + t.Errorf("expected 'html-to-markdown' default, got %q", chain.extractors[0].Name()) + } +} + +func TestResolveExtractorChain_UnknownExtractor(t *testing.T) { + settings := `{"extractors":[{"name":"unknown-thing","enabled":true},{"name":"html-to-markdown","enabled":true}]}` + ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{ + "web_fetch": json.RawMessage(settings), + }) + tool := NewWebFetchTool(WebFetchConfig{}) + chain := ResolveExtractorChain(ctx, tool) + if len(chain.extractors) != 1 { + t.Fatalf("expected 1 extractor (unknown skipped), got %d", len(chain.extractors)) + } +} + +func TestResolveExtractorChain_MalformedJSON(t *testing.T) { + ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{ + "web_fetch": []byte(`not valid json`), + }) + tool := NewWebFetchTool(WebFetchConfig{}) + chain := ResolveExtractorChain(ctx, tool) + // Malformed JSON falls through to default InProcess chain. + if chain == nil { + t.Fatal("expected default chain on malformed JSON") + } + if len(chain.extractors) != 1 || chain.extractors[0].Name() != "html-to-markdown" { + t.Errorf("expected single html-to-markdown fallback, got %d extractors", len(chain.extractors)) + } +} + +// --- formatFetchResult Tests --- + +func TestFormatFetchResult_Basic(t *testing.T) { + result := formatFetchResult("hello world", "defuddle", "https://example.com", 60000, context.Background()) + if !strings.Contains(result, "URL: https://example.com") { + t.Error("missing URL in result") + } + if !strings.Contains(result, "Extractor: defuddle") { + t.Error("missing extractor name in result") + } + if !strings.Contains(result, "hello world") { + t.Error("missing content in result") + } +} + +func TestFormatFetchResult_Truncation(t *testing.T) { + longContent := strings.Repeat("x", 200) + result := formatFetchResult(longContent, "test", "https://example.com", 100, context.Background()) + if !strings.Contains(result, "Truncated: true") && !strings.Contains(result, "Content-Length:") { + if !strings.Contains(result, "Content truncated") { + t.Error("expected truncation indicator in result") + } + } +} diff --git a/ui/web/src/i18n/locales/en/tools.json b/ui/web/src/i18n/locales/en/tools.json index 80766217..26b9d321 100644 --- a/ui/web/src/i18n/locales/en/tools.json +++ b/ui/web/src/i18n/locales/en/tools.json @@ -132,6 +132,21 @@ "save": "Save", "saving": "Saving..." }, + "extractorChain": { + "title": "Web Fetch — Extractor Chain", + "description": "Ordered list of content extractors. Each URL is tried top-to-bottom until one returns quality content.", + "timeout": "Timeout", + "baseUrl": "Base URL", + "save": "Save", + "saving": "Saving…", + "cancel": "Cancel", + "extractors": { + "defuddle": "Defuddle (Cloudflare Worker)", + "defuddleDesc": "Extracts clean markdown via fetch.goclaw.sh using the Defuddle library.", + "html-to-markdown": "Built-in HTML→Markdown", + "html-to-markdownDesc": "In-process DOM-based converter with hidden element detection." + } + }, "mediaChain": { "providerChainSuffix": "— Provider Chain", "description": "Configure and order provider fallbacks. Drag to reorder — the first enabled provider is tried first.", diff --git a/ui/web/src/i18n/locales/vi/tools.json b/ui/web/src/i18n/locales/vi/tools.json index 62b7025b..72f6289a 100644 --- a/ui/web/src/i18n/locales/vi/tools.json +++ b/ui/web/src/i18n/locales/vi/tools.json @@ -132,6 +132,21 @@ "save": "Lưu", "saving": "Đang lưu..." }, + "extractorChain": { + "title": "Web Fetch — Chuỗi bộ trích xuất", + "description": "Danh sách bộ trích xuất nội dung theo thứ tự. Mỗi URL được thử từ trên xuống cho đến khi một bộ trả về nội dung chất lượng.", + "timeout": "Thời gian chờ", + "baseUrl": "URL cơ sở", + "save": "Lưu", + "saving": "Đang lưu…", + "cancel": "Hủy", + "extractors": { + "defuddle": "Defuddle (Cloudflare Worker)", + "defuddleDesc": "Trích xuất markdown sạch qua fetch.goclaw.sh sử dụng thư viện Defuddle.", + "html-to-markdown": "HTML→Markdown tích hợp", + "html-to-markdownDesc": "Bộ chuyển đổi dựa trên DOM trong tiến trình với khả năng phát hiện phần tử ẩn." + } + }, "mediaChain": { "providerChainSuffix": "— Chuỗi provider", "description": "Cấu hình và sắp xếp các provider dự phòng. Kéo để sắp xếp lại — provider đầu tiên được thử trước.", diff --git a/ui/web/src/i18n/locales/zh/tools.json b/ui/web/src/i18n/locales/zh/tools.json index 3d2e5b81..1dfacd07 100644 --- a/ui/web/src/i18n/locales/zh/tools.json +++ b/ui/web/src/i18n/locales/zh/tools.json @@ -46,6 +46,21 @@ "saving": "保存中...", "title": "知识图谱设置" }, + "extractorChain": { + "title": "Web Fetch — 提取器链", + "description": "有序内容提取器列表。每个 URL 从上到下依次尝试，直到某个提取器返回高质量内容。", + "timeout": "超时", + "baseUrl": "基础 URL", + "save": "保存", + "saving": "保存中…", + "cancel": "取消", + "extractors": { + "defuddle": "Defuddle（Cloudflare Worker）", + "defuddleDesc": "通过 fetch.goclaw.sh 使用 Defuddle 库提取干净的 Markdown。", + "html-to-markdown": "内置 HTML→Markdown", + "html-to-markdownDesc": "进程内基于 DOM 的转换器，具备隐藏元素检测功能。" + } + }, "mediaChain": { "addProvider": "添加Provider", "cancel": "取消", diff --git a/ui/web/src/pages/builtin-tools/builtin-tool-settings-dialog.tsx b/ui/web/src/pages/builtin-tools/builtin-tool-settings-dialog.tsx index ea1f987c..b6c1e51c 100644 --- a/ui/web/src/pages/builtin-tools/builtin-tool-settings-dialog.tsx +++ b/ui/web/src/pages/builtin-tools/builtin-tool-settings-dialog.tsx @@ -14,8 +14,10 @@ import type { BuiltinToolData } from "./hooks/use-builtin-tools"; import { MEDIA_TOOLS } from "./media-provider-params-schema"; import { MediaProviderChainForm } from "./media-provider-chain-form"; import { KGSettingsForm } from "./kg-settings-form"; +import { WebFetchExtractorChainForm } from "./web-fetch-extractor-chain-form"; const KG_TOOL = "knowledge_graph_search"; +const WEB_FETCH_TOOL = "web_fetch"; interface Props { tool: BuiltinToolData | null; @@ -27,12 +29,19 @@ interface Props { export function BuiltinToolSettingsDialog({ tool, open, onOpenChange, onSave }: Props) { const isMedia = tool ? MEDIA_TOOLS.has(tool.name) : false; const isKG = tool?.name === KG_TOOL; - const wide = isMedia || isKG; + const isWebFetch = tool?.name === WEB_FETCH_TOOL; + const wide = isMedia || isKG || isWebFetch; return ( - {isMedia && tool ? ( + {isWebFetch && tool ? ( + onSave(tool.name, settings).then(() => onOpenChange(false))} + onCancel={() => onOpenChange(false)} + /> + ) : isMedia && tool ? ( ; + onSave: (settings: Record) => Promise; + onCancel: () => void; +} + +const FIXED_EXTRACTORS = ["defuddle", "html-to-markdown"] as const; + +function parseInitialEntries(settings: Record): ExtractorEntry[] { + const raw = Array.isArray(settings.extractors) + ? (settings.extractors as Record[]) + : []; + + // Build a map from name → raw entry for fast lookup + const byName = new Map(raw.map((e) => [String(e.name ?? ""), e])); + + return FIXED_EXTRACTORS.map((name) => { + const e = byName.get(name) ?? {}; + return { + id: uniqueId(), + name, + enabled: Boolean(e.enabled ?? true), + timeout: Number(e.timeout ?? 0), + base_url: String(e.base_url ?? ""), + }; + }); +} + +interface SortableCardProps { + entry: ExtractorEntry; + index: number; + onUpdate: (id: string, patch: Partial) => void; +} + +function SortableExtractorCard({ entry, index, onUpdate }: SortableCardProps) { + const { t } = useTranslation("tools"); + const { attributes, listeners, setNodeRef, transform, transition, isDragging } = useSortable({ + id: entry.id, + }); + + const style = { + transform: CSS.Transform.toString(transform), + transition, + opacity: isDragging ? 0.5 : 1, + }; + + const displayName = t(`builtin.extractorChain.extractors.${entry.name}`, { + defaultValue: entry.name, + }); + const desc = t(`builtin.extractorChain.extractors.${entry.name}Desc`, { + defaultValue: "", + }); + const showTimeout = entry.timeout > 0 || entry.name === "defuddle"; + const showBaseUrl = entry.name === "defuddle"; + + return ( +
+ {/* Row 1: drag handle, index, toggle, name */} +
+ + + + #{index + 1} + + + onUpdate(entry.id, { enabled: v })} + /> + +
+ {displayName} + {desc && ( +

{desc}

+ )} +
+
+ + {/* Optional fields */} + {(showTimeout || showBaseUrl) && ( +
+ {showTimeout && ( +
+ +
+ onUpdate(entry.id, { timeout: Number(e.target.value) })} + className="h-7 w-20 text-base md:text-sm pr-5" + /> + + s + +
+
+ )} + {showBaseUrl && ( +
+ + onUpdate(entry.id, { base_url: e.target.value })} + placeholder="https://fetch.goclaw.sh/" + className="h-7 text-base md:text-sm" + /> +
+ )} +
+ )} +
+ ); +} + +export function WebFetchExtractorChainForm({ initialSettings, onSave, onCancel }: Props) { + const { t } = useTranslation("tools"); + const [entries, setEntries] = useState(() => + parseInitialEntries(initialSettings), + ); + const [saving, setSaving] = useState(false); + + const sensors = useSensors( + useSensor(PointerSensor), + useSensor(KeyboardSensor, { coordinateGetter: sortableKeyboardCoordinates }), + ); + + const handleDragEnd = useCallback((event: DragEndEvent) => { + const { active, over } = event; + if (over && active.id !== over.id) { + setEntries((prev) => { + const oldIndex = prev.findIndex((e) => e.id === active.id); + const newIndex = prev.findIndex((e) => e.id === over.id); + return arrayMove(prev, oldIndex, newIndex); + }); + } + }, []); + + const handleUpdate = useCallback((id: string, patch: Partial) => { + setEntries((prev) => prev.map((e) => (e.id === id ? { ...e, ...patch } : e))); + }, []); + + const handleSave = async () => { + setSaving(true); + try { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const serialized = entries.map(({ id: _id, ...rest }) => rest); + await onSave({ extractors: serialized }); + } finally { + setSaving(false); + } + }; + + return ( +
+ + {t("builtin.extractorChain.title")} + {t("builtin.extractorChain.description")} + + +
+ + e.id)} strategy={verticalListSortingStrategy}> + {entries.map((entry, index) => ( + + ))} + + + +
+ + + + + +
+ ); +}