mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-14 20:17:20 +00:00
feat(tools): add Defuddle extractor chain for web_fetch (#296)
Add Cloudflare Worker (fetch.goclaw.sh) as primary markdown extractor with waterfall fallback to built-in HTML→Markdown converter. Architecture: - ExtractorChain pattern with quality gate (min 100 chars, 10 words) - Settings stored in builtin_tools DB table (not config.json5) - ResolveExtractorChain reads chain from context per-request - InProcessExtractor delegates to fetchRawContent (full SSRF + domain policy checks on redirects) - DefuddleExtractor with configurable base_url + timeout - Seed default chain [defuddle, html-to-markdown] for new deployments - Backfill migration for existing deployments Web UI: - Dedicated DnD extractor chain form on builtin tools page - Drag-and-drop ordering, enable/disable per extractor - Timeout + base URL config for Defuddle - i18n support (en/vi/zh)
This commit is contained in:
1 parent
b2330086ad
commit
38dfcf8bb0
12 files changed
+1072
-22
No files matched your search
@@ -402,6 +402,7 @@ func runGateway() {
|
||||
if pgStores.BuiltinTools != nil {
|
||||
seedBuiltinTools(context.Background(), pgStores.BuiltinTools)
|
||||
migrateBuiltinToolSettings(context.Background(), pgStores.BuiltinTools)
|
||||
backfillWebFetchSettings(context.Background(), pgStores.BuiltinTools)
|
||||
applyBuiltinToolDisables(context.Background(), pgStores.BuiltinTools, toolsReg)
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,9 @@ func builtinToolSeedData() []store.BuiltinToolDef {
|
||||
{Name: "web_search", DisplayName: "Web Search", Description: "Search the web for information using a search engine (Brave or DuckDuckGo)", Category: "web", Enabled: true,
|
||||
Metadata: json.RawMessage(`{"config_hint":"Config → Tools → Web Search"}`),
|
||||
},
|
||||
{Name: "web_fetch", DisplayName: "Web Fetch", Description: "Fetch a web page or API endpoint and extract its text content", Category: "web", Enabled: true},
|
||||
{Name: "web_fetch", DisplayName: "Web Fetch", Description: "Fetch a web page or API endpoint and extract its text content", Category: "web", Enabled: true,
|
||||
Settings: json.RawMessage(`{"extractors":[{"name":"defuddle","enabled":true,"base_url":"https://fetch.goclaw.sh/"},{"name":"html-to-markdown","enabled":true}]}`),
|
||||
},
|
||||
|
||||
// memory
|
||||
{Name: "memory_search", DisplayName: "Memory Search", Description: "Search through the agent's long-term memory using semantic similarity", Category: "memory", Enabled: true,
|
||||
@@ -200,6 +202,25 @@ func migrateBuiltinToolSettings(ctx context.Context, bts store.BuiltinToolStore)
|
||||
}
|
||||
}
|
||||
|
||||
// backfillWebFetchSettings ensures the web_fetch tool has extractor chain settings.
|
||||
// Existing deployments may have a web_fetch row with null/empty settings from a prior seed.
|
||||
// This backfills the default chain so Defuddle is available out of the box.
|
||||
func backfillWebFetchSettings(ctx context.Context, bts store.BuiltinToolStore) {
|
||||
t, err := bts.Get(ctx, "web_fetch")
|
||||
if err != nil || t == nil {
|
||||
return // not seeded yet, will be populated by next seed
|
||||
}
|
||||
if len(t.Settings) > 0 && string(t.Settings) != "{}" && string(t.Settings) != "null" {
|
||||
return // already has settings, don't overwrite
|
||||
}
|
||||
defaultSettings := json.RawMessage(`{"extractors":[{"name":"defuddle","enabled":true,"base_url":"https://fetch.goclaw.sh/"},{"name":"html-to-markdown","enabled":true}]}`)
|
||||
if err := bts.Update(ctx, "web_fetch", map[string]any{"settings": defaultSettings}); err != nil {
|
||||
slog.Warn("builtin_tools: failed to backfill web_fetch settings", "error", err)
|
||||
return
|
||||
}
|
||||
slog.Info("builtin_tools: backfilled web_fetch extractor chain settings")
|
||||
}
|
||||
|
||||
// applyBuiltinToolDisables unregisters disabled builtin tools from the registry.
|
||||
// Called at startup and on cache invalidation.
|
||||
func applyBuiltinToolDisables(ctx context.Context, bts store.BuiltinToolStore, toolsReg *tools.Registry) {
|
||||
|
||||
+71
-19
@@ -213,9 +213,40 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *Result
|
||||
}
|
||||
|
||||
func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) {
|
||||
// For markdown mode, use the extractor chain (Defuddle → InProcess waterfall)
|
||||
// resolved from builtin_tools settings stored in context.
|
||||
// InProcessExtractor delegates to fetchRawContent (same path as doDirectFetch),
|
||||
// so no fallthrough is needed — it would just retry the same request.
|
||||
if extractMode == "markdown" {
|
||||
chain := ResolveExtractorChain(ctx, t)
|
||||
if chain != nil {
|
||||
result, err := chain.Extract(ctx, rawURL)
|
||||
if err == nil {
|
||||
return formatFetchResult(result.Content, result.Extractor, rawURL, maxChars, ctx), nil
|
||||
}
|
||||
return "", fmt.Errorf("all extractors failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Text mode or no chain available — use direct HTTP fetch.
|
||||
return t.doDirectFetch(ctx, rawURL, extractMode, maxChars, policy)
|
||||
}
|
||||
|
||||
// fetchRawResult holds the output from fetchRawContent.
|
||||
type fetchRawResult struct {
|
||||
content string
|
||||
extractor string
|
||||
finalURL string
|
||||
statusCode int
|
||||
}
|
||||
|
||||
// fetchRawContent performs HTTP GET with full security checks (SSRF, domain policy on
|
||||
// redirects) and routes content by type. Returns raw extracted content without formatting.
|
||||
// Used by both doDirectFetch (text mode) and InProcessExtractor (chain fallback).
|
||||
func (t *WebFetchTool) fetchRawContent(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (fetchRawResult, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create request: %w", err)
|
||||
return fetchRawResult{}, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", fetchUserAgent)
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
@@ -234,16 +265,13 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
|
||||
if redirectCount > defaultFetchMaxRedirect {
|
||||
return fmt.Errorf("stopped after %d redirects", defaultFetchMaxRedirect)
|
||||
}
|
||||
// Check SSRF on redirect target
|
||||
if err := CheckSSRF(req.URL.String()); err != nil {
|
||||
return fmt.Errorf("redirect SSRF protection: %w", err)
|
||||
}
|
||||
// Check domain blocklist on redirect target
|
||||
redirectHost := req.URL.Hostname()
|
||||
if t.isDomainBlocked(redirectHost) {
|
||||
return fmt.Errorf("redirect to %q blocked: domain is in blocklist", redirectHost)
|
||||
}
|
||||
// Check domain allowlist on redirect target
|
||||
if policy == "allowlist" && !t.isDomainAllowed(redirectHost) {
|
||||
return fmt.Errorf("redirect to %q blocked: domain not in allowlist", redirectHost)
|
||||
}
|
||||
@@ -253,16 +281,14 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return fetchRawResult{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read enough HTML to reach <body> content — pages often have 30-50KB+ <head> sections.
|
||||
readLimit := int64(max(maxChars*10, 512*1024))
|
||||
limitReader := io.LimitReader(resp.Body, readLimit)
|
||||
body, err := io.ReadAll(limitReader)
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, readLimit))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read body: %w", err)
|
||||
return fetchRawResult{}, fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
@@ -301,21 +327,49 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
|
||||
extractor = "raw"
|
||||
}
|
||||
|
||||
// Format response metadata
|
||||
return fetchRawResult{
|
||||
content: text,
|
||||
extractor: extractor,
|
||||
finalURL: finalURL,
|
||||
statusCode: resp.StatusCode,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// doDirectFetch wraps fetchRawContent with full HTTP metadata formatting.
|
||||
// Used for text mode extraction and as ultimate fallback.
|
||||
func (t *WebFetchTool) doDirectFetch(ctx context.Context, rawURL, extractMode string, maxChars int, policy string) (string, error) {
|
||||
raw, err := t.fetchRawContent(ctx, rawURL, extractMode, maxChars, policy)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("URL: %s\n", finalURL))
|
||||
if finalURL != rawURL {
|
||||
sb.WriteString(fmt.Sprintf("URL: %s\n", raw.finalURL))
|
||||
if raw.finalURL != rawURL {
|
||||
sb.WriteString(fmt.Sprintf("Redirected from: %s\n", rawURL))
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("Status: %d\n", resp.StatusCode))
|
||||
sb.WriteString(fmt.Sprintf("Extractor: %s\n", extractor))
|
||||
sb.WriteString(fmt.Sprintf("Status: %d\n", raw.statusCode))
|
||||
sb.WriteString(fmt.Sprintf("Extractor: %s\n", raw.extractor))
|
||||
appendContent(&sb, raw.content, maxChars, raw.finalURL, ctx)
|
||||
|
||||
// If content exceeds maxChars, save full content to a file in workspace
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
||||
// formatFetchResult builds the metadata-prefixed response for chain-extracted content.
|
||||
func formatFetchResult(content, extractorName, rawURL string, maxChars int, ctx context.Context) string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("URL: %s\n", rawURL))
|
||||
sb.WriteString(fmt.Sprintf("Extractor: %s\n", extractorName))
|
||||
appendContent(&sb, content, maxChars, rawURL, ctx)
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// appendContent writes content to the builder, handling truncation and temp file overflow.
|
||||
func appendContent(sb *strings.Builder, text string, maxChars int, sourceURL string, ctx context.Context) {
|
||||
if len(text) > maxChars {
|
||||
workspace := ToolWorkspaceFromCtx(ctx)
|
||||
tmpPath, writeErr := writeWebFetchTempFile(workspace, text, finalURL)
|
||||
tmpPath, writeErr := writeWebFetchTempFile(workspace, text, sourceURL)
|
||||
if writeErr != nil {
|
||||
// Fallback: truncate inline if temp file write fails
|
||||
slog.Warn("web_fetch: failed to write temp file, falling back to truncation", "error", writeErr)
|
||||
text = text[:maxChars]
|
||||
sb.WriteString(fmt.Sprintf("Truncated: true (limit: %d chars)\n", maxChars))
|
||||
@@ -336,8 +390,6 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
|
||||
sb.WriteString("\n")
|
||||
sb.WriteString(text)
|
||||
}
|
||||
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
||||
// writeWebFetchTempFile saves fetched content to a file with security sanitization.
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ContentExtractor extracts readable content from a URL.
|
||||
type ContentExtractor interface {
|
||||
Extract(ctx context.Context, rawURL string) (string, error)
|
||||
Name() string
|
||||
}
|
||||
|
||||
// ExtractResult holds the output from a successful extraction.
|
||||
type ExtractResult struct {
|
||||
Content string
|
||||
Extractor string // name of the extractor that succeeded
|
||||
}
|
||||
|
||||
// ExtractorChain tries extractors in order until one returns quality content.
|
||||
type ExtractorChain struct {
|
||||
extractors []ContentExtractor
|
||||
}
|
||||
|
||||
// NewExtractorChain creates a chain from ordered extractors.
|
||||
func NewExtractorChain(extractors ...ContentExtractor) *ExtractorChain {
|
||||
return &ExtractorChain{extractors: extractors}
|
||||
}
|
||||
|
||||
// Extract runs each extractor in order, returning the first quality result.
|
||||
func (c *ExtractorChain) Extract(ctx context.Context, rawURL string) (ExtractResult, error) {
|
||||
var lastErr error
|
||||
for _, ext := range c.extractors {
|
||||
content, err := ext.Extract(ctx, rawURL)
|
||||
if err != nil {
|
||||
slog.Debug("extractor failed", "extractor", ext.Name(), "url", rawURL, "error", err)
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
if !isQualityContent(content) {
|
||||
slog.Debug("extractor returned low quality content", "extractor", ext.Name(), "url", rawURL, "chars", len(content))
|
||||
lastErr = fmt.Errorf("%s: content below quality threshold (%d chars)", ext.Name(), len(content))
|
||||
continue
|
||||
}
|
||||
return ExtractResult{Content: content, Extractor: ext.Name()}, nil
|
||||
}
|
||||
if lastErr != nil {
|
||||
return ExtractResult{}, fmt.Errorf("all extractors failed for %s: %w", rawURL, lastErr)
|
||||
}
|
||||
return ExtractResult{}, fmt.Errorf("no extractors configured")
|
||||
}
|
||||
|
||||
// isQualityContent checks if extracted content meets minimum quality thresholds.
|
||||
// Returns false for empty, very short (<100 chars), or low word count (<10 words) content.
|
||||
func isQualityContent(content string) bool {
|
||||
trimmed := strings.TrimSpace(content)
|
||||
if len(trimmed) < 100 {
|
||||
return false
|
||||
}
|
||||
return len(strings.Fields(trimmed)) >= 10
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Extractor chain settings — stored in builtin_tools.settings for web_fetch
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// ExtractorEntry represents a single extractor in the chain settings JSON.
|
||||
type ExtractorEntry struct {
|
||||
Name string `json:"name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Timeout int `json:"timeout,omitempty"` // seconds, 0 = use extractor default
|
||||
BaseURL string `json:"base_url,omitempty"` // for defuddle: CF Worker URL
|
||||
}
|
||||
|
||||
// extractorChainSettings is the JSON schema for web_fetch builtin_tools.settings.
|
||||
type extractorChainSettings struct {
|
||||
Extractors []ExtractorEntry `json:"extractors,omitempty"`
|
||||
}
|
||||
|
||||
// ResolveExtractorChain parses builtin_tools.settings from context and builds
|
||||
// an ordered ExtractorChain. Returns nil if no extractors are enabled.
|
||||
func ResolveExtractorChain(ctx context.Context, tool *WebFetchTool) *ExtractorChain {
|
||||
if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil {
|
||||
if raw, ok := settings["web_fetch"]; ok && len(raw) > 0 {
|
||||
chain := parseExtractorChainSettings(raw, tool)
|
||||
if chain != nil {
|
||||
return chain
|
||||
}
|
||||
}
|
||||
}
|
||||
// Default fallback: InProcess only (no external extractors).
|
||||
return NewExtractorChain(&InProcessExtractor{tool: tool})
|
||||
}
|
||||
|
||||
// parseExtractorChainSettings parses the settings JSON and builds a chain.
|
||||
func parseExtractorChainSettings(raw []byte, tool *WebFetchTool) *ExtractorChain {
|
||||
var settings extractorChainSettings
|
||||
if err := json.Unmarshal(raw, &settings); err != nil {
|
||||
slog.Warn("web_fetch: failed to parse extractor chain settings", "error", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
var extractors []ContentExtractor
|
||||
for _, entry := range settings.Extractors {
|
||||
if !entry.Enabled || entry.Name == "" {
|
||||
continue
|
||||
}
|
||||
switch entry.Name {
|
||||
case "defuddle":
|
||||
extractors = append(extractors, NewDefuddleExtractorFromEntry(entry))
|
||||
case "html-to-markdown":
|
||||
extractors = append(extractors, &InProcessExtractor{tool: tool})
|
||||
default:
|
||||
slog.Warn("web_fetch: unknown extractor in chain, skipping", "name", entry.Name)
|
||||
}
|
||||
}
|
||||
if len(extractors) == 0 {
|
||||
return nil
|
||||
}
|
||||
return NewExtractorChain(extractors...)
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defuddleBaseURL = "https://fetch.goclaw.sh/"
|
||||
defuddleTimeout = 10 * time.Second
|
||||
defuddleMaxBody = 1 << 20 // 1MB
|
||||
)
|
||||
|
||||
// DefuddleExtractor calls the fetch.goclaw.sh Cloudflare Worker to extract
|
||||
// clean markdown via Defuddle. The CF Worker handles HTTP fetch + content extraction.
|
||||
type DefuddleExtractor struct {
|
||||
baseURL string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewDefuddleExtractorFromEntry creates a DefuddleExtractor from chain settings.
|
||||
func NewDefuddleExtractorFromEntry(entry ExtractorEntry) *DefuddleExtractor {
|
||||
baseURL := entry.BaseURL
|
||||
if baseURL == "" {
|
||||
baseURL = defuddleBaseURL
|
||||
}
|
||||
// Ensure trailing slash for URL construction.
|
||||
if !strings.HasSuffix(baseURL, "/") {
|
||||
baseURL += "/"
|
||||
}
|
||||
timeout := time.Duration(entry.Timeout) * time.Second
|
||||
if timeout <= 0 {
|
||||
timeout = defuddleTimeout
|
||||
}
|
||||
return newDefuddleExtractor(baseURL, timeout)
|
||||
}
|
||||
|
||||
// newDefuddleExtractor creates a DefuddleExtractor with custom base URL and timeout (for testing).
|
||||
func newDefuddleExtractor(baseURL string, timeout time.Duration) *DefuddleExtractor {
|
||||
return &DefuddleExtractor{
|
||||
baseURL: baseURL,
|
||||
client: &http.Client{
|
||||
Timeout: timeout,
|
||||
Transport: &http.Transport{
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 5,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *DefuddleExtractor) Name() string { return "defuddle" }
|
||||
|
||||
// Extract sends a GET request to fetch.goclaw.sh/<domain>/<path> (no scheme)
|
||||
// and returns the plain markdown response.
|
||||
func (d *DefuddleExtractor) Extract(ctx context.Context, rawURL string) (string, error) {
|
||||
// Strip scheme: https://example.com/path → example.com/path
|
||||
target := strings.TrimPrefix(strings.TrimPrefix(rawURL, "https://"), "http://")
|
||||
fetchURL := d.baseURL + target
|
||||
|
||||
// Context timeout ensures cancellation propagates even if transport ignores client.Timeout.
|
||||
ctx, cancel := context.WithTimeout(ctx, d.client.Timeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fetchURL, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create defuddle request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", fetchUserAgent)
|
||||
|
||||
resp, err := d.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("defuddle fetch: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("defuddle returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, defuddleMaxBody))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read defuddle response: %w", err)
|
||||
}
|
||||
|
||||
return string(body), nil
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// InProcessExtractor delegates to WebFetchTool.fetchRawContent for HTML→markdown
|
||||
// extraction with full security checks (SSRF, domain policy on redirects).
|
||||
// This is the fallback when external extractors (Defuddle) are unavailable.
|
||||
type InProcessExtractor struct {
|
||||
tool *WebFetchTool
|
||||
}
|
||||
|
||||
func (e *InProcessExtractor) Name() string { return "html-to-markdown" }
|
||||
|
||||
// Extract fetches the URL via the tool's fetchRawContent (full security checks)
|
||||
// and returns the raw extracted markdown content.
|
||||
func (e *InProcessExtractor) Extract(ctx context.Context, rawURL string) (string, error) {
|
||||
e.tool.mu.RLock()
|
||||
policy := e.tool.policy
|
||||
e.tool.mu.RUnlock()
|
||||
|
||||
raw, err := e.tool.fetchRawContent(ctx, rawURL, "markdown", defaultFetchMaxChars, policy)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return raw.content, nil
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// --- Quality Gate Tests ---
|
||||
|
||||
func TestIsQualityContent(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
want bool
|
||||
}{
|
||||
{"empty", "", false},
|
||||
{"short", "hello world", false},
|
||||
{"under_100_chars", strings.Repeat("x", 99), false},
|
||||
{"100_chars_few_words", strings.Repeat("x", 100), false}, // 1 word
|
||||
{"whitespace_heavy", strings.Repeat(" a ", 30), true}, // 30 words, >100 chars
|
||||
{"valid_paragraph", "The quick brown fox jumps over the lazy dog. This is a sample paragraph with enough words and characters to pass the quality threshold for content extraction.", true},
|
||||
{"exactly_10_words_short", "one two three four five six seven eight nine ten", false}, // <100 chars
|
||||
{"10_words_long_enough", "word1_padding word2_padding word3_padding word4_padding word5_padding word6_padding word7_padding word8_padding word9_padding word10_padding", true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := isQualityContent(tt.content)
|
||||
if got != tt.want {
|
||||
t.Errorf("isQualityContent(%d chars, %d words) = %v, want %v",
|
||||
len(tt.content), len(strings.Fields(tt.content)), got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- Mock Extractor ---
|
||||
|
||||
type mockExtractor struct {
|
||||
name string
|
||||
content string
|
||||
err error
|
||||
}
|
||||
|
||||
func (m *mockExtractor) Name() string { return m.name }
|
||||
func (m *mockExtractor) Extract(_ context.Context, _ string) (string, error) {
|
||||
if m.err != nil {
|
||||
return "", m.err
|
||||
}
|
||||
return m.content, nil
|
||||
}
|
||||
|
||||
// qualityContent returns a string that passes isQualityContent.
|
||||
func qualityContent() string {
|
||||
return "This is a sufficiently long paragraph with more than ten words and well over one hundred characters to satisfy the quality content threshold check in the extractor chain."
|
||||
}
|
||||
|
||||
// --- ExtractorChain Tests ---
|
||||
|
||||
func TestExtractorChain_FirstSuccess(t *testing.T) {
|
||||
chain := NewExtractorChain(
|
||||
&mockExtractor{name: "first", content: qualityContent()},
|
||||
&mockExtractor{name: "second", content: "should not reach"},
|
||||
)
|
||||
result, err := chain.Extract(context.Background(), "https://example.com")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if result.Extractor != "first" {
|
||||
t.Errorf("expected extractor 'first', got %q", result.Extractor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorChain_FirstFailsSecondSucceeds(t *testing.T) {
|
||||
chain := NewExtractorChain(
|
||||
&mockExtractor{name: "first", err: fmt.Errorf("network error")},
|
||||
&mockExtractor{name: "second", content: qualityContent()},
|
||||
)
|
||||
result, err := chain.Extract(context.Background(), "https://example.com")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if result.Extractor != "second" {
|
||||
t.Errorf("expected extractor 'second', got %q", result.Extractor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorChain_FirstLowQualitySecondSucceeds(t *testing.T) {
|
||||
chain := NewExtractorChain(
|
||||
&mockExtractor{name: "first", content: "too short"},
|
||||
&mockExtractor{name: "second", content: qualityContent()},
|
||||
)
|
||||
result, err := chain.Extract(context.Background(), "https://example.com")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if result.Extractor != "second" {
|
||||
t.Errorf("expected extractor 'second', got %q", result.Extractor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorChain_AllFail(t *testing.T) {
|
||||
chain := NewExtractorChain(
|
||||
&mockExtractor{name: "first", err: fmt.Errorf("fail1")},
|
||||
&mockExtractor{name: "second", err: fmt.Errorf("fail2")},
|
||||
)
|
||||
_, err := chain.Extract(context.Background(), "https://example.com")
|
||||
if err == nil {
|
||||
t.Fatal("expected error when all extractors fail")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "all extractors failed") {
|
||||
t.Errorf("unexpected error message: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorChain_AllLowQuality(t *testing.T) {
|
||||
chain := NewExtractorChain(
|
||||
&mockExtractor{name: "first", content: "short"},
|
||||
&mockExtractor{name: "second", content: "also short"},
|
||||
)
|
||||
_, err := chain.Extract(context.Background(), "https://example.com")
|
||||
if err == nil {
|
||||
t.Fatal("expected error when all extractors return low quality")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorChain_Empty(t *testing.T) {
|
||||
chain := NewExtractorChain()
|
||||
_, err := chain.Extract(context.Background(), "https://example.com")
|
||||
if err == nil {
|
||||
t.Fatal("expected error with empty chain")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no extractors configured") {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorChain_SingleSuccess(t *testing.T) {
|
||||
chain := NewExtractorChain(
|
||||
&mockExtractor{name: "only", content: qualityContent()},
|
||||
)
|
||||
result, err := chain.Extract(context.Background(), "https://example.com")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if result.Extractor != "only" {
|
||||
t.Errorf("expected 'only', got %q", result.Extractor)
|
||||
}
|
||||
}
|
||||
|
||||
// --- DefuddleExtractor Tests ---
|
||||
|
||||
// newTestDefuddleExtractor creates a DefuddleExtractor pointing at a test server.
|
||||
func newTestDefuddleExtractor(serverURL string) *DefuddleExtractor {
|
||||
return newDefuddleExtractor(serverURL+"/", defuddleTimeout)
|
||||
}
|
||||
|
||||
func TestDefuddleExtractor_Success(t *testing.T) {
|
||||
expected := qualityContent()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/example.com/page" {
|
||||
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/markdown")
|
||||
w.Write([]byte(expected))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
ext := newTestDefuddleExtractor(server.URL)
|
||||
result, err := ext.Extract(context.Background(), "https://example.com/page")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if result != expected {
|
||||
t.Errorf("content mismatch: got %d chars, want %d chars", len(result), len(expected))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefuddleExtractor_URLConstruction(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expectedPath string
|
||||
}{
|
||||
{"https://example.com/page", "/example.com/page"},
|
||||
{"http://example.com/page", "/example.com/page"},
|
||||
{"https://x.com/user/status/123", "/x.com/user/status/123"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
var gotPath string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
w.Write([]byte(qualityContent()))
|
||||
}))
|
||||
ext := newTestDefuddleExtractor(server.URL)
|
||||
_, _ = ext.Extract(context.Background(), tt.input)
|
||||
server.Close()
|
||||
|
||||
if gotPath != tt.expectedPath {
|
||||
t.Errorf("input=%q: got path %q, want %q", tt.input, gotPath, tt.expectedPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefuddleExtractor_HTTP404(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
ext := newTestDefuddleExtractor(server.URL)
|
||||
_, err := ext.Extract(context.Background(), "https://example.com/missing")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for 404 response")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "status 404") {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefuddleExtractor_HTTP500(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
ext := newTestDefuddleExtractor(server.URL)
|
||||
_, err := ext.Extract(context.Background(), "https://example.com/error")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for 500 response")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "status 500") {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefuddleExtractor_EmptyBody(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
ext := newTestDefuddleExtractor(server.URL)
|
||||
result, err := ext.Extract(context.Background(), "https://example.com/empty")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if result != "" {
|
||||
t.Errorf("expected empty result, got %q", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefuddleExtractorFromEntry(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Write([]byte(qualityContent()))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
ext := NewDefuddleExtractorFromEntry(ExtractorEntry{
|
||||
Name: "defuddle",
|
||||
Enabled: true,
|
||||
Timeout: 5,
|
||||
BaseURL: server.URL + "/",
|
||||
})
|
||||
result, err := ext.Extract(context.Background(), "https://example.com/page")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if result != qualityContent() {
|
||||
t.Errorf("content mismatch")
|
||||
}
|
||||
// Verify timeout was applied
|
||||
if ext.client.Timeout != 5*time.Second {
|
||||
t.Errorf("expected 5s timeout, got %v", ext.client.Timeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefuddleExtractorFromEntry_Defaults(t *testing.T) {
|
||||
ext := NewDefuddleExtractorFromEntry(ExtractorEntry{Name: "defuddle", Enabled: true})
|
||||
if ext.baseURL != defuddleBaseURL {
|
||||
t.Errorf("expected default base URL %q, got %q", defuddleBaseURL, ext.baseURL)
|
||||
}
|
||||
if ext.client.Timeout != defuddleTimeout {
|
||||
t.Errorf("expected default timeout %v, got %v", defuddleTimeout, ext.client.Timeout)
|
||||
}
|
||||
}
|
||||
|
||||
// --- InProcessExtractor Tests (delegate to fetchRawContent) ---
|
||||
|
||||
func TestInProcessExtractor_HTML(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
w.Write([]byte(`<html><body><h1>Hello World</h1><p>This is a test paragraph with enough content to be meaningful and pass quality checks.</p></body></html>`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tool := NewWebFetchTool(WebFetchConfig{})
|
||||
ext := &InProcessExtractor{tool: tool}
|
||||
result, err := ext.Extract(context.Background(), server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !strings.Contains(result, "Hello World") {
|
||||
t.Errorf("expected heading content, got: %s", result)
|
||||
}
|
||||
if !strings.Contains(result, "test paragraph") {
|
||||
t.Errorf("expected paragraph content, got: %s", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInProcessExtractor_JSON(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Write([]byte(`{"key":"value","nested":{"a":1}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tool := NewWebFetchTool(WebFetchConfig{})
|
||||
ext := &InProcessExtractor{tool: tool}
|
||||
result, err := ext.Extract(context.Background(), server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !strings.Contains(result, `"key": "value"`) {
|
||||
t.Errorf("expected pretty-printed JSON, got: %s", result)
|
||||
}
|
||||
}
|
||||
|
||||
// --- ResolveExtractorChain Tests ---
|
||||
|
||||
func TestResolveExtractorChain_FromSettings(t *testing.T) {
|
||||
settings := `{"extractors":[{"name":"defuddle","enabled":true,"timeout":5,"base_url":"https://test.example.com/"},{"name":"html-to-markdown","enabled":true}]}`
|
||||
ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{
|
||||
"web_fetch": json.RawMessage(settings),
|
||||
})
|
||||
tool := NewWebFetchTool(WebFetchConfig{})
|
||||
chain := ResolveExtractorChain(ctx, tool)
|
||||
if chain == nil {
|
||||
t.Fatal("expected non-nil chain")
|
||||
}
|
||||
if len(chain.extractors) != 2 {
|
||||
t.Fatalf("expected 2 extractors, got %d", len(chain.extractors))
|
||||
}
|
||||
if chain.extractors[0].Name() != "defuddle" {
|
||||
t.Errorf("first extractor should be 'defuddle', got %q", chain.extractors[0].Name())
|
||||
}
|
||||
if chain.extractors[1].Name() != "html-to-markdown" {
|
||||
t.Errorf("second extractor should be 'html-to-markdown', got %q", chain.extractors[1].Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveExtractorChain_DisabledExtractor(t *testing.T) {
|
||||
settings := `{"extractors":[{"name":"defuddle","enabled":false},{"name":"html-to-markdown","enabled":true}]}`
|
||||
ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{
|
||||
"web_fetch": json.RawMessage(settings),
|
||||
})
|
||||
tool := NewWebFetchTool(WebFetchConfig{})
|
||||
chain := ResolveExtractorChain(ctx, tool)
|
||||
if chain == nil {
|
||||
t.Fatal("expected non-nil chain")
|
||||
}
|
||||
if len(chain.extractors) != 1 {
|
||||
t.Fatalf("expected 1 extractor (defuddle disabled), got %d", len(chain.extractors))
|
||||
}
|
||||
if chain.extractors[0].Name() != "html-to-markdown" {
|
||||
t.Errorf("expected 'html-to-markdown', got %q", chain.extractors[0].Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveExtractorChain_NoSettings(t *testing.T) {
|
||||
tool := NewWebFetchTool(WebFetchConfig{})
|
||||
chain := ResolveExtractorChain(context.Background(), tool)
|
||||
if chain == nil {
|
||||
t.Fatal("expected default chain")
|
||||
}
|
||||
if len(chain.extractors) != 1 {
|
||||
t.Fatalf("expected 1 default extractor, got %d", len(chain.extractors))
|
||||
}
|
||||
if chain.extractors[0].Name() != "html-to-markdown" {
|
||||
t.Errorf("expected 'html-to-markdown' default, got %q", chain.extractors[0].Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveExtractorChain_UnknownExtractor(t *testing.T) {
|
||||
settings := `{"extractors":[{"name":"unknown-thing","enabled":true},{"name":"html-to-markdown","enabled":true}]}`
|
||||
ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{
|
||||
"web_fetch": json.RawMessage(settings),
|
||||
})
|
||||
tool := NewWebFetchTool(WebFetchConfig{})
|
||||
chain := ResolveExtractorChain(ctx, tool)
|
||||
if len(chain.extractors) != 1 {
|
||||
t.Fatalf("expected 1 extractor (unknown skipped), got %d", len(chain.extractors))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveExtractorChain_MalformedJSON(t *testing.T) {
|
||||
ctx := WithBuiltinToolSettings(context.Background(), BuiltinToolSettings{
|
||||
"web_fetch": []byte(`not valid json`),
|
||||
})
|
||||
tool := NewWebFetchTool(WebFetchConfig{})
|
||||
chain := ResolveExtractorChain(ctx, tool)
|
||||
// Malformed JSON falls through to default InProcess chain.
|
||||
if chain == nil {
|
||||
t.Fatal("expected default chain on malformed JSON")
|
||||
}
|
||||
if len(chain.extractors) != 1 || chain.extractors[0].Name() != "html-to-markdown" {
|
||||
t.Errorf("expected single html-to-markdown fallback, got %d extractors", len(chain.extractors))
|
||||
}
|
||||
}
|
||||
|
||||
// --- formatFetchResult Tests ---
|
||||
|
||||
func TestFormatFetchResult_Basic(t *testing.T) {
|
||||
result := formatFetchResult("hello world", "defuddle", "https://example.com", 60000, context.Background())
|
||||
if !strings.Contains(result, "URL: https://example.com") {
|
||||
t.Error("missing URL in result")
|
||||
}
|
||||
if !strings.Contains(result, "Extractor: defuddle") {
|
||||
t.Error("missing extractor name in result")
|
||||
}
|
||||
if !strings.Contains(result, "hello world") {
|
||||
t.Error("missing content in result")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatFetchResult_Truncation(t *testing.T) {
|
||||
longContent := strings.Repeat("x", 200)
|
||||
result := formatFetchResult(longContent, "test", "https://example.com", 100, context.Background())
|
||||
if !strings.Contains(result, "Truncated: true") && !strings.Contains(result, "Content-Length:") {
|
||||
if !strings.Contains(result, "Content truncated") {
|
||||
t.Error("expected truncation indicator in result")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,6 +132,21 @@
|
||||
"save": "Save",
|
||||
"saving": "Saving..."
|
||||
},
|
||||
"extractorChain": {
|
||||
"title": "Web Fetch — Extractor Chain",
|
||||
"description": "Ordered list of content extractors. Each URL is tried top-to-bottom until one returns quality content.",
|
||||
"timeout": "Timeout",
|
||||
"baseUrl": "Base URL",
|
||||
"save": "Save",
|
||||
"saving": "Saving…",
|
||||
"cancel": "Cancel",
|
||||
"extractors": {
|
||||
"defuddle": "Defuddle (Cloudflare Worker)",
|
||||
"defuddleDesc": "Extracts clean markdown via fetch.goclaw.sh using the Defuddle library.",
|
||||
"html-to-markdown": "Built-in HTML→Markdown",
|
||||
"html-to-markdownDesc": "In-process DOM-based converter with hidden element detection."
|
||||
}
|
||||
},
|
||||
"mediaChain": {
|
||||
"providerChainSuffix": "— Provider Chain",
|
||||
"description": "Configure and order provider fallbacks. Drag to reorder — the first enabled provider is tried first.",
|
||||
|
||||
@@ -132,6 +132,21 @@
|
||||
"save": "Lưu",
|
||||
"saving": "Đang lưu..."
|
||||
},
|
||||
"extractorChain": {
|
||||
"title": "Web Fetch — Chuỗi bộ trích xuất",
|
||||
"description": "Danh sách bộ trích xuất nội dung theo thứ tự. Mỗi URL được thử từ trên xuống cho đến khi một bộ trả về nội dung chất lượng.",
|
||||
"timeout": "Thời gian chờ",
|
||||
"baseUrl": "URL cơ sở",
|
||||
"save": "Lưu",
|
||||
"saving": "Đang lưu…",
|
||||
"cancel": "Hủy",
|
||||
"extractors": {
|
||||
"defuddle": "Defuddle (Cloudflare Worker)",
|
||||
"defuddleDesc": "Trích xuất markdown sạch qua fetch.goclaw.sh sử dụng thư viện Defuddle.",
|
||||
"html-to-markdown": "HTML→Markdown tích hợp",
|
||||
"html-to-markdownDesc": "Bộ chuyển đổi dựa trên DOM trong tiến trình với khả năng phát hiện phần tử ẩn."
|
||||
}
|
||||
},
|
||||
"mediaChain": {
|
||||
"providerChainSuffix": "— Chuỗi provider",
|
||||
"description": "Cấu hình và sắp xếp các provider dự phòng. Kéo để sắp xếp lại — provider đầu tiên được thử trước.",
|
||||
|
||||
@@ -46,6 +46,21 @@
|
||||
"saving": "保存中...",
|
||||
"title": "知识图谱设置"
|
||||
},
|
||||
"extractorChain": {
|
||||
"title": "Web Fetch — 提取器链",
|
||||
"description": "有序内容提取器列表。每个 URL 从上到下依次尝试,直到某个提取器返回高质量内容。",
|
||||
"timeout": "超时",
|
||||
"baseUrl": "基础 URL",
|
||||
"save": "保存",
|
||||
"saving": "保存中…",
|
||||
"cancel": "取消",
|
||||
"extractors": {
|
||||
"defuddle": "Defuddle(Cloudflare Worker)",
|
||||
"defuddleDesc": "通过 fetch.goclaw.sh 使用 Defuddle 库提取干净的 Markdown。",
|
||||
"html-to-markdown": "内置 HTML→Markdown",
|
||||
"html-to-markdownDesc": "进程内基于 DOM 的转换器,具备隐藏元素检测功能。"
|
||||
}
|
||||
},
|
||||
"mediaChain": {
|
||||
"addProvider": "添加Provider",
|
||||
"cancel": "取消",
|
||||
|
||||
@@ -14,8 +14,10 @@ import type { BuiltinToolData } from "./hooks/use-builtin-tools";
|
||||
import { MEDIA_TOOLS } from "./media-provider-params-schema";
|
||||
import { MediaProviderChainForm } from "./media-provider-chain-form";
|
||||
import { KGSettingsForm } from "./kg-settings-form";
|
||||
import { WebFetchExtractorChainForm } from "./web-fetch-extractor-chain-form";
|
||||
|
||||
const KG_TOOL = "knowledge_graph_search";
|
||||
const WEB_FETCH_TOOL = "web_fetch";
|
||||
|
||||
interface Props {
|
||||
tool: BuiltinToolData | null;
|
||||
@@ -27,12 +29,19 @@ interface Props {
|
||||
export function BuiltinToolSettingsDialog({ tool, open, onOpenChange, onSave }: Props) {
|
||||
const isMedia = tool ? MEDIA_TOOLS.has(tool.name) : false;
|
||||
const isKG = tool?.name === KG_TOOL;
|
||||
const wide = isMedia || isKG;
|
||||
const isWebFetch = tool?.name === WEB_FETCH_TOOL;
|
||||
const wide = isMedia || isKG || isWebFetch;
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||||
<DialogContent className={wide ? "sm:max-w-2xl" : "sm:max-w-md"}>
|
||||
{isMedia && tool ? (
|
||||
{isWebFetch && tool ? (
|
||||
<WebFetchExtractorChainForm
|
||||
initialSettings={tool.settings ?? {}}
|
||||
onSave={(settings) => onSave(tool.name, settings).then(() => onOpenChange(false))}
|
||||
onCancel={() => onOpenChange(false)}
|
||||
/>
|
||||
) : isMedia && tool ? (
|
||||
<MediaProviderChainForm
|
||||
toolName={tool.name}
|
||||
initialSettings={tool.settings ?? {}}
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
import { useState, useCallback } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import {
|
||||
DndContext,
|
||||
closestCenter,
|
||||
KeyboardSensor,
|
||||
PointerSensor,
|
||||
useSensor,
|
||||
useSensors,
|
||||
type DragEndEvent,
|
||||
} from "@dnd-kit/core";
|
||||
import {
|
||||
SortableContext,
|
||||
verticalListSortingStrategy,
|
||||
useSortable,
|
||||
arrayMove,
|
||||
sortableKeyboardCoordinates,
|
||||
} from "@dnd-kit/sortable";
|
||||
import { CSS } from "@dnd-kit/utilities";
|
||||
import { GripVertical } from "lucide-react";
|
||||
import { uniqueId } from "@/lib/utils";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { DialogHeader, DialogTitle, DialogDescription, DialogFooter } from "@/components/ui/dialog";
|
||||
|
||||
interface ExtractorEntry {
|
||||
id: string;
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
timeout: number;
|
||||
base_url: string;
|
||||
}
|
||||
|
||||
interface Props {
|
||||
initialSettings: Record<string, unknown>;
|
||||
onSave: (settings: Record<string, unknown>) => Promise<void>;
|
||||
onCancel: () => void;
|
||||
}
|
||||
|
||||
const FIXED_EXTRACTORS = ["defuddle", "html-to-markdown"] as const;
|
||||
|
||||
function parseInitialEntries(settings: Record<string, unknown>): ExtractorEntry[] {
|
||||
const raw = Array.isArray(settings.extractors)
|
||||
? (settings.extractors as Record<string, unknown>[])
|
||||
: [];
|
||||
|
||||
// Build a map from name → raw entry for fast lookup
|
||||
const byName = new Map(raw.map((e) => [String(e.name ?? ""), e]));
|
||||
|
||||
return FIXED_EXTRACTORS.map((name) => {
|
||||
const e = byName.get(name) ?? {};
|
||||
return {
|
||||
id: uniqueId(),
|
||||
name,
|
||||
enabled: Boolean(e.enabled ?? true),
|
||||
timeout: Number(e.timeout ?? 0),
|
||||
base_url: String(e.base_url ?? ""),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
interface SortableCardProps {
|
||||
entry: ExtractorEntry;
|
||||
index: number;
|
||||
onUpdate: (id: string, patch: Partial<ExtractorEntry>) => void;
|
||||
}
|
||||
|
||||
function SortableExtractorCard({ entry, index, onUpdate }: SortableCardProps) {
|
||||
const { t } = useTranslation("tools");
|
||||
const { attributes, listeners, setNodeRef, transform, transition, isDragging } = useSortable({
|
||||
id: entry.id,
|
||||
});
|
||||
|
||||
const style = {
|
||||
transform: CSS.Transform.toString(transform),
|
||||
transition,
|
||||
opacity: isDragging ? 0.5 : 1,
|
||||
};
|
||||
|
||||
const displayName = t(`builtin.extractorChain.extractors.${entry.name}`, {
|
||||
defaultValue: entry.name,
|
||||
});
|
||||
const desc = t(`builtin.extractorChain.extractors.${entry.name}Desc`, {
|
||||
defaultValue: "",
|
||||
});
|
||||
const showTimeout = entry.timeout > 0 || entry.name === "defuddle";
|
||||
const showBaseUrl = entry.name === "defuddle";
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={setNodeRef}
|
||||
style={style}
|
||||
className={`border rounded-lg bg-card ${!entry.enabled ? "opacity-60" : ""}`}
|
||||
>
|
||||
{/* Row 1: drag handle, index, toggle, name */}
|
||||
<div className="flex items-center gap-2 px-3 pt-3 pb-1">
|
||||
<button
|
||||
type="button"
|
||||
className="cursor-grab text-muted-foreground hover:text-foreground shrink-0"
|
||||
{...attributes}
|
||||
{...listeners}
|
||||
>
|
||||
<GripVertical className="size-4" />
|
||||
</button>
|
||||
|
||||
<span className="text-xs text-muted-foreground font-mono shrink-0">
|
||||
#{index + 1}
|
||||
</span>
|
||||
|
||||
<Switch
|
||||
size="sm"
|
||||
checked={entry.enabled}
|
||||
onCheckedChange={(v) => onUpdate(entry.id, { enabled: v })}
|
||||
/>
|
||||
|
||||
<div className="flex-1 min-w-0">
|
||||
<span className="text-sm font-medium">{displayName}</span>
|
||||
{desc && (
|
||||
<p className="text-xs text-muted-foreground mt-0.5 truncate">{desc}</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Optional fields */}
|
||||
{(showTimeout || showBaseUrl) && (
|
||||
<div className="flex flex-wrap items-center gap-3 px-3 pb-3 pt-1">
|
||||
{showTimeout && (
|
||||
<div className="flex items-center gap-1.5">
|
||||
<Label className="text-xs text-muted-foreground whitespace-nowrap">
|
||||
{t("builtin.extractorChain.timeout")}
|
||||
</Label>
|
||||
<div className="relative">
|
||||
<Input
|
||||
type="number"
|
||||
min={0}
|
||||
max={600}
|
||||
value={entry.timeout}
|
||||
onChange={(e) => onUpdate(entry.id, { timeout: Number(e.target.value) })}
|
||||
className="h-7 w-20 text-base md:text-sm pr-5"
|
||||
/>
|
||||
<span className="absolute right-2 top-1/2 -translate-y-1/2 text-xs text-muted-foreground pointer-events-none">
|
||||
s
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{showBaseUrl && (
|
||||
<div className="flex items-center gap-1.5 flex-1 min-w-[200px]">
|
||||
<Label className="text-xs text-muted-foreground whitespace-nowrap">
|
||||
{t("builtin.extractorChain.baseUrl")}
|
||||
</Label>
|
||||
<Input
|
||||
value={entry.base_url}
|
||||
onChange={(e) => onUpdate(entry.id, { base_url: e.target.value })}
|
||||
placeholder="https://fetch.goclaw.sh/"
|
||||
className="h-7 text-base md:text-sm"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function WebFetchExtractorChainForm({ initialSettings, onSave, onCancel }: Props) {
|
||||
const { t } = useTranslation("tools");
|
||||
const [entries, setEntries] = useState<ExtractorEntry[]>(() =>
|
||||
parseInitialEntries(initialSettings),
|
||||
);
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
const sensors = useSensors(
|
||||
useSensor(PointerSensor),
|
||||
useSensor(KeyboardSensor, { coordinateGetter: sortableKeyboardCoordinates }),
|
||||
);
|
||||
|
||||
const handleDragEnd = useCallback((event: DragEndEvent) => {
|
||||
const { active, over } = event;
|
||||
if (over && active.id !== over.id) {
|
||||
setEntries((prev) => {
|
||||
const oldIndex = prev.findIndex((e) => e.id === active.id);
|
||||
const newIndex = prev.findIndex((e) => e.id === over.id);
|
||||
return arrayMove(prev, oldIndex, newIndex);
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handleUpdate = useCallback((id: string, patch: Partial<ExtractorEntry>) => {
|
||||
setEntries((prev) => prev.map((e) => (e.id === id ? { ...e, ...patch } : e)));
|
||||
}, []);
|
||||
|
||||
const handleSave = async () => {
|
||||
setSaving(true);
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
const serialized = entries.map(({ id: _id, ...rest }) => rest);
|
||||
await onSave({ extractors: serialized });
|
||||
} finally {
|
||||
setSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div>
|
||||
<DialogHeader>
|
||||
<DialogTitle>{t("builtin.extractorChain.title")}</DialogTitle>
|
||||
<DialogDescription>{t("builtin.extractorChain.description")}</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="space-y-2 max-h-[60vh] overflow-y-auto pr-1 my-4">
|
||||
<DndContext sensors={sensors} collisionDetection={closestCenter} onDragEnd={handleDragEnd}>
|
||||
<SortableContext items={entries.map((e) => e.id)} strategy={verticalListSortingStrategy}>
|
||||
{entries.map((entry, index) => (
|
||||
<SortableExtractorCard
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
index={index}
|
||||
onUpdate={handleUpdate}
|
||||
/>
|
||||
))}
|
||||
</SortableContext>
|
||||
</DndContext>
|
||||
|
||||
</div>
|
||||
|
||||
<DialogFooter>
|
||||
<Button variant="outline" onClick={onCancel}>
|
||||
{t("builtin.extractorChain.cancel")}
|
||||
</Button>
|
||||
<Button onClick={handleSave} disabled={saving}>
|
||||
{saving ? t("builtin.extractorChain.saving") : t("builtin.extractorChain.save")}
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user