diff --git a/cmd/gateway_builtin_tools.go b/cmd/gateway_builtin_tools.go index 9a182d61..fa98b727 100644 --- a/cmd/gateway_builtin_tools.go +++ b/cmd/gateway_builtin_tools.go @@ -29,7 +29,7 @@ func builtinToolSeedData() []store.BuiltinToolDef { Metadata: json.RawMessage(`{"config_hint":"Config → Tools → Web Search"}`), }, {Name: "web_fetch", DisplayName: "Web Fetch", Description: "Fetch a web page or API endpoint and extract its text content", Category: "web", Enabled: true, - Settings: json.RawMessage(`{"extractors":[{"name":"defuddle","enabled":true,"base_url":"https://fetch.goclaw.sh/"},{"name":"html-to-markdown","enabled":true}]}`), + Settings: json.RawMessage(`{"extractors":[{"name":"defuddle","enabled":true,"base_url":"https://fetch.goclaw.sh/","max_retries":2},{"name":"html-to-markdown","enabled":true}]}`), }, // memory @@ -210,7 +210,7 @@ func backfillWebFetchSettings(ctx context.Context, bts store.BuiltinToolStore) { if len(t.Settings) > 0 && string(t.Settings) != "{}" && string(t.Settings) != "null" { return // already has settings, don't overwrite } - defaultSettings := json.RawMessage(`{"extractors":[{"name":"defuddle","enabled":true,"base_url":"https://fetch.goclaw.sh/"},{"name":"html-to-markdown","enabled":true}]}`) + defaultSettings := json.RawMessage(`{"extractors":[{"name":"defuddle","enabled":true,"base_url":"https://fetch.goclaw.sh/","max_retries":2},{"name":"html-to-markdown","enabled":true}]}`) if err := bts.Update(ctx, "web_fetch", map[string]any{"settings": defaultSettings}); err != nil { slog.Warn("builtin_tools: failed to backfill web_fetch settings", "error", err) return diff --git a/internal/tools/media_provider_chain.go b/internal/tools/media_provider_chain.go index 72bc7d6c..9e9d04f0 100644 --- a/internal/tools/media_provider_chain.go +++ b/internal/tools/media_provider_chain.go @@ -298,7 +298,6 @@ type typedProvider interface { // used by callProvider switch statements. var dbTypeToMediaType = map[string]string{ "gemini_native": "gemini", - "openai_compat": "openai_compat", "openrouter": "openrouter", "minimax_native": "minimax", "dashscope": "dashscope", @@ -310,16 +309,20 @@ var dbTypeToMediaType = map[string]string{ // ResolveProviderType returns the media routing type for a provider. // It first checks the provider's DB type (via typedProvider interface), // then falls back to name-based heuristics for config-registered providers. +// Generic DB types like "openai_compat" are skipped in favor of name-based +// inference, since different openai_compat providers (OpenRouter, etc.) +// need different media API endpoints. func ResolveProviderType(p providers.Provider) string { - // Prefer the actual DB provider_type when available + // Prefer the actual DB provider_type when available, + // but skip generic types that don't distinguish media routing. if tp, ok := p.(typedProvider); ok { - if pt := tp.ProviderType(); pt != "" { + if pt := tp.ProviderType(); pt != "" && pt != "openai_compat" { if mt, found := dbTypeToMediaType[pt]; found { return mt } } } - // Fallback: infer from provider name (for config-registered providers) + // Fallback: infer from provider name (for config-registered and openai_compat providers) return providerTypeFromName(p.Name()) } diff --git a/internal/tools/web_fetch_extractor.go b/internal/tools/web_fetch_extractor.go index 041b8f79..1aa6a56a 100644 --- a/internal/tools/web_fetch_extractor.go +++ b/internal/tools/web_fetch_extractor.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "strings" + "time" ) // ContentExtractor extracts readable content from a URL. @@ -23,29 +24,63 @@ type ExtractResult struct { // ExtractorChain tries extractors in order until one returns quality content. type ExtractorChain struct { extractors []ContentExtractor + maxRetries []int // per-extractor max attempts (default 1) + timeouts []time.Duration // per-extractor chain-level timeout (0 = no chain timeout) } -// NewExtractorChain creates a chain from ordered extractors. +// NewExtractorChain creates a chain from ordered extractors with default settings (1 attempt, no chain timeout). func NewExtractorChain(extractors ...ContentExtractor) *ExtractorChain { - return &ExtractorChain{extractors: extractors} + maxRetries := make([]int, len(extractors)) + timeouts := make([]time.Duration, len(extractors)) + for i := range extractors { + maxRetries[i] = 1 + } + return &ExtractorChain{extractors: extractors, maxRetries: maxRetries, timeouts: timeouts} } -// Extract runs each extractor in order, returning the first quality result. +// Extract runs each extractor in order with per-entry retry and optional timeout. +// Returns the first quality result or cascades to the next extractor. func (c *ExtractorChain) Extract(ctx context.Context, rawURL string) (ExtractResult, error) { var lastErr error - for _, ext := range c.extractors { - content, err := ext.Extract(ctx, rawURL) - if err != nil { - slog.Debug("extractor failed", "extractor", ext.Name(), "url", rawURL, "error", err) - lastErr = err - continue + for i, ext := range c.extractors { + maxRetries := c.maxRetries[i] + + for attempt := 1; attempt <= maxRetries; attempt++ { + // Apply chain-level timeout if configured. + callCtx, cancel := ctx, context.CancelFunc(nil) + if timeout := c.timeouts[i]; timeout > 0 { + callCtx, cancel = context.WithTimeout(ctx, timeout) + } + + content, err := ext.Extract(callCtx, rawURL) + if cancel != nil { + cancel() + } + if err != nil { + lastErr = err + if ctx.Err() != nil { + return ExtractResult{}, fmt.Errorf("context cancelled: %w", lastErr) + } + if attempt < maxRetries { + slog.Warn("extractor_chain: attempt failed, retrying", + "extractor", ext.Name(), "url", rawURL, + "attempt", attempt, "max_retries", maxRetries, + "error", err) + } else { + slog.Debug("extractor failed", "extractor", ext.Name(), "url", rawURL, "error", err) + } + continue + } + if !isQualityContent(content) { + slog.Debug("extractor returned low quality content", "extractor", ext.Name(), "url", rawURL, "chars", len(content)) + lastErr = fmt.Errorf("%s: content below quality threshold (%d chars)", ext.Name(), len(content)) + break // low quality is not transient — don't retry, cascade to next + } + return ExtractResult{Content: content, Extractor: ext.Name()}, nil } - if !isQualityContent(content) { - slog.Debug("extractor returned low quality content", "extractor", ext.Name(), "url", rawURL, "chars", len(content)) - lastErr = fmt.Errorf("%s: content below quality threshold (%d chars)", ext.Name(), len(content)) - continue - } - return ExtractResult{Content: content, Extractor: ext.Name()}, nil + + slog.Debug("extractor_chain: extractor exhausted, moving to next", + "extractor", ext.Name(), "max_retries", maxRetries) } if lastErr != nil { return ExtractResult{}, fmt.Errorf("all extractors failed for %s: %w", rawURL, lastErr) @@ -69,10 +104,11 @@ func isQualityContent(content string) bool { // ExtractorEntry represents a single extractor in the chain settings JSON. type ExtractorEntry struct { - Name string `json:"name"` - Enabled bool `json:"enabled"` - Timeout int `json:"timeout,omitempty"` // seconds, 0 = use extractor default - BaseURL string `json:"base_url,omitempty"` // for defuddle: CF Worker URL + Name string `json:"name"` + Enabled bool `json:"enabled"` + Timeout int `json:"timeout,omitempty"` // seconds, 0 = use extractor default + MaxRetries int `json:"max_retries,omitempty"` // default 1 (no retry) + BaseURL string `json:"base_url,omitempty"` // for defuddle: CF Worker URL } // extractorChainSettings is the JSON schema for web_fetch builtin_tools.settings. @@ -104,6 +140,8 @@ func parseExtractorChainSettings(raw []byte, tool *WebFetchTool) *ExtractorChain } var extractors []ContentExtractor + var maxRetries []int + var timeouts []time.Duration for _, entry := range settings.Extractors { if !entry.Enabled || entry.Name == "" { continue @@ -115,10 +153,17 @@ func parseExtractorChainSettings(raw []byte, tool *WebFetchTool) *ExtractorChain extractors = append(extractors, &InProcessExtractor{tool: tool}) default: slog.Warn("web_fetch: unknown extractor in chain, skipping", "name", entry.Name) + continue } + retries := entry.MaxRetries + if retries <= 0 { + retries = 1 + } + maxRetries = append(maxRetries, retries) + timeouts = append(timeouts, time.Duration(entry.Timeout)*time.Second) } if len(extractors) == 0 { return nil } - return NewExtractorChain(extractors...) + return &ExtractorChain{extractors: extractors, maxRetries: maxRetries, timeouts: timeouts} }