diff --git a/internal/http/builtin_tools.go b/internal/http/builtin_tools.go index c218d909..939e3992 100644 --- a/internal/http/builtin_tools.go +++ b/internal/http/builtin_tools.go @@ -40,6 +40,7 @@ var toolSecretKeys = map[string]map[string]string{ "exa.api_key": "tools.web.exa.api_key", "tavily.api_key": "tools.web.tavily.api_key", "brave.api_key": "tools.web.brave.api_key", + "serply.api_key": "tools.web.serply.api_key", }, } diff --git a/internal/tools/web_search.go b/internal/tools/web_search.go index ea6698d1..1257aab0 100644 --- a/internal/tools/web_search.go +++ b/internal/tools/web_search.go @@ -23,6 +23,7 @@ const ( braveSearchEndpoint = "https://api.search.brave.com/res/v1/web/search" exaSearchEndpoint = "https://api.exa.ai/search" tavilySearchEndpoint = "https://api.tavily.com/search" + serplySearchEndpoint = "https://api.serply.io/v1/search" webSearchUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" ) @@ -31,6 +32,7 @@ const ( searchProviderTavily = "tavily" searchProviderBrave = "brave" searchProviderParallel = "parallel" + searchProviderSerply = "serply" searchProviderDuckDuckGo = "duckduckgo" ) @@ -38,6 +40,7 @@ var defaultSearchProviderOrder = []string{ searchProviderExa, searchProviderTavily, searchProviderBrave, + searchProviderSerply, searchProviderDuckDuckGo, } @@ -166,7 +169,7 @@ func (t *WebSearchTool) Parameters() map[string]any { }, "provider": map[string]any{ "type": "string", - "description": "Optional: force a specific provider (e.g., 'tavily', 'exa', 'brave', 'parallel', 'duckduckgo'). When omitted, the tenant's configured provider chain is used (first-success-wins). Use this to force cross-engine corroboration — call once with each provider and compare results.", + "description": "Optional: force a specific provider (e.g., 'tavily', 'exa', 'brave', 'parallel', 'serply', 'duckduckgo'). When omitted, the tenant's configured provider chain is used (first-success-wins). Use this to force cross-engine corroboration — call once with each provider and compare results.", }, }, "required": []string{"query"}, diff --git a/internal/tools/web_search_chain_test.go b/internal/tools/web_search_chain_test.go index cfc13df5..2b638b17 100644 --- a/internal/tools/web_search_chain_test.go +++ b/internal/tools/web_search_chain_test.go @@ -189,6 +189,24 @@ func TestBuildChainFromStorage(t *testing.T) { wantNames: []string{"parallel", "duckduckgo"}, wantLen: 2, }, + { + name: "Serply key present → sits before DDG in the default order", + tenantID: uuid.New(), + override: "", + secrets: map[string]string{ + "tools.web.serply.api_key": "test-key-serply", + }, + wantNames: []string{"serply", "duckduckgo"}, + wantLen: 2, + }, + { + name: "Serply without a key is skipped despite the default order", + tenantID: uuid.New(), + override: "", + secrets: map[string]string{}, + wantNames: []string{"duckduckgo"}, + wantLen: 1, + }, { name: "explicitly disabled Parallel is skipped", tenantID: uuid.New(), diff --git a/internal/tools/web_search_config.go b/internal/tools/web_search_config.go index bacbf24c..a5439e57 100644 --- a/internal/tools/web_search_config.go +++ b/internal/tools/web_search_config.go @@ -21,6 +21,8 @@ func buildProviderByName(name, apiKey string, maxResults int) SearchProvider { return newBraveSearchProvider(apiKey, maxResults) case searchProviderParallel: return newParallelSearchProvider(maxResults) + case searchProviderSerply: + return newSerplySearchProvider(apiKey, maxResults) case searchProviderDuckDuckGo: return newDuckDuckGoSearchProvider(maxResults) default: diff --git a/internal/tools/web_search_serply.go b/internal/tools/web_search_serply.go new file mode 100644 index 00000000..84dfa58b --- /dev/null +++ b/internal/tools/web_search_serply.go @@ -0,0 +1,122 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// --- Serply Search Provider --- +// +// Serply returns Google web results as JSON on a tenant-supplied key. Requests +// are GET with the key in an X-Api-Key header; results arrive under "results" +// with title/link/description fields. The API caps a response at 10 results, +// which is already maxSearchCount here, so no extra clamping is needed. + +type serplySearchProvider struct { + apiKey string + maxResults int + client *http.Client + endpoint string +} + +func newSerplySearchProvider(apiKey string, maxResults int) *serplySearchProvider { + return &serplySearchProvider{ + apiKey: apiKey, + maxResults: normalizeProviderMaxResults(maxResults), + client: &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second}, + endpoint: serplySearchEndpoint, + } +} + +func (p *serplySearchProvider) Name() string { return searchProviderSerply } + +// serplyFreshnessTBS maps the tool's freshness shortcuts onto Google's tbs +// recency filter, which Serply forwards upstream unchanged. +// +// normalizeFreshness also accepts a YYYY-MM-DDtoYYYY-MM-DD range. That form is +// deliberately absent: upstream ignores the equivalent cdr range and answers +// with unfiltered results, so sending it would report a date filter the +// results do not actually honour. A range therefore leaves tbs unset. +var serplyFreshnessTBS = map[string]string{ + "pd": "qdr:d", + "pw": "qdr:w", + "pm": "qdr:m", + "py": "qdr:y", +} + +func (p *serplySearchProvider) Search(ctx context.Context, params searchParams) ([]searchResult, error) { + limit := clampProviderResultCount(params.Count, p.maxResults) + + q := url.Values{} + q.Set("q", params.Query) + q.Set("num", strconv.Itoa(limit)) + + if params.Country != "" { + q.Set("gl", params.Country) + } + if lang := coalesceSearchText(params.SearchLang, params.UILang); lang != "" { + q.Set("hl", lang) + } + if tbs, ok := serplyFreshnessTBS[normalizeFreshness(params.Freshness)]; ok { + q.Set("tbs", tbs) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.endpoint+"?"+q.Encode(), nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("X-Api-Key", p.apiKey) + // Serply attributes usage by client, so identify the gateway rather than + // sending the browser webSearchUserAgent the scraping providers need. + req.Header.Set("User-Agent", "goclaw") + + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("serply API returned %d: %s", resp.StatusCode, truncateStr(string(body), 200)) + } + + var serplyResp struct { + Results []struct { + Title string `json:"title"` + Link string `json:"link"` + Description string `json:"description"` + } `json:"results"` + } + if err := json.Unmarshal(body, &serplyResp); err != nil { + return nil, fmt.Errorf("parse response: %w", err) + } + + results := make([]searchResult, 0, len(serplyResp.Results)) + for _, r := range serplyResp.Results { + if len(results) >= limit { + break + } + link := strings.TrimSpace(r.Link) + if link == "" { + continue + } + results = append(results, searchResult{ + Title: coalesceSearchText(r.Title, link, "Untitled"), + URL: link, + Description: truncateStr(strings.TrimSpace(r.Description), 240), + }) + } + return results, nil +} diff --git a/internal/tools/web_search_serply_test.go b/internal/tools/web_search_serply_test.go new file mode 100644 index 00000000..f82b30e1 --- /dev/null +++ b/internal/tools/web_search_serply_test.go @@ -0,0 +1,195 @@ +package tools + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "testing" + "time" +) + +func TestSerplySearchRequestAndResponse(t *testing.T) { + t.Parallel() + var got url.Values + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Errorf("method = %q, want GET", r.Method) + } + if key := r.Header.Get("X-Api-Key"); key != "test-key-serply" { + t.Errorf("X-Api-Key = %q, want test-key-serply", key) + } + got = r.URL.Query() + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"results":[ + {"title":"Go releases","link":"https://go.dev/doc/devel/release","description":" Current release notes. "}, + {"title":"","link":"https://go.dev/","description":""} + ]}`) + })) + defer server.Close() + + provider := &serplySearchProvider{apiKey: "test-key-serply", maxResults: 5, client: server.Client(), endpoint: server.URL} + results, err := provider.Search(context.Background(), searchParams{ + Query: "current Go release", Count: 2, Country: "DE", SearchLang: "de", Freshness: "pw", + }) + if err != nil { + t.Fatal(err) + } + + for _, tc := range []struct{ key, want string }{ + {"q", "current Go release"}, + {"num", "2"}, + {"gl", "DE"}, + {"hl", "de"}, + {"tbs", "qdr:w"}, + } { + if v := got.Get(tc.key); v != tc.want { + t.Errorf("query %s = %q, want %q", tc.key, v, tc.want) + } + } + + want := []searchResult{ + {Title: "Go releases", URL: "https://go.dev/doc/devel/release", Description: "Current release notes."}, + {Title: "https://go.dev/", URL: "https://go.dev/"}, + } + if !reflect.DeepEqual(results, want) { + t.Fatalf("results = %#v, want %#v", results, want) + } +} + +// Serply forwards tbs to Google. The date-range freshness form has no working +// equivalent upstream, so it must not be sent as a recency filter. +func TestSerplySearchFreshnessMapping(t *testing.T) { + t.Parallel() + tests := []struct { + freshness string + wantTBS string + }{ + {freshness: "pd", wantTBS: "qdr:d"}, + {freshness: "pw", wantTBS: "qdr:w"}, + {freshness: "pm", wantTBS: "qdr:m"}, + {freshness: "py", wantTBS: "qdr:y"}, + {freshness: "2024-01-01to2024-01-31", wantTBS: ""}, + {freshness: "nonsense", wantTBS: ""}, + {freshness: "", wantTBS: ""}, + } + for _, tc := range tests { + t.Run(tc.freshness, func(t *testing.T) { + t.Parallel() + var gotTBS string + var tbsPresent bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotTBS = r.URL.Query().Get("tbs") + _, tbsPresent = r.URL.Query()["tbs"] + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"results":[]}`) + })) + defer server.Close() + + provider := &serplySearchProvider{maxResults: 5, client: server.Client(), endpoint: server.URL} + if _, err := provider.Search(context.Background(), searchParams{Query: "q", Count: 1, Freshness: tc.freshness}); err != nil { + t.Fatal(err) + } + if gotTBS != tc.wantTBS { + t.Errorf("tbs = %q, want %q", gotTBS, tc.wantTBS) + } + if tc.wantTBS == "" && tbsPresent { + t.Errorf("tbs sent as empty param, want omitted entirely") + } + }) + } +} + +func TestSerplySearchAppliesLimitAndSkipsMissingLink(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"results":[ + {"title":"missing link","link":"","description":"skip"}, + {"title":"one","link":"https://example.com/1","description":"one"}, + {"title":"two","link":"https://example.com/2","description":"two"} + ]}`) + })) + defer server.Close() + + provider := &serplySearchProvider{maxResults: 1, client: server.Client(), endpoint: server.URL} + results, err := provider.Search(context.Background(), searchParams{Query: "q", Count: 5}) + if err != nil { + t.Fatal(err) + } + if len(results) != 1 || results[0].Title != "one" { + t.Fatalf("results = %#v, want first valid result", results) + } +} + +func TestSerplySearchResponseErrors(t *testing.T) { + t.Parallel() + tests := []struct { + name string + status int + body string + want string + }{ + {name: "invalid key", status: http.StatusUnauthorized, body: `{"detail":"Invalid API key"}`, want: "returned 401"}, + {name: "server error", status: http.StatusServiceUnavailable, body: ``, want: "returned 503"}, + {name: "malformed body", status: http.StatusOK, body: `{`, want: "parse response"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.status) + fmt.Fprint(w, tc.body) + })) + defer server.Close() + + provider := &serplySearchProvider{maxResults: 1, client: server.Client(), endpoint: server.URL} + _, err := provider.Search(context.Background(), searchParams{Query: "q", Count: 1}) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("error = %v, want containing %q", err, tc.want) + } + }) + } +} + +func TestSerplySearchPropagatesCancellation(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + client := &http.Client{Transport: serplyRoundTripFunc(func(req *http.Request) (*http.Response, error) { + <-req.Context().Done() + return nil, req.Context().Err() + })} + provider := &serplySearchProvider{maxResults: 1, client: client, endpoint: "https://api.serply.io/v1/search"} + _, err := provider.Search(ctx, searchParams{Query: "q", Count: 1}) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("error = %v, want deadline exceeded", err) + } +} + +// Serply is key-gated, so unlike Parallel it belongs in the default order: a +// tenant with no key is skipped by BuildChainFromStorage rather than by being +// absent from the order. +func TestSerplyProviderIsInDefaultOrder(t *testing.T) { + if got := NormalizeWebSearchProviderOrder(nil); !containsString(got, searchProviderSerply) { + t.Fatalf("default provider order missing serply: %v", got) + } + got := NormalizeWebSearchProviderOrder([]string{searchProviderSerply}) + if len(got) == 0 || got[0] != searchProviderSerply { + t.Fatalf("explicit provider order = %v, want serply first", got) + } + if provider := buildProviderByName(searchProviderSerply, "k", 3); provider == nil || provider.Name() != searchProviderSerply { + t.Fatalf("serply provider construction failed: %T", provider) + } +} + +type serplyRoundTripFunc func(*http.Request) (*http.Response, error) + +func (f serplyRoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} diff --git a/ui/web/src/i18n/locales/en/tools.json b/ui/web/src/i18n/locales/en/tools.json index 3a1d5bc6..10ae3d05 100644 --- a/ui/web/src/i18n/locales/en/tools.json +++ b/ui/web/src/i18n/locales/en/tools.json @@ -167,6 +167,7 @@ "tavily": "Tavily", "brave": "Brave Search", "parallel": "Parallel", + "serply": "Serply", "duckduckgo": "DuckDuckGo" }, "apiKey": "API Key", diff --git a/ui/web/src/i18n/locales/ru/tools.json b/ui/web/src/i18n/locales/ru/tools.json index 51053dad..a6a311ee 100644 --- a/ui/web/src/i18n/locales/ru/tools.json +++ b/ui/web/src/i18n/locales/ru/tools.json @@ -166,6 +166,7 @@ "tavily": "Tavily", "brave": "Brave Search", "parallel": "Parallel", + "serply": "Serply", "duckduckgo": "DuckDuckGo" }, "apiKey": "Ключ API", diff --git a/ui/web/src/i18n/locales/vi/tools.json b/ui/web/src/i18n/locales/vi/tools.json index 759813d0..24db0879 100644 --- a/ui/web/src/i18n/locales/vi/tools.json +++ b/ui/web/src/i18n/locales/vi/tools.json @@ -167,6 +167,7 @@ "tavily": "Tavily", "brave": "Brave Search", "parallel": "Parallel", + "serply": "Serply", "duckduckgo": "DuckDuckGo" }, "apiKey": "API Key", diff --git a/ui/web/src/i18n/locales/zh/tools.json b/ui/web/src/i18n/locales/zh/tools.json index 1e824272..932ddfa3 100644 --- a/ui/web/src/i18n/locales/zh/tools.json +++ b/ui/web/src/i18n/locales/zh/tools.json @@ -81,6 +81,7 @@ "tavily": "Tavily", "brave": "Brave Search", "parallel": "Parallel", + "serply": "Serply", "duckduckgo": "DuckDuckGo" }, "apiKey": "API Key", diff --git a/ui/web/src/pages/builtin-tools/web-search-chain-form.tsx b/ui/web/src/pages/builtin-tools/web-search-chain-form.tsx index b8aa129e..90741674 100644 --- a/ui/web/src/pages/builtin-tools/web-search-chain-form.tsx +++ b/ui/web/src/pages/builtin-tools/web-search-chain-form.tsx @@ -25,7 +25,7 @@ import { Label } from "@/components/ui/label"; import { Switch } from "@/components/ui/switch"; import { DialogHeader, DialogTitle, DialogDescription, DialogFooter } from "@/components/ui/dialog"; -type ProviderKey = "exa" | "tavily" | "brave" | "parallel" | "duckduckgo"; +type ProviderKey = "exa" | "tavily" | "brave" | "parallel" | "serply" | "duckduckgo"; interface ProviderEntry { id: string; @@ -44,15 +44,16 @@ interface Props { onCancel: () => void; } -const SORTABLE_PROVIDERS: ProviderKey[] = ["exa", "tavily", "brave", "parallel"]; +const SORTABLE_PROVIDERS: ProviderKey[] = ["exa", "tavily", "brave", "parallel", "serply"]; const LOCKED_PROVIDER: ProviderKey = "duckduckgo"; -const DEFAULT_ORDER: ProviderKey[] = ["exa", "tavily", "brave", "parallel"]; +const DEFAULT_ORDER: ProviderKey[] = ["exa", "tavily", "brave", "parallel", "serply"]; const RAIL_COLOR: Record = { exa: "bg-blue-600", tavily: "bg-cyan-500", brave: "bg-orange-500", parallel: "bg-violet-500", + serply: "bg-emerald-500", duckduckgo: "bg-slate-500", }; @@ -62,9 +63,12 @@ function parseInitialEntries(settings: Record): ProviderEntry[] SORTABLE_PROVIDERS.includes(p as ProviderKey), ) : DEFAULT_ORDER; - const rawOrder: ProviderKey[] = savedOrder.includes("parallel") - ? savedOrder - : [...savedOrder, "parallel"]; + // Append providers added after this tenant last saved an order, so a stored + // provider_order never hides a newer entry from the form. + const rawOrder: ProviderKey[] = [ + ...savedOrder, + ...SORTABLE_PROVIDERS.filter((p) => !savedOrder.includes(p)), + ]; return rawOrder.map((name) => { const cfg = (settings[name] ?? {}) as Record; @@ -183,7 +187,13 @@ function SortableProviderCard({ entry, index, secretsSet, onUpdate }: SortableCa ); } -function LockedDuckDuckGoCard({ settings }: { settings: Record }) { +function LockedDuckDuckGoCard({ + settings, + index, +}: { + settings: Record; + index: number; +}) { const { t } = useTranslation("tools"); const cfg = (settings[LOCKED_PROVIDER] ?? {}) as Record; const enabled = Boolean(cfg.enabled ?? true); @@ -194,7 +204,7 @@ function LockedDuckDuckGoCard({ settings }: { settings: Record
- #5 + #{index + 1} {t("builtin.searchChain.providers.duckduckgo")} @@ -277,7 +287,7 @@ export function WebSearchChainForm({ initialSettings, secretsSet, onSave, onCanc ))} - +