feat(tools): add Serply as a web_search provider (#1567)

* feat(tools): add Serply as a web_search provider

Serply is a SERP API returning Google web results on a tenant-supplied
key. It joins the existing chain the same way Exa, Tavily and Brave do:
keyed from config_secrets, skipped silently when no key is stored, and
selectable through the provider parameter for cross-engine corroboration.

Freshness maps onto Google's tbs recency filter, which Serply forwards
upstream. Only the pd/pw/pm/py shortcuts are sent: upstream ignores the
equivalent cdr date range and answers with unfiltered results, so a
range leaves tbs unset rather than naming a filter the results do not
honour.

The settings form gained a fifth sortable provider, so the locked
DuckDuckGo card now derives its position number instead of hardcoding
it, and a stored provider_order is reconciled against the full provider
list rather than against Parallel alone.

* test(tools): use a locally named round trip stub in the serply test

The serply cancellation test borrowed parallelRoundTripFunc from
web_search_parallel_test.go. Define serplyRoundTripFunc alongside the
test that uses it so the file does not depend on another provider's
test helper.

internal/http/builtin_tools.go

@@ -40,6 +40,7 @@ var toolSecretKeys = map[string]map[string]string{
 		"exa.api_key":    "tools.web.exa.api_key",
 		"tavily.api_key": "tools.web.tavily.api_key",
 		"brave.api_key":  "tools.web.brave.api_key",
+		"serply.api_key": "tools.web.serply.api_key",
 	},
 }
 

internal/tools/web_search.go

@@ -23,6 +23,7 @@ const (
 	braveSearchEndpoint  = "https://api.search.brave.com/res/v1/web/search"
 	exaSearchEndpoint    = "https://api.exa.ai/search"
 	tavilySearchEndpoint = "https://api.tavily.com/search"
+	serplySearchEndpoint = "https://api.serply.io/v1/search"
 	webSearchUserAgent   = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
 )
 
@@ -31,6 +32,7 @@ const (
 	searchProviderTavily     = "tavily"
 	searchProviderBrave      = "brave"
 	searchProviderParallel   = "parallel"
+	searchProviderSerply     = "serply"
 	searchProviderDuckDuckGo = "duckduckgo"
 )
 
@@ -38,6 +40,7 @@ var defaultSearchProviderOrder = []string{
 	searchProviderExa,
 	searchProviderTavily,
 	searchProviderBrave,
+	searchProviderSerply,
 	searchProviderDuckDuckGo,
 }
 
@@ -166,7 +169,7 @@ func (t *WebSearchTool) Parameters() map[string]any {
 			},
 			"provider": map[string]any{
 				"type":        "string",
-				"description": "Optional: force a specific provider (e.g., 'tavily', 'exa', 'brave', 'parallel', 'duckduckgo'). When omitted, the tenant's configured provider chain is used (first-success-wins). Use this to force cross-engine corroboration — call once with each provider and compare results.",
+				"description": "Optional: force a specific provider (e.g., 'tavily', 'exa', 'brave', 'parallel', 'serply', 'duckduckgo'). When omitted, the tenant's configured provider chain is used (first-success-wins). Use this to force cross-engine corroboration — call once with each provider and compare results.",
 			},
 		},
 		"required": []string{"query"},

internal/tools/web_search_chain_test.go

@@ -189,6 +189,24 @@ func TestBuildChainFromStorage(t *testing.T) {
 			wantNames: []string{"parallel", "duckduckgo"},
 			wantLen:   2,
 		},
+		{
+			name:     "Serply key present → sits before DDG in the default order",
+			tenantID: uuid.New(),
+			override: "",
+			secrets: map[string]string{
+				"tools.web.serply.api_key": "test-key-serply",
+			},
+			wantNames: []string{"serply", "duckduckgo"},
+			wantLen:   2,
+		},
+		{
+			name:      "Serply without a key is skipped despite the default order",
+			tenantID:  uuid.New(),
+			override:  "",
+			secrets:   map[string]string{},
+			wantNames: []string{"duckduckgo"},
+			wantLen:   1,
+		},
 		{
 			name:      "explicitly disabled Parallel is skipped",
 			tenantID:  uuid.New(),

internal/tools/web_search_config.go

@@ -21,6 +21,8 @@ func buildProviderByName(name, apiKey string, maxResults int) SearchProvider {
 		return newBraveSearchProvider(apiKey, maxResults)
 	case searchProviderParallel:
 		return newParallelSearchProvider(maxResults)
+	case searchProviderSerply:
+		return newSerplySearchProvider(apiKey, maxResults)
 	case searchProviderDuckDuckGo:
 		return newDuckDuckGoSearchProvider(maxResults)
 	default:

internal/tools/web_search_serply.go

@@ -0,0 +1,122 @@
+package tools
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// --- Serply Search Provider ---
+//
+// Serply returns Google web results as JSON on a tenant-supplied key. Requests
+// are GET with the key in an X-Api-Key header; results arrive under "results"
+// with title/link/description fields. The API caps a response at 10 results,
+// which is already maxSearchCount here, so no extra clamping is needed.
+
+type serplySearchProvider struct {
+	apiKey     string
+	maxResults int
+	client     *http.Client
+	endpoint   string
+}
+
+func newSerplySearchProvider(apiKey string, maxResults int) *serplySearchProvider {
+	return &serplySearchProvider{
+		apiKey:     apiKey,
+		maxResults: normalizeProviderMaxResults(maxResults),
+		client:     &http.Client{Timeout: time.Duration(searchTimeoutSeconds) * time.Second},
+		endpoint:   serplySearchEndpoint,
+	}
+}
+
+func (p *serplySearchProvider) Name() string { return searchProviderSerply }
+
+// serplyFreshnessTBS maps the tool's freshness shortcuts onto Google's tbs
+// recency filter, which Serply forwards upstream unchanged.
+//
+// normalizeFreshness also accepts a YYYY-MM-DDtoYYYY-MM-DD range. That form is
+// deliberately absent: upstream ignores the equivalent cdr range and answers
+// with unfiltered results, so sending it would report a date filter the
+// results do not actually honour. A range therefore leaves tbs unset.
+var serplyFreshnessTBS = map[string]string{
+	"pd": "qdr:d",
+	"pw": "qdr:w",
+	"pm": "qdr:m",
+	"py": "qdr:y",
+}
+
+func (p *serplySearchProvider) Search(ctx context.Context, params searchParams) ([]searchResult, error) {
+	limit := clampProviderResultCount(params.Count, p.maxResults)
+
+	q := url.Values{}
+	q.Set("q", params.Query)
+	q.Set("num", strconv.Itoa(limit))
+
+	if params.Country != "" {
+		q.Set("gl", params.Country)
+	}
+	if lang := coalesceSearchText(params.SearchLang, params.UILang); lang != "" {
+		q.Set("hl", lang)
+	}
+	if tbs, ok := serplyFreshnessTBS[normalizeFreshness(params.Freshness)]; ok {
+		q.Set("tbs", tbs)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.endpoint+"?"+q.Encode(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("X-Api-Key", p.apiKey)
+	// Serply attributes usage by client, so identify the gateway rather than
+	// sending the browser webSearchUserAgent the scraping providers need.
+	req.Header.Set("User-Agent", "goclaw")
+
+	resp, err := p.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read response: %w", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("serply API returned %d: %s", resp.StatusCode, truncateStr(string(body), 200))
+	}
+
+	var serplyResp struct {
+		Results []struct {
+			Title       string `json:"title"`
+			Link        string `json:"link"`
+			Description string `json:"description"`
+		} `json:"results"`
+	}
+	if err := json.Unmarshal(body, &serplyResp); err != nil {
+		return nil, fmt.Errorf("parse response: %w", err)
+	}
+
+	results := make([]searchResult, 0, len(serplyResp.Results))
+	for _, r := range serplyResp.Results {
+		if len(results) >= limit {
+			break
+		}
+		link := strings.TrimSpace(r.Link)
+		if link == "" {
+			continue
+		}
+		results = append(results, searchResult{
+			Title:       coalesceSearchText(r.Title, link, "Untitled"),
+			URL:         link,
+			Description: truncateStr(strings.TrimSpace(r.Description), 240),
+		})
+	}
+	return results, nil
+}

internal/tools/web_search_serply_test.go

@@ -0,0 +1,195 @@
+package tools
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestSerplySearchRequestAndResponse(t *testing.T) {
+	t.Parallel()
+	var got url.Values
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodGet {
+			t.Errorf("method = %q, want GET", r.Method)
+		}
+		if key := r.Header.Get("X-Api-Key"); key != "test-key-serply" {
+			t.Errorf("X-Api-Key = %q, want test-key-serply", key)
+		}
+		got = r.URL.Query()
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprint(w, `{"results":[
+			{"title":"Go releases","link":"https://go.dev/doc/devel/release","description":" Current release notes. "},
+			{"title":"","link":"https://go.dev/","description":""}
+		]}`)
+	}))
+	defer server.Close()
+
+	provider := &serplySearchProvider{apiKey: "test-key-serply", maxResults: 5, client: server.Client(), endpoint: server.URL}
+	results, err := provider.Search(context.Background(), searchParams{
+		Query: "current Go release", Count: 2, Country: "DE", SearchLang: "de", Freshness: "pw",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, tc := range []struct{ key, want string }{
+		{"q", "current Go release"},
+		{"num", "2"},
+		{"gl", "DE"},
+		{"hl", "de"},
+		{"tbs", "qdr:w"},
+	} {
+		if v := got.Get(tc.key); v != tc.want {
+			t.Errorf("query %s = %q, want %q", tc.key, v, tc.want)
+		}
+	}
+
+	want := []searchResult{
+		{Title: "Go releases", URL: "https://go.dev/doc/devel/release", Description: "Current release notes."},
+		{Title: "https://go.dev/", URL: "https://go.dev/"},
+	}
+	if !reflect.DeepEqual(results, want) {
+		t.Fatalf("results = %#v, want %#v", results, want)
+	}
+}
+
+// Serply forwards tbs to Google. The date-range freshness form has no working
+// equivalent upstream, so it must not be sent as a recency filter.
+func TestSerplySearchFreshnessMapping(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		freshness string
+		wantTBS   string
+	}{
+		{freshness: "pd", wantTBS: "qdr:d"},
+		{freshness: "pw", wantTBS: "qdr:w"},
+		{freshness: "pm", wantTBS: "qdr:m"},
+		{freshness: "py", wantTBS: "qdr:y"},
+		{freshness: "2024-01-01to2024-01-31", wantTBS: ""},
+		{freshness: "nonsense", wantTBS: ""},
+		{freshness: "", wantTBS: ""},
+	}
+	for _, tc := range tests {
+		t.Run(tc.freshness, func(t *testing.T) {
+			t.Parallel()
+			var gotTBS string
+			var tbsPresent bool
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				gotTBS = r.URL.Query().Get("tbs")
+				_, tbsPresent = r.URL.Query()["tbs"]
+				w.Header().Set("Content-Type", "application/json")
+				fmt.Fprint(w, `{"results":[]}`)
+			}))
+			defer server.Close()
+
+			provider := &serplySearchProvider{maxResults: 5, client: server.Client(), endpoint: server.URL}
+			if _, err := provider.Search(context.Background(), searchParams{Query: "q", Count: 1, Freshness: tc.freshness}); err != nil {
+				t.Fatal(err)
+			}
+			if gotTBS != tc.wantTBS {
+				t.Errorf("tbs = %q, want %q", gotTBS, tc.wantTBS)
+			}
+			if tc.wantTBS == "" && tbsPresent {
+				t.Errorf("tbs sent as empty param, want omitted entirely")
+			}
+		})
+	}
+}
+
+func TestSerplySearchAppliesLimitAndSkipsMissingLink(t *testing.T) {
+	t.Parallel()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprint(w, `{"results":[
+			{"title":"missing link","link":"","description":"skip"},
+			{"title":"one","link":"https://example.com/1","description":"one"},
+			{"title":"two","link":"https://example.com/2","description":"two"}
+		]}`)
+	}))
+	defer server.Close()
+
+	provider := &serplySearchProvider{maxResults: 1, client: server.Client(), endpoint: server.URL}
+	results, err := provider.Search(context.Background(), searchParams{Query: "q", Count: 5})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(results) != 1 || results[0].Title != "one" {
+		t.Fatalf("results = %#v, want first valid result", results)
+	}
+}
+
+func TestSerplySearchResponseErrors(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name   string
+		status int
+		body   string
+		want   string
+	}{
+		{name: "invalid key", status: http.StatusUnauthorized, body: `{"detail":"Invalid API key"}`, want: "returned 401"},
+		{name: "server error", status: http.StatusServiceUnavailable, body: ``, want: "returned 503"},
+		{name: "malformed body", status: http.StatusOK, body: `{`, want: "parse response"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(tc.status)
+				fmt.Fprint(w, tc.body)
+			}))
+			defer server.Close()
+
+			provider := &serplySearchProvider{maxResults: 1, client: server.Client(), endpoint: server.URL}
+			_, err := provider.Search(context.Background(), searchParams{Query: "q", Count: 1})
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("error = %v, want containing %q", err, tc.want)
+			}
+		})
+	}
+}
+
+func TestSerplySearchPropagatesCancellation(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	client := &http.Client{Transport: serplyRoundTripFunc(func(req *http.Request) (*http.Response, error) {
+		<-req.Context().Done()
+		return nil, req.Context().Err()
+	})}
+	provider := &serplySearchProvider{maxResults: 1, client: client, endpoint: "https://api.serply.io/v1/search"}
+	_, err := provider.Search(ctx, searchParams{Query: "q", Count: 1})
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("error = %v, want deadline exceeded", err)
+	}
+}
+
+// Serply is key-gated, so unlike Parallel it belongs in the default order: a
+// tenant with no key is skipped by BuildChainFromStorage rather than by being
+// absent from the order.
+func TestSerplyProviderIsInDefaultOrder(t *testing.T) {
+	if got := NormalizeWebSearchProviderOrder(nil); !containsString(got, searchProviderSerply) {
+		t.Fatalf("default provider order missing serply: %v", got)
+	}
+	got := NormalizeWebSearchProviderOrder([]string{searchProviderSerply})
+	if len(got) == 0 || got[0] != searchProviderSerply {
+		t.Fatalf("explicit provider order = %v, want serply first", got)
+	}
+	if provider := buildProviderByName(searchProviderSerply, "k", 3); provider == nil || provider.Name() != searchProviderSerply {
+		t.Fatalf("serply provider construction failed: %T", provider)
+	}
+}
+
+type serplyRoundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f serplyRoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req)
+}

ui/web/src/i18n/locales/en/tools.json

@@ -167,6 +167,7 @@
         "tavily": "Tavily",
         "brave": "Brave Search",
         "parallel": "Parallel",
+        "serply": "Serply",
         "duckduckgo": "DuckDuckGo"
       },
       "apiKey": "API Key",

ui/web/src/i18n/locales/ru/tools.json

@@ -166,6 +166,7 @@
         "tavily": "Tavily",
         "brave": "Brave Search",
         "parallel": "Parallel",
+        "serply": "Serply",
         "duckduckgo": "DuckDuckGo"
       },
       "apiKey": "Ключ API",

ui/web/src/i18n/locales/vi/tools.json

@@ -167,6 +167,7 @@
         "tavily": "Tavily",
         "brave": "Brave Search",
         "parallel": "Parallel",
+        "serply": "Serply",
         "duckduckgo": "DuckDuckGo"
       },
       "apiKey": "API Key",

ui/web/src/i18n/locales/zh/tools.json

@@ -81,6 +81,7 @@
         "tavily": "Tavily",
         "brave": "Brave Search",
         "parallel": "Parallel",
+        "serply": "Serply",
         "duckduckgo": "DuckDuckGo"
       },
       "apiKey": "API Key",

ui/web/src/pages/builtin-tools/web-search-chain-form.tsx

@@ -25,7 +25,7 @@ import { Label } from "@/components/ui/label";
 import { Switch } from "@/components/ui/switch";
 import { DialogHeader, DialogTitle, DialogDescription, DialogFooter } from "@/components/ui/dialog";
 
-type ProviderKey = "exa" | "tavily" | "brave" | "parallel" | "duckduckgo";
+type ProviderKey = "exa" | "tavily" | "brave" | "parallel" | "serply" | "duckduckgo";
 
 interface ProviderEntry {
   id: string;
@@ -44,15 +44,16 @@ interface Props {
   onCancel: () => void;
 }
 
-const SORTABLE_PROVIDERS: ProviderKey[] = ["exa", "tavily", "brave", "parallel"];
+const SORTABLE_PROVIDERS: ProviderKey[] = ["exa", "tavily", "brave", "parallel", "serply"];
 const LOCKED_PROVIDER: ProviderKey = "duckduckgo";
-const DEFAULT_ORDER: ProviderKey[] = ["exa", "tavily", "brave", "parallel"];
+const DEFAULT_ORDER: ProviderKey[] = ["exa", "tavily", "brave", "parallel", "serply"];
 
 const RAIL_COLOR: Record<ProviderKey, string> = {
   exa: "bg-blue-600",
   tavily: "bg-cyan-500",
   brave: "bg-orange-500",
   parallel: "bg-violet-500",
+  serply: "bg-emerald-500",
   duckduckgo: "bg-slate-500",
 };
 
@@ -62,9 +63,12 @@ function parseInitialEntries(settings: Record<string, unknown>): ProviderEntry[]
         SORTABLE_PROVIDERS.includes(p as ProviderKey),
       )
     : DEFAULT_ORDER;
-  const rawOrder: ProviderKey[] = savedOrder.includes("parallel")
-    ? savedOrder
-    : [...savedOrder, "parallel"];
+  // Append providers added after this tenant last saved an order, so a stored
+  // provider_order never hides a newer entry from the form.
+  const rawOrder: ProviderKey[] = [
+    ...savedOrder,
+    ...SORTABLE_PROVIDERS.filter((p) => !savedOrder.includes(p)),
+  ];
 
   return rawOrder.map((name) => {
     const cfg = (settings[name] ?? {}) as Record<string, unknown>;
@@ -183,7 +187,13 @@ function SortableProviderCard({ entry, index, secretsSet, onUpdate }: SortableCa
   );
 }
 
-function LockedDuckDuckGoCard({ settings }: { settings: Record<string, unknown> }) {
+function LockedDuckDuckGoCard({
+  settings,
+  index,
+}: {
+  settings: Record<string, unknown>;
+  index: number;
+}) {
   const { t } = useTranslation("tools");
   const cfg = (settings[LOCKED_PROVIDER] ?? {}) as Record<string, unknown>;
   const enabled = Boolean(cfg.enabled ?? true);
@@ -194,7 +204,7 @@ function LockedDuckDuckGoCard({ settings }: { settings: Record<string, unknown>
       <div className="flex-1 px-3 py-3">
         <div className="flex items-center gap-2">
           <Lock className="size-4 text-muted-foreground shrink-0" />
-          <span className="text-xs text-muted-foreground font-mono shrink-0">#5</span>
+          <span className="text-xs text-muted-foreground font-mono shrink-0">#{index + 1}</span>
           <Switch size="sm" checked disabled />
           <span className="text-sm font-medium flex-1">
             {t("builtin.searchChain.providers.duckduckgo")}
@@ -277,7 +287,7 @@ export function WebSearchChainForm({ initialSettings, secretsSet, onSave, onCanc
             ))}
           </SortableContext>
         </DndContext>
-        <LockedDuckDuckGoCard settings={initialSettings} />
+        <LockedDuckDuckGoCard settings={initialSettings} index={entries.length} />
       </div>
 
       <DialogFooter>