feat(mcp): hybrid search mode — keep first 40 tools inline, defer rest

Instead of all-or-nothing when MCP tool count exceeds threshold,
keep first 40 tools registered inline and only defer the excess
to BM25 search via mcp_tool_search. System prompt now shows both
inline descriptions and search guidance in hybrid mode.

Also raises skill inline count from 40 to 60 (token limit is the
real bottleneck for skills).

internal/agent/loop_history.go

@@ -202,10 +202,9 @@ func (l *Loop) buildMessages(ctx context.Context, history []providers.Message, s
 		}
 		toolNames = filtered
 	}
-	var mcpToolDescs map[string]string
-	if !hasMCPToolSearch {
-		mcpToolDescs = l.buildMCPToolDescs(toolNames)
-	}
+	// Always build MCP tool descriptions for inline tools — in hybrid search
+	// mode the kept inline tools still need descriptions in the system prompt.
+	mcpToolDescs := l.buildMCPToolDescs(toolNames)
 
 	// Bootstrap DM mode: only restrict tools for open agents (identity being created).
 	// Predefined agents keep full capabilities — BOOTSTRAP.md guides behavior.
@@ -366,7 +365,7 @@ func filterBootstrapTools(toolNames []string) []string {
 // these limits, inline all skills as XML in the system prompt (like TS).
 // Above these limits, only include skill_search instructions.
 const (
-	skillInlineMaxCount  = 40   // max skills to inline
+	skillInlineMaxCount  = 60   // max skills to inline
 	skillInlineMaxTokens = 5000 // max estimated tokens for skill descriptions
 )
 

internal/agent/systemprompt.go

@@ -235,10 +235,11 @@ func BuildSystemPrompt(cfg SystemPromptConfig) string {
 
 	// 4.5. ## MCP Tools (full only) — skip during bootstrap
 	if !isMinimal && !cfg.IsBootstrap {
+		if len(cfg.MCPToolDescs) > 0 {
+			lines = append(lines, buildMCPToolsInlineSection(cfg.MCPToolDescs)...)
+		}
 		if cfg.HasMCPToolSearch {
 			lines = append(lines, buildMCPToolsSearchSection()...)
-		} else if len(cfg.MCPToolDescs) > 0 {
-			lines = append(lines, buildMCPToolsInlineSection(cfg.MCPToolDescs)...)
 		}
 	}
 

internal/agent/systemprompt_sections.go

@@ -11,25 +11,35 @@ import (
 	"github.com/nextlevelbuilder/goclaw/internal/store"
 )
 
+// mcpOptionalParamInstruction is the shared instruction for MCP tool optional parameters.
+// Includes a concrete WRONG/RIGHT example because some models (GPT-5.4) ignore prose-only guidance
+// and fill every optional field with hallucinated values.
+const mcpOptionalParamInstruction = "**Optional parameters:** Only include parameters where you have a SPECIFIC value from the user. " +
+	"Do NOT fill in optional fields with guessed values, empty strings, or placeholder text like \"optional\". " +
+	"If unsure, OMIT the field — the tool will use sensible defaults.\n" +
+	"WRONG: {\"url\": \"https://example.com\", \"debug\": true, \"timeout\": 10000, \"format\": \"bullet\"}\n" +
+	"RIGHT: {\"url\": \"https://example.com\"}"
+
 // mcpToolDescMaxLen is the max character length for MCP tool descriptions
 // in the system prompt inline section. ~200 chars ≈ ~50 tokens, balancing
 // discoverability with prompt budget.
 const mcpToolDescMaxLen = 200
 
-// buildMCPToolsSearchSection generates the MCP tools instruction block for search mode.
-// Shown when mcp_tool_search is registered instead of individual MCP tools.
+// buildMCPToolsSearchSection generates the MCP tools search instruction block.
+// Shown when mcp_tool_search is registered — may appear alongside the inline
+// section in hybrid mode (some tools inline, rest discoverable via search).
 func buildMCPToolsSearchSection() []string {
 	return []string{
-		"## MCP Tools (mandatory — prefer over core tools)",
+		"## Additional MCP Tools (use mcp_tool_search to discover)",
 		"",
-		"You have access to external tool integrations (MCP servers) with many specialized tools.",
-		"Not all tools are loaded by default — use `mcp_tool_search` to discover them.",
+		"Additional external tool integrations are available beyond those listed above.",
+		"Use `mcp_tool_search` to discover them.",
 		"**When an MCP tool overlaps with a core tool (e.g. database query, file ops, messaging), always prefer the MCP tool** — it has richer context and tighter integration.",
 		"1. Before performing external operations (database, API, file management, messaging), run `mcp_tool_search` with descriptive English keywords.",
 		"2. Matching tools are activated immediately and can be called right away in the same turn.",
 		"3. If no match found, proceed with other available tools.",
 		"",
-		"**Optional parameters:** Only include if you have a concrete value from user context. Do not send empty strings or placeholders — omit the field entirely. The tool will use sensible defaults.",
+		mcpOptionalParamInstruction,
 		"",
 	}
 }
@@ -42,7 +52,7 @@ func buildMCPToolsInlineSection(descs map[string]string) []string {
 		"",
 		"External tool integrations (MCP servers). **When an MCP tool overlaps with a core tool, always prefer the MCP tool.**",
 		"",
-		"**Optional parameters:** Only include if you have a concrete value from user context. Do not send empty strings or placeholders — omit the field entirely. The tool will use sensible defaults.",
+		mcpOptionalParamInstruction,
 		"",
 	}
 	for name, desc := range descs {

internal/mcp/manager.go

@@ -21,6 +21,7 @@ import (
 
 const (
 	healthCheckInterval  = 30 * time.Second
+	healthFailThreshold  = 3 // consecutive ping failures before marking disconnected
 	initialBackoff       = 2 * time.Second
 	maxBackoff           = 60 * time.Second
 	maxReconnectAttempts = 10
@@ -52,6 +53,7 @@ type serverState struct {
 
 	mu              sync.Mutex
 	reconnAttempts  int
+	healthFailures  int // consecutive ping failures (resets on success)
 	lastErr         string
 }
 
@@ -61,9 +63,10 @@ type serverState struct {
 //   - DB-backed: queries MCPServerStore per agent+user for permission-filtered servers
 //
 // When total MCP tool count exceeds mcpToolInlineMaxCount, the manager
-// enters "search mode": tools are kept in deferredTools instead of the
-// registry, and only mcp_tool_search is registered. Tools are activated
-// on demand via ActivateTools().
+// enters hybrid search mode: the first mcpToolInlineMaxCount tools stay
+// registered inline, while excess tools move to deferredTools and are
+// discovered via mcp_tool_search. Tools are activated on demand via
+// ActivateTools().
 type Manager struct {
 	mu       sync.RWMutex
 	servers  map[string]*serverState
@@ -309,21 +312,28 @@ func (m *Manager) LoadForAgent(ctx context.Context, agentID uuid.UUID, userID st
 	return nil
 }
 
-// maybeEnterSearchMode moves all registered BridgeTools to deferredTools
-// if total count exceeds the inline threshold.
+// maybeEnterSearchMode partially defers MCP tools when total count exceeds
+// the inline threshold. The first mcpToolInlineMaxCount tools stay registered
+// inline; the rest are moved to deferredTools and discovered via mcp_tool_search.
 func (m *Manager) maybeEnterSearchMode() {
 	allNames := m.ToolNames()
 	if len(allNames) <= mcpToolInlineMaxCount {
 		return
 	}
 
+	// Build a set of names to defer (everything beyond the threshold).
+	deferSet := make(map[string]struct{}, len(allNames)-mcpToolInlineMaxCount)
+	for _, name := range allNames[mcpToolInlineMaxCount:] {
+		deferSet[name] = struct{}{}
+	}
+
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
-	m.deferredTools = make(map[string]*BridgeTool, len(allNames))
+	m.deferredTools = make(map[string]*BridgeTool, len(deferSet))
 	m.activatedTools = make(map[string]struct{})
 
-	// Move all tools to deferred — handle both pool-backed and standalone
+	// Move only excess tools to deferred, keep the rest inline.
 	for serverName := range m.servers {
 		var toolNames []string
 		if _, isPool := m.poolServers[serverName]; isPool {
@@ -332,7 +342,12 @@ func (m *Manager) maybeEnterSearchMode() {
 			toolNames = m.servers[serverName].toolNames
 		}
 
+		var kept []string
 		for _, name := range toolNames {
+			if _, shouldDefer := deferSet[name]; !shouldDefer {
+				kept = append(kept, name)
+				continue
+			}
 			if bt, ok := m.registry.Get(name); ok {
 				if bridge, ok := bt.(*BridgeTool); ok {
 					m.deferredTools[name] = bridge
@@ -341,18 +356,21 @@ func (m *Manager) maybeEnterSearchMode() {
 			}
 		}
 
-		// Clear tool names
+		// Update per-server tool names to only the kept inline tools.
 		if _, isPool := m.poolServers[serverName]; isPool {
-			m.poolToolNames[serverName] = nil
+			m.poolToolNames[serverName] = kept
 		} else {
-			m.servers[serverName].toolNames = nil
+			m.servers[serverName].toolNames = kept
 		}
 	}
 
-	tools.UnregisterToolGroup("mcp")
+	// Update "mcp" group to only the kept inline names.
+	inlineNames := allNames[:mcpToolInlineMaxCount]
+	tools.RegisterToolGroup("mcp", inlineNames)
 	m.searchMode = true
 
 	slog.Info("mcp.search_mode.enabled",
+		"inline_tools", len(inlineNames),
 		"deferred_tools", len(m.deferredTools),
 		"threshold", mcpToolInlineMaxCount)
 }