// Package agent — response sanitization pipeline. // // Matching TS sanitization chain: // // extractAssistantText() → per-block: // 1. stripMinimaxToolCallXml() → Go: stripGarbledToolXML() // 2. stripDowngradedToolCallText() → Go: stripDowngradedToolCallText() // 3. stripThinkingTagsFromText() → Go: stripThinkingTags() // then: // 4. sanitizeUserFacingText() → Go: sanitizeUserFacingText() // - stripFinalTagsFromText() → Go: stripFinalTags() // - collapseConsecutiveDuplicateBlocks() // // Additional Go-specific: // 5. stripEchoedSystemMessages() → strip hallucinated [System Message] blocks // 6. stripGarbledToolXML() → strip garbled XML from models like DeepSeek package agent import ( "log/slog" "regexp" "strings" "github.com/nextlevelbuilder/goclaw/internal/store" ) // SanitizeAssistantContent applies the full sanitization pipeline to assistant // response text before saving to session and sending to user. // Matching TS extractAssistantText() + sanitizeUserFacingText(). func SanitizeAssistantContent(content string) string { if content == "" { return content } original := content // 1. Strip garbled tool-call XML (DeepSeek, GLM, Minimax) content = stripGarbledToolXML(content) if content == "" { return "" } // 2. Strip downgraded tool call text ([Tool Call: ...], [Tool Result ...]) content = stripDowngradedToolCallText(content) // 3. Strip thinking/reasoning tags (, , , ) content = stripThinkingTags(content) // 4. Strip tags (keep content inside) content = stripFinalTags(content) // 5. Strip echoed [System Message] blocks content = stripEchoedSystemMessages(content) // 6. Collapse consecutive duplicate blocks content = collapseConsecutiveDuplicateBlocks(content) // 7. Strip MEDIA: paths from LLM output (media delivered separately) content = stripMediaPaths(content) // 8. Strip leading blank lines (preserve indentation) content = stripLeadingBlankLines(content) content = strings.TrimSpace(content) if content != original { slog.Debug("sanitized assistant content", "original_len", len(original), "cleaned_len", len(content), ) } return content } // --- 1. Garbled tool-call XML --- // garbledToolXMLPattern matches XML-like tool call artifacts that some models // (DeepSeek, GLM, etc.) emit as text content instead of proper tool calls. var garbledToolXMLPattern = regexp.MustCompile( `(?s)]*>`, ) // fullToolCallBlockPattern matches a COMPLETE Anthropic-style tool-call block // (`...`) that a model emitted as text instead // of invoking it natively. Unlike garbledToolXMLPattern — which removes only the // tags — this captures the whole block, tags plus the inner text, so // a stray block is removed cleanly rather than leaving the argument values // orphaned in the user-facing reply. var fullToolCallBlockPattern = regexp.MustCompile( `(?is).*?`, ) // invokeNamePattern extracts the tool name from an `` tag so a // dropped text-encoded tool call can be logged with the tool it tried to call. var invokeNamePattern = regexp.MustCompile(`(?i)...` // block emitted as text WITHOUT the surrounding wrapper. Some // models — and the claude-cli proxy under a degraded session — drop the wrapper // and emit just the invoke block, which fullToolCallBlockPattern misses; the // tag-only strip would then leak the inner text. Applied after the // wrapped form so wrapper-nested invokes are already gone. var bareInvokeBlockPattern = regexp.MustCompile( `(?is)`, ) var garbledToolXMLIndicators = []string{ "invfunction_calls", "functioninvoke", " values mangled into the reply. // Remove whole blocks — ... wrappers first, // then any bare ... left without a wrapper — and log the // attempted tool name(s) at WARN so this otherwise-silent no-op is diagnosable. var droppedTools []string var droppedBlocks int for _, re := range []*regexp.Regexp{fullToolCallBlockPattern, bareInvokeBlockPattern} { blocks := re.FindAllString(content, -1) if len(blocks) == 0 { continue } droppedBlocks += len(blocks) for _, b := range blocks { for _, m := range invokeNamePattern.FindAllStringSubmatch(b, -1) { droppedTools = append(droppedTools, m[1]) } } content = re.ReplaceAllString(content, "") } if droppedBlocks > 0 { slog.Warn("dropped text-encoded tool call from response", "tools", droppedTools, "blocks", droppedBlocks, "hint", "model wrote a tool call as text instead of invoking it; the tool did not run", ) } // Strip any remaining stray tags (partial DeepSeek/GLM/Minimax artifacts). cleaned := strings.TrimSpace(garbledToolXMLPattern.ReplaceAllString(content, "")) if cleaned == "" { slog.Warn("stripped entire response as garbled tool XML", "original_len", len(original)) return "" } if cleaned != original { slog.Warn("stripped garbled tool call XML from response", "original_len", len(original), "remaining_len", len(cleaned), ) } return cleaned } // --- 2. Downgraded tool call text --- // stripDowngradedToolCallText removes [Tool Call: ...], [Tool Result ...], // and [Historical context: ...] blocks that some models emit as text. // Matching TS stripDowngradedToolCallText(). // Uses line-by-line scanning (Go regexp doesn't support lookahead). func stripDowngradedToolCallText(content string) string { if !strings.Contains(content, "[Tool Call:") && !strings.Contains(content, "[Tool Result") && !strings.Contains(content, "[Historical context:") { return content } lines := strings.Split(content, "\n") var result []string skipping := false for _, line := range lines { trimmed := strings.TrimSpace(line) // Start skipping on these markers if strings.HasPrefix(trimmed, "[Tool Call:") || strings.HasPrefix(trimmed, "[Tool Result") || strings.HasPrefix(trimmed, "[Historical context:") { skipping = true continue } // Stop skipping on non-indented, non-empty line that isn't part of the block if skipping { // Arguments JSON and tool output are typically indented or empty if trimmed == "" || strings.HasPrefix(trimmed, "Arguments:") || strings.HasPrefix(trimmed, "{") || strings.HasPrefix(trimmed, "}") { continue } // Non-tool-block line → stop skipping skipping = false } result = append(result, line) } return strings.TrimSpace(strings.Join(result, "\n")) } // --- 3. Thinking/reasoning tags --- // Matches TS stripThinkingTagsFromText() with strict mode. // Strips: ..., ..., // // ..., ..., // ... // // Go regexp doesn't support backreferences, so we use separate patterns. var thinkingTagPatterns = []*regexp.Regexp{ regexp.MustCompile(`(?is)]*>.*?`), regexp.MustCompile(`(?is)]*>.*?`), regexp.MustCompile(`(?is)]*>.*?`), regexp.MustCompile(`(?is)]*>.*?`), regexp.MustCompile(`(?is)]*>.*?`), regexp.MustCompile(`(?is)]*>.*?`), } func stripThinkingTags(content string) string { lower := strings.ToLower(content) if !strings.Contains(lower, " tags --- // Matches TS stripFinalTagsFromText(). Removes and tags // but keeps the content inside. var finalTagPattern = regexp.MustCompile(`(?i)<\s*/?\s*final\s*>`) func stripFinalTags(content string) string { if !strings.Contains(strings.ToLower(content), "final") { return content } return finalTagPattern.ReplaceAllString(content, "") } // --- 5. Echoed [System Message] --- // stripEchoedSystemMessages removes "[System Message] ..." blocks that LLMs // hallucinate/echo in their response text. // Uses line-based scanning (Go regexp doesn't support lookahead). func stripEchoedSystemMessages(content string) string { if !strings.Contains(content, "[System Message]") { return content } lines := strings.Split(content, "\n") var result []string skipping := false for _, line := range lines { if strings.HasPrefix(strings.TrimSpace(line), "[System Message]") { skipping = true continue } if skipping { // Empty line ends the system message block if strings.TrimSpace(line) == "" { skipping = false continue } // Still part of the system message block (Stats:, reply instructions, etc.) continue } result = append(result, line) } cleaned := strings.TrimSpace(strings.Join(result, "\n")) if cleaned != strings.TrimSpace(content) { slog.Warn("stripped echoed [System Message] from assistant response", "original_len", len(content), "cleaned_len", len(cleaned), ) } return cleaned } // --- 6. Collapse consecutive duplicate blocks --- // collapseConsecutiveDuplicateBlocks removes repeated paragraph blocks. // Matching TS collapseConsecutiveDuplicateBlocks(). func collapseConsecutiveDuplicateBlocks(content string) string { blocks := strings.Split(content, "\n\n") if len(blocks) <= 1 { return content } var result []string for i, block := range blocks { trimmed := strings.TrimSpace(block) if trimmed == "" { continue } if i > 0 && len(result) > 0 && trimmed == strings.TrimSpace(result[len(result)-1]) { continue // skip duplicate } result = append(result, block) } collapsed := strings.Join(result, "\n\n") if collapsed != content { slog.Debug("collapsed duplicate blocks", "original_blocks", len(blocks), "result_blocks", len(result), ) } return collapsed } // --- 7. Strip MEDIA: paths --- // mediaPathPattern matches "MEDIA:" followed by a path (absolute or relative). var mediaPathPattern = regexp.MustCompile(`MEDIA:\S+`) // stripMediaPaths removes lines containing MEDIA:/path references from LLM output. // These are tool result artifacts that should not appear in user-facing text // (media files are delivered separately via OutboundMessage.Media). func stripMediaPaths(content string) string { if !strings.Contains(content, "MEDIA:") { return content } lines := strings.Split(content, "\n") var result []string for _, line := range lines { trimmed := strings.TrimSpace(line) if strings.HasPrefix(trimmed, "[[audio_as_voice]]") { continue } // Strip any line containing a MEDIA: path reference, regardless of wrapping format. // LLMs echo these in many forms: bare "MEDIA:/path", markdown "![alt](MEDIA:relative/path)", // JSON '{"image":"MEDIA:/path"}', etc. Match MEDIA: followed by any non-space path char. if mediaPathPattern.MatchString(trimmed) { continue } result = append(result, line) } return strings.TrimSpace(strings.Join(result, "\n")) } // --- 8. Strip leading blank lines --- var leadingBlankLinesPattern = regexp.MustCompile(`^(?:[ \t]*\r?\n)+`) func stripLeadingBlankLines(content string) string { return leadingBlankLinesPattern.ReplaceAllString(content, "") } // --- 9. Config leak detection (predefined agents) --- // configLeakFileNames are internal file names that should not appear in user-facing output // when a predefined agent describes its procedures or configuration. var configLeakFileNames = []string{ "SOUL.md", "IDENTITY.md", "AGENTS.md", "BOOTSTRAP.md", "internal_config", "system prompt", } // Patterns to strip markdown code from content before config leak detection. // Mentions inside code blocks/inline code are typically architecture docs, not leaks. var fencedCodeBlockPattern = regexp.MustCompile("(?s)```[^`]*```") var inlineCodePattern = regexp.MustCompile("`[^`\n]+`") // stripMarkdownCode removes fenced code blocks and inline code from text. func stripMarkdownCode(s string) string { s = fencedCodeBlockPattern.ReplaceAllString(s, "") s = inlineCodePattern.ReplaceAllString(s, "") return s } // StripConfigLeak detects when a predefined agent dumps its internal configuration // (e.g. referencing SOUL.md, AGENTS.md, IDENTITY.md) and replaces the entire // response with a friendly decline. // // Only active for predefined agents. Single-gate detection: // 3+ distinct internal file names mentioned in plain text → replace entire response. // Mentions inside markdown code blocks and inline code are excluded from counting, // as they typically appear in architecture explanations rather than actual leaks. func StripConfigLeak(content, agentType string) string { if agentType != store.AgentTypePredefined || content == "" { return content } // Count hits only in plain text (outside code blocks/inline code) plain := stripMarkdownCode(content) hits := 0 for _, name := range configLeakFileNames { if strings.Contains(plain, name) { hits++ } } if hits < 3 { return content } slog.Warn("security.config_leak_stripped", "file_hits", hits, "original_len", len(content), ) return "🔒 Security check not passed." } // --- NO_REPLY detection --- // IsSilentReply checks if the text contains a standalone NO_REPLY token. // // Divergent from TS isSilentReplyText() (exact-match only) — we match broadly so // both prefix forms (`NO_REPLY because offline`) and terminal sentinel forms // (`not for me. NO_REPLY`) suppress delivery. The token must not be glued to // another word (`NO_REPLYING` and `XNO_REPLY` are not silent). Case-insensitive. func IsSilentReply(text string) bool { trimmed := strings.TrimSpace(text) if trimmed == "" { return false } // Strip decorative wrappers from both ends (quotes, markdown emphasis, punctuation). stripped := strings.Trim(trimmed, "_ \t\n\r.,:;!?\"'`*~#>-()[]{}") const token = "NO_REPLY" return containsStandaloneNoReplyToken(stripped, token) } func containsStandaloneNoReplyToken(text, token string) bool { if len(text) < len(token) { return false } for i := 0; i+len(token) <= len(text); i++ { if !strings.EqualFold(text[i:i+len(token)], token) { continue } beforeOK := i == 0 || !isAlphaNumByte(text[i-1]) after := i + len(token) afterOK := after == len(text) || !isAlphaNumByte(text[after]) if beforeOK && afterOK { return true } } return false } func isAlphaNumByte(b byte) bool { return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') } func isAlphaNum(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') } func isWordChar(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' } // --- Message Directives ([[name:value]]) --- // messageDirectivePattern matches structured routing tags: [[word]] or [[word:value]]. // Single-line only (no (?s) dotall). Does NOT match arbitrary [[...]] content. var messageDirectivePattern = regexp.MustCompile(`\[\[\w+(?::[^\]\n]+)?\]\]`) // StripMessageDirectives removes internal [[...]] routing tags from user-facing text, // preserving [[tts...]] tags needed by the TTS auto-apply pipeline. func StripMessageDirectives(content string) string { if !strings.Contains(content, "[[") { return content } result := messageDirectivePattern.ReplaceAllStringFunc(content, func(match string) string { inner := match[2 : len(match)-2] // strip [[ and ]] if strings.HasPrefix(inner, "tts") { return match // preserve for TTS AutoTagged mode } return "" }) return strings.TrimSpace(result) }