fix: guardrail fixes

This commit is contained in:
Alex committed 2026-08-11 13:33:52 +01:00
1 parent e44cff2e91
commit 9446c16108
21 files changed
+772 -122

No files matched your search

+1 -27
View File
@@ -2,10 +2,7 @@ import logging
from typing import Dict, Generator, Optional
from application.agents.base import BaseAgent
from application.agents.tools.internal_search import (
INTERNAL_TOOL_ID,
add_internal_search_tool,
)
from application.agents.tools.internal_search import add_internal_search_tool
from application.agents.tools.wiki import add_wiki_tool
from application.logging import LogContext
@@ -59,26 +56,3 @@ class AgenticAgent(BaseAgent):
log_context.stacks.append(
{"component": "agent", "data": {"tool_calls": self.tool_calls.copy()}}
)
def _collect_internal_sources(self):
"""Merge the cached InternalSearchTool's docs into ``retrieved_docs``,
deduped, preserving any pre-fetched docs so a mixed-exposure agent cites
both pre-fetched and tool-retrieved sources (not just the tool's)."""
cache_key = f"internal_search:{INTERNAL_TOOL_ID}:{self.user or ''}"
tool = self.tool_executor._loaded_tools.get(cache_key)
if not (tool and getattr(tool, "retrieved_docs", None)):
return
def _key(d):
if isinstance(d, dict):
return (d.get("source"), d.get("title"), d.get("text"))
return id(d)
merged = list(self.retrieved_docs or [])
seen = {_key(d) for d in merged}
for doc in tool.retrieved_docs:
k = _key(doc)
if k not in seen:
seen.add(k)
merged.append(doc)
self.retrieved_docs = merged
+153 -30
View File
@@ -24,7 +24,7 @@ from application.guardrails.runtime import (
resolve_config as resolve_guardrails_config,
)
from application.guardrails.stream import StreamingOutputGuard
from application.guardrails.types import Stage
from application.guardrails.types import Action, Stage, resolve_tool_result
from application.llm.handlers.handler_creator import LLMHandlerCreator
from application.llm.llm_creator import LLMCreator
from application.logging import build_stack_data, log_activity, LogContext
@@ -189,22 +189,21 @@ class BaseAgent(ABC):
engine = self.guardrails
if engine is None or not engine.has_stage(stage):
return None
# _build_document_block runs once per document shed by the token
# budget, so an over-budget turn would otherwise pay for N identical
# scans and write N sets of duplicate audit rows.
cache_key = (stage, hash(text))
# The same text is scanned twice in two places: the route runs the
# input stage before it persists the question and ``gen`` runs it
# again, and the token-shed loop rebuilds the document block. Keyed on
# the text itself rather than its hash — a decision carries the
# redacted text, so serving one for a colliding key would substitute
# the wrong turn's output.
cache_key = (stage, text)
cache = getattr(self, "_guardrail_cache", None)
if cache is None:
cache = self._guardrail_cache = {}
cached = cache.get(cache_key)
if cached is not None:
return cached
# Keep the scan context pointed at this turn's documents; retrieval can
# land after the engine was built.
engine.context.retrieved_docs = getattr(self, "retrieved_docs", None) or []
decision = engine.evaluate(text, stage)
if stage is Stage.RETRIEVAL:
cache[cache_key] = decision
cache[cache_key] = decision
return decision
def bind_guardrail_log_context(self, log_context) -> None:
@@ -224,15 +223,22 @@ class BaseAgent(ABC):
"""Apply tool-result controls to a string, returning what may be used."""
if not isinstance(text, str) or not text:
return text
decision = self._guardrail_stage(text, Stage.TOOL_RESULT)
if decision is None:
return text
if decision.blocked:
return (
"[Tool result withheld by a content policy. Tell the user the "
"result could not be used; do not speculate about its contents.]"
)
return decision.text if decision.redacted else text
return resolve_tool_result(
text, self._guardrail_stage(text, Stage.TOOL_RESULT)
)
def bind_guardrail_message_id(self, message_id: Optional[str]) -> None:
"""Tell the recorder which message its rows belong to.
Set as soon as the row is reserved, so a flush that happens before the
caller's own flush — an input block returns from ``gen`` immediately —
still lands linked instead of orphaned with a NULL ``message_id``.
"""
if not message_id:
return
recorder = getattr(self.guardrails, "recorder", None)
if recorder is not None:
recorder.message_id = message_id
def flush_guardrail_audit(self, message_id: Optional[str] = None) -> None:
engine = self._guardrail_engine
@@ -240,6 +246,19 @@ class BaseAgent(ABC):
if recorder is not None and hasattr(recorder, "flush"):
recorder.flush(message_id)
def apply_input_guardrails(self, query: str):
"""Run input controls once and return ``(query, decision)``.
The query comes back redacted when a redact control fired, so callers
that persist or log the question store what the control produced
rather than the raw text. ``gen`` and the route both call this; the
stage cache makes the second call free.
"""
decision = self._guardrail_stage(query, Stage.INPUT)
if decision is not None and decision.redacted:
return decision.text, decision
return query, decision
@staticmethod
def _guardrail_block_event(decision, message: str) -> Dict:
"""The terminal payload for a blocked turn.
@@ -263,16 +282,13 @@ class BaseAgent(ABC):
self, query: str, log_context: LogContext = None
) -> Generator[Dict, None, None]:
self.bind_guardrail_log_context(log_context)
decision = self._guardrail_stage(query, Stage.INPUT)
if decision is not None:
if decision.blocked:
yield self._guardrail_block_event(
decision, decision.block_message or GUARDRAIL_DEFAULT_MESSAGE
)
self.flush_guardrail_audit()
return
if decision.redacted:
query = decision.text
query, decision = self.apply_input_guardrails(query)
if decision is not None and decision.blocked:
yield self._guardrail_block_event(
decision, decision.block_message or GUARDRAIL_DEFAULT_MESSAGE
)
self.flush_guardrail_audit()
return
yield from self._gen_inner(query, log_context)
yield from self._emit_responses_metadata()
@@ -762,6 +778,46 @@ class BaseAgent(ABC):
formatted = decision.text
return f"<documents>\n{formatted}\n</documents>\n{self.DOCUMENT_GUARD}"
def _guard_embedded_documents(self, system_prompt: str) -> str:
"""Scan documents that a custom prompt interpolates itself.
``_build_document_block`` returns early for these agents because the
rendered prompt already carries the documents, which left retrieval
controls scanning nothing at all — and text inside the system prompt
arrives with system authority, the worst place for unscanned,
attacker-influenceable material. The prompt was rendered before the
agent ran, so the verdict is applied by patching it here.
"""
from application.api.answer.services.prompt_renderer import (
format_docs_for_prompt,
)
formatted = format_docs_for_prompt(getattr(self, "retrieved_docs", None))
if not formatted:
return system_prompt
decision = self._guardrail_stage(formatted, Stage.RETRIEVAL)
if decision is None or not (decision.blocked or decision.redacted):
return system_prompt
self._apply_retrieval_decision(decision)
if formatted in system_prompt:
replacement = (
self.RETRIEVAL_BLOCKED_NOTE if decision.blocked else decision.text
)
return system_prompt.replace(formatted, replacement)
# The template placed the documents somewhere this cannot reach. Fail
# the turn rather than send the model text a control just rejected.
if decision.blocked:
raise ValueError(
"Retrieved sources were withheld by a content policy and this "
"agent's prompt embeds them directly, so the request cannot be "
"completed."
)
logger.warning(
"Retrieval redaction could not be applied to an embedding prompt; "
"the sources shown to the user were scrubbed but the prompt was not"
)
return system_prompt
def _apply_retrieval_decision(self, decision) -> None:
"""Mirror a retrieval verdict onto the documents the client will see."""
docs = getattr(self, "retrieved_docs", None) or []
@@ -776,17 +832,69 @@ class BaseAgent(ABC):
engine = self.guardrails
if engine is None:
return
# Only a redacting control can change a document, and a remote judge
# cannot redact at all — it reports no spans. Narrowing the per-document
# pass is what keeps an 8-chunk retrieval from turning one stage
# evaluation into nine, each with its own judge call.
redacting = [
control
for control in engine.config.controls_for(Stage.RETRIEVAL)
if control.action is Action.REDACT
]
if not redacting:
return
scrubbed = []
for doc in docs:
if not isinstance(doc, dict) or not doc.get("text"):
scrubbed.append(doc)
continue
per_doc = engine.evaluate(str(doc["text"]), Stage.RETRIEVAL)
per_doc = engine.evaluate(
str(doc["text"]), Stage.RETRIEVAL, controls=redacting
)
scrubbed.append(
{**doc, "text": per_doc.text} if per_doc.redacted else doc
)
self.retrieved_docs = scrubbed
def _collect_internal_sources(self) -> None:
"""Merge the cached InternalSearchTool's docs into ``retrieved_docs``,
deduped, preserving any pre-fetched docs so a mixed-exposure agent cites
both pre-fetched and tool-retrieved sources (not just the tool's)."""
from application.agents.tools.internal_search import INTERNAL_TOOL_ID
executor = getattr(self, "tool_executor", None)
loaded = getattr(executor, "_loaded_tools", None) or {}
tool = loaded.get(f"internal_search:{INTERNAL_TOOL_ID}:{self.user or ''}")
if not (tool and getattr(tool, "retrieved_docs", None)):
return
def _key(d):
if isinstance(d, dict):
return (d.get("source"), d.get("title"), d.get("text"))
return id(d)
merged = list(self.retrieved_docs or [])
seen = {_key(d) for d in merged}
for doc in tool.retrieved_docs:
k = _key(doc)
if k not in seen:
seen.add(k)
merged.append(doc)
self.retrieved_docs = merged
def _refresh_sources_before_output(self) -> None:
"""Pull tool-retrieved documents in before output controls run.
``internal_search`` results land when the tool loop finishes, which is
after the answer starts streaming. Groundedness judges the answer
against ``retrieved_docs``, so without this it sees an empty list and
reports every tool-retrieved answer as unsourced.
"""
try:
self._collect_internal_sources()
except Exception:
logger.debug("Could not refresh sources before output guarding")
def _compose_user_turn(self, document_block: str, query: str) -> str:
"""Combine the document block and the question into one user message."""
return f"{document_block}\n\n{query}" if document_block else query
@@ -800,6 +908,12 @@ class BaseAgent(ABC):
from application.core.model_utils import get_token_limit
from application.utils import num_tokens_from_string
# Retrieval controls run inside _build_document_block for the usual
# path; a prompt that embeds the documents skips that block entirely,
# so its scan happens here instead.
if getattr(self, "prompt_embeds_documents", False):
system_prompt = self._guard_embedded_documents(system_prompt)
if self.compressed_summary:
compression_context = (
"\n\n---\n\n"
@@ -1128,9 +1242,13 @@ class BaseAgent(ABC):
guarding = engine is not None and engine.has_stage(Stage.OUTPUT)
if isinstance(response, str):
if guarding:
self._refresh_sources_before_output()
yield from self._guarded_complete_answer(response, answer_event)
return
if hasattr(response, "message") and getattr(response.message, "content", None):
if guarding:
self._refresh_sources_before_output()
yield from self._guarded_complete_answer(response.message.content, answer_event)
return
@@ -1164,6 +1282,7 @@ class BaseAgent(ABC):
buffered.append(text)
elif isinstance(event, dict) and "type" in event:
yield event
self._refresh_sources_before_output()
yield from self._guarded_complete_answer("".join(buffered), answer_event)
return
@@ -1182,6 +1301,10 @@ class BaseAgent(ABC):
step.decisions[-1], step.block_message or GUARDRAIL_DEFAULT_MESSAGE
)
return
# The tool loop has finished by the time the generator is exhausted, so
# this is the last point at which the deferred checks in ``flush`` can
# still be given the documents the answer was actually built from.
self._refresh_sources_before_output()
step = guard.flush()
if step.emit:
yield answer_event(step.emit)
+1 -27
View File
@@ -2,10 +2,7 @@ import logging
from typing import Dict, Generator, Optional
from application.agents.base import BaseAgent
from application.agents.tools.internal_search import (
INTERNAL_TOOL_ID,
add_internal_search_tool,
)
from application.agents.tools.internal_search import add_internal_search_tool
from application.agents.tools.wiki import add_wiki_tool
from application.logging import LogContext
@@ -61,26 +58,3 @@ class ClassicAgent(BaseAgent):
log_context.stacks.append(
{"component": "agent", "data": {"tool_calls": self.tool_calls.copy()}}
)
def _collect_internal_sources(self):
"""Merge the cached InternalSearchTool's docs into ``retrieved_docs``,
deduped, preserving any pre-fetched docs so a mixed-exposure agent cites
both pre-fetched and tool-retrieved sources (not just the tool's)."""
cache_key = f"internal_search:{INTERNAL_TOOL_ID}:{self.user or ''}"
tool = self.tool_executor._loaded_tools.get(cache_key)
if not (tool and getattr(tool, "retrieved_docs", None)):
return
def _key(d):
if isinstance(d, dict):
return (d.get("source"), d.get("title"), d.get("text"))
return id(d)
merged = list(self.retrieved_docs or [])
seen = {_key(d) for d in merged}
for doc in tool.retrieved_docs:
k = _key(doc)
if k not in seen:
seen.add(k)
merged.append(doc)
self.retrieved_docs = merged
+2 -7
View File
@@ -13,7 +13,7 @@ from application.agents.default_tools import (
)
from application.agents.tools.tool_action_parser import ToolActionParser
from application.agents.tools.tool_manager import ToolManager
from application.guardrails.types import Stage as GuardrailStage
from application.guardrails.types import Stage as GuardrailStage, resolve_tool_result
from application.security.encryption import decrypt_credentials
from application.storage.db.base_repository import looks_like_uuid
from application.storage.db.repositories.agents import AgentsRepository
@@ -621,12 +621,7 @@ class ToolExecutor:
except Exception:
logger.exception("Tool-result guardrail failed for %s.%s", tool_name, action_name)
return result
if decision.blocked:
return (
"[Tool result withheld by a content policy. Tell the user the "
"result could not be used; do not speculate about its contents.]"
)
return decision.text if decision.redacted else result
return resolve_tool_result(result, decision)
def check_pause(self, tools_dict: Dict, call, llm_class_name: str) -> Optional[Dict]:
"""Return a pending-action dict (approval / client / headless_denied) or None.
+7 -11
View File
@@ -6,7 +6,6 @@ from application.agents.base import BaseAgent
from application.guardrails.config import (
DEFAULT_BLOCK_MESSAGE as GUARDRAIL_DEFAULT_MESSAGE,
)
from application.guardrails.types import Stage
from application.agents.workflows.schemas import (
ExecutionStatus,
Workflow,
@@ -60,16 +59,13 @@ class WorkflowAgent(BaseAgent):
# here or a workflow agent would accept guardrail config in the builder
# and silently enforce none of it.
self.bind_guardrail_log_context(log_context)
decision = self._guardrail_stage(query, Stage.INPUT)
if decision is not None:
if decision.blocked:
yield self._guardrail_block_event(
decision, decision.block_message or GUARDRAIL_DEFAULT_MESSAGE
)
self.flush_guardrail_audit()
return
if decision.redacted:
query = decision.text
query, decision = self.apply_input_guardrails(query)
if decision is not None and decision.blocked:
yield self._guardrail_block_event(
decision, decision.block_message or GUARDRAIL_DEFAULT_MESSAGE
)
self.flush_guardrail_audit()
return
yield from self._gen_inner(query, log_context)
def _gen_inner(self, query: str, log_context: LogContext) -> Generator[Dict[str, str], None, None]:
@@ -369,6 +369,11 @@ class WorkflowEngine:
if doc_manifest:
node_prompt = f"{node_prompt}\n\n{doc_manifest}" if node_prompt else doc_manifest
# No ``agent_config`` here, deliberately: per-agent guardrails are not
# wired through workflows yet, so a node runs the instance floor (which
# ``resolve_config(None)`` still applies) but none of the parent
# agent's own controls. Passing it would need an aggregate output guard
# in WorkflowAgent to be meaningful.
factory_kwargs = {
"agent_type": node_config.agent_type,
"endpoint": self.agent.endpoint,
+32 -1
View File
@@ -301,6 +301,23 @@ class BaseAnswerResource:
# Intentional: a continuation round reserves no new WAL row, so on the
# stateless ``/v1`` path the intermediate tool rounds aren't persisted
# (only the first turn + the final answer turn are). Accepted as-is.
# Input controls have to run before the question is stored, not only
# inside ``gen``: this frame is what writes ``conversation_messages``
# and ``user_logs``, so a redaction that reached the model prompt alone
# would still leave the raw text — the PII the control exists to keep
# out of storage — in both. ``gen`` re-runs the stage against the
# original question and hits the agent's stage cache, so the scan is
# paid for once.
raw_question = question
guard_input = getattr(agent, "apply_input_guardrails", None)
if callable(guard_input) and not _continuation:
try:
question, _ = guard_input(question)
except Exception:
logger.exception(
"Input guardrail scan failed; persisting the question unredacted"
)
wal_eligible = should_persist and not _continuation
if wal_eligible:
try:
@@ -328,6 +345,16 @@ class BaseAnswerResource:
elif _continuation and _continuation.get("reserved_message_id"):
reserved_message_id = _continuation["reserved_message_id"]
# Bind the row now so an audit flush that happens before the ``finally``
# below — an input block returns from ``gen`` immediately and flushes
# there — still writes rows linked to their message instead of orphans.
bind_message_id = getattr(agent, "bind_guardrail_message_id", None)
if callable(bind_message_id):
try:
bind_message_id(reserved_message_id)
except Exception:
logger.exception("Could not bind guardrail audit to the message row")
primary_llm = getattr(agent, "llm", None)
if primary_llm is not None:
primary_llm._request_id = request_id
@@ -572,7 +599,11 @@ class BaseAnswerResource:
reasoning_content=_continuation.get("reasoning_content", ""),
)
else:
gen_iter = agent.gen(query=question)
# The original text: ``gen`` runs the input stage itself and
# applies the redaction to what it sends the model. Handing it
# the already-redacted question would make that a second scan
# over different text, and a remote check would be paid twice.
gen_iter = agent.gen(query=raw_question)
# Seed a liveness heartbeat the moment generation starts, before
# the first chunk. The row is still ``pending`` here; this stamps a
+25 -1
View File
@@ -15,6 +15,7 @@ class ScanContext:
self,
query: Optional[str] = None,
retrieved_docs: Optional[list] = None,
docs_provider=None,
tool_name: Optional[str] = None,
action_name: Optional[str] = None,
tool_args: Optional[Dict[str, Any]] = None,
@@ -23,7 +24,14 @@ class ScanContext:
user: Optional[str] = None,
):
self.query = query
self.retrieved_docs = retrieved_docs or []
# Retrieval can land after the engine is built, and an agent rebinds
# ``retrieved_docs`` rather than mutating it (tool collection, token
# shedding, redaction), so a snapshot taken here goes stale. Reading
# through a provider keeps every stage — including the streaming
# output guard, which does not go through ``_guardrail_stage`` — on
# this turn's actual documents.
self._docs_provider = docs_provider
self._retrieved_docs = retrieved_docs or []
self.tool_name = tool_name
self.action_name = action_name
self.tool_args = tool_args or {}
@@ -31,6 +39,22 @@ class ScanContext:
self.agent_id = agent_id
self.user = user
@property
def retrieved_docs(self) -> list:
if self._docs_provider is not None:
try:
return self._docs_provider() or []
except Exception:
return []
return self._retrieved_docs
@retrieved_docs.setter
def retrieved_docs(self, value: Optional[list]) -> None:
# An explicit assignment is authoritative: it replaces the provider so
# a caller that pins documents is not silently overridden by the agent.
self._docs_provider = None
self._retrieved_docs = value or []
class GuardrailCheck(ABC):
"""A detector. Stateless per scan; constructed once per control."""
+1 -1
View File
@@ -253,7 +253,7 @@ def build_engine(agent, log_context=None) -> Optional[GuardrailEngine]:
fail_open=config.fail_open,
)
context = ScanContext(
retrieved_docs=getattr(agent, "retrieved_docs", None),
docs_provider=lambda: getattr(agent, "retrieved_docs", None),
llm_factory=_judge_factory(agent),
agent_id=str(agent.agent_id) if getattr(agent, "agent_id", None) else None,
user=getattr(agent, "user", None),
+29 -2
View File
@@ -73,13 +73,21 @@ class StreamingOutputGuard:
self._incremental = [c for c in controls if not self._complete_only(c)]
self._deferred = [c for c in controls if self._complete_only(c)]
self._has_remote = any(self._attr(c, "remote", False) for c in self._incremental)
# monitor_only degrades every action to FLAG, so no verdict reached
# here can change a byte of the answer. Withholding output to wait for
# one would buy nothing and cost the user a frozen stream — with a
# remote control, the whole answer when it is shorter than a segment.
# Pass the stream through untouched and scan the finished answer once,
# purely for the audit trail.
self._monitor_only = engine.config.mode != "scan_all"
# The window must cover the longest match any active check can report,
# or that check silently stops working the moment output is streamed.
self.lookback = (
max(0, lookback) if lookback is not None else self._required_window()
)
# Complete-text checks need the finished answer, not the tail.
self._full = "" if self._deferred else None
# Complete-text checks need the finished answer, not the tail; so does
# the monitor_only end-of-stream scan.
self._full = "" if (self._deferred or self._monitor_only) else None
@staticmethod
def _attr(control, name: str, default):
@@ -127,6 +135,9 @@ class StreamingOutputGuard:
return StreamChunk()
if not self.active:
return StreamChunk(emit=text)
if self._monitor_only:
self._full += text
return StreamChunk(emit=text)
combined = self._held + text
emit_end = self._release_point(combined)
@@ -145,6 +156,8 @@ class StreamingOutputGuard:
if not self.active:
tail, self._held = self._held, ""
return StreamChunk(emit=tail)
if self._monitor_only:
return self._monitor_scan()
step = (
self._scan_and_split(self._held, force=True, final=True)
if self._held
@@ -154,6 +167,20 @@ class StreamingOutputGuard:
return step
return self._run_deferred(step)
def _monitor_scan(self) -> StreamChunk:
"""Scan the finished answer once, for the journal only.
One scan of the whole answer rather than the per-chunk sweep the
enforcing path runs: the verdict cannot act, so the only thing that
matters is that it is recorded.
"""
controls = self._incremental + self._deferred
if not controls or not self._full:
return StreamChunk()
decision = self.engine.evaluate(self._full, Stage.OUTPUT, controls=controls)
self.decisions.append(decision)
return StreamChunk(decisions=[decision])
def _run_deferred(self, step: StreamChunk) -> StreamChunk:
"""Run complete-text checks over the finished answer.
+19
View File
@@ -157,6 +157,25 @@ class StageDecision:
}
TOOL_RESULT_BLOCKED_NOTE = (
"[Tool result withheld by a content policy. Tell the user the "
"result could not be used; do not speculate about its contents.]"
)
def resolve_tool_result(result: str, decision: Optional[StageDecision]) -> str:
"""Reduce a tool-result verdict to the text the caller may use.
Shared by the executor path and the client-resume path so a result that
was blocked server-side and one blocked on resume read identically.
"""
if decision is None:
return result
if decision.blocked:
return TOOL_RESULT_BLOCKED_NOTE
return decision.text if decision.redacted else result
def apply_spans(text: str, spans: List[Span]) -> str:
"""Replace ``spans`` in ``text``, right-to-left so offsets stay valid.