mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-15 10:14:32 +00:00
fix: guardrail fixes
This commit is contained in:
1 parent
e44cff2e91
commit
9446c16108
21 files changed
+772
-122
No files matched your search
@@ -2,10 +2,7 @@ import logging
|
||||
from typing import Dict, Generator, Optional
|
||||
|
||||
from application.agents.base import BaseAgent
|
||||
from application.agents.tools.internal_search import (
|
||||
INTERNAL_TOOL_ID,
|
||||
add_internal_search_tool,
|
||||
)
|
||||
from application.agents.tools.internal_search import add_internal_search_tool
|
||||
from application.agents.tools.wiki import add_wiki_tool
|
||||
from application.logging import LogContext
|
||||
|
||||
@@ -59,26 +56,3 @@ class AgenticAgent(BaseAgent):
|
||||
log_context.stacks.append(
|
||||
{"component": "agent", "data": {"tool_calls": self.tool_calls.copy()}}
|
||||
)
|
||||
|
||||
def _collect_internal_sources(self):
|
||||
"""Merge the cached InternalSearchTool's docs into ``retrieved_docs``,
|
||||
deduped, preserving any pre-fetched docs so a mixed-exposure agent cites
|
||||
both pre-fetched and tool-retrieved sources (not just the tool's)."""
|
||||
cache_key = f"internal_search:{INTERNAL_TOOL_ID}:{self.user or ''}"
|
||||
tool = self.tool_executor._loaded_tools.get(cache_key)
|
||||
if not (tool and getattr(tool, "retrieved_docs", None)):
|
||||
return
|
||||
|
||||
def _key(d):
|
||||
if isinstance(d, dict):
|
||||
return (d.get("source"), d.get("title"), d.get("text"))
|
||||
return id(d)
|
||||
|
||||
merged = list(self.retrieved_docs or [])
|
||||
seen = {_key(d) for d in merged}
|
||||
for doc in tool.retrieved_docs:
|
||||
k = _key(doc)
|
||||
if k not in seen:
|
||||
seen.add(k)
|
||||
merged.append(doc)
|
||||
self.retrieved_docs = merged
|
||||
+153
-30
@@ -24,7 +24,7 @@ from application.guardrails.runtime import (
|
||||
resolve_config as resolve_guardrails_config,
|
||||
)
|
||||
from application.guardrails.stream import StreamingOutputGuard
|
||||
from application.guardrails.types import Stage
|
||||
from application.guardrails.types import Action, Stage, resolve_tool_result
|
||||
from application.llm.handlers.handler_creator import LLMHandlerCreator
|
||||
from application.llm.llm_creator import LLMCreator
|
||||
from application.logging import build_stack_data, log_activity, LogContext
|
||||
@@ -189,22 +189,21 @@ class BaseAgent(ABC):
|
||||
engine = self.guardrails
|
||||
if engine is None or not engine.has_stage(stage):
|
||||
return None
|
||||
# _build_document_block runs once per document shed by the token
|
||||
# budget, so an over-budget turn would otherwise pay for N identical
|
||||
# scans and write N sets of duplicate audit rows.
|
||||
cache_key = (stage, hash(text))
|
||||
# The same text is scanned twice in two places: the route runs the
|
||||
# input stage before it persists the question and ``gen`` runs it
|
||||
# again, and the token-shed loop rebuilds the document block. Keyed on
|
||||
# the text itself rather than its hash — a decision carries the
|
||||
# redacted text, so serving one for a colliding key would substitute
|
||||
# the wrong turn's output.
|
||||
cache_key = (stage, text)
|
||||
cache = getattr(self, "_guardrail_cache", None)
|
||||
if cache is None:
|
||||
cache = self._guardrail_cache = {}
|
||||
cached = cache.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
# Keep the scan context pointed at this turn's documents; retrieval can
|
||||
# land after the engine was built.
|
||||
engine.context.retrieved_docs = getattr(self, "retrieved_docs", None) or []
|
||||
decision = engine.evaluate(text, stage)
|
||||
if stage is Stage.RETRIEVAL:
|
||||
cache[cache_key] = decision
|
||||
cache[cache_key] = decision
|
||||
return decision
|
||||
|
||||
def bind_guardrail_log_context(self, log_context) -> None:
|
||||
@@ -224,15 +223,22 @@ class BaseAgent(ABC):
|
||||
"""Apply tool-result controls to a string, returning what may be used."""
|
||||
if not isinstance(text, str) or not text:
|
||||
return text
|
||||
decision = self._guardrail_stage(text, Stage.TOOL_RESULT)
|
||||
if decision is None:
|
||||
return text
|
||||
if decision.blocked:
|
||||
return (
|
||||
"[Tool result withheld by a content policy. Tell the user the "
|
||||
"result could not be used; do not speculate about its contents.]"
|
||||
)
|
||||
return decision.text if decision.redacted else text
|
||||
return resolve_tool_result(
|
||||
text, self._guardrail_stage(text, Stage.TOOL_RESULT)
|
||||
)
|
||||
|
||||
def bind_guardrail_message_id(self, message_id: Optional[str]) -> None:
|
||||
"""Tell the recorder which message its rows belong to.
|
||||
|
||||
Set as soon as the row is reserved, so a flush that happens before the
|
||||
caller's own flush — an input block returns from ``gen`` immediately —
|
||||
still lands linked instead of orphaned with a NULL ``message_id``.
|
||||
"""
|
||||
if not message_id:
|
||||
return
|
||||
recorder = getattr(self.guardrails, "recorder", None)
|
||||
if recorder is not None:
|
||||
recorder.message_id = message_id
|
||||
|
||||
def flush_guardrail_audit(self, message_id: Optional[str] = None) -> None:
|
||||
engine = self._guardrail_engine
|
||||
@@ -240,6 +246,19 @@ class BaseAgent(ABC):
|
||||
if recorder is not None and hasattr(recorder, "flush"):
|
||||
recorder.flush(message_id)
|
||||
|
||||
def apply_input_guardrails(self, query: str):
|
||||
"""Run input controls once and return ``(query, decision)``.
|
||||
|
||||
The query comes back redacted when a redact control fired, so callers
|
||||
that persist or log the question store what the control produced
|
||||
rather than the raw text. ``gen`` and the route both call this; the
|
||||
stage cache makes the second call free.
|
||||
"""
|
||||
decision = self._guardrail_stage(query, Stage.INPUT)
|
||||
if decision is not None and decision.redacted:
|
||||
return decision.text, decision
|
||||
return query, decision
|
||||
|
||||
@staticmethod
|
||||
def _guardrail_block_event(decision, message: str) -> Dict:
|
||||
"""The terminal payload for a blocked turn.
|
||||
@@ -263,16 +282,13 @@ class BaseAgent(ABC):
|
||||
self, query: str, log_context: LogContext = None
|
||||
) -> Generator[Dict, None, None]:
|
||||
self.bind_guardrail_log_context(log_context)
|
||||
decision = self._guardrail_stage(query, Stage.INPUT)
|
||||
if decision is not None:
|
||||
if decision.blocked:
|
||||
yield self._guardrail_block_event(
|
||||
decision, decision.block_message or GUARDRAIL_DEFAULT_MESSAGE
|
||||
)
|
||||
self.flush_guardrail_audit()
|
||||
return
|
||||
if decision.redacted:
|
||||
query = decision.text
|
||||
query, decision = self.apply_input_guardrails(query)
|
||||
if decision is not None and decision.blocked:
|
||||
yield self._guardrail_block_event(
|
||||
decision, decision.block_message or GUARDRAIL_DEFAULT_MESSAGE
|
||||
)
|
||||
self.flush_guardrail_audit()
|
||||
return
|
||||
yield from self._gen_inner(query, log_context)
|
||||
yield from self._emit_responses_metadata()
|
||||
|
||||
@@ -762,6 +778,46 @@ class BaseAgent(ABC):
|
||||
formatted = decision.text
|
||||
return f"<documents>\n{formatted}\n</documents>\n{self.DOCUMENT_GUARD}"
|
||||
|
||||
def _guard_embedded_documents(self, system_prompt: str) -> str:
|
||||
"""Scan documents that a custom prompt interpolates itself.
|
||||
|
||||
``_build_document_block`` returns early for these agents because the
|
||||
rendered prompt already carries the documents, which left retrieval
|
||||
controls scanning nothing at all — and text inside the system prompt
|
||||
arrives with system authority, the worst place for unscanned,
|
||||
attacker-influenceable material. The prompt was rendered before the
|
||||
agent ran, so the verdict is applied by patching it here.
|
||||
"""
|
||||
from application.api.answer.services.prompt_renderer import (
|
||||
format_docs_for_prompt,
|
||||
)
|
||||
|
||||
formatted = format_docs_for_prompt(getattr(self, "retrieved_docs", None))
|
||||
if not formatted:
|
||||
return system_prompt
|
||||
decision = self._guardrail_stage(formatted, Stage.RETRIEVAL)
|
||||
if decision is None or not (decision.blocked or decision.redacted):
|
||||
return system_prompt
|
||||
self._apply_retrieval_decision(decision)
|
||||
if formatted in system_prompt:
|
||||
replacement = (
|
||||
self.RETRIEVAL_BLOCKED_NOTE if decision.blocked else decision.text
|
||||
)
|
||||
return system_prompt.replace(formatted, replacement)
|
||||
# The template placed the documents somewhere this cannot reach. Fail
|
||||
# the turn rather than send the model text a control just rejected.
|
||||
if decision.blocked:
|
||||
raise ValueError(
|
||||
"Retrieved sources were withheld by a content policy and this "
|
||||
"agent's prompt embeds them directly, so the request cannot be "
|
||||
"completed."
|
||||
)
|
||||
logger.warning(
|
||||
"Retrieval redaction could not be applied to an embedding prompt; "
|
||||
"the sources shown to the user were scrubbed but the prompt was not"
|
||||
)
|
||||
return system_prompt
|
||||
|
||||
def _apply_retrieval_decision(self, decision) -> None:
|
||||
"""Mirror a retrieval verdict onto the documents the client will see."""
|
||||
docs = getattr(self, "retrieved_docs", None) or []
|
||||
@@ -776,17 +832,69 @@ class BaseAgent(ABC):
|
||||
engine = self.guardrails
|
||||
if engine is None:
|
||||
return
|
||||
# Only a redacting control can change a document, and a remote judge
|
||||
# cannot redact at all — it reports no spans. Narrowing the per-document
|
||||
# pass is what keeps an 8-chunk retrieval from turning one stage
|
||||
# evaluation into nine, each with its own judge call.
|
||||
redacting = [
|
||||
control
|
||||
for control in engine.config.controls_for(Stage.RETRIEVAL)
|
||||
if control.action is Action.REDACT
|
||||
]
|
||||
if not redacting:
|
||||
return
|
||||
scrubbed = []
|
||||
for doc in docs:
|
||||
if not isinstance(doc, dict) or not doc.get("text"):
|
||||
scrubbed.append(doc)
|
||||
continue
|
||||
per_doc = engine.evaluate(str(doc["text"]), Stage.RETRIEVAL)
|
||||
per_doc = engine.evaluate(
|
||||
str(doc["text"]), Stage.RETRIEVAL, controls=redacting
|
||||
)
|
||||
scrubbed.append(
|
||||
{**doc, "text": per_doc.text} if per_doc.redacted else doc
|
||||
)
|
||||
self.retrieved_docs = scrubbed
|
||||
|
||||
def _collect_internal_sources(self) -> None:
|
||||
"""Merge the cached InternalSearchTool's docs into ``retrieved_docs``,
|
||||
deduped, preserving any pre-fetched docs so a mixed-exposure agent cites
|
||||
both pre-fetched and tool-retrieved sources (not just the tool's)."""
|
||||
from application.agents.tools.internal_search import INTERNAL_TOOL_ID
|
||||
|
||||
executor = getattr(self, "tool_executor", None)
|
||||
loaded = getattr(executor, "_loaded_tools", None) or {}
|
||||
tool = loaded.get(f"internal_search:{INTERNAL_TOOL_ID}:{self.user or ''}")
|
||||
if not (tool and getattr(tool, "retrieved_docs", None)):
|
||||
return
|
||||
|
||||
def _key(d):
|
||||
if isinstance(d, dict):
|
||||
return (d.get("source"), d.get("title"), d.get("text"))
|
||||
return id(d)
|
||||
|
||||
merged = list(self.retrieved_docs or [])
|
||||
seen = {_key(d) for d in merged}
|
||||
for doc in tool.retrieved_docs:
|
||||
k = _key(doc)
|
||||
if k not in seen:
|
||||
seen.add(k)
|
||||
merged.append(doc)
|
||||
self.retrieved_docs = merged
|
||||
|
||||
def _refresh_sources_before_output(self) -> None:
|
||||
"""Pull tool-retrieved documents in before output controls run.
|
||||
|
||||
``internal_search`` results land when the tool loop finishes, which is
|
||||
after the answer starts streaming. Groundedness judges the answer
|
||||
against ``retrieved_docs``, so without this it sees an empty list and
|
||||
reports every tool-retrieved answer as unsourced.
|
||||
"""
|
||||
try:
|
||||
self._collect_internal_sources()
|
||||
except Exception:
|
||||
logger.debug("Could not refresh sources before output guarding")
|
||||
|
||||
def _compose_user_turn(self, document_block: str, query: str) -> str:
|
||||
"""Combine the document block and the question into one user message."""
|
||||
return f"{document_block}\n\n{query}" if document_block else query
|
||||
@@ -800,6 +908,12 @@ class BaseAgent(ABC):
|
||||
from application.core.model_utils import get_token_limit
|
||||
from application.utils import num_tokens_from_string
|
||||
|
||||
# Retrieval controls run inside _build_document_block for the usual
|
||||
# path; a prompt that embeds the documents skips that block entirely,
|
||||
# so its scan happens here instead.
|
||||
if getattr(self, "prompt_embeds_documents", False):
|
||||
system_prompt = self._guard_embedded_documents(system_prompt)
|
||||
|
||||
if self.compressed_summary:
|
||||
compression_context = (
|
||||
"\n\n---\n\n"
|
||||
@@ -1128,9 +1242,13 @@ class BaseAgent(ABC):
|
||||
guarding = engine is not None and engine.has_stage(Stage.OUTPUT)
|
||||
|
||||
if isinstance(response, str):
|
||||
if guarding:
|
||||
self._refresh_sources_before_output()
|
||||
yield from self._guarded_complete_answer(response, answer_event)
|
||||
return
|
||||
if hasattr(response, "message") and getattr(response.message, "content", None):
|
||||
if guarding:
|
||||
self._refresh_sources_before_output()
|
||||
yield from self._guarded_complete_answer(response.message.content, answer_event)
|
||||
return
|
||||
|
||||
@@ -1164,6 +1282,7 @@ class BaseAgent(ABC):
|
||||
buffered.append(text)
|
||||
elif isinstance(event, dict) and "type" in event:
|
||||
yield event
|
||||
self._refresh_sources_before_output()
|
||||
yield from self._guarded_complete_answer("".join(buffered), answer_event)
|
||||
return
|
||||
|
||||
@@ -1182,6 +1301,10 @@ class BaseAgent(ABC):
|
||||
step.decisions[-1], step.block_message or GUARDRAIL_DEFAULT_MESSAGE
|
||||
)
|
||||
return
|
||||
# The tool loop has finished by the time the generator is exhausted, so
|
||||
# this is the last point at which the deferred checks in ``flush`` can
|
||||
# still be given the documents the answer was actually built from.
|
||||
self._refresh_sources_before_output()
|
||||
step = guard.flush()
|
||||
if step.emit:
|
||||
yield answer_event(step.emit)
|
||||
|
||||
@@ -2,10 +2,7 @@ import logging
|
||||
from typing import Dict, Generator, Optional
|
||||
|
||||
from application.agents.base import BaseAgent
|
||||
from application.agents.tools.internal_search import (
|
||||
INTERNAL_TOOL_ID,
|
||||
add_internal_search_tool,
|
||||
)
|
||||
from application.agents.tools.internal_search import add_internal_search_tool
|
||||
from application.agents.tools.wiki import add_wiki_tool
|
||||
from application.logging import LogContext
|
||||
|
||||
@@ -61,26 +58,3 @@ class ClassicAgent(BaseAgent):
|
||||
log_context.stacks.append(
|
||||
{"component": "agent", "data": {"tool_calls": self.tool_calls.copy()}}
|
||||
)
|
||||
|
||||
def _collect_internal_sources(self):
|
||||
"""Merge the cached InternalSearchTool's docs into ``retrieved_docs``,
|
||||
deduped, preserving any pre-fetched docs so a mixed-exposure agent cites
|
||||
both pre-fetched and tool-retrieved sources (not just the tool's)."""
|
||||
cache_key = f"internal_search:{INTERNAL_TOOL_ID}:{self.user or ''}"
|
||||
tool = self.tool_executor._loaded_tools.get(cache_key)
|
||||
if not (tool and getattr(tool, "retrieved_docs", None)):
|
||||
return
|
||||
|
||||
def _key(d):
|
||||
if isinstance(d, dict):
|
||||
return (d.get("source"), d.get("title"), d.get("text"))
|
||||
return id(d)
|
||||
|
||||
merged = list(self.retrieved_docs or [])
|
||||
seen = {_key(d) for d in merged}
|
||||
for doc in tool.retrieved_docs:
|
||||
k = _key(doc)
|
||||
if k not in seen:
|
||||
seen.add(k)
|
||||
merged.append(doc)
|
||||
self.retrieved_docs = merged
|
||||
@@ -13,7 +13,7 @@ from application.agents.default_tools import (
|
||||
)
|
||||
from application.agents.tools.tool_action_parser import ToolActionParser
|
||||
from application.agents.tools.tool_manager import ToolManager
|
||||
from application.guardrails.types import Stage as GuardrailStage
|
||||
from application.guardrails.types import Stage as GuardrailStage, resolve_tool_result
|
||||
from application.security.encryption import decrypt_credentials
|
||||
from application.storage.db.base_repository import looks_like_uuid
|
||||
from application.storage.db.repositories.agents import AgentsRepository
|
||||
@@ -621,12 +621,7 @@ class ToolExecutor:
|
||||
except Exception:
|
||||
logger.exception("Tool-result guardrail failed for %s.%s", tool_name, action_name)
|
||||
return result
|
||||
if decision.blocked:
|
||||
return (
|
||||
"[Tool result withheld by a content policy. Tell the user the "
|
||||
"result could not be used; do not speculate about its contents.]"
|
||||
)
|
||||
return decision.text if decision.redacted else result
|
||||
return resolve_tool_result(result, decision)
|
||||
|
||||
def check_pause(self, tools_dict: Dict, call, llm_class_name: str) -> Optional[Dict]:
|
||||
"""Return a pending-action dict (approval / client / headless_denied) or None.
|
||||
|
||||
@@ -6,7 +6,6 @@ from application.agents.base import BaseAgent
|
||||
from application.guardrails.config import (
|
||||
DEFAULT_BLOCK_MESSAGE as GUARDRAIL_DEFAULT_MESSAGE,
|
||||
)
|
||||
from application.guardrails.types import Stage
|
||||
from application.agents.workflows.schemas import (
|
||||
ExecutionStatus,
|
||||
Workflow,
|
||||
@@ -60,16 +59,13 @@ class WorkflowAgent(BaseAgent):
|
||||
# here or a workflow agent would accept guardrail config in the builder
|
||||
# and silently enforce none of it.
|
||||
self.bind_guardrail_log_context(log_context)
|
||||
decision = self._guardrail_stage(query, Stage.INPUT)
|
||||
if decision is not None:
|
||||
if decision.blocked:
|
||||
yield self._guardrail_block_event(
|
||||
decision, decision.block_message or GUARDRAIL_DEFAULT_MESSAGE
|
||||
)
|
||||
self.flush_guardrail_audit()
|
||||
return
|
||||
if decision.redacted:
|
||||
query = decision.text
|
||||
query, decision = self.apply_input_guardrails(query)
|
||||
if decision is not None and decision.blocked:
|
||||
yield self._guardrail_block_event(
|
||||
decision, decision.block_message or GUARDRAIL_DEFAULT_MESSAGE
|
||||
)
|
||||
self.flush_guardrail_audit()
|
||||
return
|
||||
yield from self._gen_inner(query, log_context)
|
||||
|
||||
def _gen_inner(self, query: str, log_context: LogContext) -> Generator[Dict[str, str], None, None]:
|
||||
|
||||
@@ -369,6 +369,11 @@ class WorkflowEngine:
|
||||
if doc_manifest:
|
||||
node_prompt = f"{node_prompt}\n\n{doc_manifest}" if node_prompt else doc_manifest
|
||||
|
||||
# No ``agent_config`` here, deliberately: per-agent guardrails are not
|
||||
# wired through workflows yet, so a node runs the instance floor (which
|
||||
# ``resolve_config(None)`` still applies) but none of the parent
|
||||
# agent's own controls. Passing it would need an aggregate output guard
|
||||
# in WorkflowAgent to be meaningful.
|
||||
factory_kwargs = {
|
||||
"agent_type": node_config.agent_type,
|
||||
"endpoint": self.agent.endpoint,
|
||||
|
||||
@@ -301,6 +301,23 @@ class BaseAnswerResource:
|
||||
# Intentional: a continuation round reserves no new WAL row, so on the
|
||||
# stateless ``/v1`` path the intermediate tool rounds aren't persisted
|
||||
# (only the first turn + the final answer turn are). Accepted as-is.
|
||||
# Input controls have to run before the question is stored, not only
|
||||
# inside ``gen``: this frame is what writes ``conversation_messages``
|
||||
# and ``user_logs``, so a redaction that reached the model prompt alone
|
||||
# would still leave the raw text — the PII the control exists to keep
|
||||
# out of storage — in both. ``gen`` re-runs the stage against the
|
||||
# original question and hits the agent's stage cache, so the scan is
|
||||
# paid for once.
|
||||
raw_question = question
|
||||
guard_input = getattr(agent, "apply_input_guardrails", None)
|
||||
if callable(guard_input) and not _continuation:
|
||||
try:
|
||||
question, _ = guard_input(question)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Input guardrail scan failed; persisting the question unredacted"
|
||||
)
|
||||
|
||||
wal_eligible = should_persist and not _continuation
|
||||
if wal_eligible:
|
||||
try:
|
||||
@@ -328,6 +345,16 @@ class BaseAnswerResource:
|
||||
elif _continuation and _continuation.get("reserved_message_id"):
|
||||
reserved_message_id = _continuation["reserved_message_id"]
|
||||
|
||||
# Bind the row now so an audit flush that happens before the ``finally``
|
||||
# below — an input block returns from ``gen`` immediately and flushes
|
||||
# there — still writes rows linked to their message instead of orphans.
|
||||
bind_message_id = getattr(agent, "bind_guardrail_message_id", None)
|
||||
if callable(bind_message_id):
|
||||
try:
|
||||
bind_message_id(reserved_message_id)
|
||||
except Exception:
|
||||
logger.exception("Could not bind guardrail audit to the message row")
|
||||
|
||||
primary_llm = getattr(agent, "llm", None)
|
||||
if primary_llm is not None:
|
||||
primary_llm._request_id = request_id
|
||||
@@ -572,7 +599,11 @@ class BaseAnswerResource:
|
||||
reasoning_content=_continuation.get("reasoning_content", ""),
|
||||
)
|
||||
else:
|
||||
gen_iter = agent.gen(query=question)
|
||||
# The original text: ``gen`` runs the input stage itself and
|
||||
# applies the redaction to what it sends the model. Handing it
|
||||
# the already-redacted question would make that a second scan
|
||||
# over different text, and a remote check would be paid twice.
|
||||
gen_iter = agent.gen(query=raw_question)
|
||||
|
||||
# Seed a liveness heartbeat the moment generation starts, before
|
||||
# the first chunk. The row is still ``pending`` here; this stamps a
|
||||
|
||||
@@ -15,6 +15,7 @@ class ScanContext:
|
||||
self,
|
||||
query: Optional[str] = None,
|
||||
retrieved_docs: Optional[list] = None,
|
||||
docs_provider=None,
|
||||
tool_name: Optional[str] = None,
|
||||
action_name: Optional[str] = None,
|
||||
tool_args: Optional[Dict[str, Any]] = None,
|
||||
@@ -23,7 +24,14 @@ class ScanContext:
|
||||
user: Optional[str] = None,
|
||||
):
|
||||
self.query = query
|
||||
self.retrieved_docs = retrieved_docs or []
|
||||
# Retrieval can land after the engine is built, and an agent rebinds
|
||||
# ``retrieved_docs`` rather than mutating it (tool collection, token
|
||||
# shedding, redaction), so a snapshot taken here goes stale. Reading
|
||||
# through a provider keeps every stage — including the streaming
|
||||
# output guard, which does not go through ``_guardrail_stage`` — on
|
||||
# this turn's actual documents.
|
||||
self._docs_provider = docs_provider
|
||||
self._retrieved_docs = retrieved_docs or []
|
||||
self.tool_name = tool_name
|
||||
self.action_name = action_name
|
||||
self.tool_args = tool_args or {}
|
||||
@@ -31,6 +39,22 @@ class ScanContext:
|
||||
self.agent_id = agent_id
|
||||
self.user = user
|
||||
|
||||
@property
|
||||
def retrieved_docs(self) -> list:
|
||||
if self._docs_provider is not None:
|
||||
try:
|
||||
return self._docs_provider() or []
|
||||
except Exception:
|
||||
return []
|
||||
return self._retrieved_docs
|
||||
|
||||
@retrieved_docs.setter
|
||||
def retrieved_docs(self, value: Optional[list]) -> None:
|
||||
# An explicit assignment is authoritative: it replaces the provider so
|
||||
# a caller that pins documents is not silently overridden by the agent.
|
||||
self._docs_provider = None
|
||||
self._retrieved_docs = value or []
|
||||
|
||||
|
||||
class GuardrailCheck(ABC):
|
||||
"""A detector. Stateless per scan; constructed once per control."""
|
||||
|
||||
@@ -253,7 +253,7 @@ def build_engine(agent, log_context=None) -> Optional[GuardrailEngine]:
|
||||
fail_open=config.fail_open,
|
||||
)
|
||||
context = ScanContext(
|
||||
retrieved_docs=getattr(agent, "retrieved_docs", None),
|
||||
docs_provider=lambda: getattr(agent, "retrieved_docs", None),
|
||||
llm_factory=_judge_factory(agent),
|
||||
agent_id=str(agent.agent_id) if getattr(agent, "agent_id", None) else None,
|
||||
user=getattr(agent, "user", None),
|
||||
|
||||
@@ -73,13 +73,21 @@ class StreamingOutputGuard:
|
||||
self._incremental = [c for c in controls if not self._complete_only(c)]
|
||||
self._deferred = [c for c in controls if self._complete_only(c)]
|
||||
self._has_remote = any(self._attr(c, "remote", False) for c in self._incremental)
|
||||
# monitor_only degrades every action to FLAG, so no verdict reached
|
||||
# here can change a byte of the answer. Withholding output to wait for
|
||||
# one would buy nothing and cost the user a frozen stream — with a
|
||||
# remote control, the whole answer when it is shorter than a segment.
|
||||
# Pass the stream through untouched and scan the finished answer once,
|
||||
# purely for the audit trail.
|
||||
self._monitor_only = engine.config.mode != "scan_all"
|
||||
# The window must cover the longest match any active check can report,
|
||||
# or that check silently stops working the moment output is streamed.
|
||||
self.lookback = (
|
||||
max(0, lookback) if lookback is not None else self._required_window()
|
||||
)
|
||||
# Complete-text checks need the finished answer, not the tail.
|
||||
self._full = "" if self._deferred else None
|
||||
# Complete-text checks need the finished answer, not the tail; so does
|
||||
# the monitor_only end-of-stream scan.
|
||||
self._full = "" if (self._deferred or self._monitor_only) else None
|
||||
|
||||
@staticmethod
|
||||
def _attr(control, name: str, default):
|
||||
@@ -127,6 +135,9 @@ class StreamingOutputGuard:
|
||||
return StreamChunk()
|
||||
if not self.active:
|
||||
return StreamChunk(emit=text)
|
||||
if self._monitor_only:
|
||||
self._full += text
|
||||
return StreamChunk(emit=text)
|
||||
|
||||
combined = self._held + text
|
||||
emit_end = self._release_point(combined)
|
||||
@@ -145,6 +156,8 @@ class StreamingOutputGuard:
|
||||
if not self.active:
|
||||
tail, self._held = self._held, ""
|
||||
return StreamChunk(emit=tail)
|
||||
if self._monitor_only:
|
||||
return self._monitor_scan()
|
||||
step = (
|
||||
self._scan_and_split(self._held, force=True, final=True)
|
||||
if self._held
|
||||
@@ -154,6 +167,20 @@ class StreamingOutputGuard:
|
||||
return step
|
||||
return self._run_deferred(step)
|
||||
|
||||
def _monitor_scan(self) -> StreamChunk:
|
||||
"""Scan the finished answer once, for the journal only.
|
||||
|
||||
One scan of the whole answer rather than the per-chunk sweep the
|
||||
enforcing path runs: the verdict cannot act, so the only thing that
|
||||
matters is that it is recorded.
|
||||
"""
|
||||
controls = self._incremental + self._deferred
|
||||
if not controls or not self._full:
|
||||
return StreamChunk()
|
||||
decision = self.engine.evaluate(self._full, Stage.OUTPUT, controls=controls)
|
||||
self.decisions.append(decision)
|
||||
return StreamChunk(decisions=[decision])
|
||||
|
||||
def _run_deferred(self, step: StreamChunk) -> StreamChunk:
|
||||
"""Run complete-text checks over the finished answer.
|
||||
|
||||
|
||||
@@ -157,6 +157,25 @@ class StageDecision:
|
||||
}
|
||||
|
||||
|
||||
TOOL_RESULT_BLOCKED_NOTE = (
|
||||
"[Tool result withheld by a content policy. Tell the user the "
|
||||
"result could not be used; do not speculate about its contents.]"
|
||||
)
|
||||
|
||||
|
||||
def resolve_tool_result(result: str, decision: Optional[StageDecision]) -> str:
|
||||
"""Reduce a tool-result verdict to the text the caller may use.
|
||||
|
||||
Shared by the executor path and the client-resume path so a result that
|
||||
was blocked server-side and one blocked on resume read identically.
|
||||
"""
|
||||
if decision is None:
|
||||
return result
|
||||
if decision.blocked:
|
||||
return TOOL_RESULT_BLOCKED_NOTE
|
||||
return decision.text if decision.redacted else result
|
||||
|
||||
|
||||
def apply_spans(text: str, spans: List[Span]) -> str:
|
||||
"""Replace ``spans`` in ``text``, right-to-left so offsets stay valid.
|
||||
|
||||
|
||||
Reference in new issue
Block a user