feat: better sandbox handling

This commit is contained in:
Alex committed 2026-08-21 13:04:57 +01:00
1 parent baf45447db
commit 4b0d82a6c1
28 files changed
+1574 -49

No files matched your search

+166 -11
View File
@@ -1,3 +1,4 @@
import json
import logging
import re
import uuid
@@ -7,6 +8,7 @@ from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.exc import IntegrityError
from application.agents.default_tools import (
BUILTIN_AGENT_TOOLS,
is_headless_excluded_tool,
resolve_tool_by_id,
synthesized_default_tools,
@@ -41,6 +43,19 @@ def _is_foreign_key_violation(exc: BaseException) -> bool:
_MAX_LLM_NAME_LEN = 64
def _dedupable_tool_names() -> frozenset:
"""Builtin tool names whose duplicate registrations may be collapsed.
A builtin can resolve through both the default-tool and the builtin-agent
registry, so the same row can arrive twice under different synthetic ids.
Only these are safe to collapse — an MCP or user-added tool may
legitimately appear more than once under a single name.
"""
from application.core.settings import settings
return frozenset(BUILTIN_AGENT_TOOLS) | frozenset(getattr(settings, "DEFAULT_CHAT_TOOLS", None) or [])
def _sanitize_tool_prefix(tool_name: Optional[str]) -> str:
"""Reduce a tool name to characters allowed in function-call names."""
return re.sub(r"[^a-zA-Z0-9_-]+", "_", str(tool_name or "")).strip("_")
@@ -370,6 +385,9 @@ class ToolExecutor:
self.attachments: List[Dict] = []
self.client_tools: Optional[List[Dict]] = None
self._name_to_tool: Dict[str, Tuple[str, str]] = {}
# Signatures of unresolvable calls that already failed this turn, so an
# invented tool name cannot be retried until MAX_TOOL_ITERATIONS.
self._unresolvable_calls: Dict[str, int] = {}
self._tool_to_name: Dict[Tuple[str, str], str] = {}
# Filled by the LLMHandler.handle_tool_calls headless loop.
self.headless_denials: List[Dict] = []
@@ -525,6 +543,19 @@ class ToolExecutor:
# (tool_id, tool_name, action_name, action, is_client)
entries: List[Tuple[str, str, str, Dict, bool]] = []
name_counts: Counter = Counter()
# A builtin can arrive twice: once as the user's stored row (keyed by
# list index in ``_get_user_tools``) and once as the synthesized default
# (keyed by uuid5). Pass 2 then hands the model two indistinguishable
# copies with mangled names (``artifact_generator_create_artifact`` +
# ``…_1``). The two rows are NOT byte-identical — a stored row has been
# through ``transform_actions``, which stamps ``active``/``filled_by_llm``
# onto every action — so the key is (tool name, action name) and the
# first occurrence wins, which is the stored row. Safe only for
# builtins: they carry no per-row config
# (``get_config_requirements() == {}``), while two MCP rows can
# legitimately share a name and must stay distinct.
seen_actions: set = set()
dedupable = _dedupable_tool_names()
for tool_id, tool in tools_dict.items():
is_api = tool["name"] == "api_tool"
@@ -540,7 +571,17 @@ class ToolExecutor:
for action in actions:
if not action.get("active", True):
continue
entries.append((tool_id, tool.get("name", ""), action["name"], action, is_client))
tool_name = tool.get("name", "")
if tool_name in dedupable:
fingerprint = (tool_name, action["name"], is_client)
if fingerprint in seen_actions:
logger.debug(
"duplicate_tool_registration_collapsed",
extra={"tool_name": tool_name, "action_name": action["name"]},
)
continue
seen_actions.add(fingerprint)
entries.append((tool_id, tool_name, action["name"], action, is_client))
name_counts[action["name"]] += 1
# Pass 2: assign LLM-visible names and build mappings
@@ -804,6 +845,34 @@ class ToolExecutor:
)
return True
def _available_tool_names(self, tools_dict: Dict) -> str:
"""Render the names the model can actually call, for a correctable error.
Prefer the LLM-visible action names assigned by
:meth:`prepare_tools_for_llm` — those are the strings the model puts in
a tool call. Fall back to tool names when the mapping has not been built
(headless paths, tests). Never the internal tool ids: quoting those
tells the model nothing about what to call instead.
"""
names = sorted(self._tool_to_name.values())
if not names:
names = sorted(
{(tool or {}).get("name") or tool_id for tool_id, tool in (tools_dict or {}).items()}
)
return ", ".join(str(name) for name in names) if names else "(none available)"
@staticmethod
def _call_signature(llm_name: str, call_args: Any) -> str:
"""Stable key for "the model just made this exact call again"."""
try:
rendered = json.dumps(call_args, sort_keys=True, default=str)
except (TypeError, ValueError):
rendered = str(call_args)
return f"{llm_name}::{rendered}"
# After this many identical unresolvable calls, refuse rather than re-run.
UNRESOLVABLE_CALL_LIMIT = 2
def execute(self, tools_dict: Dict, call, llm_class_name: str):
"""Execute a tool call. Yields status events, returns (result, call_id)."""
parser = ToolActionParser(llm_class_name, name_mapping=self._name_to_tool)
@@ -811,8 +880,64 @@ class ToolExecutor:
llm_name = getattr(call, "name", "unknown")
call_id = getattr(call, "id", None) or str(uuid.uuid4())
unresolvable = tool_id is None or action_name is None or tool_id not in tools_dict
# A tool the model invented will never resolve, so re-running it just
# burns the turn's iteration budget on an identical failure. Answer the
# third attempt from here, without touching the tool layer.
if unresolvable:
signature = self._call_signature(llm_name, call_args)
failures = self._unresolvable_calls.get(signature, 0)
if failures >= self.UNRESOLVABLE_CALL_LIMIT:
repeated = (
f"'{llm_name}' has already failed {failures} times with these arguments and "
f"will keep failing. Stop calling it and either use a different tool "
f"({self._available_tool_names(tools_dict)}) or answer without one."
)
logger.warning(
"tool_call_repeated_failure",
extra={"llm_tool_name": llm_name, "call_id": call_id, "failures": failures},
)
tool_call_data = {
"tool_name": "unknown",
"call_id": call_id,
"action_name": llm_name,
"arguments": call_args if isinstance(call_args, dict) else {},
"result": repeated,
"status": "error",
}
# Journal it like the branches below, so a hallucination storm
# is not under-counted in the analytics used to size it.
if _record_proposed(
call_id,
"unknown",
llm_name or "unknown",
call_args if isinstance(call_args, dict) else {},
message_id=self.message_id,
user_id=self.user,
agent_id=self.agent_id,
):
_mark_failed(
call_id,
repeated,
message_id=self.message_id,
user_id=self.user,
)
yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
self.tool_calls.append(tool_call_data)
return repeated, call_id
self._unresolvable_calls[signature] = failures + 1
if tool_id is None or action_name is None:
# Say which half actually failed. Reporting a name problem for a
# registered tool whose arguments were merely malformed is what sent
# one production investigation down the wrong path.
name_is_known = llm_name in self._name_to_tool
parse_reason = (
"its arguments were not a valid JSON object"
if name_is_known
else "the tool name could not be resolved and its arguments were not a JSON object"
)
error_message = f"Error: Failed to parse LLM tool call. Tool name: {llm_name}"
logger.error(
"tool_call_parse_failed",
@@ -828,7 +953,10 @@ class ToolExecutor:
"call_id": call_id,
"action_name": llm_name,
"arguments": call_args or {},
"result": f"Failed to parse tool call. Invalid tool name format: {llm_name}",
"result": (
f"Could not run '{llm_name}': {parse_reason}. "
f"Available tools: {self._available_tool_names(tools_dict)}."
),
"status": "error",
}
# Journal the malformed call so it still shows up in tool analytics.
@@ -849,7 +977,7 @@ class ToolExecutor:
)
yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
self.tool_calls.append(tool_call_data)
return "Failed to parse tool call.", call_id
return tool_call_data["result"], call_id
if tool_id not in tools_dict:
error_message = f"Error: Tool ID '{tool_id}' extracted from LLM call not found in available tools_dict. Available IDs: {list(tools_dict.keys())}"
@@ -868,7 +996,10 @@ class ToolExecutor:
"call_id": call_id,
"action_name": llm_name,
"arguments": call_args,
"result": f"Tool with ID {tool_id} not found. Available tools: {list(tools_dict.keys())}",
"result": (
f"Could not run '{llm_name}': no such tool. "
f"Available tools: {self._available_tool_names(tools_dict)}."
),
"status": "error",
}
# Journal the unresolvable call so it still shows up in tool analytics.
@@ -883,13 +1014,13 @@ class ToolExecutor:
):
_mark_failed(
call_id,
f"Tool with ID {tool_id} not found.",
tool_call_data["result"],
message_id=self.message_id,
user_id=self.user,
)
yield {"type": "tool_call", "data": {**tool_call_data, "status": "error"}}
self.tool_calls.append(tool_call_data)
return f"Tool with ID {tool_id} not found.", call_id
return tool_call_data["result"], call_id
tool_call_data = {
"tool_name": tools_dict[tool_id]["name"],
@@ -1059,7 +1190,11 @@ class ToolExecutor:
# Normalize inside the guard: a tool returning an unexpected
# shape must not break the call it just completed.
artifacts = [
{"id": str(a["id"]).strip(), "filename": a.get("filename")}
{
"id": str(a["id"]).strip(),
"filename": a.get("filename"),
"ref": a.get("ref"),
}
for a in (get_artifacts(action_name, **parameters) or [])
if isinstance(a, dict) and a.get("id")
]
@@ -1213,9 +1348,25 @@ class ToolExecutor:
return tool
# Keys the client needs that are not part of the fixed shape below. They are
# small and optional, and are copied only when present so an ordinary tool
# call does not grow null columns in every persisted row.
_PRESERVED_TOOL_CALL_KEYS = ("artifacts", "device_id")
def get_truncated_tool_calls(self) -> List[Dict]:
return [
{
"""Project tool calls into the shape that is streamed and persisted.
This is what the client reloads, so anything dropped here is live-only
and vanishes when the conversation is reopened. ``result_full`` and
``resolved_arguments`` are shed deliberately — they are the untruncated
copies this projection exists to remove — but ``artifacts`` and
``device_id`` were omitted by oversight, which cost a multi-file
``run_code`` all but its first download chip on reload and left the
remote-device approval UI without the id it keys its sticky action on.
"""
projected = []
for tool_call in self.tool_calls:
entry = {
"tool_name": tool_call.get("tool_name"),
"call_id": tool_call.get("call_id"),
"action_name": tool_call.get("action_name"),
@@ -1224,5 +1375,9 @@ class ToolExecutor:
"result": truncate_tool_result(tool_call.get("result")),
"status": tool_call.get("status", "completed"),
}
for tool_call in self.tool_calls
]
for key in self._PRESERVED_TOOL_CALL_KEYS:
value = tool_call.get(key)
if value:
entry[key] = value
projected.append(entry)
return projected
+68 -7
View File
@@ -142,6 +142,10 @@ _SCHEMAS: Dict[str, Dict[str, Any]] = {
"properties": {
"type": {"type": "string", "enum": ["heading", "paragraph"]},
"text": {"type": "string"},
# ``level`` is legal on an html heading and the synopsis
# lists both block shapes side by side, so models carry
# it over. Rejecting it cost the whole spec.
"level": {"type": "integer", "minimum": 1, "maximum": 3},
},
"required": ["type", "text"],
},
@@ -226,7 +230,7 @@ _SPEC_SYNOPSIS = (
'presentation {"title"?, "slides": [{"title"?, "bullets"?: [str], "notes"?}]} · '
'document {"title"?, "sections": [{"heading"?, "paragraphs"?: [str]}]} · '
'spreadsheet {"sheets": [{"name"?, "rows": [[cell, ...]]}]} · '
'pdf {"title"?, "blocks": [{"type": "heading"|"paragraph", "text"}]} · '
'pdf {"title"?, "blocks": [{"type": "heading"|"paragraph", "text", "level"?: 1-3}]} · '
'html {"title"?, "blocks": [...]} where each block is '
'{"type": "heading", "text", "level"?: 1-3} | {"type": "paragraph", "text"} | '
'{"type": "list", "items": [str], "ordered"?: bool} | '
@@ -318,7 +322,14 @@ _RENDERERS: Dict[str, str] = {
" story.append(Paragraph(escape(str(title)), styles['Title']))\n"
" story.append(Spacer(1, 12))\n"
"for block in spec.get('blocks', []):\n"
" style = styles['Heading1'] if block.get('type') == 'heading' else styles['BodyText']\n"
" if block.get('type') == 'heading':\n"
" try:\n"
" level = int(block.get('level') or 1)\n"
" except (TypeError, ValueError):\n"
" level = 1\n"
" style = styles['Heading%d' % min(max(level, 1), 3)]\n"
" else:\n"
" style = styles['BodyText']\n"
" story.append(Paragraph(escape(str(block.get('text', ''))), style))\n"
" story.append(Spacer(1, 6))\n"
"SimpleDocTemplate({out_path!r}, pagesize=letter).build(story)\n"
@@ -436,6 +447,7 @@ class ArtifactGeneratorTool(Tool):
self.message_id: Optional[str] = self.config.get("message_id")
self._last_artifact_id: Optional[str] = None
self._last_filename: Optional[str] = None
self._last_ref: Optional[str] = None
# ------------------------------------------------------------------
# Tool ABC
@@ -454,6 +466,10 @@ class ArtifactGeneratorTool(Tool):
"or HTML file: it gives them a downloadable, versioned file. Never paste a "
"whole file into the chat instead, and never claim a file was produced unless "
"this tool returned a ref.\n"
"The file is surfaced to the user as a download button the moment this call "
"returns: name it in your answer and say it is ready, but do NOT write a link "
"or path to it. Any URL you write for it is dead and reads to the user as a "
"failed download.\n"
"Do NOT use it for a short snippet the user only wants to read inline, or to "
"change a file you already made — use edit_artifact for that."
),
@@ -538,10 +554,20 @@ class ArtifactGeneratorTool(Tool):
return self._last_artifact_id
def get_artifacts(self, action_name: str, **kwargs: Any) -> List[Dict[str, Any]]:
"""Return the produced artifact with its filename, for UI labelling."""
"""Return the produced artifact with its filename and ref, for the UI.
``ref`` is the model-facing handle (``A1``); the UI needs it to resolve
a ref the model typed into its answer back to this artifact.
"""
if not self._last_artifact_id:
return []
return [{"id": self._last_artifact_id, "filename": self._last_filename}]
return [
{
"id": self._last_artifact_id,
"filename": self._last_filename,
"ref": self._last_ref,
}
]
# ------------------------------------------------------------------
# Dispatch
@@ -550,6 +576,7 @@ class ArtifactGeneratorTool(Tool):
"""Dispatch a create/edit/rewrite action."""
self._last_artifact_id = None
self._last_filename = None
self._last_ref = None
if not self.user_id:
return {"status": "error", "error": "artifact_generator requires a valid user_id."}
if self.conversation_id is None and self.workflow_run_id is None:
@@ -570,7 +597,7 @@ class ArtifactGeneratorTool(Tool):
def _create(self, **kwargs: Any) -> Dict[str, Any]:
"""Validate, render, and persist a new artifact at version 1."""
kind = kwargs.get("kind")
spec = kwargs.get("spec")
spec = self._coerce_spec(kwargs.get("spec"))
title = kwargs.get("title")
if kind not in _KIND_INFO:
return {"status": "error", "error": f"unsupported kind: {kind!r}; expected one of {sorted(_KIND_INFO)}."}
@@ -604,12 +631,16 @@ class ArtifactGeneratorTool(Tool):
return {"status": "error", "error": "failed to persist artifact."}
self._last_artifact_id = ref["artifact_id"]
self._last_filename = ref.get("filename")
self._last_ref = ref.get("ref")
return {"status": "ok", **ref}
def _edit(self, **kwargs: Any) -> Dict[str, Any]:
"""Merge-patch and/or list-append the current spec, re-render, and append a version."""
spec_patch = kwargs.get("spec_patch")
spec_append = kwargs.get("spec_append")
# Same stringification as ``spec``: these are declared ``{"type":
# "object"}`` and sent without ``strict`` too, so a model that
# stringifies one stringifies all three.
spec_patch = self._coerce_spec(kwargs.get("spec_patch"))
spec_append = self._coerce_spec(kwargs.get("spec_append"))
if spec_patch is None and spec_append is None:
return {"status": "error", "error": "edit_artifact needs spec_patch and/or spec_append."}
if spec_patch is not None and not isinstance(spec_patch, dict):
@@ -643,6 +674,7 @@ class ArtifactGeneratorTool(Tool):
self, artifact_id: str, kind: str, spec: Any, action: str, title: Optional[str] = None
) -> Dict[str, Any]:
"""Validate the new spec, re-render, and append the next version of an existing artifact."""
spec = self._coerce_spec(spec)
valid = self._validate(kind, spec)
if valid is not None:
return valid
@@ -671,13 +703,42 @@ class ArtifactGeneratorTool(Tool):
return {"status": "error", "error": "failed to persist artifact version."}
self._last_artifact_id = ref["artifact_id"]
self._last_filename = ref.get("filename")
self._last_ref = ref.get("ref")
return {"status": "ok", **ref}
# ------------------------------------------------------------------
# Spec / render helpers
# ------------------------------------------------------------------
@staticmethod
def _coerce_spec(spec: Any) -> Any:
"""Parse a JSON-encoded spec string into the object the schemas expect.
``spec`` is declared ``{"type": "object"}`` in the action metadata, but
tool definitions go out without ``strict`` (many ``openai_compatible``
endpoints reject it, and the schemas are not strict-conformant), so
nothing makes a model honour that. Weaker models — DeepSeek-V4-Flash
most of all — send ``"spec": "{\"blocks\": [...]}"`` instead. Rejecting
that outright taught the model the artifact tool was broken and sent it
off to hand-write a renderer through ``code_executor``.
Args:
spec: The spec as received from the tool call.
Returns:
The parsed object when ``spec`` is a JSON string encoding one,
otherwise ``spec`` unchanged so the caller still rejects it.
"""
if not isinstance(spec, str):
return spec
try:
parsed = json.loads(spec)
except (TypeError, ValueError):
return spec
return parsed if isinstance(parsed, dict) else spec
def _validate(self, kind: str, spec: Any) -> Optional[Dict[str, Any]]:
"""Return an error payload when ``spec`` is invalid for ``kind``, else None."""
spec = self._coerce_spec(spec)
if not isinstance(spec, dict):
return {"status": "error", "error": "spec must be a JSON object."}
try:
+5 -1
View File
@@ -119,6 +119,8 @@ class CodeExecutorTool(Tool):
"Files written by the code are saved as downloadable artifacts (write throwaway "
"files under `tmp/`, or pass `outputs` to save only specific files); only a compact "
"summary (output tail + artifact references) is returned, never raw bytes. "
"Each saved file appears to the user as a download button: name it in your answer, "
"never write a link or sandbox path to it. "
"Each call is capped at ~60s of wall-clock; for longer work, start it in the "
"background and poll with additional run_code calls (use persist=true to keep state). "
+ self._environment_note()
@@ -416,7 +418,9 @@ class CodeExecutorTool(Tool):
if captured:
self._last_artifact_id = captured[0]["artifact_id"]
self._last_artifacts = [
{"id": a["artifact_id"], "filename": a.get("filename")}
# ``ref`` is the model-facing handle (``A1``); the UI needs it to
# resolve a ref the model typed into its answer.
{"id": a["artifact_id"], "filename": a.get("filename"), "ref": a.get("ref")}
for a in captured
if a.get("artifact_id")
]
+46 -15
View File
@@ -23,9 +23,51 @@ class ToolActionParser:
return self.name_mapping[call_name]
return None
@staticmethod
def _decode_arguments(raw):
"""Decode a tool call's ``arguments`` payload into a dict.
A zero-parameter action (``note_view``, ``note_delete``, the todo/
scheduler list actions, parameterless MCP tools) arrives with
``arguments`` as ``""`` or absent, which is not JSON. Treating that as a
parse failure discarded the *name* too, so a registered tool was
reported as "Invalid tool name format" and the model was handed an
error it could not act on.
Args:
raw: The provider's ``arguments`` value — a JSON string, a dict, or
empty/None for a call that takes no parameters.
Returns:
The decoded dict, or ``None`` when the payload is genuinely
malformed.
"""
if raw is None:
return {}
if isinstance(raw, dict):
return raw
if isinstance(raw, str):
if not raw.strip():
return {}
try:
decoded = json.loads(raw)
except (TypeError, ValueError):
return None
return decoded if isinstance(decoded, dict) else None
return None
def _parse_openai_llm(self, call):
try:
call_args = json.loads(call.arguments)
# An empty payload is a zero-parameter call, not a parse failure —
# treating it as one used to discard the (valid, registered) name
# along with it. Only genuinely malformed arguments fail here.
call_args = self._decode_arguments(call.arguments)
if call_args is None:
logger.error(
"Error parsing OpenAI LLM call: arguments are not a JSON object (%s)",
getattr(call, "name", "<unknown>"),
)
return None, None, None
resolved = self._resolve_via_mapping(call.name)
if resolved:
@@ -56,27 +98,16 @@ class ToolActionParser:
def _parse_google_llm(self, call):
try:
call_args = call.arguments
# Gemini's SDK natively returns ``args`` as a dict, but the
# resume path (``gen_continuation``) stringifies it for the
# assistant message. Coerce a JSON string back into a dict;
# fall back to an empty dict on malformed input so downstream
# ``call_args.items()`` doesn't crash the stream.
if isinstance(call_args, str):
try:
call_args = json.loads(call_args)
except (json.JSONDecodeError, TypeError):
logger.warning(
"Google call.arguments was not valid JSON; "
"falling back to empty args for %s",
getattr(call, "name", "<unknown>"),
)
call_args = {}
if not isinstance(call_args, dict):
call_args = self._decode_arguments(call.arguments)
if call_args is None:
logger.warning(
"Google call.arguments has unexpected type %s; "
"Google call.arguments was not a JSON object; "
"falling back to empty args for %s",
type(call_args).__name__,
getattr(call, "name", "<unknown>"),
)
call_args = {}
@@ -1,4 +1,5 @@
## Formatting
- Use markdown. Put code in fenced blocks with a language tag.
- Only use a mermaid diagram when the user asks for one or a diagram is clearly the best way to answer, and make sure the syntax is valid.
- Never write a download link or file path for a file you produced. Generated files are attached to the message automatically and the user can already see and download them; refer to one by its filename in plain text. A URL you invent for a file is always dead.