Add document I/O to workflows and surface run artifacts

Let workflow runs consume and produce documents end to end: bridge uploaded
attachments into run-scoped artifacts so nodes receive the input documents
(with a per-run cap and server-computed size/sha256, and the run row pre-created
so produced artifacts are authorized during the run); emit the run id to the
client and add a builder panel that lists, previews, and downloads a run's
artifacts; and allow attaching documents to a Preview run via the existing
upload flow.

Also fixes issues a compliance workflow surfaced: attachment ownership now keys
on the raw identity instead of a sanitized one (the sanitized form could not be
read back and could collide across users); workflow code nodes read prior state
from a state.json data file instead of templating it into the program, so
untrusted document content can never be interpolated into executed code;
structured node output wrapped in code fences is recovered; and the live
speech-to-text ownership check compares the raw identity.

tests/agents/test_workflow_agent_graph.py

@@ -1,6 +1,6 @@
 """Tests for application/agents/workflow_agent.py graph loading and saving.
 
-Tests _parse_embedded_workflow, _load_from_database, and _save_workflow_run
+Tests _parse_embedded_workflow, _load_from_database, and _finalize_workflow_run
 against the ephemeral ``pg_conn`` fixture. Agent construction is bypassed via
 ``__new__`` so we avoid the BaseAgent's LLM/tool wiring.
 """
@@ -20,6 +20,7 @@ def _make_agent(*, workflow_id=None, workflow=None, workflow_owner=None,
     agent.workflow_owner = workflow_owner
     agent._workflow_data = workflow
     agent._engine = None
+    agent._run_persisted = False
     agent.decoded_token = decoded_token or {}
     return agent
 
@@ -184,7 +185,7 @@ class TestSaveWorkflowRun:
     def test_returns_when_no_engine(self):
         agent = _make_agent(workflow_id="x", workflow_owner="u")
         # _engine is None
-        agent._save_workflow_run("query")
+        agent._finalize_workflow_run(agent.workflow_owner, None, "query")
         # should not raise
 
     def test_returns_when_no_workflow_id(self):
@@ -193,7 +194,7 @@ class TestSaveWorkflowRun:
         agent._engine.execution_log = []
         agent._engine.state = {}
         agent._engine.get_execution_summary.return_value = []
-        agent._save_workflow_run("query")
+        agent._finalize_workflow_run(agent.workflow_owner, None, "query")
 
     def test_returns_when_workflow_missing_in_db(self, pg_conn):
         agent = _make_agent(
@@ -206,7 +207,7 @@ class TestSaveWorkflowRun:
         agent._engine.get_execution_summary.return_value = []
         with _patch_db(pg_conn):
             # Should just return None since workflow not found in DB
-            agent._save_workflow_run("q")
+            agent._finalize_workflow_run(agent.workflow_owner, None, "q")
 
     def test_creates_run_row(self, pg_conn):
         from application.storage.db.repositories.workflows import (
@@ -233,7 +234,7 @@ class TestSaveWorkflowRun:
         agent._engine.workflow_run_id = run_id
 
         with _patch_db(pg_conn):
-            agent._save_workflow_run("my query")
+            agent._finalize_workflow_run(agent.workflow_owner, None, "my query")
 
         runs = WorkflowRunsRepository(pg_conn).list_for_workflow(str(wf["id"]))
         assert len(runs) >= 1
@@ -255,7 +256,7 @@ class TestSaveWorkflowRun:
             "application.agents.workflow_agent.db_session", _broken
         ):
             # Should not raise
-            agent._save_workflow_run("q")
+            agent._finalize_workflow_run(agent.workflow_owner, None, "q")
 
 
 class TestDetermineRunStatus:

tests/agents/test_workflow_agent_pg_write.py

@@ -1,4 +1,4 @@
-"""Tests for the Postgres dual-write path inside WorkflowAgent._save_workflow_run.
+"""Tests for the Postgres write path inside WorkflowAgent._finalize_workflow_run.
 
 Specifically verifies the inner ``_pg_write`` closure that:
 1. Calls WorkflowsRepository.get_by_legacy_id() to resolve the Mongo workflow id.
@@ -64,7 +64,7 @@ def _stub_mongo(agent, insert_id=None):
 
 
 # ---------------------------------------------------------------------------
-# _save_workflow_run — PG dual-write logic
+# _finalize_workflow_run — PG write logic
 # ---------------------------------------------------------------------------
 
 

tests/agents/test_workflow_code_node.py

@@ -5,6 +5,7 @@ patched persistence helper (no live gateway / DB / storage), plus the
 serialization round-trip and CEL branching on an artifact reference.
 """
 
+import json
 from types import SimpleNamespace
 
 import pytest
@@ -61,19 +62,25 @@ class _Result:
 
 
 class _FakeManager:
-    """Records open/close and returns a fixed exec result; no real sandbox."""
+    """Records open/close/put_file/exec and returns a fixed result; no real sandbox."""
 
     def __init__(self, result):
         self._result = result
         self.opened = []
         self.closed = []
+        self.put_files = []
+        self.last_code = None
 
     def open(self, session_id, ttl=None):
         self.opened.append(session_id)
         return session_id
 
+    def put_file(self, session_id, dest_path, data):
+        self.put_files.append((dest_path, data))
+
     def exec(self, session_id, code, timeout=None):
         self.last_timeout = timeout
+        self.last_code = code
         return self._result
 
     def close(self, session_id):
@@ -141,6 +148,40 @@ def test_code_node_no_artifacts_still_writes_status(patch_sandbox):
     assert engine.state["out"] == {"artifacts": [], "status": "ok"}
 
 
+def test_code_node_reads_prior_state_from_state_json(patch_sandbox):
+    # Prior state is staged as DATA in state.json (workspace root = kernel cwd) so
+    # node code reads it with json.load(open("state.json")) -- e.g. state["decision"].
+    engine = _engine()
+    engine.state["decision"] = {"pass": True, "score": 7}
+    node = _code_node(
+        output_variable="out",
+        code="import json\nd = json.load(open('state.json'))\nprint(d['decision'])\n",
+    )
+
+    list(engine._execute_code_node(node))
+
+    manager = patch_sandbox["manager_holder"]["manager"]
+    staged = dict(manager.put_files)
+    assert "state.json" in staged
+    payload = json.loads(staged["state.json"].decode("utf-8"))
+    assert payload["decision"] == {"pass": True, "score": 7}
+
+
+def test_code_node_literal_braces_passed_verbatim_not_templated(patch_sandbox):
+    # Proves code nodes are NOT Jinja-rendered: a literal ``{{ ... }}`` in the code
+    # reaches exec() byte-for-byte (no injection path that interpolates state).
+    engine = _engine()
+    engine.state["decision"] = "INJECTED"
+    literal = "x = '{{ agent.decision }}'\nprint(x)\n"
+    node = _code_node(output_variable="out", code=literal)
+
+    list(engine._execute_code_node(node))
+
+    manager = patch_sandbox["manager_holder"]["manager"]
+    assert manager.last_code == literal
+    assert "INJECTED" not in manager.last_code
+
+
 def test_code_node_failure_raises(patch_sandbox):
     engine = _engine()
     patch_sandbox["result"] = _Result(ok=False, error_name="ValueError", error_value="boom")

tests/agents/test_workflow_engine_coverage.py

@@ -65,6 +65,21 @@ class TestExecuteLoop:
         events = list(engine.execute({}, "query"))
         assert any(e.get("type") == "error" and "start node" in e.get("error", "") for e in events)
 
+    @pytest.mark.unit
+    def test_emits_workflow_run_id_first(self):
+        """A ``workflow_run`` event carrying the run id precedes any step event."""
+        nodes = [
+            _make_node("n1", NodeType.START, "Start"),
+            _make_node("n2", NodeType.END, "End", config={"config": {}}),
+        ]
+        edges = [_make_edge("e1", "n1", "n2")]
+        engine = WorkflowEngine(_make_graph(nodes, edges), _make_agent())
+        events = list(engine.execute({}, "hello"))
+        run_events = [e for e in events if e.get("type") == "workflow_run"]
+        assert len(run_events) == 1
+        assert run_events[0]["workflow_run_id"] == engine.workflow_run_id
+        assert events[0]["type"] == "workflow_run"
+
     @pytest.mark.unit
     def test_start_to_end(self):
         nodes = [

tests/agents/test_workflow_engine_structured_recovery.py

@@ -0,0 +1,44 @@
+"""Engine robustness: fenced/prose structured output recovery in _parse_structured_output."""
+
+from __future__ import annotations
+
+from application.agents.workflows.workflow_engine import WorkflowEngine
+
+
+def _engine() -> WorkflowEngine:
+    return WorkflowEngine.__new__(WorkflowEngine)
+
+
+def test_parse_structured_output_bare_json():
+    ok, val = _engine()._parse_structured_output('{"x": true}')
+    assert ok and val == {"x": True}
+
+
+def test_parse_structured_output_json_fence():
+    ok, val = _engine()._parse_structured_output('```json\n{"a": 1, "b": null}\n```')
+    assert ok and val == {"a": 1, "b": None}
+
+
+def test_parse_structured_output_unlabelled_fence():
+    ok, val = _engine()._parse_structured_output("text\n```\n{\"y\": [1, 2]}\n```\nmore")
+    assert ok and val == {"y": [1, 2]}
+
+
+def test_parse_structured_output_prose_wrapped_object():
+    ok, val = _engine()._parse_structured_output('Result: {"z": "v"} done')
+    assert ok and val == {"z": "v"}
+
+
+def test_parse_structured_output_non_json():
+    ok, val = _engine()._parse_structured_output("not json at all")
+    assert ok is False and val is None
+
+
+def test_parse_structured_output_empty():
+    ok, val = _engine()._parse_structured_output("   ")
+    assert ok is False and val is None
+
+
+def test_strip_json_fence_prefers_fence_over_braces():
+    out = WorkflowEngine._strip_json_fence('lead {"outer": 1} ```json\n{"in": 2}\n```')
+    assert out == '{"in": 2}'

tests/agents/test_workflow_input_documents.py

@@ -0,0 +1,280 @@
+"""Workflow input-document bridge: uploaded attachments become run-scoped artifacts.
+
+The agent pre-creates the ``workflow_runs`` row, re-persists each attachment's bytes
+through the canonical artifact path (server-side size/sha256/storage key), and passes
+the resulting references into the run as ``initial_inputs["input_documents"]`` so nodes
+can read ``agent.input_documents``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import uuid
+
+import pytest
+from sqlalchemy import text
+
+from application.agents.workflow_agent import WorkflowAgent, _MAX_INPUT_DOCUMENTS
+from application.agents.workflows.workflow_engine import WorkflowEngine
+from application.storage.db.repositories.artifacts import ArtifactsRepository
+from application.storage.db.repositories.workflow_runs import WorkflowRunsRepository
+from application.storage.local import LocalStorage
+from application.storage.storage_creator import StorageCreator
+
+pytestmark = pytest.mark.integration
+
+OWNER = "user-bridge"
+
+
+def _wire(pg_engine, tmp_path, monkeypatch) -> LocalStorage:
+    """Point storage + the db session at the ephemeral fixtures."""
+    storage = LocalStorage(base_dir=str(tmp_path))
+    monkeypatch.setattr(StorageCreator, "_instance", storage, raising=False)
+    monkeypatch.setattr("application.storage.db.session.get_engine", lambda: pg_engine)
+    return storage
+
+
+def _make_workflow(pg_engine, owner: str = OWNER) -> str:
+    """Insert an owned workflow row and return its id."""
+    wf_id = str(uuid.uuid4())
+    with pg_engine.begin() as conn:
+        conn.execute(
+            text(
+                "INSERT INTO workflows (id, user_id, name, current_graph_version) "
+                "VALUES (CAST(:id AS uuid), :uid, :name, 1)"
+            ),
+            {"id": wf_id, "uid": owner, "name": "Bridge WF"},
+        )
+    return wf_id
+
+
+def _stage_attachment(storage: LocalStorage, data: bytes, filename: str, mime: str) -> dict:
+    """Write attachment bytes to storage and return the attachment dict shape."""
+    upload_path = f"inputs/{OWNER}/attachments/{uuid.uuid4()}_{filename}"
+    storage.save_file(io.BytesIO(data), upload_path)
+    return {
+        "id": str(uuid.uuid4()),
+        "filename": filename,
+        "upload_path": upload_path,
+        "path": upload_path,
+        "mime_type": mime,
+        "size": len(data),
+        "user_id": OWNER,
+    }
+
+
+def _agent(workflow_id, attachments, owner: str = OWNER) -> WorkflowAgent:
+    """Build a WorkflowAgent without invoking the LLM-creating base __init__."""
+    agent = WorkflowAgent.__new__(WorkflowAgent)
+    agent.workflow_id = workflow_id
+    agent.workflow_owner = owner
+    agent.decoded_token = {"sub": owner}
+    agent.attachments = attachments
+    agent.chat_history = []
+    agent.retrieved_docs = []
+    agent._workflow_data = None
+    agent._engine = None
+    agent._run_persisted = False
+    return agent
+
+
+_EMBEDDED_GRAPH = {
+    "name": "Draft",
+    "nodes": [
+        {"id": "n1", "type": "start", "title": "Start"},
+        {"id": "n2", "type": "end", "title": "End", "data": {}},
+    ],
+    "edges": [{"id": "e1", "source": "n1", "target": "n2"}],
+}
+
+
+class _RecordingEngine(WorkflowEngine):
+    """Engine that records initial_inputs and runs the run-row existence probe."""
+
+    probe = None
+    instances: list = []
+
+    def __init__(self, graph, agent, workflow_run_id=None):
+        super().__init__(graph, agent, workflow_run_id=workflow_run_id)
+        self.captured_inputs = None
+        _RecordingEngine.instances.append(self)
+
+    def execute(self, initial_inputs, query):
+        self.captured_inputs = initial_inputs
+        if _RecordingEngine.probe is not None:
+            _RecordingEngine.probe(self.workflow_run_id)
+        self._initialize_state(initial_inputs, query)
+        return iter(())
+
+
+def _patch_engine(monkeypatch, probe=None) -> None:
+    """Make ``_gen_inner`` build the recording engine and reset its capture state."""
+    _RecordingEngine.instances = []
+    _RecordingEngine.probe = probe
+    monkeypatch.setattr(
+        "application.agents.workflow_agent.WorkflowEngine", _RecordingEngine
+    )
+
+
+def test_attachments_bridge_to_run_scoped_artifacts(pg_engine, tmp_path, monkeypatch):
+    """N attachments -> N run-scoped artifacts + input_documents refs; nodes can read them."""
+    storage = _wire(pg_engine, tmp_path, monkeypatch)
+    wf_id = _make_workflow(pg_engine)
+
+    a1 = b"report-one-bytes"
+    a2 = b"second attachment payload"
+    attachments = [
+        _stage_attachment(storage, a1, "report.txt", "text/plain"),
+        _stage_attachment(storage, a2, "data.csv", "text/csv"),
+    ]
+    agent = _agent(wf_id, attachments)
+
+    run_seen = {}
+
+    def _probe(run_id):
+        with pg_engine.connect() as conn:
+            run_seen["row"] = WorkflowRunsRepository(conn).get(run_id)
+
+    _patch_engine(monkeypatch, probe=_probe)
+
+    list(agent._gen_inner("summarize", log_context=None))
+    engine = _RecordingEngine.instances[-1]
+
+    # The run row existed BEFORE execute (so a mid-run download would authz).
+    assert run_seen["row"] is not None
+    assert run_seen["row"]["user_id"] == OWNER
+
+    # initial_inputs carried the refs into the run.
+    refs = engine.captured_inputs["input_documents"]
+    assert len(refs) == 2
+    assert {r["filename"] for r in refs} == {"report.txt", "data.csv"}
+    assert all(r["artifact_id"] for r in refs)
+    assert refs[0]["ref"] == "A1"
+    assert refs[1]["ref"] == "A2"
+
+    # N run-scoped artifacts persisted, parented to THIS run, server-computed size/sha256.
+    run_id = engine.workflow_run_id
+    with pg_engine.connect() as conn:
+        repo = ArtifactsRepository(conn)
+        by_name = {}
+        for ref, payload in zip(refs, (a1, a2)):
+            artifact = repo.get_artifact_in_parent(ref["artifact_id"], workflow_run_id=run_id)
+            assert artifact is not None
+            assert artifact["kind"] == "file"
+            version = repo.get_version(ref["artifact_id"], 1)
+            assert version["size"] == len(payload)
+            assert version["sha256"] == hashlib.sha256(payload).hexdigest()
+            by_name[version["filename"]] = version
+    assert set(by_name) == {"report.txt", "data.csv"}
+    assert by_name["report.txt"]["size"] == len(a1)
+
+    # A node/template can read agent.input_documents from the engine state.
+    context = engine._build_template_context()
+    assert context["agent"]["input_documents"] == refs
+    assert len(context["agent"]["input_documents"]) == 2
+
+    # The bytes round-trip from storage (never entered state).
+    with pg_engine.connect() as conn:
+        v = ArtifactsRepository(conn).get_version(refs[0]["artifact_id"], 1)
+    assert storage.get_file(v["storage_path"]).read() == a1
+
+
+def test_attachments_capped_per_run(pg_engine, tmp_path, monkeypatch):
+    """More than the cap of attachments bridges only the cap; the rest are dropped."""
+    storage = _wire(pg_engine, tmp_path, monkeypatch)
+    wf_id = _make_workflow(pg_engine)
+
+    over = _MAX_INPUT_DOCUMENTS + 5
+    attachments = [
+        _stage_attachment(storage, f"doc-{i}".encode(), f"f{i}.txt", "text/plain")
+        for i in range(over)
+    ]
+    agent = _agent(wf_id, attachments)
+    _patch_engine(monkeypatch)
+
+    list(agent._gen_inner("summarize", log_context=None))
+    engine = _RecordingEngine.instances[-1]
+
+    refs = engine.captured_inputs["input_documents"]
+    assert len(refs) == _MAX_INPUT_DOCUMENTS
+
+    run_id = engine.workflow_run_id
+    with pg_engine.connect() as conn:
+        n = conn.execute(
+            text(
+                "SELECT count(*) FROM artifacts WHERE workflow_run_id = CAST(:r AS uuid)"
+            ),
+            {"r": run_id},
+        ).scalar()
+    assert n == _MAX_INPUT_DOCUMENTS
+
+
+def test_run_row_precreated_before_execute(pg_engine, tmp_path, monkeypatch):
+    """An owned workflow pre-inserts the run row keyed by the engine run id."""
+    _wire(pg_engine, tmp_path, monkeypatch)
+    wf_id = _make_workflow(pg_engine)
+    agent = _agent(wf_id, [])
+    _patch_engine(monkeypatch)
+
+    list(agent._gen_inner("go", log_context=None))
+    engine = _RecordingEngine.instances[-1]
+
+    with pg_engine.connect() as conn:
+        run = WorkflowRunsRepository(conn).get(engine.workflow_run_id)
+    assert run is not None
+    assert run["user_id"] == OWNER
+    assert str(run["workflow_id"]) == wf_id
+    # Finalized to a terminal status after the run completes.
+    assert run["status"] == "completed"
+    assert run["ended_at"] is not None
+
+
+def test_unowned_workflow_creates_no_run_row(pg_engine, tmp_path, monkeypatch):
+    """A draft/unowned workflow id never persists a run row and skips the bridge."""
+    storage = _wire(pg_engine, tmp_path, monkeypatch)
+    # Embedded (draft) graph whose id is NOT an owned workflow row: the run
+    # executes but no run row is persisted and the bridge is skipped.
+    attachments = [_stage_attachment(storage, b"x", "f.txt", "text/plain")]
+    agent = _agent(str(uuid.uuid4()), attachments)
+    agent._workflow_data = _EMBEDDED_GRAPH
+    _patch_engine(monkeypatch)
+
+    list(agent._gen_inner("go", log_context=None))
+    engine = _RecordingEngine.instances[-1]
+
+    with pg_engine.connect() as conn:
+        run = WorkflowRunsRepository(conn).get(engine.workflow_run_id)
+        # No bridged artifacts either (would be orphaned without a parent row).
+        n = conn.execute(
+            text(
+                "SELECT count(*) FROM artifacts WHERE workflow_run_id = CAST(:r AS uuid)"
+            ),
+            {"r": engine.workflow_run_id},
+        ).scalar()
+    assert run is None
+    assert n == 0
+    assert engine.captured_inputs["input_documents"] == []
+
+
+def test_no_attachments_run_still_works(pg_engine, tmp_path, monkeypatch):
+    """A run with no attachments produces empty input_documents and no artifacts."""
+    _wire(pg_engine, tmp_path, monkeypatch)
+    wf_id = _make_workflow(pg_engine)
+    agent = _agent(wf_id, [])
+    _patch_engine(monkeypatch)
+
+    list(agent._gen_inner("go", log_context=None))
+    engine = _RecordingEngine.instances[-1]
+
+    assert engine.captured_inputs["input_documents"] == []
+    with pg_engine.connect() as conn:
+        run = WorkflowRunsRepository(conn).get(engine.workflow_run_id)
+        n = conn.execute(
+            text(
+                "SELECT count(*) FROM artifacts WHERE workflow_run_id = CAST(:r AS uuid)"
+            ),
+            {"r": engine.workflow_run_id},
+        ).scalar()
+    assert run is not None
+    assert n == 0