Store execution traces in a new request_traces table

One row per execution holding its span tree as JSONB, linked to Logs rows
by request, message, activity and workflow-run ids. message_id cascades so
deleting or truncating a conversation drops its traces; a daily beat task
enforces TRACES_RETENTION_DAYS.

docsgpt/alembic/versions/0037_request_traces.py

@@ -0,0 +1,84 @@
+"""0037 request_traces — one execution trace per request for the Logs UI.
+
+Each row is the full span tree of one execution (a chat turn, a tool-approval
+continuation, a scheduled or webhook run, a search, a graph extraction):
+agent runs, LLM calls, tool calls, retrieval and embeddings with their
+timings, stored as a ``spans`` JSONB array. The Logs UI loads a trace whole,
+so one row per trace keeps the write to a single INSERT and lets retention
+and conversation deletion remove a trace in one step.
+
+``message_id`` cascades: deleting a conversation, or truncating it when a
+turn is superseded, removes that turn's traces with it. The link ids
+(``request_id``, ``activity_id``, ``workflow_run_id``) carry partial indexes
+because only the Logs rows that have them look traces up by them.
+
+``status`` is the only CHECK; span kinds and sources are validated in code so
+new ones need no migration.
+
+Idempotent both ways.
+
+Revision ID: 0037_request_traces
+Revises: 0036_device_audit_created_idx
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+revision: str = "0037_request_traces"
+down_revision: Union[str, None] = "0036_device_audit_created_idx"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS request_traces (
+            id               UUID PRIMARY KEY,
+            request_id       TEXT,
+            message_id       UUID REFERENCES conversation_messages(id) ON DELETE CASCADE,
+            conversation_id  UUID,
+            activity_id      TEXT,
+            workflow_run_id  UUID,
+            user_id          TEXT,
+            agent_id         UUID,
+            source           TEXT NOT NULL,
+            name             TEXT,
+            status           TEXT NOT NULL
+                CONSTRAINT request_traces_status_chk
+                CHECK (status IN ('ok', 'error', 'paused', 'cancelled')),
+            started_at       TIMESTAMPTZ NOT NULL,
+            duration_ms      INTEGER NOT NULL DEFAULT 0,
+            span_count       INTEGER NOT NULL DEFAULT 0,
+            dropped_spans    INTEGER NOT NULL DEFAULT 0,
+            summary          JSONB NOT NULL DEFAULT '{}'::jsonb,
+            spans            JSONB NOT NULL DEFAULT '[]'::jsonb,
+            otel_trace_id    TEXT,
+            created_at       TIMESTAMPTZ NOT NULL DEFAULT now()
+        );
+        """
+    )
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS request_traces_request_idx
+            ON request_traces (request_id) WHERE request_id IS NOT NULL;
+        CREATE INDEX IF NOT EXISTS request_traces_message_idx
+            ON request_traces (message_id) WHERE message_id IS NOT NULL;
+        CREATE INDEX IF NOT EXISTS request_traces_activity_idx
+            ON request_traces (activity_id) WHERE activity_id IS NOT NULL;
+        CREATE INDEX IF NOT EXISTS request_traces_workflow_run_idx
+            ON request_traces (workflow_run_id) WHERE workflow_run_id IS NOT NULL;
+        CREATE INDEX IF NOT EXISTS request_traces_user_started_idx
+            ON request_traces (user_id, started_at DESC);
+        CREATE INDEX IF NOT EXISTS request_traces_agent_started_idx
+            ON request_traces (agent_id, started_at DESC) WHERE agent_id IS NOT NULL;
+        CREATE INDEX IF NOT EXISTS request_traces_created_idx
+            ON request_traces (created_at);
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS request_traces;")

docsgpt/api/user/tasks.py

@@ -608,6 +608,11 @@ def setup_periodic_tasks(sender, **kwargs):
         cleanup_guardrail_events.s(),
         name="cleanup-guardrail-events",
     )
+    sender.add_periodic_task(
+        timedelta(hours=24),
+        cleanup_traces.s(),
+        name="cleanup-traces",
+    )
     sender.add_periodic_task(
         timedelta(hours=24),
         cleanup_orphan_memories.s(),
@@ -827,6 +832,30 @@ def cleanup_guardrail_events(self):
     return {"deleted": deleted, "ttl_days": ttl_days}
 
 
+@celery.task(bind=True, acks_late=False)
+def cleanup_traces(self):
+    """Delete ``request_traces`` rows older than ``TRACES_RETENTION_DAYS``.
+
+    Every chat turn, scheduled run and search writes a trace, and each one
+    carries content previews, so the table is bounded by a retention window
+    like the other per-request journals.
+    """
+    from docsgpt.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {"deleted": 0, "skipped": "POSTGRES_URI not set"}
+
+    from docsgpt.storage.db.engine import get_engine
+    from docsgpt.storage.db.repositories.request_traces import (
+        RequestTracesRepository,
+    )
+
+    ttl_days = settings.TRACES_RETENTION_DAYS
+    engine = get_engine()
+    with engine.begin() as conn:
+        deleted = RequestTracesRepository(conn).purge_older_than(ttl_days)
+    return {"deleted": deleted, "ttl_days": ttl_days}
+
+
 @celery.task(bind=True, acks_late=False)
 def cleanup_orphan_memories(self):
     """Sweep orphan memories left by the 0009 FK-to-trigger orphan window.

docsgpt/storage/db/models.py

@@ -825,6 +825,42 @@ Index(
 Index("ix_guardrail_events_message", guardrail_events_table.c.message_id)
 Index("ix_guardrail_events_created", guardrail_events_table.c.created_at)
 
+# One execution trace per request (chat turn, continuation, scheduled or
+# webhook run, search, graph extraction): the span tree as a JSONB array,
+# rendered as a waterfall in the Logs UI. ``message_id`` cascades so deleting
+# or truncating a conversation removes its traces. Migration 0037.
+request_traces_table = Table(
+    "request_traces",
+    metadata,
+    Column("id", UUID(as_uuid=True), primary_key=True),
+    Column("request_id", Text),
+    Column(
+        "message_id",
+        UUID(as_uuid=True),
+        ForeignKey("conversation_messages.id", ondelete="CASCADE"),
+    ),
+    Column("conversation_id", UUID(as_uuid=True)),
+    Column("activity_id", Text),
+    Column("workflow_run_id", UUID(as_uuid=True)),
+    Column("user_id", Text),
+    Column("agent_id", UUID(as_uuid=True)),
+    Column("source", Text, nullable=False),
+    Column("name", Text),
+    # ok | error | paused | cancelled
+    Column("status", Text, nullable=False),
+    Column("started_at", DateTime(timezone=True), nullable=False),
+    Column("duration_ms", Integer, nullable=False, server_default="0"),
+    Column("span_count", Integer, nullable=False, server_default="0"),
+    Column("dropped_spans", Integer, nullable=False, server_default="0"),
+    Column("summary", JSONB, nullable=False, server_default="{}"),
+    Column("spans", JSONB, nullable=False, server_default="[]"),
+    Column("otel_trace_id", Text),
+    Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
+)
+
+Index("request_traces_user_started_idx", request_traces_table.c.user_id, request_traces_table.c.started_at)
+Index("request_traces_created_idx", request_traces_table.c.created_at)
+
 tool_call_attempts_table = Table(
     "tool_call_attempts",
     metadata,

docsgpt/storage/db/repositories/request_traces.py

@@ -0,0 +1,219 @@
+"""Repository for ``request_traces``: one stored execution trace per request."""
+
+from __future__ import annotations
+
+import datetime
+import json
+import logging
+from typing import Any, Dict, Iterable, List, Optional
+
+from sqlalchemy import Connection, text
+from sqlalchemy.exc import IntegrityError
+
+from docsgpt.storage.db.base_repository import looks_like_uuid, row_to_dict
+from docsgpt.storage.db.serialization import PGNativeJSONEncoder
+from docsgpt.utils import strip_null_bytes
+
+logger = logging.getLogger(__name__)
+
+#: Columns a trace can be looked up by, mapped to their SQL type.
+REF_FIELDS: Dict[str, str] = {
+    "id": "uuid",
+    "request_id": "text",
+    "message_id": "uuid",
+    "activity_id": "text",
+    "workflow_run_id": "uuid",
+}
+
+_SUMMARY_COLUMNS = (
+    "id, request_id, message_id, conversation_id, activity_id, workflow_run_id, "
+    "user_id, agent_id, source, name, status, started_at, duration_ms, "
+    "span_count, dropped_spans, summary, otel_trace_id"
+)
+
+
+def _dump_jsonb(value: Any) -> str:
+    return json.dumps(strip_null_bytes(value), cls=PGNativeJSONEncoder)
+
+
+def _uuid_or_none(value: Optional[str]) -> Optional[str]:
+    return str(value) if value and looks_like_uuid(str(value)) else None
+
+
+def _started_at(record: Dict[str, Any]) -> datetime.datetime:
+    started_ns = record.get("started_at_ns")
+    if started_ns:
+        return datetime.datetime.fromtimestamp(started_ns / 1e9, tz=datetime.timezone.utc)
+    return datetime.datetime.now(datetime.timezone.utc)
+
+
+def _scope_clause(user_id: Optional[str], agent_id: Optional[str]) -> tuple[str, Dict[str, Any]]:
+    """Owner scoping: an owned agent's traces, else the caller's own traces."""
+    if agent_id:
+        return "agent_id = CAST(:scope_agent AS uuid)", {"scope_agent": str(agent_id)}
+    return "user_id = :scope_user", {"scope_user": user_id}
+
+
+class RequestTracesRepository:
+    def __init__(self, conn: Connection) -> None:
+        self._conn = conn
+
+    def insert(self, record: Dict[str, Any]) -> bool:
+        """Insert one finished trace (the dict from ``Trace.to_record``).
+
+        A foreign-key failure means the message was deleted before the trace
+        flushed (e.g. the turn was superseded); the trace describes a turn the
+        user discarded, so it is dropped rather than raised. The insert runs
+        in a savepoint so that drop leaves the caller's transaction usable.
+
+        Args:
+            record: The trace record.
+
+        Returns:
+            True when the row was written.
+        """
+        params = {
+            "id": record["id"],
+            "request_id": record.get("request_id"),
+            "message_id": _uuid_or_none(record.get("message_id")),
+            "conversation_id": _uuid_or_none(record.get("conversation_id")),
+            "activity_id": record.get("activity_id"),
+            "workflow_run_id": _uuid_or_none(record.get("workflow_run_id")),
+            "user_id": record.get("user_id"),
+            "agent_id": _uuid_or_none(record.get("agent_id")),
+            "source": record.get("source") or "unknown",
+            "name": strip_null_bytes(record.get("name")),
+            "status": record.get("status") or "ok",
+            "started_at": _started_at(record),
+            "duration_ms": int(record.get("duration_ms") or 0),
+            "span_count": int(record.get("span_count") or 0),
+            "dropped_spans": int(record.get("dropped_spans") or 0),
+            "summary": _dump_jsonb(record.get("summary") or {}),
+            "spans": _dump_jsonb(record.get("spans") or []),
+            "otel_trace_id": record.get("otel_trace_id"),
+        }
+        statement = text(
+            """
+            INSERT INTO request_traces
+                (id, request_id, message_id, conversation_id, activity_id,
+                 workflow_run_id, user_id, agent_id, source, name, status,
+                 started_at, duration_ms, span_count, dropped_spans, summary,
+                 spans, otel_trace_id)
+            VALUES
+                (CAST(:id AS uuid), :request_id, CAST(:message_id AS uuid),
+                 CAST(:conversation_id AS uuid), :activity_id,
+                 CAST(:workflow_run_id AS uuid), :user_id, CAST(:agent_id AS uuid),
+                 :source, :name, :status, :started_at, :duration_ms, :span_count,
+                 :dropped_spans, CAST(:summary AS jsonb), CAST(:spans AS jsonb),
+                 :otel_trace_id)
+            ON CONFLICT (id) DO NOTHING
+            """
+        )
+        try:
+            with self._conn.begin_nested():
+                result = self._conn.execute(statement, params)
+        except IntegrityError as exc:
+            sqlstate = getattr(getattr(exc, "orig", None), "sqlstate", None)
+            if sqlstate != "23503":
+                raise
+            logger.info("Dropped trace %s: its message no longer exists", record["id"])
+            return False
+        return (result.rowcount or 0) > 0
+
+    def list_by_ref(
+        self,
+        field: str,
+        value: str,
+        *,
+        user_id: Optional[str],
+        agent_id: Optional[str] = None,
+        limit: int = 20,
+    ) -> List[dict]:
+        """Full traces matching ``field = value``, oldest first, owner-scoped.
+
+        Args:
+            field: One of :data:`REF_FIELDS`.
+            value: The id to match.
+            user_id: The caller; used when ``agent_id`` is not given.
+            agent_id: An agent the caller owns; scopes to that agent's traces.
+            limit: Maximum traces returned.
+
+        Returns:
+            Trace dicts including ``spans``; empty for an unknown field or id.
+        """
+        sql_type = REF_FIELDS.get(field)
+        if sql_type is None or not value:
+            return []
+        if sql_type == "uuid" and not looks_like_uuid(str(value)):
+            return []
+        if not agent_id and not user_id:
+            return []
+        scope, scope_params = _scope_clause(user_id, agent_id)
+        result = self._conn.execute(
+            text(
+                f"""
+                SELECT {_SUMMARY_COLUMNS}, spans FROM request_traces
+                WHERE {field} = CAST(:value AS {sql_type}) AND {scope}
+                ORDER BY started_at
+                LIMIT :limit
+                """
+            ),
+            {"value": str(value), "limit": max(1, min(int(limit), 100)), **scope_params},
+        )
+        return [row_to_dict(row) for row in result.fetchall()]
+
+    def summaries_for_refs(
+        self,
+        refs: Dict[str, Iterable[str]],
+        *,
+        user_id: Optional[str],
+        agent_id: Optional[str] = None,
+    ) -> Dict[str, Dict[str, List[dict]]]:
+        """Trace summaries (no spans) for a page of Logs rows.
+
+        Args:
+            refs: ``{field: [ids...]}`` for fields in :data:`REF_FIELDS`.
+            user_id: The caller; used when ``agent_id`` is not given.
+            agent_id: An agent the caller owns.
+
+        Returns:
+            ``{field: {id: [summary, ...]}}`` with summaries oldest first.
+        """
+        out: Dict[str, Dict[str, List[dict]]] = {}
+        if not agent_id and not user_id:
+            return out
+        scope, scope_params = _scope_clause(user_id, agent_id)
+        for field, values in refs.items():
+            sql_type = REF_FIELDS.get(field)
+            if sql_type is None:
+                continue
+            ids = sorted({str(v) for v in values if v})
+            if sql_type == "uuid":
+                ids = [v for v in ids if looks_like_uuid(v)]
+            if not ids:
+                continue
+            result = self._conn.execute(
+                text(
+                    f"""
+                    SELECT {_SUMMARY_COLUMNS} FROM request_traces
+                    WHERE {field} = ANY(CAST(:ids AS {sql_type}[])) AND {scope}
+                    ORDER BY started_at
+                    """
+                ),
+                {"ids": ids, **scope_params},
+            )
+            for row in result.fetchall():
+                data = row_to_dict(row)
+                out.setdefault(field, {}).setdefault(str(data[field]), []).append(data)
+        return out
+
+    def purge_older_than(self, days: int) -> int:
+        """Delete traces older than ``days`` (the retention window)."""
+        result = self._conn.execute(
+            text(
+                "DELETE FROM request_traces "
+                "WHERE created_at < NOW() - CAST(:days || ' days' AS interval)"
+            ),
+            {"days": str(max(1, days))},
+        )
+        return result.rowcount or 0