refactor: rename the application package to docsgpt

The backend import package is now docsgpt, the name it will carry on PyPI;
application was far too generic to install into anyone's site-packages.
git mv plus a mechanical rewrite of every import, dotted string and path
reference: 734 Python files, the compose files, Dockerfile, workflows, docs,
setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage
config, .gitignore. Behaviour is unchanged.

Kept for one release:
- A top-level application package whose meta-path finder resolves
  application.x.y to the already-imported docsgpt.x.y object, so old imports
  and entry points (celery -A application.app.celery,
  uvicorn application.asgi:asgi_app) keep working with a FutureWarning.
- Celery registers every application.* task name as an alias of its
  docsgpt.* task on start-up, so messages queued by the previous release still
  run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries
  the previous release wrote are left unread instead of firing twice.

The backend image builds from the repository root (docker build -f
docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore
allow-lists docsgpt/ and application/ and keeps caches, local data, .env
files, the sample index files and the Dockerfile out. Compose and the image
workflows point at the new context.

docsgpt/api/user/__init__.py

@@ -0,0 +1,5 @@
+"""User API module - provides all user-related API endpoints"""
+
+from .routes import user
+
+__all__ = ["user"]

docsgpt/api/user/agents/__init__.py

@@ -0,0 +1,17 @@
+"""Agents module."""
+
+from .guardrails import agents_guardrails_ns
+from .routes import agents_ns
+from .sharing import agents_sharing_ns
+from .webhooks import agents_webhooks_ns
+from .folders import agents_folders_ns
+from .portability import agents_portability_ns
+
+__all__ = [
+    "agents_guardrails_ns",
+    "agents_ns",
+    "agents_sharing_ns",
+    "agents_webhooks_ns",
+    "agents_folders_ns",
+    "agents_portability_ns",
+]

docsgpt/api/user/agents/folders.py

@@ -0,0 +1,366 @@
+"""
+Agent folders management routes.
+Provides virtual folder organization for agents (Google Drive-like structure).
+"""
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import Namespace, Resource, fields
+from sqlalchemy import text as _sql_text
+
+from docsgpt.api import api
+from docsgpt.storage.db.base_repository import looks_like_uuid
+from docsgpt.storage.db.repositories.agent_folders import AgentFoldersRepository
+from docsgpt.storage.db.repositories.agents import AgentsRepository
+from docsgpt.storage.db.session import db_readonly, db_session
+
+
+agents_folders_ns = Namespace(
+    "agents_folders", description="Agent folder management", path="/api/agents/folders"
+)
+
+
+def _resolve_folder_id(repo: AgentFoldersRepository, folder_id: str, user: str):
+    """Resolve a folder id that may be either a UUID or legacy Mongo ObjectId."""
+    if not folder_id:
+        return None
+    if looks_like_uuid(folder_id):
+        row = repo.get(folder_id, user)
+        if row is not None:
+            return row
+    return repo.get_by_legacy_id(folder_id, user)
+
+
+def _folder_error_response(message: str, err: Exception):
+    current_app.logger.error(f"{message}: {err}", exc_info=True)
+    return make_response(jsonify({"success": False, "message": message}), 400)
+
+
+def _serialize_folder(f: dict) -> dict:
+    created_at = f.get("created_at")
+    updated_at = f.get("updated_at")
+    return {
+        "id": str(f["id"]),
+        "name": f.get("name"),
+        "parent_id": str(f["parent_id"]) if f.get("parent_id") else None,
+        "created_at": created_at.isoformat() if hasattr(created_at, "isoformat") else created_at,
+        "updated_at": updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at,
+    }
+
+
+@agents_folders_ns.route("/")
+class AgentFolders(Resource):
+    @api.doc(description="Get all folders for the user")
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                folders = AgentFoldersRepository(conn).list_for_user(user)
+            result = [_serialize_folder(f) for f in folders]
+            return make_response(jsonify({"folders": result}), 200)
+        except Exception as err:
+            return _folder_error_response("Failed to fetch folders", err)
+
+    @api.doc(description="Create a new folder")
+    @api.expect(
+        api.model(
+            "CreateFolder",
+            {
+                "name": fields.String(required=True, description="Folder name"),
+                "parent_id": fields.String(required=False, description="Parent folder ID"),
+            },
+        )
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        if not data or not data.get("name"):
+            return make_response(jsonify({"success": False, "message": "Folder name is required"}), 400)
+
+        parent_id_input = data.get("parent_id")
+        description = data.get("description")
+
+        try:
+            with db_session() as conn:
+                repo = AgentFoldersRepository(conn)
+                pg_parent_id = None
+                if parent_id_input:
+                    parent = _resolve_folder_id(repo, parent_id_input, user)
+                    if not parent:
+                        return make_response(
+                            jsonify({"success": False, "message": "Parent folder not found"}),
+                            404,
+                        )
+                    pg_parent_id = str(parent["id"])
+                folder = repo.create(
+                    user, data["name"],
+                    description=description,
+                    parent_id=pg_parent_id,
+                )
+            return make_response(
+                jsonify(
+                    {
+                        "id": str(folder["id"]),
+                        "name": folder["name"],
+                        "parent_id": pg_parent_id,
+                    }
+                ),
+                201,
+            )
+        except Exception as err:
+            return _folder_error_response("Failed to create folder", err)
+
+
+@agents_folders_ns.route("/<string:folder_id>")
+class AgentFolder(Resource):
+    @api.doc(description="Get a specific folder with its agents")
+    def get(self, folder_id):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                folders_repo = AgentFoldersRepository(conn)
+                folder = _resolve_folder_id(folders_repo, folder_id, user)
+                if not folder:
+                    return make_response(
+                        jsonify({"success": False, "message": "Folder not found"}),
+                        404,
+                    )
+                pg_folder_id = str(folder["id"])
+
+                agents_rows = conn.execute(
+                    _sql_text(
+                        "SELECT id, name, description FROM agents "
+                        "WHERE user_id = :user_id AND folder_id = CAST(:fid AS uuid) "
+                        "ORDER BY created_at DESC"
+                    ),
+                    {"user_id": user, "fid": pg_folder_id},
+                ).fetchall()
+                agents_list = [
+                    {
+                        "id": str(row._mapping["id"]),
+                        "name": row._mapping["name"],
+                        "description": row._mapping.get("description", "") or "",
+                    }
+                    for row in agents_rows
+                ]
+
+                subfolders = folders_repo.list_children(pg_folder_id, user)
+                subfolders_list = [
+                    {"id": str(sf["id"]), "name": sf["name"]}
+                    for sf in subfolders
+                ]
+
+            return make_response(
+                jsonify(
+                    {
+                        "id": pg_folder_id,
+                        "name": folder["name"],
+                        "parent_id": (
+                            str(folder["parent_id"]) if folder.get("parent_id") else None
+                        ),
+                        "agents": agents_list,
+                        "subfolders": subfolders_list,
+                    }
+                ),
+                200,
+            )
+        except Exception as err:
+            return _folder_error_response("Failed to fetch folder", err)
+
+    @api.doc(description="Update a folder")
+    def put(self, folder_id):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        if not data:
+            return make_response(jsonify({"success": False, "message": "No data provided"}), 400)
+
+        try:
+            with db_session() as conn:
+                repo = AgentFoldersRepository(conn)
+                folder = _resolve_folder_id(repo, folder_id, user)
+                if not folder:
+                    return make_response(
+                        jsonify({"success": False, "message": "Folder not found"}),
+                        404,
+                    )
+                pg_folder_id = str(folder["id"])
+
+                update_fields: dict = {}
+                if "name" in data:
+                    update_fields["name"] = data["name"]
+                if "description" in data:
+                    update_fields["description"] = data["description"]
+                if "parent_id" in data:
+                    parent_input = data.get("parent_id")
+                    if parent_input:
+                        if parent_input == folder_id or parent_input == pg_folder_id:
+                            return make_response(
+                                jsonify(
+                                    {
+                                        "success": False,
+                                        "message": "Cannot set folder as its own parent",
+                                    }
+                                ),
+                                400,
+                            )
+                        parent = _resolve_folder_id(repo, parent_input, user)
+                        if not parent:
+                            return make_response(
+                                jsonify({"success": False, "message": "Parent folder not found"}),
+                                404,
+                            )
+                        if str(parent["id"]) == pg_folder_id:
+                            return make_response(
+                                jsonify(
+                                    {
+                                        "success": False,
+                                        "message": "Cannot set folder as its own parent",
+                                    }
+                                ),
+                                400,
+                            )
+                        update_fields["parent_id"] = str(parent["id"])
+                    else:
+                        update_fields["parent_id"] = None
+
+                if update_fields:
+                    repo.update(pg_folder_id, user, update_fields)
+
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            return _folder_error_response("Failed to update folder", err)
+
+    @api.doc(description="Delete a folder")
+    def delete(self, folder_id):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            with db_session() as conn:
+                repo = AgentFoldersRepository(conn)
+                folder = _resolve_folder_id(repo, folder_id, user)
+                if not folder:
+                    return make_response(
+                        jsonify({"success": False, "message": "Folder not found"}),
+                        404,
+                    )
+                pg_folder_id = str(folder["id"])
+                # Clear folder assignments from agents; self-FK
+                # ``ON DELETE SET NULL`` handles child folders.
+                AgentsRepository(conn).clear_folder_for_all(pg_folder_id, user)
+                deleted = repo.delete(pg_folder_id, user)
+            if not deleted:
+                return make_response(
+                    jsonify({"success": False, "message": "Folder not found"}),
+                    404,
+                )
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            return _folder_error_response("Failed to delete folder", err)
+
+
+@agents_folders_ns.route("/move_agent")
+class MoveAgentToFolder(Resource):
+    @api.doc(description="Move an agent to a folder or remove from folder")
+    @api.expect(
+        api.model(
+            "MoveAgent",
+            {
+                "agent_id": fields.String(required=True, description="Agent ID to move"),
+                "folder_id": fields.String(required=False, description="Target folder ID (null to remove from folder)"),
+            },
+        )
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        if not data or not data.get("agent_id"):
+            return make_response(jsonify({"success": False, "message": "Agent ID is required"}), 400)
+
+        agent_id_input = data["agent_id"]
+        folder_id_input = data.get("folder_id")
+
+        try:
+            with db_session() as conn:
+                agents_repo = AgentsRepository(conn)
+                agent = agents_repo.get_any(agent_id_input, user)
+                if not agent:
+                    return make_response(
+                        jsonify({"success": False, "message": "Agent not found"}),
+                        404,
+                    )
+                pg_folder_id = None
+                if folder_id_input:
+                    folders_repo = AgentFoldersRepository(conn)
+                    folder = _resolve_folder_id(folders_repo, folder_id_input, user)
+                    if not folder:
+                        return make_response(
+                            jsonify({"success": False, "message": "Folder not found"}),
+                            404,
+                        )
+                    pg_folder_id = str(folder["id"])
+                agents_repo.set_folder(str(agent["id"]), user, pg_folder_id)
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            return _folder_error_response("Failed to move agent", err)
+
+
+@agents_folders_ns.route("/bulk_move")
+class BulkMoveAgents(Resource):
+    @api.doc(description="Move multiple agents to a folder")
+    @api.expect(
+        api.model(
+            "BulkMoveAgents",
+            {
+                "agent_ids": fields.List(fields.String, required=True, description="List of agent IDs"),
+                "folder_id": fields.String(required=False, description="Target folder ID"),
+            },
+        )
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        if not data or not data.get("agent_ids"):
+            return make_response(jsonify({"success": False, "message": "Agent IDs are required"}), 400)
+
+        agent_ids = data["agent_ids"]
+        folder_id_input = data.get("folder_id")
+
+        try:
+            with db_session() as conn:
+                agents_repo = AgentsRepository(conn)
+                pg_folder_id = None
+                if folder_id_input:
+                    folders_repo = AgentFoldersRepository(conn)
+                    folder = _resolve_folder_id(folders_repo, folder_id_input, user)
+                    if not folder:
+                        return make_response(
+                            jsonify({"success": False, "message": "Folder not found"}),
+                            404,
+                        )
+                    pg_folder_id = str(folder["id"])
+                for agent_id_input in agent_ids:
+                    agent = agents_repo.get_any(agent_id_input, user)
+                    if agent is not None:
+                        agents_repo.set_folder(str(agent["id"]), user, pg_folder_id)
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            return _folder_error_response("Failed to move agents", err)

docsgpt/api/user/agents/guardrails.py

@@ -0,0 +1,156 @@
+"""Guardrails catalog and decision-journal routes."""
+
+from flask import jsonify, make_response, request
+from flask_restx import Namespace, Resource
+
+from docsgpt.api import api
+from docsgpt.api.user.team_sharing import team_access_for
+from docsgpt.core.settings import settings
+from docsgpt.guardrails.checks.patterns import DEFAULT_PII_ENTITIES, PII_PATTERNS
+from docsgpt.guardrails.config import DEFAULT_BLOCK_MESSAGE, MODES
+from docsgpt.guardrails.guardrail_creator import GuardrailCreator
+from docsgpt.guardrails import runtime as guardrails_runtime
+from docsgpt.guardrails.types import ACTIONS_BY_STAGE, Stage
+from docsgpt.storage.db.repositories.agents import AgentsRepository
+from docsgpt.storage.db.repositories.guardrail_events import (
+    GuardrailEventsRepository,
+)
+from docsgpt.storage.db.session import db_readonly
+
+agents_guardrails_ns = Namespace(
+    "guardrails", description="Agent guardrail configuration and audit", path="/api"
+)
+
+
+@agents_guardrails_ns.route("/guardrails/catalog")
+class GuardrailCatalog(Resource):
+    @api.doc(description="List available guardrail checks and their capabilities")
+    def get(self):
+        if not request.decoded_token:
+            return {"success": False}, 401
+        floor = guardrails_runtime.instance_floor()
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "enabled": bool(settings.GUARDRAILS_ENABLED),
+                    "checks": GuardrailCreator.catalog(),
+                    "stages": [s.value for s in Stage],
+                    "modes": list(MODES),
+                    "actions_by_stage": {
+                        stage.value: sorted(a.value for a in actions)
+                        for stage, actions in ACTIONS_BY_STAGE.items()
+                    },
+                    "default_block_message": DEFAULT_BLOCK_MESSAGE,
+                    "pii_entities": sorted(PII_PATTERNS),
+                    "default_pii_entities": DEFAULT_PII_ENTITIES,
+                    # Only which (check, stage) pairs the floor claims, and the
+                    # action it imposes. The settings stay server-side: handing
+                    # every authenticated user the banned-term list and the
+                    # policy prompts makes evading them trivial.
+                    "floor": (
+                        {
+                            "mode": floor.mode,
+                            "fail_open": floor.fail_open,
+                            "controls": [
+                                {
+                                    "check": c.check,
+                                    "stage": c.stage.value,
+                                    "action": c.action.value,
+                                }
+                                for c in floor.controls
+                            ],
+                        }
+                        if floor and floor.enabled
+                        else None
+                    ),
+                }
+            ),
+            200,
+        )
+
+
+def _readable_agent(conn, agent_id: str, user: str):
+    """Return the agent row when the caller may read it, else None."""
+    repo = AgentsRepository(conn)
+    agent = repo.get_any(agent_id, user)
+    if agent:
+        return agent
+    if team_access_for(conn, user, "agent", agent_id):
+        return repo.get_by_id(agent_id)
+    return None
+
+
+@agents_guardrails_ns.route("/guardrails/events")
+class GuardrailEvents(Resource):
+    @api.doc(
+        params={"agent_id": "Agent ID", "limit": "Max rows (default 100)",
+                "offset": "Row offset"},
+        description="List guardrail decisions recorded for an agent",
+    )
+    def get(self):
+        if not (decoded_token := request.decoded_token):
+            return {"success": False}, 401
+        user = decoded_token["sub"]
+        agent_id = request.args.get("agent_id")
+        if not agent_id:
+            return make_response(
+                jsonify({"success": False, "message": "agent_id required"}), 400
+            )
+        try:
+            limit = int(request.args.get("limit", 100))
+            offset = int(request.args.get("offset", 0))
+        except (TypeError, ValueError):
+            return make_response(
+                jsonify({"success": False, "message": "limit/offset must be integers"}),
+                400,
+            )
+        with db_readonly() as conn:
+            agent = _readable_agent(conn, agent_id, user)
+            if not agent:
+                return make_response(
+                    jsonify({"success": False, "message": "Agent not found"}), 404
+                )
+            # Query on the row's UUID, not the caller's argument: a legacy
+            # 24-hex Mongo id resolves fine above but would blow up the cast.
+            # Rows stay scoped to the requesting user even on a shared agent —
+            # another member's blocked prompts are not this caller's to read.
+            events = GuardrailEventsRepository(conn).list_for_agent(
+                str(agent["id"]), user, limit=limit, offset=offset
+            )
+        return make_response(jsonify({"success": True, "events": events}), 200)
+
+
+@agents_guardrails_ns.route("/guardrails/summary")
+class GuardrailSummary(Resource):
+    @api.doc(
+        params={
+            "days": "Trailing window in days (default 30)",
+            "agent_id": "Scope the aggregate to one agent (optional)",
+        },
+        description="Aggregate guardrail activity for the caller",
+    )
+    def get(self):
+        if not (decoded_token := request.decoded_token):
+            return {"success": False}, 401
+        user = decoded_token["sub"]
+        try:
+            days = int(request.args.get("days", 30))
+        except (TypeError, ValueError):
+            return make_response(
+                jsonify({"success": False, "message": "days must be an integer"}), 400
+            )
+        agent_id = request.args.get("agent_id")
+        with db_readonly() as conn:
+            scoped_id = None
+            if agent_id:
+                agent = _readable_agent(conn, agent_id, user)
+                if not agent:
+                    return make_response(
+                        jsonify({"success": False, "message": "Agent not found"}), 404
+                    )
+                scoped_id = str(agent["id"])
+            summary = GuardrailEventsRepository(conn).summary_for_user(
+                user, days=days, agent_id=scoped_id
+            )
+        return make_response(jsonify({"success": True, **summary}), 200)

docsgpt/api/user/agents/sharing.py

@@ -0,0 +1,282 @@
+"""Agent management sharing functionality."""
+
+import datetime
+import secrets
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+from sqlalchemy import text as _sql_text
+
+from docsgpt.api import api
+from docsgpt.core.settings import settings
+from docsgpt.api.user.base import resolve_tool_details
+from docsgpt.storage.db.base_repository import looks_like_uuid
+from docsgpt.storage.db.repositories.agents import AgentsRepository
+from docsgpt.storage.db.repositories.users import UsersRepository
+from docsgpt.storage.db.session import db_readonly, db_session
+from docsgpt.utils import generate_image_url
+
+agents_sharing_ns = Namespace(
+    "agents", description="Agent management operations", path="/api"
+)
+
+
+def _serialize_agent_basic(agent: dict) -> dict:
+    """Shape a PG agent row into the API response dict."""
+    source_id = agent.get("source_id")
+    return {
+        "id": str(agent["id"]),
+        "user": agent.get("user_id", ""),
+        "name": agent.get("name", ""),
+        "image": (
+            generate_image_url(
+                agent["image"], agent["id"], agent.get("user_id")
+            )
+            if agent.get("image")
+            else ""
+        ),
+        "description": agent.get("description", ""),
+        "source": str(source_id) if source_id else "",
+        "chunks": str(agent["chunks"]) if agent.get("chunks") is not None else "0",
+        "retriever": agent.get("retriever", "classic") or "classic",
+        "prompt_id": str(agent["prompt_id"]) if agent.get("prompt_id") else "default",
+        "tools": agent.get("tools", []) or [],
+        "tool_details": resolve_tool_details(agent.get("tools", []) or []),
+        "agent_type": agent.get("agent_type", "") or "",
+        "status": agent.get("status", "") or "",
+        "json_schema": agent.get("json_schema"),
+        "limited_token_mode": agent.get("limited_token_mode", False),
+        "token_limit": agent.get("token_limit") or settings.DEFAULT_AGENT_LIMITS["token_limit"],
+        "limited_request_mode": agent.get("limited_request_mode", False),
+        "request_limit": agent.get("request_limit") or settings.DEFAULT_AGENT_LIMITS["request_limit"],
+        "created_at": agent.get("created_at", ""),
+        "updated_at": agent.get("updated_at", ""),
+        "shared": bool(agent.get("shared", False)),
+        "shared_token": agent.get("shared_token", "") or "",
+        "shared_metadata": agent.get("shared_metadata", {}) or {},
+    }
+
+
+@agents_sharing_ns.route("/shared_agent")
+class SharedAgent(Resource):
+    @api.doc(
+        params={
+            "token": "Shared token of the agent",
+        },
+        description="Get a shared agent by token or ID",
+    )
+    def get(self):
+        shared_token = request.args.get("token")
+
+        if not shared_token:
+            return make_response(
+                jsonify({"success": False, "message": "Token or ID is required"}), 400
+            )
+        try:
+            with db_readonly() as conn:
+                shared_agent = AgentsRepository(conn).find_by_shared_token(
+                    shared_token,
+                )
+            if not shared_agent:
+                return make_response(
+                    jsonify({"success": False, "message": "Shared agent not found"}),
+                    404,
+                )
+            agent_id = str(shared_agent["id"])
+            data = _serialize_agent_basic(shared_agent)
+
+            if data["tools"]:
+                enriched_tools = []
+                for detail in data["tool_details"]:
+                    enriched_tools.append(detail.get("name", ""))
+                data["tools"] = enriched_tools
+            decoded_token = getattr(request, "decoded_token", None)
+            if decoded_token:
+                user_id = decoded_token.get("sub")
+                owner_id = shared_agent.get("user_id")
+
+                if user_id != owner_id:
+                    with db_session() as conn:
+                        users_repo = UsersRepository(conn)
+                        users_repo.upsert(user_id)
+                        users_repo.add_shared(user_id, agent_id)
+            return make_response(jsonify(data), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving shared agent: {err}")
+            return make_response(jsonify({"success": False}), 400)
+
+
+@agents_sharing_ns.route("/shared_agents")
+class SharedAgents(Resource):
+    @api.doc(description="Get shared agents explicitly shared with the user")
+    def get(self):
+        try:
+            decoded_token = request.decoded_token
+            if not decoded_token:
+                return make_response(jsonify({"success": False}), 401)
+            user_id = decoded_token.get("sub")
+
+            with db_session() as conn:
+                users_repo = UsersRepository(conn)
+                user_doc = users_repo.upsert(user_id)
+                shared_with_ids = (
+                    user_doc.get("agent_preferences", {}).get("shared_with_me", [])
+                    if isinstance(user_doc.get("agent_preferences"), dict)
+                    else []
+                )
+                # Keep only UUID-shaped ids; ObjectId leftovers are stripped below.
+                uuid_ids = [sid for sid in shared_with_ids if looks_like_uuid(sid)]
+                non_uuid_ids = [sid for sid in shared_with_ids if not looks_like_uuid(sid)]
+
+                if uuid_ids:
+                    result = conn.execute(
+                        _sql_text(
+                            "SELECT * FROM agents "
+                            "WHERE id = ANY(CAST(:ids AS uuid[])) "
+                            "AND shared = true"
+                        ),
+                        {"ids": uuid_ids},
+                    )
+                    shared_agents = [dict(row._mapping) for row in result.fetchall()]
+                else:
+                    shared_agents = []
+
+                found_ids_set = {str(agent["id"]) for agent in shared_agents}
+                stale_ids = [sid for sid in uuid_ids if sid not in found_ids_set]
+                stale_ids.extend(non_uuid_ids)
+                if stale_ids:
+                    users_repo.remove_shared_bulk(user_id, stale_ids)
+
+                pinned_ids = set(
+                    user_doc.get("agent_preferences", {}).get("pinned", [])
+                    if isinstance(user_doc.get("agent_preferences"), dict)
+                    else []
+                )
+
+            list_shared_agents = []
+            for agent in shared_agents:
+                agent_id_str = str(agent["id"])
+                list_shared_agents.append(
+                    {
+                        "id": agent_id_str,
+                        "name": agent.get("name", ""),
+                        "description": agent.get("description", ""),
+                        "image": (
+                            generate_image_url(
+                                agent["image"], agent["id"], agent.get("user_id")
+                            )
+                            if agent.get("image")
+                            else ""
+                        ),
+                        "tools": agent.get("tools", []) or [],
+                        "tool_details": resolve_tool_details(
+                            agent.get("tools", []) or []
+                        ),
+                        "agent_type": agent.get("agent_type", "") or "",
+                        "status": agent.get("status", "") or "",
+                        "json_schema": agent.get("json_schema"),
+                        "limited_token_mode": agent.get("limited_token_mode", False),
+                        "token_limit": agent.get("token_limit") or settings.DEFAULT_AGENT_LIMITS["token_limit"],
+                        "limited_request_mode": agent.get("limited_request_mode", False),
+                        "request_limit": agent.get("request_limit") or settings.DEFAULT_AGENT_LIMITS["request_limit"],
+                        "created_at": agent.get("created_at", ""),
+                        "updated_at": agent.get("updated_at", ""),
+                        "pinned": agent_id_str in pinned_ids,
+                        "shared": bool(agent.get("shared", False)),
+                        "shared_token": agent.get("shared_token", "") or "",
+                        "shared_metadata": agent.get("shared_metadata", {}) or {},
+                    }
+                )
+
+            return make_response(jsonify(list_shared_agents), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving shared agents: {err}")
+            return make_response(jsonify({"success": False}), 400)
+
+
+@agents_sharing_ns.route("/share_agent")
+class ShareAgent(Resource):
+    @api.expect(
+        api.model(
+            "ShareAgentModel",
+            {
+                "id": fields.String(required=True, description="ID of the agent"),
+                "shared": fields.Boolean(
+                    required=True, description="Share or unshare the agent"
+                ),
+                "username": fields.String(
+                    required=False, description="Name of the user"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Share or unshare an agent")
+    def put(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+
+        data = request.get_json()
+        if not data:
+            return make_response(
+                jsonify({"success": False, "message": "Missing JSON body"}), 400
+            )
+        agent_id = data.get("id")
+        shared = data.get("shared")
+        username = data.get("username", "")
+
+        if not agent_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        if shared is None:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Shared parameter is required and must be true or false",
+                    }
+                ),
+                400,
+            )
+        shared_token = None
+        try:
+            with db_session() as conn:
+                repo = AgentsRepository(conn)
+                agent = repo.get_any(agent_id, user)
+                if not agent:
+                    return make_response(
+                        jsonify({"success": False, "message": "Agent not found"}), 404
+                    )
+                if shared:
+                    shared_metadata = {
+                        "shared_by": username,
+                        "shared_at": datetime.datetime.now(
+                            datetime.timezone.utc
+                        ).isoformat(),
+                    }
+                    shared_token = secrets.token_urlsafe(32)
+                    repo.update(
+                        str(agent["id"]), user,
+                        {
+                            "shared": True,
+                            "shared_token": shared_token,
+                            "shared_metadata": shared_metadata,
+                        },
+                    )
+                else:
+                    repo.update(
+                        str(agent["id"]), user,
+                        {
+                            "shared": False,
+                            "shared_token": None,
+                            "shared_metadata": None,
+                        },
+                    )
+        except Exception as err:
+            current_app.logger.error(f"Error sharing/unsharing agent: {err}", exc_info=True)
+            return make_response(jsonify({"success": False, "error": "Failed to update agent sharing status"}), 400)
+        return make_response(
+            jsonify({"success": True, "shared_token": shared_token}), 200
+        )

docsgpt/api/user/agents/webhooks.py

@@ -0,0 +1,232 @@
+"""Agent management webhook handlers."""
+
+import secrets
+import uuid
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import Namespace, Resource
+from sqlalchemy import text as sql_text
+
+from docsgpt.api import api
+from docsgpt.api.user.base import require_agent
+from docsgpt.api.user.tasks import process_agent_webhook
+from docsgpt.core.settings import settings
+from docsgpt.storage.db.base_repository import looks_like_uuid
+from docsgpt.storage.db.repositories.agents import AgentsRepository
+from docsgpt.storage.db.repositories.idempotency import IdempotencyRepository
+from docsgpt.storage.db.session import db_readonly, db_session
+
+
+agents_webhooks_ns = Namespace(
+    "agents", description="Agent management operations", path="/api"
+)
+
+
+_IDEMPOTENCY_KEY_MAX_LEN = 256
+
+
+def _read_idempotency_key():
+    """Return (key, error_response). Empty header → (None, None); oversized → (None, 400)."""
+    key = request.headers.get("Idempotency-Key")
+    if not key:
+        return None, None
+    if len(key) > _IDEMPOTENCY_KEY_MAX_LEN:
+        return None, make_response(
+            jsonify(
+                {
+                    "success": False,
+                    "message": (
+                        f"Idempotency-Key exceeds maximum length of "
+                        f"{_IDEMPOTENCY_KEY_MAX_LEN} characters"
+                    ),
+                }
+            ),
+            400,
+        )
+    return key, None
+
+
+def _scoped_idempotency_key(idempotency_key, scope):
+    """``{scope}:{key}`` so different agents can't collide on the same key."""
+    if not idempotency_key or not scope:
+        return None
+    return f"{scope}:{idempotency_key}"
+
+
+@agents_webhooks_ns.route("/agent_webhook")
+class AgentWebhook(Resource):
+    @api.doc(
+        params={"id": "ID of the agent"},
+        description="Generate webhook URL for the agent",
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        agent_id = request.args.get("id")
+        if not agent_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        try:
+            with db_readonly() as conn:
+                agent = AgentsRepository(conn).get_any(agent_id, user)
+            if not agent:
+                return make_response(
+                    jsonify({"success": False, "message": "Agent not found"}), 404
+                )
+            webhook_token = agent.get("incoming_webhook_token")
+            if not webhook_token:
+                webhook_token = secrets.token_urlsafe(32)
+                with db_session() as conn:
+                    AgentsRepository(conn).update(
+                        str(agent["id"]), user,
+                        {"incoming_webhook_token": webhook_token},
+                    )
+            base_url = settings.API_URL.rstrip("/")
+            full_webhook_url = f"{base_url}/api/webhooks/agents/{webhook_token}"
+        except Exception as err:
+            current_app.logger.error(
+                f"Error generating webhook URL: {err}", exc_info=True
+            )
+            return make_response(
+                jsonify({"success": False, "message": "Error generating webhook URL"}),
+                400,
+            )
+        return make_response(
+            jsonify({"success": True, "webhook_url": full_webhook_url}), 200
+        )
+
+
+@agents_webhooks_ns.route("/webhooks/agents/<string:webhook_token>")
+class AgentWebhookListener(Resource):
+    method_decorators = [require_agent]
+
+    def _enqueue_webhook_task(self, agent_id_str, payload, source_method, agent=None):
+        if not payload:
+            current_app.logger.warning(
+                f"Webhook ({source_method}) received for agent {agent_id_str} with empty payload."
+            )
+        current_app.logger.info(
+            f"Incoming {source_method} webhook for agent {agent_id_str}. Enqueuing task with payload: {payload}"
+        )
+
+        idempotency_key, key_error = _read_idempotency_key()
+        if key_error is not None:
+            return key_error
+        # Resolve to PG UUID first so dedup writes don't crash on legacy ids.
+        agent_uuid = None
+        if agent is not None:
+            candidate = str(agent.get("id") or "")
+            if looks_like_uuid(candidate):
+                agent_uuid = candidate
+        if idempotency_key and agent_uuid is None:
+            current_app.logger.warning(
+                "Skipping webhook idempotency dedup: agent %s has non-UUID id",
+                agent_id_str,
+            )
+            idempotency_key = None
+        # Agent-scoped (webhooks have no user_id).
+        scoped_key = _scoped_idempotency_key(idempotency_key, agent_uuid)
+        # Claim before enqueue; the loser returns the winner's task_id.
+        predetermined_task_id = None
+        if scoped_key:
+            predetermined_task_id = str(uuid.uuid4())
+            with db_session() as conn:
+                claimed = IdempotencyRepository(conn).record_webhook(
+                    key=scoped_key,
+                    agent_id=agent_uuid,
+                    task_id=predetermined_task_id,
+                    response_json={
+                        "success": True, "task_id": predetermined_task_id,
+                    },
+                )
+            if claimed is None:
+                with db_readonly() as conn:
+                    cached = IdempotencyRepository(conn).get_webhook(scoped_key)
+                if cached is not None:
+                    return make_response(jsonify(cached["response_json"]), 200)
+                return make_response(
+                    jsonify({"success": True, "task_id": "deduplicated"}), 200
+                )
+
+        try:
+            apply_kwargs = dict(
+                kwargs={
+                    "agent_id": agent_id_str,
+                    "payload": payload,
+                    # Scoped so the worker dedup row matches the HTTP claim.
+                    "idempotency_key": scoped_key or idempotency_key,
+                },
+            )
+            if predetermined_task_id is not None:
+                apply_kwargs["task_id"] = predetermined_task_id
+            task = process_agent_webhook.apply_async(**apply_kwargs)
+            current_app.logger.info(
+                f"Task {task.id} enqueued for agent {agent_id_str} ({source_method})."
+            )
+            response_payload = {"success": True, "task_id": task.id}
+            return make_response(jsonify(response_payload), 200)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error enqueuing webhook task ({source_method}) for agent {agent_id_str}: {err}",
+                exc_info=True,
+            )
+            if scoped_key:
+                # Roll back the claim so a retry can succeed.
+                try:
+                    with db_session() as conn:
+                        conn.execute(
+                            sql_text(
+                                "DELETE FROM webhook_dedup "
+                                "WHERE idempotency_key = :k"
+                            ),
+                            {"k": scoped_key},
+                        )
+                except Exception:
+                    current_app.logger.exception(
+                        "Failed to release webhook_dedup claim for key=%s",
+                        scoped_key,
+                    )
+            return make_response(
+                jsonify({"success": False, "message": "Error processing webhook"}), 500
+            )
+
+    @api.doc(
+        description=(
+            "Webhook listener for agent events (POST). Expects JSON payload, which "
+            "is used to trigger processing. Honors an optional ``Idempotency-Key`` "
+            "header: a repeat request with the same key within 24h returns the "
+            "original cached response and does not re-enqueue the task."
+        ),
+    )
+    def post(self, webhook_token, agent, agent_id_str):
+        payload = request.get_json()
+        if payload is None:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Invalid or missing JSON data in request body",
+                    }
+                ),
+                400,
+            )
+        return self._enqueue_webhook_task(
+            agent_id_str, payload, source_method="POST", agent=agent,
+        )
+
+    @api.doc(
+        description=(
+            "Webhook listener for agent events (GET). Uses URL query parameters as "
+            "payload to trigger processing. Honors an optional ``Idempotency-Key`` "
+            "header: a repeat request with the same key within 24h returns the "
+            "original cached response and does not re-enqueue the task."
+        ),
+    )
+    def get(self, webhook_token, agent, agent_id_str):
+        payload = request.args.to_dict(flat=True)
+        return self._enqueue_webhook_task(
+            agent_id_str, payload, source_method="GET", agent=agent,
+        )

docsgpt/api/user/analytics/__init__.py

@@ -0,0 +1,5 @@
+"""Analytics module."""
+
+from .routes import analytics_ns
+
+__all__ = ["analytics_ns"]

docsgpt/api/user/artifacts/__init__.py

@@ -0,0 +1,5 @@
+"""Artifacts module."""
+
+from .routes import artifacts_ns
+
+__all__ = ["artifacts_ns"]

docsgpt/api/user/artifacts/authz.py

@@ -0,0 +1,168 @@
+"""Parent-derived authorization helpers for artifact access."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+from flask import request
+
+from docsgpt.storage.db.repositories.agents import AgentsRepository
+from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
+from docsgpt.storage.db.repositories.conversations import ConversationsRepository
+from docsgpt.storage.db.repositories.shared_conversations import (
+    SharedConversationsRepository,
+)
+from docsgpt.storage.db.repositories.workflow_runs import WorkflowRunsRepository
+from docsgpt.storage.db.session import db_readonly
+
+
+@dataclass(frozen=True)
+class Principal:
+    """The resolved caller of an artifact route.
+
+    A decoded JWT is a full *owner* session (``agent_id`` is ``None``). An
+    ``api_key`` is a low-trust, publicly-distributed *agent* credential -- it is
+    embedded client-side in the widget and accepted from the query string -- so
+    it resolves to the owning ``user_id`` but stays **agent-scoped**: it may reach
+    an artifact only when the request also carries that artifact's parent
+    ``conversation_id`` (the unguessable per-visitor secret), confining it to a
+    single conversation rather than the agent's whole corpus, and may not perform
+    mutating operations.
+    """
+
+    user_id: Optional[str] = None
+    agent_id: Optional[str] = None
+
+    @property
+    def is_agent_scoped(self) -> bool:
+        """True for an api_key (agent) principal that must be confined to its agent."""
+        return self.agent_id is not None
+
+
+def resolve_principal() -> Principal:
+    """Resolve the caller to a :class:`Principal`.
+
+    Priority: a decoded JWT (owner session) first, then an ``api_key`` query/form
+    param (agent-scoped). An unresolvable caller yields an anonymous principal
+    (both fields ``None``) that only a valid ``share_token`` can authorize.
+    """
+    decoded_token = getattr(request, "decoded_token", None)
+    if decoded_token:
+        return Principal(user_id=decoded_token.get("sub"))
+
+    api_key = request.args.get("api_key") or request.form.get("api_key")
+    if api_key:
+        with db_readonly() as conn:
+            agent = AgentsRepository(conn).find_by_key(api_key)
+        if agent and agent.get("user_id") and agent.get("id"):
+            return Principal(user_id=str(agent["user_id"]), agent_id=str(agent["id"]))
+    return Principal()
+
+
+def _shared_row_for(conn, conversation_id, share_token):
+    """Return the shared_conversations row iff share_token grants this conversation, else None."""
+    if not share_token:
+        return None
+    shared = SharedConversationsRepository(conn).find_by_uuid(share_token)
+    if shared and str(shared.get("conversation_id")) == str(conversation_id):
+        return shared
+    return None
+
+
+def user_can_access_conversation(
+    conn, conversation_id: str, user_id: Optional[str], share_token: Optional[str]
+) -> bool:
+    """Allow if the caller owns/shares the conversation, or holds a valid share token.
+
+    Conversation-level gate only. Reuses ``ConversationsRepository.get`` (owner OR
+    ``shared_with``) so artifact access tracks message access, and honours a valid
+    share token. A share-token caller reaches the conversation here but must still
+    be snapshot-scoped per-artifact by the caller (see ``authorize_artifact``): a
+    valid token does NOT imply access to every artifact in the conversation.
+    """
+    if user_id:
+        if ConversationsRepository(conn).get(conversation_id, user_id) is not None:
+            return True
+    return _shared_row_for(conn, conversation_id, share_token) is not None
+
+
+def authorize_artifact(conn, artifact: dict, principal: Principal) -> bool:
+    """Authorize a READ of ``artifact`` for ``principal``; missing parent fails closed.
+
+    A low-trust agent api_key is confined to a single conversation it proves by
+    carrying that artifact's parent ``conversation_id`` query param (owner match +
+    matching conversation + agent scope) and never inherits share-link access. A
+    JWT owner or ``shared_with`` collaborator gets full access to every artifact of
+    the parent; a share-token holder is confined to the shared ``first_n_queries``
+    snapshot (an artifact whose ``message_id`` is outside it, or NULL, is denied).
+    """
+    if principal.is_agent_scoped:
+        # An agent key is not the owner's session and is embedded in public widget
+        # JS, so require the artifact to belong to the owner AND the request to
+        # carry the parent conversation_id (the unguessable per-visitor secret):
+        # the key alone, without that id, cannot download a known artifact.
+        if str(artifact.get("user_id")) != str(principal.user_id):
+            return False
+        req_conv = request.args.get("conversation_id")
+        parent = artifact.get("conversation_id")
+        if not req_conv or parent is None or str(parent) != str(req_conv):
+            return False
+        return ArtifactsRepository(conn).artifact_in_agent_scope(
+            str(artifact.get("id")), str(principal.agent_id)
+        )
+
+    conversation_id = artifact.get("conversation_id")
+    workflow_run_id = artifact.get("workflow_run_id")
+    share_token = request.args.get("share_token")
+
+    if conversation_id is not None:
+        # Owner or shared_with collaborator: full access to every artifact.
+        if principal.user_id and ConversationsRepository(conn).get(
+            str(conversation_id), principal.user_id
+        ) is not None:
+            return True
+        # Share-token holder: only artifacts inside the shared first_n_queries snapshot.
+        shared = _shared_row_for(conn, str(conversation_id), share_token)
+        if shared is None:
+            return False
+        message_id = artifact.get("message_id")
+        if not message_id:
+            return False
+        first_n = int(shared.get("first_n_queries") or 0)
+        return ConversationsRepository(conn).message_in_first_n(
+            str(conversation_id), str(message_id), first_n
+        )
+    if workflow_run_id is not None:
+        if not principal.user_id:
+            return False
+        run = WorkflowRunsRepository(conn).get(str(workflow_run_id))
+        return run is not None and run.get("user_id") == principal.user_id
+    # No parent row reachable -> deny (e.g. deleted conversation/run).
+    return False
+
+
+def authorize_artifact_write(conn, artifact: dict, principal: Principal) -> bool:
+    """Authorize a *mutating* artifact operation (e.g. delete / restore).
+
+    Stricter than :func:`authorize_artifact`: a write requires an authenticated
+    *owner* of the parent. Low-trust agent api_keys, share links, and
+    ``shared_with`` collaborators inherit read/download access only, so an
+    agent-scoped, anonymous, or share-token-only caller is denied -- they can read
+    an artifact but never mutate it.
+    """
+    if principal.is_agent_scoped or not principal.user_id:
+        return False
+    conversation_id = artifact.get("conversation_id")
+    workflow_run_id = artifact.get("workflow_run_id")
+
+    if conversation_id is not None:
+        return (
+            ConversationsRepository(conn).get_owned(str(conversation_id), principal.user_id)
+            is not None
+        )
+    if workflow_run_id is not None:
+        run = WorkflowRunsRepository(conn).get(str(workflow_run_id))
+        return run is not None and run.get("user_id") == principal.user_id
+    # No parent row reachable -> deny (e.g. deleted conversation/run).
+    return False

docsgpt/api/user/artifacts/routes.py

@@ -0,0 +1,402 @@
+"""Artifact metadata and download routes (parent-derived authz)."""
+
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+from flask import (
+    Response,
+    current_app,
+    jsonify,
+    make_response,
+    redirect,
+    request,
+    stream_with_context,
+)
+from flask_restx import Namespace, Resource
+
+from docsgpt.api import api
+from docsgpt.api.user.artifacts.authz import (
+    _shared_row_for,
+    authorize_artifact,
+    authorize_artifact_write,
+    resolve_principal,
+    user_can_access_conversation,
+)
+from docsgpt.core.settings import settings
+from docsgpt.storage.db.base_repository import looks_like_uuid
+from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
+from docsgpt.storage.db.repositories.conversations import ConversationsRepository
+from docsgpt.storage.db.repositories.workflow_runs import WorkflowRunsRepository
+from docsgpt.storage.db.session import db_readonly, db_session
+from docsgpt.storage.storage_creator import StorageCreator
+
+artifacts_ns = Namespace("artifacts", description="Artifact operations", path="/api")
+
+# Presigned-URL TTL for private S3 artifact downloads (seconds).
+_PRESIGNED_URL_TTL = 300
+
+
+_ARTIFACT_URL_ENVELOPE_MIME = "application/vnd.docsgpt.artifact-url+json"
+
+
+def _sanitize_header_filename(filename: Optional[str], fallback: str) -> str:
+    """Strip CRLF / quotes from a display filename for a Content-Disposition header."""
+    if not filename:
+        return fallback
+    cleaned = re.sub(r'[\r\n"]', "", str(filename)).strip()
+    return cleaned or fallback
+
+
+def _artifact_summary(artifact: dict) -> dict:
+    """Project an artifact identity row to its API metadata shape (owner id withheld)."""
+    return {
+        "id": str(artifact.get("id")),
+        "conversation_id": (str(artifact["conversation_id"]) if artifact.get("conversation_id") is not None else None),
+        "workflow_run_id": (str(artifact["workflow_run_id"]) if artifact.get("workflow_run_id") is not None else None),
+        "message_id": (str(artifact["message_id"]) if artifact.get("message_id") is not None else None),
+        "kind": artifact.get("kind"),
+        "title": artifact.get("title"),
+        "metadata": artifact.get("metadata"),
+        "current_version": artifact.get("current_version"),
+        "created_at": _iso(artifact.get("created_at")),
+        "updated_at": _iso(artifact.get("updated_at")),
+    }
+
+
+def _version_summary(version: dict, *, include_spec: bool = False) -> dict:
+    """Project a version row to its API metadata shape (storage_path withheld)."""
+    out = {
+        "version": version.get("version"),
+        "mime_type": version.get("mime_type"),
+        "filename": version.get("filename"),
+        "size": version.get("size"),
+        "sha256": version.get("sha256"),
+        "preview_text": version.get("preview_text"),
+        "produced_by": version.get("produced_by"),
+        "created_at": _iso(version.get("created_at")),
+    }
+    if include_spec:
+        out["spec"] = version.get("spec")
+    return out
+
+
+def _iso(value):
+    """ISO-format a datetime, passing through other values unchanged."""
+    return value.isoformat() if hasattr(value, "isoformat") else value
+
+
+@artifacts_ns.route("/artifacts")
+class ListArtifacts(Resource):
+    @api.doc(description="List artifacts for a conversation, workflow run, or the caller")
+    def get(self):
+        principal = resolve_principal()
+        user_id = principal.user_id
+        conversation_id = request.args.get("conversation_id")
+        workflow_run_id = request.args.get("workflow_run_id")
+        share_token = request.args.get("share_token")
+
+        if not user_id and not share_token:
+            return make_response(jsonify({"success": False, "message": "Authentication required"}), 401)
+
+        # Gate UUID-shape before any CAST(:id AS uuid) reaches the repo, so a
+        # malformed id is rejected cleanly instead of poisoning the transaction.
+        if conversation_id and not looks_like_uuid(conversation_id):
+            return make_response(jsonify({"success": False, "message": "Invalid conversation_id"}), 400)
+        if workflow_run_id and not looks_like_uuid(workflow_run_id):
+            return make_response(jsonify({"success": False, "message": "Invalid workflow_run_id"}), 400)
+
+        try:
+            with db_readonly() as conn:
+                repo = ArtifactsRepository(conn)
+                if principal.is_agent_scoped:
+                    # A low-trust agent api_key is embedded in public widget JS and
+                    # shared by every anonymous visitor, so it may NOT enumerate the
+                    # agent's artifacts. The unguessable conversation_id (UUIDv4) is
+                    # the only per-visitor secret: require it as a bearer capability
+                    # and scope the query to it. Workflow runs are not agent-scoped,
+                    # so reject that filter.
+                    if workflow_run_id:
+                        return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                    if not conversation_id:
+                        return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                    rows = repo.list_artifacts_for_agent(principal.agent_id, user_id, conversation_id=conversation_id)
+                elif conversation_id:
+                    if not user_can_access_conversation(conn, conversation_id, user_id, share_token):
+                        return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                    rows = repo.list_artifacts(conversation_id=conversation_id)
+                    # Owner / shared_with collaborator sees every artifact; a
+                    # share-token caller is confined to the shared first_n_queries
+                    # snapshot (drop artifacts whose message is outside it or NULL).
+                    conv_repo = ConversationsRepository(conn)
+                    is_owner = bool(user_id and conv_repo.get(conversation_id, user_id) is not None)
+                    if not is_owner:
+                        shared = _shared_row_for(conn, conversation_id, share_token)
+                        first_n = int(shared.get("first_n_queries") or 0) if shared else 0
+                        snapshot_ids = conv_repo.first_n_message_ids(conversation_id, first_n)
+                        rows = [
+                            r
+                            for r in rows
+                            if r.get("message_id") is not None and str(r.get("message_id")) in snapshot_ids
+                        ]
+                elif workflow_run_id:
+                    run = WorkflowRunsRepository(conn).get(workflow_run_id)
+                    if run is None or run.get("user_id") != user_id:
+                        return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                    rows = repo.list_artifacts(workflow_run_id=workflow_run_id)
+                else:
+                    if not user_id:
+                        return make_response(
+                            jsonify({"success": False, "message": "Authentication required"}),
+                            401,
+                        )
+                    rows = repo.list_artifacts(user_id=user_id)
+
+            return make_response(
+                jsonify({"success": True, "artifacts": [_artifact_summary(r) for r in rows]}),
+                200,
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error listing artifacts: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+
+
+@artifacts_ns.route("/artifacts/<artifact_id>")
+class GetArtifact(Resource):
+    @api.doc(description="Get an artifact's metadata, version list, and current spec")
+    def get(self, artifact_id: str):
+        if not looks_like_uuid(artifact_id):
+            return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+        principal = resolve_principal()
+        try:
+            with db_readonly() as conn:
+                repo = ArtifactsRepository(conn)
+                artifact = repo.get_artifact(artifact_id)
+                if artifact is None:
+                    return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+                if not authorize_artifact(conn, artifact, principal):
+                    return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                versions = repo.list_versions(artifact_id)
+                current = repo.get_version(artifact_id, artifact.get("current_version"))
+
+            payload = _artifact_summary(artifact)
+            payload["versions"] = [_version_summary(v) for v in versions]
+            payload["spec"] = current.get("spec") if current else None
+            return make_response(jsonify({"success": True, "artifact": payload}), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving artifact: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+
+    @api.doc(description="Delete an artifact and all its versions (owner only)")
+    def delete(self, artifact_id: str):
+        if not looks_like_uuid(artifact_id):
+            return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+        principal = resolve_principal()
+        try:
+            with db_session() as conn:
+                repo = ArtifactsRepository(conn)
+                artifact = repo.get_artifact(artifact_id)
+                if artifact is None:
+                    return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+                # Delete is a WRITE: only the parent owner may delete; share
+                # links / read-only collaborators are denied (read access only).
+                if not authorize_artifact_write(conn, artifact, principal):
+                    return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                storage_paths = repo.delete_artifact(artifact_id)
+            # Reap the bytes best-effort AFTER the row delete commits.
+            _reap_storage(storage_paths)
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error deleting artifact: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)
+
+
+def _reap_storage(paths: list) -> None:
+    """Best-effort delete artifact bytes after the DB rows are gone; never raises."""
+    if not paths:
+        return
+    try:
+        storage = StorageCreator.get_storage()
+    except Exception:
+        current_app.logger.warning("artifact delete: storage unavailable", exc_info=True)
+        return
+    for path in paths:
+        try:
+            storage.delete_file(path)
+        except Exception:
+            current_app.logger.warning("artifact delete: failed to delete bytes %s", path, exc_info=True)
+
+
+@artifacts_ns.route("/artifacts/<artifact_id>/versions/<int:version>")
+class GetArtifactVersion(Resource):
+    @api.doc(description="Get a single artifact version's metadata and spec")
+    def get(self, artifact_id: str, version: int):
+        if not looks_like_uuid(artifact_id):
+            return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+        principal = resolve_principal()
+        try:
+            with db_readonly() as conn:
+                repo = ArtifactsRepository(conn)
+                artifact = repo.get_artifact(artifact_id)
+                if artifact is None:
+                    return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+                if not authorize_artifact(conn, artifact, principal):
+                    return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                version_row = repo.get_version(artifact_id, version)
+            if version_row is None:
+                return make_response(jsonify({"success": False, "message": "Version not found"}), 404)
+            return make_response(
+                jsonify({"success": True, "version": _version_summary(version_row, include_spec=True)}),
+                200,
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving artifact version: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+
+
+@artifacts_ns.route("/artifacts/<artifact_id>/download")
+class DownloadArtifact(Resource):
+    @api.doc(description="Download an artifact's bytes (302 to a presigned URL on S3)")
+    def get(self, artifact_id: str):
+        if not looks_like_uuid(artifact_id):
+            return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+        principal = resolve_principal()
+        version_arg = request.args.get("version")
+        try:
+            with db_readonly() as conn:
+                repo = ArtifactsRepository(conn)
+                artifact = repo.get_artifact(artifact_id)
+                if artifact is None:
+                    return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+                if not authorize_artifact(conn, artifact, principal):
+                    return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                version = artifact.get("current_version")
+                if version_arg is not None:
+                    try:
+                        version = int(version_arg)
+                    except ValueError:
+                        return make_response(jsonify({"success": False, "message": "Invalid version"}), 400)
+                version_row = repo.get_version(artifact_id, version)
+
+            if version_row is None:
+                return make_response(jsonify({"success": False, "message": "Version not found"}), 404)
+            # The object key is derived only from the stored path, never client input.
+            storage_path = version_row.get("storage_path")
+            if not storage_path:
+                return make_response(jsonify({"success": False, "message": "No file for this version"}), 404)
+
+            filename = _sanitize_header_filename(version_row.get("filename"), f"artifact-{artifact_id}")
+            mime_type = version_row.get("mime_type") or "application/octet-stream"
+            storage = StorageCreator.get_storage()
+
+            # With URL_STRATEGY=="s3" the contract is to hand back a presigned
+            # URL. If the active backend can't mint one, that's a config error:
+            # surface a 500 rather than silently proxying bytes from a backend
+            # the operator expected to be off the hot path.
+            if getattr(settings, "URL_STRATEGY", "backend") == "s3":
+                try:
+                    url = storage.generate_presigned_url(storage_path, expires_in=_PRESIGNED_URL_TTL)
+                except NotImplementedError:
+                    current_app.logger.error(
+                        "URL_STRATEGY=s3 but %s cannot mint presigned URLs",
+                        type(storage).__name__,
+                    )
+                    return make_response(
+                        jsonify({"success": False, "message": "Storage misconfigured"}),
+                        500,
+                    )
+                # A 302 to a cross-origin S3 URL can't be read by the app's authed
+                # fetch (the bucket has no CORS grant for the app origin). When the
+                # client opts in via ?disposition=url (or Accept: application/json)
+                # hand the presigned URL back as JSON so it can navigate to it
+                # top-level (no CORS). Default stays a 302 so nothing else breaks.
+                if request.args.get("disposition") == "url" or (
+                    "application/json" in request.headers.get("Accept", "")
+                ):
+                    resp = make_response(jsonify({"success": True, "url": url}), 200)
+                    # Tag the envelope with a distinctive media type so the client
+                    # keys off a server-set signal, not the JSON body shape. A
+                    # ``data`` artifact whose bytes happen to be
+                    # ``{"success":true,"url":"..."}`` is streamed under the backend
+                    # strategy with its own (``application/json``) content-type and
+                    # can never carry this vendor type, so it can't be mistaken for a
+                    # redirect (open-redirect gadget). Content-Type is CORS-safelisted,
+                    # so this stays readable cross-origin without expose-headers.
+                    resp.headers["Content-Type"] = _ARTIFACT_URL_ENVELOPE_MIME
+                    return resp
+                return redirect(url, code=302)
+
+            # Stream the bytes in chunks instead of buffering the whole object in
+            # worker memory (artifacts can be many MB); close the handle when done.
+            file_obj = storage.get_file(storage_path)
+
+            def _stream():
+                try:
+                    for chunk in iter(lambda: file_obj.read(65536), b""):
+                        yield chunk
+                finally:
+                    close = getattr(file_obj, "close", None)
+                    if callable(close):
+                        close()
+
+            return Response(
+                stream_with_context(_stream()),
+                mimetype=mime_type,
+                headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+            )
+        except FileNotFoundError:
+            return make_response(jsonify({"success": False, "message": "File not found"}), 404)
+        except Exception as err:
+            current_app.logger.error(f"Error downloading artifact: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+
+
+@artifacts_ns.route("/artifacts/<artifact_id>/restore")
+class RestoreArtifact(Resource):
+    @api.doc(description="Restore a prior version by appending it as the new current version")
+    def post(self, artifact_id: str):
+        if not looks_like_uuid(artifact_id):
+            return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+        principal = resolve_principal()
+        data = request.get_json(silent=True) or {}
+        target_version = data.get("version")
+        if target_version is None:
+            return make_response(jsonify({"success": False, "message": "Missing version"}), 400)
+        try:
+            target_version = int(target_version)
+        except (ValueError, TypeError):
+            return make_response(jsonify({"success": False, "message": "Invalid version"}), 400)
+
+        try:
+            with db_session() as conn:
+                repo = ArtifactsRepository(conn)
+                artifact = repo.get_artifact(artifact_id)
+                if artifact is None:
+                    return make_response(jsonify({"success": False, "message": "Artifact not found"}), 404)
+                # Restore is a WRITE (it appends a new current version); share
+                # links / shared_with collaborators inherit read access only, so
+                # gate on the stricter owner-required write check.
+                if not authorize_artifact_write(conn, artifact, principal):
+                    return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+                source = repo.get_version(artifact_id, target_version)
+                if source is None:
+                    return make_response(jsonify({"success": False, "message": "Version not found"}), 404)
+                new_version = repo.append_version(
+                    artifact_id,
+                    mime_type=source.get("mime_type"),
+                    filename=source.get("filename"),
+                    storage_path=source.get("storage_path"),
+                    size=source.get("size"),
+                    sha256=source.get("sha256"),
+                    spec=source.get("spec"),
+                    preview_text=source.get("preview_text"),
+                    produced_by=source.get("produced_by"),
+                )
+            return make_response(
+                jsonify({"success": True, "version": _version_summary(new_version, include_spec=True)}),
+                200,
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error restoring artifact: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)

docsgpt/api/user/attachments/__init__.py

@@ -0,0 +1,5 @@
+"""Attachments module."""
+
+from .routes import attachments_ns
+
+__all__ = ["attachments_ns"]

docsgpt/api/user/attachments/routes.py

@@ -0,0 +1,837 @@
+"""File attachments and media routes."""
+
+import os
+import tempfile
+from pathlib import Path
+
+import uuid
+
+from flask import Response, current_app, jsonify, make_response, redirect, request
+from flask_restx import fields, Namespace, Resource
+from werkzeug.datastructures import FileStorage
+
+from docsgpt.api import api
+from docsgpt.cache import get_redis_instance
+from docsgpt.core.settings import settings
+from docsgpt.storage.db.base_repository import looks_like_uuid
+from docsgpt.storage.db.repositories.agents import AgentsRepository
+from docsgpt.storage.db.session import db_readonly
+from docsgpt.stt.constants import (
+    SUPPORTED_AUDIO_EXTENSIONS,
+    SUPPORTED_AUDIO_MIME_TYPES,
+)
+from docsgpt.stt.upload_limits import (
+    AudioFileTooLargeError,
+    build_stt_file_size_limit_message,
+    enforce_audio_file_size_limit,
+    is_audio_filename,
+)
+from docsgpt.stt.live_session import (
+    apply_live_stt_hypothesis,
+    create_live_stt_session,
+    delete_live_stt_session,
+    finalize_live_stt_session,
+    get_live_stt_transcript_text,
+    load_live_stt_session,
+    save_live_stt_session,
+)
+from docsgpt.stt.stt_creator import STTCreator
+from docsgpt.tts.tts_creator import TTSCreator
+from docsgpt.upload_limits import (
+    copy_upload_to_path,
+    enforce_parseable_attachment,
+    is_unsupported_upload_message,
+    UnsupportedUploadTypeError,
+    unsupported_upload_message,
+    upload_limit_message,
+    UploadTooLargeError,
+)
+from docsgpt.utils import (
+    get_agent_image_content_type,
+    is_external_image_url,
+    is_safe_agent_image_path,
+    safe_filename,
+    verify_agent_image_capability,
+)
+
+
+attachments_ns = Namespace(
+    "attachments", description="File attachments and media operations", path="/api"
+)
+
+_AGENT_IMAGE_CHUNK_BYTES = 64 * 1024
+_AGENT_IMAGE_PRESIGNED_TTL_SECONDS = 300
+_AGENT_IMAGE_REDIRECT_CACHE_SECONDS = _AGENT_IMAGE_PRESIGNED_TTL_SECONDS - 60
+
+
+def _stream_file(file_obj, size_bytes: int):
+    """Yield up to the size-checked file range and always close the handle."""
+    remaining = size_bytes
+    try:
+        while remaining > 0:
+            chunk = file_obj.read(min(_AGENT_IMAGE_CHUNK_BYTES, remaining))
+            if not chunk:
+                break
+            remaining -= len(chunk)
+            yield chunk
+    finally:
+        file_obj.close()
+
+
+def _resolve_authenticated_user():
+    decoded_token = getattr(request, "decoded_token", None)
+    api_key = request.form.get("api_key") or request.args.get("api_key")
+
+    # Return the RAW user identity (not safe_filename'd): the attachment row's
+    # ``user_id`` is an identity key that must match the raw ``sub`` used by
+    # /stream (initial_user_id) and the artifact authz gate. safe_filename is
+    # applied only to the storage path component at write time. (An email-style
+    # sub like ``a@b.com`` was previously sanitized to ``abcom``, so the
+    # uploaded attachment became unreadable by its own owner on /stream.)
+    if decoded_token:
+        return decoded_token.get("sub")
+
+    if api_key:
+        with db_readonly() as conn:
+            agent = AgentsRepository(conn).find_by_key(api_key)
+        if not agent:
+            return make_response(
+                jsonify({"success": False, "message": "Invalid API key"}), 401
+            )
+        return agent.get("user_id")
+
+    return None
+
+
+def _get_uploaded_file_size(file) -> int:
+    try:
+        current_position = file.stream.tell()
+        file.stream.seek(0, os.SEEK_END)
+        size_bytes = file.stream.tell()
+        file.stream.seek(current_position)
+        return size_bytes
+    except Exception:
+        return 0
+
+
+def _is_supported_audio_mimetype(mimetype: str) -> bool:
+    if not mimetype:
+        return True
+    normalized = mimetype.split(";")[0].strip().lower()
+    return normalized.startswith("audio/") or normalized in SUPPORTED_AUDIO_MIME_TYPES
+
+
+def _enforce_uploaded_audio_size_limit(file, filename: str) -> None:
+    if not is_audio_filename(filename):
+        return
+    size_bytes = _get_uploaded_file_size(file)
+    if size_bytes:
+        enforce_audio_file_size_limit(size_bytes)
+
+
+def _get_store_attachment_user_error(exc: Exception, filename: str | None = None) -> str:
+    """Map an upload failure to a fixed client-facing message.
+
+    Every branch returns text this module composes itself. Nothing is read off
+    the exception, so no exception state — message or traceback — can reach a
+    response body (CodeQL py/stack-trace-exposure).
+
+    Args:
+        exc: The exception raised while processing one uploaded file.
+        filename: That file's name, which supplies the extension for the
+            unsupported-type message.
+
+    Returns:
+        The message to report for this failure.
+    """
+    if isinstance(exc, AudioFileTooLargeError):
+        return build_stt_file_size_limit_message()
+    if isinstance(exc, UploadTooLargeError):
+        return upload_limit_message()
+    if isinstance(exc, UnsupportedUploadTypeError):
+        return unsupported_upload_message(filename)
+    return "Failed to process file"
+
+
+def _require_live_stt_redis():
+    redis_client = get_redis_instance()
+    if redis_client:
+        return redis_client
+    return make_response(
+        jsonify({"success": False, "message": "Live transcription is unavailable"}),
+        503,
+    )
+
+
+def _parse_bool_form_value(value: str | None) -> bool:
+    if value is None:
+        return False
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+@attachments_ns.route("/store_attachment")
+class StoreAttachment(Resource):
+    @api.expect(
+        api.model(
+            "AttachmentModel",
+            {
+                "file": fields.Raw(required=True, description="File(s) to upload"),
+                "api_key": fields.String(
+                    required=False, description="API key (optional)"
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Stores one or multiple attachments without vectorization or training. Supports user or API key authentication."
+    )
+    def post(self):
+        auth_user = _resolve_authenticated_user()
+        if hasattr(auth_user, "status_code"):
+            return auth_user
+        
+        files = request.files.getlist("file")
+        if not files:
+            single_file = request.files.get("file")
+            if single_file:
+                files = [single_file]
+        
+        if not files or all(f.filename == "" for f in files):
+            return make_response(
+                jsonify({"status": "error", "message": "Missing file(s)"}),
+                400,
+            )
+        
+        user = auth_user
+        if not user:
+            return make_response(
+                jsonify({"success": False, "message": "Authentication required"}), 401
+            )
+        
+        try:
+            from docsgpt.api.user.tasks import store_attachment
+            from docsgpt.api.user.base import storage
+
+            tasks = []
+            errors = []
+            original_file_count = len(files)
+            
+            for idx, file in enumerate(files):
+                try:
+                    attachment_id = uuid.uuid4()
+                    original_filename = safe_filename(os.path.basename(file.filename))
+                    _enforce_uploaded_audio_size_limit(file, original_filename)
+                    # safe_filename only the path component; the DB user_id stays raw.
+                    path_user = safe_filename(user)
+                    relative_path = (
+                        f"{settings.UPLOAD_FOLDER}/{path_user}/attachments/"
+                        f"{str(attachment_id)}/{original_filename}"
+                    )
+
+                    # Validate a bounded local spool before storage reads it.
+                    # This protects local and S3 backends even without a
+                    # trustworthy Content-Length header.
+                    with tempfile.TemporaryDirectory() as temp_dir:
+                        staged_path = os.path.join(temp_dir, original_filename)
+                        copy_upload_to_path(file, staged_path)
+                        # Refuse here — nothing is stored or queued yet — so a
+                        # video the picker let through never reaches the
+                        # worker's reader, which would open it as text.
+                        enforce_parseable_attachment(staged_path, original_filename)
+                        with open(staged_path, "rb") as staged_stream:
+                            staged_upload = FileStorage(
+                                stream=staged_stream,
+                                filename=original_filename,
+                                content_type=file.mimetype,
+                            )
+                            metadata = storage.save_file(
+                                staged_upload, relative_path
+                            )
+                    file_info = {
+                        "filename": original_filename,
+                        "attachment_id": str(attachment_id),
+                        "path": relative_path,
+                        "metadata": metadata,
+                    }
+
+                    task = store_attachment.delay(file_info, user)
+                    tasks.append({
+                        "task_id": task.id,
+                        "filename": original_filename,
+                        "attachment_id": str(attachment_id),
+                        "upload_index": idx,
+                    })
+                except Exception as file_err:
+                    current_app.logger.error(f"Error processing file {idx} ({file.filename}): {file_err}", exc_info=True)
+                    errors.append({
+                        "upload_index": idx,
+                        "filename": file.filename,
+                        "error": _get_store_attachment_user_error(
+                            file_err, file.filename
+                        ),
+                    })
+            
+            if not tasks:
+                size_limit_messages = {
+                    build_stt_file_size_limit_message(),
+                    upload_limit_message(),
+                }
+                if errors and all(
+                    error.get("error") in size_limit_messages for error in errors
+                ):
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "message": errors[0]["error"],
+                                "errors": errors,
+                            }
+                        ),
+                        413,
+                    )
+                if errors and all(
+                    is_unsupported_upload_message(error.get("error")) for error in errors
+                ):
+                    # Tell the user which type was refused rather than the
+                    # generic copy — this is the branch a phone picker that
+                    # ignores ``accept`` lands in.
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "message": errors[0]["error"],
+                                "errors": errors,
+                            }
+                        ),
+                        400,
+                    )
+                return make_response(
+                    jsonify({"status": "error", "message": "No valid files to upload"}),
+                    400,
+                )
+            
+            if original_file_count == 1 and len(tasks) == 1:
+                current_app.logger.info("Returning single task_id response")
+                return make_response(
+                    jsonify(
+                        {
+                            "success": True,
+                            "task_id": tasks[0]["task_id"],
+                            # Surface the attachment_id so the frontend
+                            # can correlate ``attachment.*`` SSE events
+                            # to this row and skip the polling fallback.
+                            "attachment_id": tasks[0]["attachment_id"],
+                            "message": "File uploaded successfully. Processing started.",
+                        }
+                    ),
+                    200,
+                )
+            else:
+                response_data = {
+                    "success": True,
+                    "tasks": tasks,
+                    "message": f"{len(tasks)} file(s) uploaded successfully. Processing started.",
+                }
+                if errors:
+                    response_data["errors"] = errors
+                    response_data["message"] += f" {len(errors)} file(s) failed."
+                
+                return make_response(
+                    jsonify(response_data),
+                    200,
+                )
+        except Exception as err:
+            current_app.logger.error(f"Error storing attachment: {err}", exc_info=True)
+            return make_response(jsonify({"success": False, "error": "Failed to store attachment"}), 400)
+
+
+@attachments_ns.route("/stt")
+class SpeechToText(Resource):
+    @api.expect(
+        api.model(
+            "SpeechToTextModel",
+            {
+                "file": fields.Raw(required=True, description="Audio file"),
+                "language": fields.String(
+                    required=False, description="Optional transcription language hint"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Transcribe an uploaded audio file")
+    def post(self):
+        auth_user = _resolve_authenticated_user()
+        if hasattr(auth_user, "status_code"):
+            return auth_user
+        if not auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Authentication required"}),
+                401,
+            )
+
+        file = request.files.get("file")
+        if not file or file.filename == "":
+            return make_response(
+                jsonify({"success": False, "message": "Missing file"}),
+                400,
+            )
+
+        filename = safe_filename(os.path.basename(file.filename))
+        suffix = Path(filename).suffix.lower()
+        if suffix not in SUPPORTED_AUDIO_EXTENSIONS:
+            return make_response(
+                jsonify({"success": False, "message": "Unsupported audio format"}),
+                400,
+            )
+
+        if not _is_supported_audio_mimetype(file.mimetype or ""):
+            return make_response(
+                jsonify({"success": False, "message": "Unsupported audio MIME type"}),
+                400,
+            )
+
+        try:
+            _enforce_uploaded_audio_size_limit(file, filename)
+        except AudioFileTooLargeError:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": build_stt_file_size_limit_message(),
+                    }
+                ),
+                413,
+            )
+
+        temp_path = None
+        try:
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:
+                file.save(temp_file.name)
+                temp_path = Path(temp_file.name)
+
+            stt_instance = STTCreator.create_stt(settings.STT_PROVIDER)
+            transcript = stt_instance.transcribe(
+                temp_path,
+                language=request.form.get("language") or settings.STT_LANGUAGE,
+                timestamps=settings.STT_ENABLE_TIMESTAMPS,
+                diarize=settings.STT_ENABLE_DIARIZATION,
+            )
+            return make_response(jsonify({"success": True, **transcript}), 200)
+        except Exception as err:
+            current_app.logger.error(f"Error transcribing audio: {err}", exc_info=True)
+            return make_response(
+                jsonify({"success": False, "message": "Failed to transcribe audio"}),
+                400,
+            )
+        finally:
+            if temp_path and temp_path.exists():
+                temp_path.unlink()
+
+
+@attachments_ns.route("/stt/live/start")
+class LiveSpeechToTextStart(Resource):
+    @api.doc(description="Start a live speech-to-text session")
+    def post(self):
+        auth_user = _resolve_authenticated_user()
+        if hasattr(auth_user, "status_code"):
+            return auth_user
+        if not auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Authentication required"}),
+                401,
+            )
+
+        redis_client = _require_live_stt_redis()
+        if hasattr(redis_client, "status_code"):
+            return redis_client
+
+        payload = request.get_json(silent=True) or {}
+        session_state = create_live_stt_session(
+            user=auth_user,
+            language=payload.get("language") or settings.STT_LANGUAGE,
+        )
+        save_live_stt_session(redis_client, session_state)
+
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "session_id": session_state["session_id"],
+                    "language": session_state.get("language"),
+                    "committed_text": "",
+                    "mutable_text": "",
+                    "previous_hypothesis": "",
+                    "latest_hypothesis": "",
+                    "finalized_text": "",
+                    "pending_text": "",
+                    "transcript_text": "",
+                }
+            ),
+            200,
+        )
+
+
+@attachments_ns.route("/stt/live/chunk")
+class LiveSpeechToTextChunk(Resource):
+    @api.expect(
+        api.model(
+            "LiveSpeechToTextChunkModel",
+            {
+                "session_id": fields.String(
+                    required=True, description="Live transcription session ID"
+                ),
+                "chunk_index": fields.Integer(
+                    required=True, description="Sequential chunk index"
+                ),
+                "is_silence": fields.Boolean(
+                    required=False,
+                    description="Whether the latest capture window was mostly silence",
+                ),
+                "file": fields.Raw(required=True, description="Audio chunk"),
+            },
+        )
+    )
+    @api.doc(description="Transcribe a chunk for a live speech-to-text session")
+    def post(self):
+        auth_user = _resolve_authenticated_user()
+        if hasattr(auth_user, "status_code"):
+            return auth_user
+        if not auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Authentication required"}),
+                401,
+            )
+
+        redis_client = _require_live_stt_redis()
+        if hasattr(redis_client, "status_code"):
+            return redis_client
+
+        session_id = request.form.get("session_id", "").strip()
+        if not session_id:
+            return make_response(
+                jsonify({"success": False, "message": "Missing session_id"}),
+                400,
+            )
+
+        session_state = load_live_stt_session(redis_client, session_id)
+        if not session_state:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Live transcription session not found",
+                    }
+                ),
+                404,
+            )
+
+        # The stored ``user`` is the RAW sub (see _resolve_authenticated_user), and
+        # auth_user is also raw, so compare raw-vs-raw -- never safe_filename'd, or an
+        # email-style sub (a@b.com) would 403 the owner on their own session.
+        if str(session_state.get("user", "")) != auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Forbidden"}),
+                403,
+            )
+
+        chunk_index_raw = request.form.get("chunk_index", "").strip()
+        if chunk_index_raw == "":
+            return make_response(
+                jsonify({"success": False, "message": "Missing chunk_index"}),
+                400,
+            )
+
+        try:
+            chunk_index = int(chunk_index_raw)
+        except ValueError:
+            return make_response(
+                jsonify({"success": False, "message": "Invalid chunk_index"}),
+                400,
+            )
+        is_silence = _parse_bool_form_value(request.form.get("is_silence"))
+
+        file = request.files.get("file")
+        if not file or file.filename == "":
+            return make_response(
+                jsonify({"success": False, "message": "Missing file"}),
+                400,
+            )
+
+        filename = safe_filename(os.path.basename(file.filename))
+        suffix = Path(filename).suffix.lower()
+        if suffix not in SUPPORTED_AUDIO_EXTENSIONS:
+            return make_response(
+                jsonify({"success": False, "message": "Unsupported audio format"}),
+                400,
+            )
+
+        if not _is_supported_audio_mimetype(file.mimetype or ""):
+            return make_response(
+                jsonify({"success": False, "message": "Unsupported audio MIME type"}),
+                400,
+            )
+
+        try:
+            _enforce_uploaded_audio_size_limit(file, filename)
+        except AudioFileTooLargeError:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": build_stt_file_size_limit_message(),
+                    }
+                ),
+                413,
+            )
+
+        temp_path = None
+        try:
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:
+                file.save(temp_file.name)
+                temp_path = Path(temp_file.name)
+
+            session_language = session_state.get("language") or settings.STT_LANGUAGE
+            stt_instance = STTCreator.create_stt(settings.STT_PROVIDER)
+            transcript = stt_instance.transcribe(
+                temp_path,
+                language=session_language,
+                timestamps=False,
+                diarize=False,
+            )
+            if not session_state.get("language") and transcript.get("language"):
+                session_state["language"] = transcript["language"]
+
+            try:
+                apply_live_stt_hypothesis(
+                    session_state,
+                    str(transcript.get("text", "")),
+                    chunk_index,
+                    is_silence=is_silence,
+                )
+            except ValueError:
+                current_app.logger.warning(
+                    "Invalid live transcription chunk",
+                    exc_info=True,
+                )
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "message": "Invalid live transcription chunk",
+                        }
+                    ),
+                    409,
+                )
+            save_live_stt_session(redis_client, session_state)
+
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "session_id": session_id,
+                        "chunk_index": chunk_index,
+                        "chunk_text": transcript.get("text", ""),
+                        "is_silence": is_silence,
+                        "language": session_state.get("language"),
+                        "committed_text": session_state.get("committed_text", ""),
+                        "mutable_text": session_state.get("mutable_text", ""),
+                        "previous_hypothesis": session_state.get(
+                            "previous_hypothesis", ""
+                        ),
+                        "latest_hypothesis": session_state.get(
+                            "latest_hypothesis", ""
+                        ),
+                        "finalized_text": session_state.get("committed_text", ""),
+                        "pending_text": session_state.get("mutable_text", ""),
+                        "transcript_text": get_live_stt_transcript_text(session_state),
+                    }
+                ),
+                200,
+            )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error transcribing live audio chunk: {err}", exc_info=True
+            )
+            return make_response(
+                jsonify({"success": False, "message": "Failed to transcribe audio"}),
+                400,
+            )
+        finally:
+            if temp_path and temp_path.exists():
+                temp_path.unlink()
+
+
+@attachments_ns.route("/stt/live/finish")
+class LiveSpeechToTextFinish(Resource):
+    @api.doc(description="Finish a live speech-to-text session")
+    def post(self):
+        auth_user = _resolve_authenticated_user()
+        if hasattr(auth_user, "status_code"):
+            return auth_user
+        if not auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Authentication required"}),
+                401,
+            )
+
+        redis_client = _require_live_stt_redis()
+        if hasattr(redis_client, "status_code"):
+            return redis_client
+
+        payload = request.get_json(silent=True) or {}
+        session_id = str(payload.get("session_id", "")).strip()
+        if not session_id:
+            return make_response(
+                jsonify({"success": False, "message": "Missing session_id"}),
+                400,
+            )
+
+        session_state = load_live_stt_session(redis_client, session_id)
+        if not session_state:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": "Live transcription session not found",
+                    }
+                ),
+                404,
+            )
+
+        # Stored ``user`` and auth_user are both RAW subs; compare raw-vs-raw so an
+        # email-style sub owner is not 403'd on their own finish call.
+        if str(session_state.get("user", "")) != auth_user:
+            return make_response(
+                jsonify({"success": False, "message": "Forbidden"}),
+                403,
+            )
+
+        final_text = finalize_live_stt_session(session_state)
+        delete_live_stt_session(redis_client, session_id)
+
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "session_id": session_id,
+                    "language": session_state.get("language"),
+                    "text": final_text,
+                }
+            ),
+            200,
+        )
+
+
+@attachments_ns.route("/images/<string:agent_id>/<string:capability>")
+class ServeImage(Resource):
+    @api.doc(description="Serve an agent image using an opaque capability URL")
+    def get(self, agent_id, capability):
+        try:
+            if not looks_like_uuid(agent_id):
+                return make_response(
+                    jsonify({"success": False, "message": "Image not found"}), 404
+                )
+            with db_readonly() as conn:
+                agent = AgentsRepository(conn).find_image_record(agent_id)
+            if not agent:
+                return make_response(
+                    jsonify({"success": False, "message": "Image not found"}), 404
+                )
+
+            image_path = agent.get("image")
+            user_id = agent.get("user_id")
+            content_type = get_agent_image_content_type(image_path)
+            if (
+                is_external_image_url(image_path)
+                or not content_type
+                or not is_safe_agent_image_path(image_path, user_id)
+                or not verify_agent_image_capability(
+                    capability, agent_id, image_path, user_id
+                )
+            ):
+                return make_response(
+                    jsonify({"success": False, "message": "Image not found"}), 404
+                )
+
+            from docsgpt.api.user.base import storage
+
+            size_bytes = storage.get_file_size(image_path)
+            if size_bytes < 0 or size_bytes > settings.AGENT_IMAGE_MAX_BYTES:
+                current_app.logger.warning(
+                    "Rejected oversized legacy agent image for agent %s", agent_id
+                )
+                return make_response(
+                    jsonify({"success": False, "message": "Image not found"}), 404
+                )
+
+            if settings.STORAGE_TYPE == "s3":
+                image_url = storage.generate_presigned_url(
+                    image_path,
+                    expires_in=_AGENT_IMAGE_PRESIGNED_TTL_SECONDS,
+                    content_type=content_type,
+                )
+                response = redirect(image_url, code=302)
+                response.headers.set(
+                    "Cache-Control",
+                    f"private, max-age={_AGENT_IMAGE_REDIRECT_CACHE_SECONDS}",
+                )
+                response.headers.set("X-Content-Type-Options", "nosniff")
+                return response
+
+            file_obj = storage.get_file(image_path)
+            response = Response(
+                _stream_file(file_obj, size_bytes),
+                content_type=content_type,
+                direct_passthrough=True,
+            )
+            response.headers.set("Cache-Control", "public, max-age=86400, immutable")
+            response.headers.set("X-Content-Type-Options", "nosniff")
+
+            return response
+        except FileNotFoundError:
+            return make_response(
+                jsonify({"success": False, "message": "Image not found"}), 404
+            )
+        except ValueError:
+            return make_response(
+                jsonify({"success": False, "message": "Image not found"}), 404
+            )
+        except Exception as e:
+            current_app.logger.error(f"Error serving image: {e}")
+            return make_response(
+                jsonify({"success": False, "message": "Error retrieving image"}), 500
+            )
+
+
+@attachments_ns.route("/tts")
+class TextToSpeech(Resource):
+    tts_model = api.model(
+        "TextToSpeechModel",
+        {
+            "text": fields.String(
+                required=True, description="Text to be synthesized as audio"
+            ),
+        },
+    )
+
+    @api.expect(tts_model)
+    @api.doc(description="Synthesize audio speech from text")
+    def post(self):
+        data = request.get_json()
+        text = data["text"]
+        try:
+            tts_instance = TTSCreator.create_tts(settings.TTS_PROVIDER)
+            audio_base64, detected_language = tts_instance.text_to_speech(text)
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "audio_base64": audio_base64,
+                        "lang": detected_language,
+                    }
+                ),
+                200,
+            )
+        except Exception as err:
+            current_app.logger.error(f"Error synthesizing audio: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)

docsgpt/api/user/authz.py

@@ -0,0 +1,97 @@
+"""Role resolution and authorization helpers (admin/user RBAC).
+
+Roles are resolved per-request from the database plus computed overlays and are
+NEVER read from the inbound JWT: ``simple_jwt``/``session_jwt`` tokens are
+self-mintable, so trusting a ``roles`` claim would be a trivial privilege
+escalation. ``resolve_roles`` always rebuilds the set from scratch.
+
+Policy (see ``rbac-spec.md``):
+- Persisted RBAC (``user_roles``) applies only under ``AUTH_TYPE=oidc``.
+- ``AUTH_TYPE=None`` (no-auth self-host) grants admin only when
+  ``LOCAL_MODE_ADMIN`` is enabled (default off).
+- ``simple_jwt`` / ``session_jwt`` can never be admin (shared/throwaway ``sub``).
+- ``/v1`` agent keys, device tokens, and pre-auth requests are role-less.
+"""
+
+from __future__ import annotations
+
+import logging
+from functools import wraps
+
+from flask import jsonify, make_response, request
+
+from docsgpt.core.settings import settings
+from docsgpt.storage.db.repositories.user_roles import UserRolesRepository
+from docsgpt.storage.db.session import db_readonly
+
+logger = logging.getLogger(__name__)
+
+ROLE_USER = "user"
+ROLE_ADMIN = "admin"
+
+
+def resolve_roles(token: dict | None) -> list[str]:
+    """Compute the sorted role set for a request principal.
+
+    Fails open to less privilege: a DB error while reading grants demotes
+    DB-backed admins to ``user`` rather than raising (which would turn a roles
+    outage into a total-auth outage on the universal before-request path).
+    """
+    if not token:
+        return [ROLE_USER]
+    roles = {ROLE_USER}
+    sub = token.get("sub")
+    if settings.AUTH_TYPE == "oidc":
+        if sub:
+            try:
+                with db_readonly() as conn:
+                    roles.update(UserRolesRepository(conn).role_names_for(sub))
+            except Exception:
+                logger.error(
+                    "resolve_roles: user_roles read failed for sub=%s", sub, exc_info=True
+                )
+    elif settings.AUTH_TYPE is None and settings.LOCAL_MODE_ADMIN:
+        roles.add(ROLE_ADMIN)
+    # simple_jwt / session_jwt: never admin.
+    return sorted(roles)
+
+
+def has_role(token: dict | None, name: str) -> bool:
+    """True if the principal holds ``name``.
+
+    ``user`` is implicit and always true. Tolerates a ``None`` token or a token
+    that never went through ``resolve_roles`` (e.g. async/SSE or /v1 paths) —
+    those are treated as role-less and only satisfy the implicit ``user`` role.
+    """
+    if name == ROLE_USER:
+        return True
+    roles = (token or {}).get("roles") or []
+    return name in roles
+
+
+def require_role(name: str):
+    """Decorator factory: 401 when unauthenticated, 403 when lacking ``name``.
+
+    Fails closed — a missing/None token or absent ``roles`` key never raises and
+    never passes through. The frontend route guard is cosmetic; this is the
+    security boundary.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            token = getattr(request, "decoded_token", None)
+            if not token:
+                return make_response(
+                    jsonify({"success": False, "message": "Authentication required"}), 401
+                )
+            if not has_role(token, name):
+                return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+admin_required = require_role(ROLE_ADMIN)

docsgpt/api/user/base.py

@@ -0,0 +1,435 @@
+"""
+Shared utilities, database connections, and helper functions for user API routes.
+"""
+
+import datetime
+import os
+import uuid
+from functools import wraps
+from pathlib import PurePosixPath
+from typing import Optional, Tuple
+
+from flask import current_app, jsonify, make_response, Response
+from PIL import Image, UnidentifiedImageError
+from werkzeug.utils import secure_filename
+
+from sqlalchemy import text as _sql_text
+
+from docsgpt.core.settings import settings
+from docsgpt.storage.db.base_repository import looks_like_uuid, row_to_dict
+from docsgpt.storage.db.repositories.users import UsersRepository
+from docsgpt.storage.db.session import db_readonly, db_session
+from docsgpt.storage.storage_creator import StorageCreator
+from docsgpt.utils import (
+    AGENT_IMAGE_FORMATS,
+    get_agent_image_content_type,
+    is_external_image_url,
+    safe_user_storage_component,
+)
+from docsgpt.vectorstore.vector_creator import VectorCreator
+
+
+storage = StorageCreator.get_storage()
+
+
+current_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+)
+
+def generate_minute_range(start_date, end_date):
+    """Generate a dictionary with minute-level time ranges."""
+    return {
+        (start_date + datetime.timedelta(minutes=i)).strftime("%Y-%m-%d %H:%M:00"): 0
+        for i in range(int((end_date - start_date).total_seconds() // 60) + 1)
+    }
+
+
+def generate_hourly_range(start_date, end_date):
+    """Generate a dictionary with hourly time ranges."""
+    return {
+        (start_date + datetime.timedelta(hours=i)).strftime("%Y-%m-%d %H:00"): 0
+        for i in range(int((end_date - start_date).total_seconds() // 3600) + 1)
+    }
+
+
+def generate_date_range(start_date, end_date):
+    """Generate a dictionary with daily date ranges."""
+    return {
+        (start_date + datetime.timedelta(days=i)).strftime("%Y-%m-%d"): 0
+        for i in range((end_date - start_date).days + 1)
+    }
+
+
+def ensure_user_doc(user_id):
+    """
+    Ensure a Postgres ``users`` row exists for ``user_id``.
+
+    Returns the row as a dict with the shape legacy callers expect — in
+    particular ``user_id`` and ``agent_preferences`` (with ``pinned`` and
+    ``shared_with_me`` list keys always present).
+
+    Args:
+        user_id: The user ID to ensure
+
+    Returns:
+        The user document as a dict.
+    """
+    with db_session() as conn:
+        user_doc = UsersRepository(conn).upsert(user_id)
+
+    prefs = user_doc.get("agent_preferences") or {}
+    if not isinstance(prefs, dict):
+        prefs = {}
+    prefs.setdefault("pinned", [])
+    prefs.setdefault("shared_with_me", [])
+    user_doc["agent_preferences"] = prefs
+    return user_doc
+
+
+def resolve_tool_details(tool_ids):
+    """
+    Resolve tool IDs to their display details.
+
+    Accepts Postgres UUIDs, legacy Mongo ObjectId strings, or the
+    synthetic ids of default chat tools / agent-selectable builtins
+    (mixed lists are supported). Synthetic ids are resolved in memory;
+    real ids are looked up via ``get_any``. Unknown ids are silently
+    skipped.
+
+    Args:
+        tool_ids: List of tool IDs (UUIDs, legacy ObjectId strings, or
+            synthetic default-tool / builtin ids).
+
+    Returns:
+        List of tool details with ``id``, ``name``, and ``display_name``.
+    """
+    if not tool_ids:
+        return []
+
+    from docsgpt.agents.default_tools import (
+        is_synthesized_tool_id,
+        synthesize_tool_by_name,
+        synthesized_tool_name_for_id,
+    )
+
+    uuid_ids: list[str] = []
+    legacy_ids: list[str] = []
+    default_details: list[dict] = []
+    for tid in tool_ids:
+        if not tid:
+            continue
+        tid_str = str(tid)
+        if is_synthesized_tool_id(tid_str):
+            synth = synthesize_tool_by_name(synthesized_tool_name_for_id(tid_str))
+            if synth is not None:
+                default_details.append(
+                    {
+                        "id": tid_str,
+                        "name": synth.get("name", ""),
+                        "display_name": synth.get("display_name", ""),
+                    }
+                )
+            continue
+        if looks_like_uuid(tid_str):
+            uuid_ids.append(tid_str)
+        else:
+            legacy_ids.append(tid_str)
+
+    if not uuid_ids and not legacy_ids:
+        return default_details
+
+    rows: list[dict] = []
+    with db_readonly() as conn:
+        if uuid_ids:
+            result = conn.execute(
+                _sql_text(
+                    "SELECT * FROM user_tools "
+                    "WHERE id = ANY(CAST(:ids AS uuid[]))"
+                ),
+                {"ids": uuid_ids},
+            )
+            rows.extend(row_to_dict(r) for r in result.fetchall())
+        if legacy_ids:
+            result = conn.execute(
+                _sql_text(
+                    "SELECT * FROM user_tools "
+                    "WHERE legacy_mongo_id = ANY(:ids)"
+                ),
+                {"ids": legacy_ids},
+            )
+            rows.extend(row_to_dict(r) for r in result.fetchall())
+
+    return default_details + [
+        {
+            "id": str(tool.get("id") or tool.get("legacy_mongo_id") or ""),
+            "name": tool.get("name", "") or "",
+            "display_name": (
+                tool.get("custom_name")
+                or tool.get("display_name")
+                or tool.get("name", "")
+                or ""
+            ),
+        }
+        for tool in rows
+    ]
+
+
+_BUILTIN_PROMPT_NAMES = {
+    "default": "Default",
+    "creative": "Creative",
+    "strict": "Strict",
+}
+
+
+def resolve_prompt_name(prompt_id) -> Optional[str]:
+    """Resolve a prompt id to its display name by id (owner-agnostic).
+
+    Mirrors ``resolve_tool_details``: looks the prompt up by id without user
+    scoping, so a team member viewing a shared agent sees the OWNER's prompt
+    name rather than nothing. Built-in prompt sentinels map to friendly labels.
+    Returns None when the prompt is missing/unknown.
+    """
+    if not prompt_id:
+        return None
+    pid = str(prompt_id)
+    if pid in _BUILTIN_PROMPT_NAMES:
+        return _BUILTIN_PROMPT_NAMES[pid]
+    if not looks_like_uuid(pid):
+        return None
+    with db_readonly() as conn:
+        result = conn.execute(
+            _sql_text("SELECT name FROM prompts WHERE id = CAST(:id AS uuid)"),
+            {"id": pid},
+        )
+        row = result.fetchone()
+    return row[0] if row is not None else None
+
+
+def resolve_source_details(source_ids) -> list[dict]:
+    """Resolve source ids to ``[{"id", "name"}]`` by id (owner-agnostic).
+
+    Order-preserving; an id with no matching source row yields ``name: None`` so
+    the client can fall back. Lets a team member viewing a shared agent see the
+    owner's source names instead of a raw id / "External KB".
+    """
+    ids = [str(s) for s in (source_ids or []) if s and looks_like_uuid(str(s))]
+    if not ids:
+        return []
+    with db_readonly() as conn:
+        result = conn.execute(
+            _sql_text(
+                "SELECT id, name FROM sources WHERE id = ANY(CAST(:ids AS uuid[]))"
+            ),
+            {"ids": ids},
+        )
+        by_id = {str(r[0]): r[1] for r in result.fetchall()}
+    return [{"id": sid, "name": by_id.get(sid)} for sid in ids]
+
+
+def get_vector_store(source_id):
+    """
+    Get the Vector Store for a given source ID.
+
+    Args:
+        source_id (str): source id of the document
+
+    Returns:
+        Vector store instance
+    """
+    store = VectorCreator.create_vectorstore(
+        settings.VECTOR_STORE,
+        source_id=source_id,
+        embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+    )
+    return store
+
+
+def _validate_agent_image_upload(file, filename: str) -> None:
+    """Validate an uploaded agent avatar and rewind it for storage.
+
+    Args:
+        file: Werkzeug file upload object.
+        filename: Sanitized upload filename.
+
+    Raises:
+        OSError: If the upload stream cannot be inspected.
+        ValueError: If the file exceeds a limit or is not an allowed raster image.
+    """
+    extension = PurePosixPath(filename).suffix.lower()
+    image_policy = AGENT_IMAGE_FORMATS.get(extension)
+    if not filename or image_policy is None:
+        raise ValueError("Unsupported image extension")
+
+    stream = file.stream
+    stream.seek(0, os.SEEK_END)
+    size_bytes = stream.tell()
+    stream.seek(0)
+    if size_bytes > settings.AGENT_IMAGE_MAX_BYTES:
+        raise ValueError("Image exceeds the upload size limit")
+
+    with Image.open(stream) as image:
+        image_format = (image.format or "").upper()
+        if image_format not in image_policy[0]:
+            raise ValueError("Image content does not match its extension")
+        if image.width * image.height > settings.AGENT_IMAGE_MAX_PIXELS:
+            raise ValueError("Image exceeds the pixel limit")
+        image.verify()
+    stream.seek(0)
+
+
+def _safe_agent_image_filename(filename: str) -> str:
+    """Build an ASCII storage filename without discarding a Unicode file's suffix.
+
+    Args:
+        filename: Original multipart upload filename.
+
+    Returns:
+        A traversal-safe filename with a normalized, allow-listed extension.
+
+    Raises:
+        ValueError: If the original filename has no supported image extension.
+    """
+    normalized_path = filename.replace("\\", "/")
+    original_path = PurePosixPath(normalized_path)
+    extension = original_path.suffix.lower()
+    if extension not in AGENT_IMAGE_FORMATS:
+        raise ValueError("Unsupported image extension")
+
+    safe_stem = secure_filename(original_path.stem).strip("._") or "avatar"
+    return f"{safe_stem}{extension}"
+
+
+def copy_agent_image_for_user(image_path: object, user: str, storage) -> str:
+    """Copy an internal agent avatar into ``user``'s own attachments directory.
+
+    Avatar paths are validated against their owner's upload directory, so an
+    agent copied to a new owner needs its own blob rather than a reference to
+    the original owner's.
+
+    Args:
+        image_path: Source avatar storage path, or an external image URL.
+        user: Identity that will own the copy.
+        storage: Storage backend used for both the read and the write.
+
+    Returns:
+        The newly owned storage path, an unchanged external URL, or an empty
+        string when the source is unusable.
+    """
+    if not isinstance(image_path, str) or not image_path:
+        return ""
+    if is_external_image_url(image_path):
+        return image_path
+    if get_agent_image_content_type(image_path) is None:
+        return ""
+
+    source_name = PurePosixPath(image_path).name
+    prefix, separator, remainder = source_name.partition("_")
+    if separator and remainder and looks_like_uuid(prefix):
+        source_name = remainder
+    try:
+        filename = _safe_agent_image_filename(source_name)
+    except ValueError:
+        return ""
+
+    owner_component = safe_user_storage_component(user)
+    destination = (
+        f"{settings.UPLOAD_FOLDER.rstrip('/')}/{owner_component}/"
+        f"attachments/{uuid.uuid4()}_{filename}"
+    )
+    try:
+        with storage.get_file(image_path) as source_file:
+            storage.save_file(source_file, destination, storage_class="STANDARD")
+    except Exception as e:
+        current_app.logger.warning(
+            "Could not copy agent image %s for %s: %s", image_path, user, e
+        )
+        return ""
+    return destination
+
+
+def handle_image_upload(
+    request, existing_url: str, user: str, storage
+) -> Tuple[Optional[str], Optional[Response]]:
+    """
+    Handle image file upload from request.
+
+    Args:
+        request: Flask request object
+        existing_url: Existing image URL (fallback)
+        user: User ID
+        storage: Storage instance
+
+    Returns:
+        Tuple of (image_url, error_response)
+    """
+    image_url = existing_url
+
+    if "image" in request.files:
+        file = request.files["image"]
+        if file.filename != "":
+            try:
+                filename = _safe_agent_image_filename(file.filename)
+                _validate_agent_image_upload(file, filename)
+            except (
+                Image.DecompressionBombError,
+                UnidentifiedImageError,
+                ValueError,
+                OSError,
+            ) as e:
+                current_app.logger.warning(f"Invalid agent image upload: {e}")
+                return None, make_response(
+                    jsonify({"success": False, "message": "Invalid image upload"}),
+                    400,
+                )
+
+            owner_component = safe_user_storage_component(user)
+            upload_path = (
+                f"{settings.UPLOAD_FOLDER.rstrip('/')}/{owner_component}/"
+                f"attachments/{uuid.uuid4()}_{filename}"
+            )
+            try:
+                storage.save_file(file, upload_path, storage_class="STANDARD")
+                image_url = upload_path
+            except Exception as e:
+                current_app.logger.error(f"Error uploading image: {e}")
+                return None, make_response(
+                    jsonify({"success": False, "message": "Image upload failed"}),
+                    400,
+                )
+    return image_url, None
+
+
+def require_agent(func):
+    """
+    Decorator to require valid agent webhook token.
+
+    Args:
+        func: Function to decorate
+
+    Returns:
+        Wrapped function
+    """
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        from docsgpt.storage.db.repositories.agents import AgentsRepository
+
+        webhook_token = kwargs.get("webhook_token")
+        if not webhook_token:
+            return make_response(
+                jsonify({"success": False, "message": "Webhook token missing"}), 400
+            )
+        with db_readonly() as conn:
+            agent = AgentsRepository(conn).find_by_webhook_token(webhook_token)
+        if not agent:
+            current_app.logger.warning(
+                f"Webhook attempt with invalid token: {webhook_token}"
+            )
+            return make_response(
+                jsonify({"success": False, "message": "Agent not found"}), 404
+            )
+        kwargs["agent"] = agent
+        kwargs["agent_id_str"] = str(agent["id"])
+        return func(*args, **kwargs)
+
+    return wrapper

docsgpt/api/user/conversations/__init__.py

@@ -0,0 +1,5 @@
+"""Conversation management module."""
+
+from .routes import conversations_ns
+
+__all__ = ["conversations_ns"]

docsgpt/api/user/conversations/routes.py

@@ -0,0 +1,489 @@
+"""Conversation management routes."""
+
+import datetime
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+from sqlalchemy import text as sql_text
+
+from docsgpt.api import api
+from docsgpt.api.answer.services.conversation_service import (
+    TERMINATED_RESPONSE_PLACEHOLDER,
+)
+from docsgpt.storage.db.base_repository import looks_like_uuid, row_to_dict
+from docsgpt.storage.db.repositories.agents import AgentsRepository
+from docsgpt.storage.db.repositories.attachments import AttachmentsRepository
+from docsgpt.storage.db.repositories.conversations import ConversationsRepository
+from docsgpt.storage.db.repositories.message_events import MessageEventsRepository
+from docsgpt.storage.db.session import db_readonly, db_session
+from docsgpt.utils import check_required_fields
+
+conversations_ns = Namespace(
+    "conversations", description="Conversation management operations", path="/api"
+)
+
+
+@conversations_ns.route("/delete_conversation")
+class DeleteConversation(Resource):
+    @api.doc(
+        description="Deletes a conversation by ID",
+        params={"id": "The ID of the conversation to delete"},
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        conversation_id = request.args.get("id")
+        if not conversation_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        user_id = decoded_token["sub"]
+        try:
+            with db_session() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.get_any(conversation_id, user_id)
+                if conv is not None:
+                    repo.delete(str(conv["id"]), user_id)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error deleting conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@conversations_ns.route("/delete_all_conversations")
+class DeleteAllConversations(Resource):
+    @api.doc(
+        description="Deletes all conversations for a specific user",
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user_id = decoded_token.get("sub")
+        try:
+            with db_session() as conn:
+                ConversationsRepository(conn).delete_all_for_user(user_id)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error deleting all conversations: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@conversations_ns.route("/get_conversations")
+class GetConversations(Resource):
+    @api.doc(
+        description="Retrieve a list of the latest 30 sidebar conversations (visibility = listed)",
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user_id = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                conversations = ConversationsRepository(conn).list_for_user(
+                    user_id, limit=30
+                )
+            list_conversations = [
+                {
+                    "id": str(conversation["id"]),
+                    "name": conversation["name"],
+                    "agent_id": (
+                        str(conversation["agent_id"])
+                        if conversation.get("agent_id")
+                        else None
+                    ),
+                    "is_shared_usage": conversation.get("is_shared_usage", False),
+                    "shared_token": conversation.get("shared_token", None),
+                }
+                for conversation in conversations
+            ]
+        except Exception as err:
+            current_app.logger.error(
+                f"Error retrieving conversations: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify(list_conversations), 200)
+
+
+@conversations_ns.route("/search_conversations")
+class SearchConversations(Resource):
+    @staticmethod
+    def _build_match_snippet(text_value: str, query: str, radius: int = 60) -> str:
+        if not text_value:
+            return ""
+        idx = text_value.lower().find(query.lower())
+        if idx == -1:
+            snippet = text_value[: radius * 2]
+            return snippet + ("…" if len(text_value) > len(snippet) else "")
+        start = max(0, idx - radius)
+        end = min(len(text_value), idx + len(query) + radius)
+        snippet = text_value[start:end]
+        if start > 0:
+            snippet = "…" + snippet
+        if end < len(text_value):
+            snippet = snippet + "…"
+        return snippet
+
+    @api.doc(
+        description=(
+            "Search the authenticated user's conversations by name or "
+            "message content (case-insensitive substring match). Mirrors "
+            "the visibility filter and response shape of /get_conversations, "
+            "and additionally returns ``match_field`` (``name``, ``prompt`` "
+            "or ``response``) and ``match_snippet`` (a short excerpt of the "
+            "matched text centered on the query) for each result."
+        ),
+        params={
+            "q": "Search term (required)",
+            "limit": "Maximum number of results to return (default 30, max 100)",
+        },
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        query = (request.args.get("q") or "").strip()
+        if not query:
+            return make_response(
+                jsonify({"success": False, "message": "q is required"}), 400
+            )
+        try:
+            limit = int(request.args.get("limit", 30))
+        except (TypeError, ValueError):
+            limit = 30
+        limit = max(1, min(limit, 100))
+        user_id = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                conversations = ConversationsRepository(conn).search_for_user(
+                    user_id, query, limit=limit
+                )
+            list_conversations = [
+                {
+                    "id": str(conversation["id"]),
+                    "name": conversation["name"],
+                    "agent_id": (
+                        str(conversation["agent_id"])
+                        if conversation.get("agent_id")
+                        else None
+                    ),
+                    "is_shared_usage": conversation.get("is_shared_usage", False),
+                    "shared_token": conversation.get("shared_token", None),
+                    "match_field": conversation.get("match_field"),
+                    "match_snippet": self._build_match_snippet(
+                        conversation.get("match_text") or "", query
+                    ),
+                }
+                for conversation in conversations
+            ]
+        except Exception as err:
+            current_app.logger.error(
+                f"Error searching conversations: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify(list_conversations), 200)
+
+
+@conversations_ns.route("/get_single_conversation")
+class GetSingleConversation(Resource):
+    @api.doc(
+        description="Retrieve a single conversation by ID",
+        params={"id": "The conversation ID"},
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        conversation_id = request.args.get("id")
+        if not conversation_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        user_id = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                repo = ConversationsRepository(conn)
+                conversation = repo.get_any(conversation_id, user_id)
+                if not conversation:
+                    return make_response(jsonify({"status": "not found"}), 404)
+                conv_pg_id = str(conversation["id"])
+                messages = repo.get_messages(conv_pg_id)
+
+                # Resolve attachment details (id, fileName) for each message.
+                attachments_repo = AttachmentsRepository(conn)
+                queries = []
+                for msg in messages:
+                    metadata = msg.get("metadata") or {}
+                    query = {
+                        "prompt": msg.get("prompt"),
+                        "response": msg.get("response"),
+                        "thought": msg.get("thought"),
+                        "sources": msg.get("sources") or [],
+                        "tool_calls": msg.get("tool_calls") or [],
+                        "timestamp": msg.get("timestamp"),
+                        "model_id": msg.get("model_id"),
+                        # Lets the client distinguish placeholder rows from
+                        # finalised answers and tail-poll in-flight ones.
+                        "message_id": str(msg["id"]) if msg.get("id") else None,
+                        "status": msg.get("status"),
+                        "request_id": msg.get("request_id"),
+                        "last_heartbeat_at": metadata.get("last_heartbeat_at"),
+                        # Surfaced from metadata so the chat can render a
+                        # workflow run's produced artifacts on reload.
+                        "workflow_run_id": metadata.get("workflow_run_id"),
+                    }
+                    if metadata:
+                        query["metadata"] = metadata
+                    # Feedback on conversation_messages is a JSONB blob with
+                    # shape {"text": <str>, "timestamp": <iso>}. The legacy
+                    # frontend consumed a flat scalar feedback string, so
+                    # unwrap the ``text`` field for compat.
+                    feedback = msg.get("feedback")
+                    if feedback is not None:
+                        if isinstance(feedback, dict):
+                            query["feedback"] = feedback.get("text")
+                            if feedback.get("timestamp"):
+                                query["feedback_timestamp"] = feedback["timestamp"]
+                        else:
+                            query["feedback"] = feedback
+                    attachments = msg.get("attachments") or []
+                    if attachments:
+                        attachment_details = []
+                        for attachment_id in attachments:
+                            try:
+                                att = attachments_repo.get_any(
+                                    str(attachment_id), user_id
+                                )
+                                if att:
+                                    attachment_details.append(
+                                        {
+                                            "id": str(att["id"]),
+                                            "fileName": att.get(
+                                                "filename", "Unknown file"
+                                            ),
+                                        }
+                                    )
+                            except Exception as e:
+                                current_app.logger.error(
+                                    f"Error retrieving attachment {attachment_id}: {e}",
+                                    exc_info=True,
+                                )
+                        query["attachments"] = attachment_details
+                    queries.append(query)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error retrieving conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        data = {
+            "queries": queries,
+            "agent_id": (
+                str(conversation["agent_id"]) if conversation.get("agent_id") else None
+            ),
+            "is_shared_usage": conversation.get("is_shared_usage", False),
+            "shared_token": conversation.get("shared_token", None),
+        }
+        return make_response(jsonify(data), 200)
+
+
+@conversations_ns.route("/update_conversation_name")
+class UpdateConversationName(Resource):
+    @api.expect(
+        api.model(
+            "UpdateConversationModel",
+            {
+                "id": fields.String(required=True, description="Conversation ID"),
+                "name": fields.String(
+                    required=True, description="New name of the conversation"
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Updates the name of a conversation",
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        data = request.get_json()
+        required_fields = ["id", "name"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        user_id = decoded_token.get("sub")
+        try:
+            with db_session() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.get_any(data["id"], user_id)
+                if conv is not None:
+                    repo.rename(str(conv["id"]), user_id, data["name"])
+        except Exception as err:
+            current_app.logger.error(
+                f"Error updating conversation name: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@conversations_ns.route("/feedback")
+class SubmitFeedback(Resource):
+    @api.expect(
+        api.model(
+            "FeedbackModel",
+            {
+                "question": fields.String(
+                    required=False, description="The user question"
+                ),
+                "answer": fields.String(required=False, description="The AI answer"),
+                "feedback": fields.String(required=True, description="User feedback"),
+                "question_index": fields.Integer(
+                    required=True,
+                    description="The question number in that particular conversation",
+                ),
+                "conversation_id": fields.String(
+                    required=True, description="id of the particular conversation"
+                ),
+                "api_key": fields.String(description="Optional API key"),
+            },
+        )
+    )
+    @api.doc(
+        description="Submit feedback for a conversation",
+    )
+    def post(self):
+        data = request.get_json() or {}
+        decoded_token = request.decoded_token
+        # api_key callers (widgets) carry no JWT; resolve the key to its owner.
+        scoped_api_key = None
+        if not decoded_token:
+            api_key = data.get("api_key")
+            agent = None
+            if api_key:
+                with db_readonly() as conn:
+                    agent = AgentsRepository(conn).find_by_key(api_key)
+            if not agent:
+                return make_response(jsonify({"success": False}), 401)
+            decoded_token = {"sub": agent.get("user_id")}
+            scoped_api_key = api_key
+        required_fields = ["feedback", "conversation_id", "question_index"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        user_id = decoded_token.get("sub")
+        feedback_value = data["feedback"]
+        question_index = int(data["question_index"])
+        # Normalize string feedback to lowercase so analytics queries
+        # (which match 'like'/'dislike') count rows correctly. Tolerate
+        # legacy uppercase clients on ingest. Non-string values pass through.
+        if isinstance(feedback_value, str):
+            feedback_value = feedback_value.lower()
+        feedback_payload = (
+            None
+            if feedback_value is None
+            else {
+                "text": feedback_value,
+                "timestamp": datetime.datetime.now(
+                    datetime.timezone.utc
+                ).isoformat(),
+            }
+        )
+        try:
+            with db_session() as conn:
+                repo = ConversationsRepository(conn)
+                conv = repo.get_any(data["conversation_id"], user_id)
+                # A key may only rate conversations it created.
+                if conv is None or (
+                    scoped_api_key and conv.get("api_key") != scoped_api_key
+                ):
+                    return make_response(
+                        jsonify({"success": False, "message": "Not found"}), 404
+                    )
+                repo.set_feedback(str(conv["id"]), question_index, feedback_payload)
+        except Exception as err:
+            current_app.logger.error(f"Error submitting feedback: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@conversations_ns.route("/messages/<string:message_id>/tail")
+class GetMessageTail(Resource):
+    @api.doc(
+        description=(
+            "Current state of one conversation_messages row, scoped to the "
+            "authenticated user. Used to reconnect to an in-flight stream "
+            "after a refresh."
+        ),
+        params={"message_id": "Message UUID"},
+    )
+    def get(self, message_id):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        if not looks_like_uuid(message_id):
+            return make_response(
+                jsonify({"success": False, "message": "Invalid message id"}), 400
+            )
+        user_id = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                # Owner-or-shared, matching ``ConversationsRepository.get``.
+                row = conn.execute(
+                    sql_text(
+                        "SELECT m.* FROM conversation_messages m "
+                        "JOIN conversations c ON c.id = m.conversation_id "
+                        "WHERE m.id = CAST(:mid AS uuid) "
+                        "AND (c.user_id = :uid OR :uid = ANY(c.shared_with))"
+                    ),
+                    {"mid": message_id, "uid": user_id},
+                ).fetchone()
+                if row is None:
+                    return make_response(jsonify({"status": "not found"}), 404)
+                msg = row_to_dict(row)
+                # Mid-stream the row's response is the placeholder; rebuild
+                # the live partial from the journal so /tail mirrors SSE.
+                status = msg.get("status")
+                response = msg.get("response")
+                thought = msg.get("thought")
+                sources = msg.get("sources") or []
+                tool_calls = msg.get("tool_calls") or []
+                if status in ("pending", "streaming") and (
+                    response == TERMINATED_RESPONSE_PLACEHOLDER
+                ):
+                    partial = MessageEventsRepository(conn).reconstruct_partial(
+                        message_id
+                    )
+                    response = partial["response"]
+                    thought = partial["thought"] or thought
+                    if partial["sources"]:
+                        sources = partial["sources"]
+                    if partial["tool_calls"]:
+                        tool_calls = partial["tool_calls"]
+        except Exception as err:
+            current_app.logger.error(
+                f"Error tailing message {message_id}: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+        metadata = msg.get("message_metadata") or {}
+        return make_response(
+            jsonify(
+                {
+                    "message_id": str(msg["id"]),
+                    "status": status,
+                    "response": response,
+                    "thought": thought,
+                    "sources": sources,
+                    "tool_calls": tool_calls,
+                    "request_id": msg.get("request_id"),
+                    "last_heartbeat_at": metadata.get("last_heartbeat_at"),
+                    "error": metadata.get("error"),
+                }
+            ),
+            200,
+        )

docsgpt/api/user/idempotency.py

@@ -0,0 +1,294 @@
+"""Per-Celery-task idempotency wrapper backed by ``task_dedup``."""
+
+from __future__ import annotations
+
+import functools
+import inspect
+import logging
+import threading
+import uuid
+from typing import Any, Callable, Optional
+
+from docsgpt.storage.db.repositories.idempotency import IdempotencyRepository
+from docsgpt.storage.db.session import db_readonly, db_session
+
+
+logger = logging.getLogger(__name__)
+
+
+# Poison-loop cap; transient-failure headroom without infinite retry.
+MAX_TASK_ATTEMPTS = 5
+
+# 30s heartbeat / 60s TTL → ~2 missed ticks of slack before reclaim.
+LEASE_TTL_SECONDS = 60
+LEASE_HEARTBEAT_INTERVAL = 30
+
+# 10 × 60s ≈ 5 min of deferral before giving up on a held lease.
+LEASE_RETRY_MAX = 10
+
+
+def with_idempotency(
+    task_name: str,
+    *,
+    on_poison: Optional[Callable[[str, dict], None]] = None,
+) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    """Short-circuit on completed key; gate concurrent runs via a lease.
+
+    The guard key is the caller's ``idempotency_key``, or one synthesized
+    from ``source_id`` so a keyless dispatch is still poison-guarded.
+
+    Entry short-circuits:
+      - completed row → return cached result
+      - live lease held → retry(countdown=LEASE_TTL_SECONDS)
+      - attempt_count > MAX_TASK_ATTEMPTS → poison alert; ``on_poison`` fires
+    Success writes ``completed``; exceptions leave ``pending`` for
+    autoretry until the poison-loop guard trips.
+    """
+
+    def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
+        @functools.wraps(fn)
+        def wrapper(self, *args: Any, idempotency_key: Any = None, **kwargs: Any) -> Any:
+            explicit_key = (
+                idempotency_key
+                if isinstance(idempotency_key, str) and idempotency_key
+                else None
+            )
+            # A keyless dispatch still gets the guard via a synthesized key;
+            # None means no anchor exists — run unguarded, as before.
+            key = explicit_key or _synthesize_guard_key(task_name, kwargs)
+            if key is None:
+                return fn(self, *args, idempotency_key=idempotency_key, **kwargs)
+
+            cached = _lookup_completed(key)
+            if cached is not None:
+                logger.info(
+                    "idempotency hit for task=%s key=%s — returning cached result",
+                    task_name, key,
+                )
+                return cached
+
+            owner_id = str(uuid.uuid4())
+            attempt = _try_claim_lease(
+                key, task_name, _safe_task_id(self), owner_id,
+            )
+            if attempt is None:
+                # Live lease held by another worker. Re-queue and bail
+                # quickly — by the time the retry fires (LEASE_TTL
+                # seconds), Worker 1 has either finalised (we'll hit
+                # ``_lookup_completed`` and return cached) or its lease
+                # has expired and we can claim.
+                logger.info(
+                    "idempotency: live lease held; deferring task=%s key=%s",
+                    task_name, key,
+                )
+                raise self.retry(
+                    countdown=LEASE_TTL_SECONDS,
+                    max_retries=LEASE_RETRY_MAX,
+                )
+
+            if attempt > MAX_TASK_ATTEMPTS:
+                logger.error(
+                    "idempotency poison-loop guard: task=%s key=%s attempts=%s",
+                    task_name, key, attempt,
+                    extra={
+                        "alert": "idempotency_poison_loop",
+                        "task_name": task_name,
+                        "idempotency_key": key,
+                        "attempts": attempt,
+                    },
+                )
+                poisoned = {
+                    "success": False,
+                    "error": "idempotency poison-loop guard tripped",
+                    "attempts": attempt,
+                }
+                _finalize(key, poisoned, status="failed")
+                _run_poison_hook(
+                    on_poison, task_name, fn, self, args, kwargs, idempotency_key,
+                )
+                return poisoned
+
+            heartbeat_thread, heartbeat_stop = _start_lease_heartbeat(
+                key, owner_id,
+            )
+            try:
+                result = fn(self, *args, idempotency_key=idempotency_key, **kwargs)
+                _finalize(key, result, status="completed")
+                return result
+            except Exception:
+                # Drop the lease so the next retry doesn't wait LEASE_TTL.
+                _release_lease(key, owner_id)
+                raise
+            finally:
+                _stop_lease_heartbeat(heartbeat_thread, heartbeat_stop)
+
+        return wrapper
+
+    return decorator
+
+
+def _synthesize_guard_key(task_name: str, kwargs: dict) -> Optional[str]:
+    """Derive a deterministic guard key from ``source_id`` for a keyless dispatch.
+
+    ``source_id`` is stable across broker redeliveries and unique per
+    upload, so the poison-loop counter survives an OOM SIGKILL. Returns
+    ``None`` when absent — the dispatch then runs unguarded as before.
+    """
+    source_id = kwargs.get("source_id")
+    if source_id:
+        return f"auto:{task_name}:{source_id}"
+    return None
+
+
+def _run_poison_hook(
+    on_poison: Optional[Callable[[str, dict], None]],
+    task_name: str,
+    fn: Callable[..., Any],
+    task_self: Any,
+    args: tuple,
+    kwargs: dict,
+    idempotency_key: Any,
+) -> None:
+    """Invoke a task's poison-path hook with named call args; swallow failures.
+
+    A hook failure must never change the poison-guard outcome.
+    """
+    if on_poison is None:
+        return
+    try:
+        bound = inspect.signature(fn).bind_partial(
+            task_self, *args, idempotency_key=idempotency_key, **kwargs,
+        )
+        on_poison(task_name, dict(bound.arguments))
+    except Exception:
+        logger.exception(
+            "idempotency: poison hook failed for task=%s", task_name,
+        )
+
+
+def _lookup_completed(key: str) -> Any:
+    """Return cached ``result_json`` if a completed row exists for ``key``, else None."""
+    with db_readonly() as conn:
+        row = IdempotencyRepository(conn).get_task(key)
+    if row is None:
+        return None
+    if row.get("status") != "completed":
+        return None
+    return row.get("result_json")
+
+
+def _try_claim_lease(
+    key: str, task_name: str, task_id: str, owner_id: str,
+) -> Optional[int]:
+    """Atomic CAS; returns ``attempt_count`` or ``None`` when held.
+
+    DB outage → treated as ``attempt=1`` so transient failures don't
+    block all task execution; reconciler repairs the lease columns.
+    """
+    try:
+        with db_session() as conn:
+            return IdempotencyRepository(conn).try_claim_lease(
+                key=key,
+                task_name=task_name,
+                task_id=task_id,
+                owner_id=owner_id,
+                ttl_seconds=LEASE_TTL_SECONDS,
+            )
+    except Exception:
+        logger.exception(
+            "idempotency lease-claim failed for key=%s task=%s", key, task_name,
+        )
+        return 1
+
+
+def _finalize(key: str, result_json: Any, *, status: str) -> None:
+    """Best-effort terminal write. Never let DB outage fail the task."""
+    try:
+        with db_session() as conn:
+            IdempotencyRepository(conn).finalize_task(
+                key=key, result_json=result_json, status=status,
+            )
+    except Exception:
+        logger.exception(
+            "idempotency finalize failed for key=%s status=%s", key, status,
+        )
+
+
+def _release_lease(key: str, owner_id: str) -> None:
+    """Best-effort lease release on the wrapper's exception path."""
+    try:
+        with db_session() as conn:
+            IdempotencyRepository(conn).release_lease(key, owner_id)
+    except Exception:
+        logger.exception("idempotency release-lease failed for key=%s", key)
+
+
+def _start_lease_heartbeat(
+    key: str, owner_id: str,
+) -> tuple[threading.Thread, threading.Event]:
+    """Spawn a daemon thread that bumps ``lease_expires_at`` every
+    :data:`LEASE_HEARTBEAT_INTERVAL` seconds until ``stop_event`` fires.
+
+    Mirrors ``docsgpt.worker._start_ingest_heartbeat`` so the two
+    durability heartbeats share shape and cadence.
+    """
+    stop_event = threading.Event()
+    thread = threading.Thread(
+        target=_lease_heartbeat_loop,
+        args=(key, owner_id, stop_event, LEASE_HEARTBEAT_INTERVAL),
+        daemon=True,
+        name=f"idempotency-lease-heartbeat:{key[:32]}",
+    )
+    thread.start()
+    return thread, stop_event
+
+
+def _stop_lease_heartbeat(
+    thread: threading.Thread, stop_event: threading.Event,
+) -> None:
+    """Signal the heartbeat thread to exit and join with a short timeout."""
+    stop_event.set()
+    thread.join(timeout=10)
+
+
+def _lease_heartbeat_loop(
+    key: str,
+    owner_id: str,
+    stop_event: threading.Event,
+    interval: int,
+) -> None:
+    """Refresh the lease until ``stop_event`` is set or ownership is lost.
+
+    A failed refresh (rowcount 0) means another worker stole the lease
+    after expiry — at that point the damage is already possible, so we
+    log and keep ticking. Don't escalate to thread death; the main task
+    body needs to keep running so its outcome is at least *recorded*.
+    """
+    while not stop_event.wait(interval):
+        try:
+            with db_session() as conn:
+                still_owned = IdempotencyRepository(conn).refresh_lease(
+                    key=key, owner_id=owner_id, ttl_seconds=LEASE_TTL_SECONDS,
+                )
+            if not still_owned:
+                logger.warning(
+                    "idempotency lease lost mid-task for key=%s "
+                    "(another worker may have taken over)",
+                    key,
+                )
+        except Exception:
+            logger.exception(
+                "idempotency lease-heartbeat tick failed for key=%s", key,
+            )
+
+
+def _safe_task_id(task_self: Any) -> str:
+    """Best-effort extraction of ``self.request.id`` from a Celery task."""
+    try:
+        request = getattr(task_self, "request", None)
+        task_id: Optional[str] = (
+            getattr(request, "id", None) if request is not None else None
+        )
+    except Exception:
+        task_id = None
+    return task_id or "unknown"

docsgpt/api/user/me/__init__.py

@@ -0,0 +1,3 @@
+from .routes import me_ns
+
+__all__ = ["me_ns"]

docsgpt/api/user/me/routes.py

@@ -0,0 +1,34 @@
+"""Current-principal endpoint.
+
+``GET /api/user/me`` returns the caller's user id and resolved roles, sourced
+only from ``request.decoded_token`` (already populated and role-resolved by the
+auth chokepoint in ``app.py``). Auth-mode-agnostic. ``email``/``name``/
+``picture`` are OIDC-only and optional — they are echoed from the token and are
+never present for ``simple_jwt``/``session_jwt``/no-auth modes.
+"""
+
+from __future__ import annotations
+
+from flask import jsonify, make_response, request
+from flask_restx import Namespace, Resource
+
+me_ns = Namespace("me", description="Current user identity and roles", path="/api")
+
+
+@me_ns.route("/user/me")
+class MeResource(Resource):
+    def get(self):
+        """Return ``{user_id, roles, email?, name?, picture?}`` for the caller."""
+        decoded_token = getattr(request, "decoded_token", None)
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        body = {
+            "success": True,
+            "user_id": decoded_token.get("sub"),
+            "roles": decoded_token.get("roles") or ["user"],
+        }
+        for field in ("email", "name", "picture"):
+            value = decoded_token.get(field)
+            if value:
+                body[field] = value
+        return make_response(jsonify(body), 200)

docsgpt/api/user/models/__init__.py

@@ -0,0 +1,3 @@
+from .routes import models_ns
+
+__all__ = ["models_ns"]

docsgpt/api/user/models/routes.py

@@ -0,0 +1,574 @@
+"""Model routes.
+
+- ``GET /api/models`` — list available models for the current user.
+  Combines the built-in catalog with the user's BYOM records.
+- ``GET/POST/PATCH/DELETE /api/user/models[/<id>]`` — CRUD for the
+  user's own OpenAI-compatible model registrations (BYOM).
+- ``POST /api/user/models/<id>/test`` — sanity-check the upstream
+  endpoint with a tiny request.
+
+Every BYOM endpoint is user-scoped at the repository layer
+(every query filters on ``user_id`` from ``request.decoded_token``).
+"""
+
+from __future__ import annotations
+
+import logging
+
+import requests
+from flask import current_app, jsonify, make_response, request
+from flask_restx import Namespace, Resource
+
+from docsgpt.api import api
+from docsgpt.core.model_registry import ModelRegistry
+from docsgpt.security.safe_url import (
+    UnsafeUserUrlError,
+    pinned_post,
+    validate_user_base_url,
+)
+from docsgpt.storage.db.repositories.user_custom_models import (
+    UserCustomModelsRepository,
+)
+from docsgpt.storage.db.session import db_readonly, db_session
+from docsgpt.utils import check_required_fields
+
+
+logger = logging.getLogger(__name__)
+
+
+models_ns = Namespace("models", description="Available models", path="/api")
+
+
+_CONTEXT_WINDOW_MIN = 1_000
+_CONTEXT_WINDOW_MAX = 10_000_000
+
+
+def _user_id_or_401():
+    decoded_token = request.decoded_token
+    if not decoded_token:
+        return None, make_response(jsonify({"success": False}), 401)
+    user_id = decoded_token.get("sub")
+    if not user_id:
+        return None, make_response(jsonify({"success": False}), 401)
+    return user_id, None
+
+
+def _normalize_capabilities(raw) -> dict:
+    """Coerce + bound the user-supplied capabilities payload."""
+    raw = raw or {}
+    if not isinstance(raw, dict):
+        raise ValueError("'capabilities' must be an object")
+    out = {}
+    if "supports_tools" in raw:
+        out["supports_tools"] = bool(raw["supports_tools"])
+    if "supports_structured_output" in raw:
+        out["supports_structured_output"] = bool(raw["supports_structured_output"])
+    if "supports_streaming" in raw:
+        out["supports_streaming"] = bool(raw["supports_streaming"])
+    if "attachments" in raw:
+        atts = raw["attachments"] or []
+        if not isinstance(atts, list):
+            raise ValueError("'capabilities.attachments' must be a list")
+        coerced = [str(a) for a in atts]
+        # Reject unknown aliases at the API boundary so bad payloads
+        # never reach the registry layer (where lenient expansion just
+        # drops them). Raw MIME types (containing ``/``) pass through
+        # unchanged for parity with the built-in YAML schema.
+        from docsgpt.core.model_yaml import builtin_attachment_aliases
+
+        aliases = builtin_attachment_aliases()
+        for entry in coerced:
+            if "/" in entry:
+                continue
+            if entry not in aliases:
+                valid = ", ".join(sorted(aliases.keys())) or "<none defined>"
+                raise ValueError(
+                    f"unknown attachment alias '{entry}' in "
+                    f"'capabilities.attachments'. Valid aliases: {valid}, "
+                    f"or use a raw MIME type like 'image/png'."
+                )
+        out["attachments"] = coerced
+    if "context_window" in raw:
+        try:
+            cw = int(raw["context_window"])
+        except (TypeError, ValueError):
+            raise ValueError("'capabilities.context_window' must be an integer")
+        if not (_CONTEXT_WINDOW_MIN <= cw <= _CONTEXT_WINDOW_MAX):
+            raise ValueError(
+                f"'capabilities.context_window' must be between "
+                f"{_CONTEXT_WINDOW_MIN} and {_CONTEXT_WINDOW_MAX}"
+            )
+        out["context_window"] = cw
+    if "api_flavor" in raw:
+        from docsgpt.core.model_yaml import VALID_API_FLAVORS
+
+        api_flavor = raw["api_flavor"]
+        if api_flavor not in VALID_API_FLAVORS:
+            valid = ", ".join(sorted(VALID_API_FLAVORS))
+            raise ValueError(
+                f"'capabilities.api_flavor' must be one of [{valid}]"
+            )
+        out["api_flavor"] = api_flavor
+    if "reasoning_effort" in raw and raw["reasoning_effort"] is not None:
+        from docsgpt.core.model_yaml import VALID_REASONING_EFFORTS
+
+        reasoning_effort = raw["reasoning_effort"]
+        if reasoning_effort not in VALID_REASONING_EFFORTS:
+            valid = ", ".join(sorted(VALID_REASONING_EFFORTS))
+            raise ValueError(
+                f"'capabilities.reasoning_effort' must be one of [{valid}]"
+            )
+        out["reasoning_effort"] = reasoning_effort
+    return out
+
+
+def _row_to_response(row: dict) -> dict:
+    """Wire-format projection — never includes the API key."""
+    return {
+        "id": str(row["id"]),
+        "upstream_model_id": row["upstream_model_id"],
+        "display_name": row["display_name"],
+        "description": row.get("description") or "",
+        "base_url": row["base_url"],
+        "capabilities": row.get("capabilities") or {},
+        "enabled": bool(row.get("enabled", True)),
+        "source": "user",
+    }
+
+
+@models_ns.route("/models")
+class ModelsListResource(Resource):
+    def get(self):
+        """Get list of available models with their capabilities.
+
+        When the request is authenticated, the response includes the
+        user's own BYOM registrations alongside the built-in catalog.
+        """
+        try:
+            user_id = None
+            decoded_token = getattr(request, "decoded_token", None)
+            if decoded_token:
+                user_id = decoded_token.get("sub")
+
+            registry = ModelRegistry.get_instance()
+            models = registry.get_enabled_models(user_id=user_id)
+
+            response = {
+                "models": [model.to_dict() for model in models],
+                "default_model_id": registry.default_model_id,
+                "count": len(models),
+            }
+        except Exception as err:
+            current_app.logger.error(f"Error fetching models: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)
+        return make_response(jsonify(response), 200)
+
+
+@models_ns.route("/user/models")
+class UserModelsCollectionResource(Resource):
+    @api.doc(description="List the current user's BYOM custom models")
+    def get(self):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+        try:
+            with db_readonly() as conn:
+                rows = UserCustomModelsRepository(conn).list_for_user(user_id)
+            return make_response(
+                jsonify({"models": [_row_to_response(r) for r in rows]}), 200
+            )
+        except Exception as e:
+            current_app.logger.error(
+                f"Error listing user custom models: {e}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 500)
+
+    @api.doc(description="Register a new BYOM custom model")
+    def post(self):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+
+        data = request.get_json() or {}
+        missing = check_required_fields(
+            data,
+            ["upstream_model_id", "display_name", "base_url", "api_key"],
+        )
+        if missing:
+            return missing
+
+        # SECURITY: reject blank api_key — would leak instance API key
+        # to the user-supplied base_url via LLMCreator fallback.
+        for required_nonblank in (
+            "upstream_model_id",
+            "display_name",
+            "base_url",
+            "api_key",
+        ):
+            value = data.get(required_nonblank)
+            if not isinstance(value, str) or not value.strip():
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "error": f"'{required_nonblank}' must be a non-empty string",
+                        }
+                    ),
+                    400,
+                )
+
+        # SSRF guard at create time. Re-runs at dispatch time (LLMCreator)
+        # as defense in depth against DNS rebinding and pre-guard rows.
+        try:
+            validate_user_base_url(data["base_url"])
+        except UnsafeUserUrlError as e:
+            return make_response(
+                jsonify({"success": False, "error": str(e)}), 400
+            )
+
+        try:
+            capabilities = _normalize_capabilities(data.get("capabilities"))
+        except ValueError as e:
+            return make_response(
+                jsonify({"success": False, "error": str(e)}), 400
+            )
+
+        try:
+            with db_session() as conn:
+                row = UserCustomModelsRepository(conn).create(
+                    user_id=user_id,
+                    upstream_model_id=data["upstream_model_id"],
+                    display_name=data["display_name"],
+                    description=data.get("description") or "",
+                    base_url=data["base_url"],
+                    api_key_plaintext=data["api_key"],
+                    capabilities=capabilities,
+                    enabled=bool(data.get("enabled", True)),
+                )
+        except Exception as e:
+            current_app.logger.error(
+                f"Error creating user custom model: {e}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 500)
+
+        ModelRegistry.invalidate_user(user_id)
+        return make_response(jsonify(_row_to_response(row)), 201)
+
+
+@models_ns.route("/user/models/<string:model_id>")
+class UserModelResource(Resource):
+    @api.doc(description="Get one BYOM custom model")
+    def get(self, model_id):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+        try:
+            with db_readonly() as conn:
+                row = UserCustomModelsRepository(conn).get(model_id, user_id)
+        except Exception as e:
+            current_app.logger.error(
+                f"Error fetching user custom model: {e}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 500)
+        if row is None:
+            return make_response(jsonify({"success": False}), 404)
+        return make_response(jsonify(_row_to_response(row)), 200)
+
+    @api.doc(description="Update a BYOM custom model (partial)")
+    def patch(self, model_id):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+
+        data = request.get_json() or {}
+
+        # Reject present-but-blank values for fields where blank doesn't
+        # mean "no change". (The api_key special case — blank means "keep
+        # existing" — is handled below.)
+        for required_nonblank in (
+            "upstream_model_id",
+            "display_name",
+            "base_url",
+        ):
+            if required_nonblank in data:
+                value = data[required_nonblank]
+                if not isinstance(value, str) or not value.strip():
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "error": f"'{required_nonblank}' cannot be blank",
+                            }
+                        ),
+                        400,
+                    )
+
+        if "base_url" in data and data["base_url"]:
+            try:
+                validate_user_base_url(data["base_url"])
+            except UnsafeUserUrlError as e:
+                return make_response(
+                    jsonify({"success": False, "error": str(e)}), 400
+                )
+
+        update_fields: dict = {}
+        for k in (
+            "upstream_model_id",
+            "display_name",
+            "description",
+            "base_url",
+            "enabled",
+        ):
+            if k in data:
+                update_fields[k] = data[k]
+
+        if "capabilities" in data:
+            try:
+                update_fields["capabilities"] = _normalize_capabilities(
+                    data["capabilities"]
+                )
+            except ValueError as e:
+                return make_response(
+                    jsonify({"success": False, "error": str(e)}), 400
+                )
+
+        # PATCH semantics: blank/missing api_key → keep the existing
+        # ciphertext; non-empty api_key → re-encrypt and replace.
+        if data.get("api_key"):
+            update_fields["api_key_plaintext"] = data["api_key"]
+
+        if not update_fields:
+            return make_response(
+                jsonify({"success": False, "error": "no updatable fields"}), 400
+            )
+
+        try:
+            with db_session() as conn:
+                ok = UserCustomModelsRepository(conn).update(
+                    model_id, user_id, update_fields
+                )
+        except Exception as e:
+            current_app.logger.error(
+                f"Error updating user custom model: {e}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 500)
+
+        if not ok:
+            return make_response(jsonify({"success": False}), 404)
+
+        ModelRegistry.invalidate_user(user_id)
+        with db_readonly() as conn:
+            row = UserCustomModelsRepository(conn).get(model_id, user_id)
+        return make_response(jsonify(_row_to_response(row)), 200)
+
+    @api.doc(description="Delete a BYOM custom model")
+    def delete(self, model_id):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+        try:
+            with db_session() as conn:
+                ok = UserCustomModelsRepository(conn).delete(model_id, user_id)
+        except Exception as e:
+            current_app.logger.error(
+                f"Error deleting user custom model: {e}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 500)
+        if not ok:
+            return make_response(jsonify({"success": False}), 404)
+
+        ModelRegistry.invalidate_user(user_id)
+        return make_response(jsonify({"success": True}), 200)
+
+
+def _run_connection_test(
+    base_url: str,
+    api_key: str,
+    upstream_model_id: str,
+    api_flavor: str = "chat_completions",
+):
+    """Send a minimal generation request to verify a BYOM endpoint.
+
+    Returns ``(body, http_status)``. Upstream errors return 200 with
+    ``ok=False`` so the UI can render inline errors; only local SSRF
+    rejection returns 400.
+    """
+    if api_flavor == "responses":
+        url = base_url.rstrip("/") + "/responses"
+        payload = {
+            "model": upstream_model_id,
+            "input": "hi",
+            "max_output_tokens": 16,
+            "stream": False,
+            "store": False,
+        }
+    else:
+        url = base_url.rstrip("/") + "/chat/completions"
+        payload = {
+            "model": upstream_model_id,
+            "messages": [{"role": "user", "content": "hi"}],
+            "max_tokens": 1,
+            "stream": False,
+        }
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+    try:
+        # pinned_post closes the DNS-rebinding window. Redirects off
+        # because 3xx could bounce to an internal address (the SSRF
+        # guard only validates the supplied URL).
+        resp = pinned_post(
+            url,
+            json=payload,
+            headers=headers,
+            timeout=5,
+            allow_redirects=False,
+        )
+    except UnsafeUserUrlError as e:
+        return {"ok": False, "error": str(e)}, 400
+    except requests.RequestException as e:
+        return {"ok": False, "error": f"connection error: {e}"}, 200
+
+    if 300 <= resp.status_code < 400:
+        return (
+            {
+                "ok": False,
+                "error": (
+                    f"upstream returned HTTP {resp.status_code} "
+                    "redirect; refusing to follow"
+                ),
+            },
+            200,
+        )
+
+    if resp.status_code >= 400:
+        # Cap and only reflect JSON to avoid body-exfil via non-API responses.
+        content_type = (resp.headers.get("Content-Type") or "").lower()
+        if "application/json" in content_type:
+            text = (resp.text or "")[:500]
+            error_msg = f"upstream returned HTTP {resp.status_code}: {text}"
+        else:
+            error_msg = f"upstream returned HTTP {resp.status_code}"
+        return {"ok": False, "error": error_msg}, 200
+
+    return {"ok": True}, 200
+
+
+@models_ns.route("/user/models/test")
+class UserModelTestPayloadResource(Resource):
+    @api.doc(
+        description=(
+            "Test an arbitrary BYOM payload (display_name / model id / "
+            "base_url / api_key) without saving. Used by the UI's 'Test "
+            "connection' button so the user can validate before they "
+            "Save. Uses the selected API protocol with the same SSRF guard "
+            "and 5s timeout as the by-id variant."
+        )
+    )
+    def post(self):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+
+        data = request.get_json() or {}
+        missing = check_required_fields(
+            data, ["base_url", "api_key", "upstream_model_id"]
+        )
+        if missing:
+            return missing
+
+        try:
+            capabilities = _normalize_capabilities(data.get("capabilities"))
+        except ValueError as e:
+            return make_response(jsonify({"ok": False, "error": str(e)}), 400)
+        body, status = _run_connection_test(
+            data["base_url"],
+            data["api_key"],
+            data["upstream_model_id"],
+            capabilities.get("api_flavor", "chat_completions"),
+        )
+        return make_response(jsonify(body), status)
+
+
+@models_ns.route("/user/models/<string:model_id>/test")
+class UserModelTestResource(Resource):
+    @api.doc(
+        description=(
+            "Test a saved BYOM record. Defaults to the stored "
+            "base_url / upstream_model_id / encrypted api_key, but "
+            "any of those can be overridden via the request body so "
+            "the UI can test in-flight edits before saving. Used by "
+            "the 'Test connection' button in edit mode."
+        )
+    )
+    def post(self, model_id):
+        user_id, err = _user_id_or_401()
+        if err:
+            return err
+
+        data = request.get_json() or {}
+        # Per-field overrides; blank/missing falls back to stored value.
+        override_base_url = (data.get("base_url") or "").strip() or None
+        override_upstream_model_id = (
+            data.get("upstream_model_id") or ""
+        ).strip() or None
+        override_api_key = (data.get("api_key") or "").strip() or None
+
+        try:
+            with db_readonly() as conn:
+                repo = UserCustomModelsRepository(conn)
+                row = repo.get(model_id, user_id)
+                if row is None:
+                    return make_response(jsonify({"success": False}), 404)
+                stored_api_key = (
+                    repo._decrypt_api_key(
+                        row.get("api_key_encrypted", ""), user_id
+                    )
+                    if not override_api_key
+                    else None
+                )
+        except Exception as e:
+            current_app.logger.error(
+                f"Error loading user custom model for test: {e}", exc_info=True
+            )
+            return make_response(
+                jsonify({"ok": False, "error": "internal error loading model"}),
+                500,
+            )
+
+        api_key = override_api_key or stored_api_key
+        if not api_key:
+            return make_response(
+                jsonify(
+                    {
+                        "ok": False,
+                        "error": (
+                            "Stored API key could not be decrypted. The "
+                            "encryption secret may have rotated. Re-save "
+                            "the model with the API key to recover."
+                        ),
+                    }
+                ),
+                400,
+            )
+
+        base_url = override_base_url or row["base_url"]
+        upstream_model_id = (
+            override_upstream_model_id or row["upstream_model_id"]
+        )
+        try:
+            capabilities = (
+                _normalize_capabilities(data["capabilities"])
+                if "capabilities" in data
+                else (row.get("capabilities") or {})
+            )
+        except ValueError as e:
+            return make_response(jsonify({"ok": False, "error": str(e)}), 400)
+        body, status = _run_connection_test(
+            base_url,
+            api_key,
+            upstream_model_id,
+            capabilities.get("api_flavor", "chat_completions"),
+        )
+        return make_response(jsonify(body), status)

docsgpt/api/user/prompts/__init__.py

@@ -0,0 +1,5 @@
+"""Prompts module."""
+
+from .routes import prompts_ns
+
+__all__ = ["prompts_ns"]

docsgpt/api/user/prompts/routes.py

@@ -0,0 +1,233 @@
+"""Prompt management routes."""
+
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+
+from docsgpt.api import api
+from docsgpt.api.user.team_sharing import team_access_for, visible_with_access
+from docsgpt.storage.db.repositories.prompts import PromptsRepository
+from docsgpt.prompts.composer import compose_preset, is_composed_preset
+from docsgpt.storage.db.session import db_readonly, db_session
+from docsgpt.utils import check_required_fields
+
+prompts_ns = Namespace(
+    "prompts", description="Prompt management operations", path="/api"
+)
+
+
+@prompts_ns.route("/create_prompt")
+class CreatePrompt(Resource):
+    create_prompt_model = api.model(
+        "CreatePromptModel",
+        {
+            "content": fields.String(
+                required=True, description="Content of the prompt"
+            ),
+            "name": fields.String(required=True, description="Name of the prompt"),
+        },
+    )
+
+    @api.expect(create_prompt_model)
+    @api.doc(description="Create a new prompt")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        data = request.get_json()
+        required_fields = ["content", "name"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        user = decoded_token.get("sub")
+        try:
+            with db_session() as conn:
+                prompt = PromptsRepository(conn).create(user, data["name"], data["content"])
+            new_id = str(prompt["id"])
+        except Exception as err:
+            current_app.logger.error(f"Error creating prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"id": new_id}), 200)
+
+
+@prompts_ns.route("/get_prompts")
+class GetPrompts(Resource):
+    @api.doc(description="Get all prompts for the user")
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                repo = PromptsRepository(conn)
+                prompts = repo.list_for_user(user)
+                owned_ids = {str(p["id"]) for p in prompts}
+                team_shared = visible_with_access(conn, user, "prompt")
+                shared_ids = [pid for pid in team_shared if pid not in owned_ids]
+                shared_prompts = repo.list_by_ids(shared_ids)
+            list_prompts = [
+                {"id": "default", "name": "default", "type": "public"},
+                {"id": "creative", "name": "creative", "type": "public"},
+                {"id": "strict", "name": "strict", "type": "public"},
+            ]
+            for prompt in prompts:
+                list_prompts.append(
+                    {
+                        "id": str(prompt["id"]),
+                        "name": prompt["name"],
+                        "type": "private",
+                    }
+                )
+            for prompt in shared_prompts:
+                list_prompts.append(
+                    {
+                        "id": str(prompt["id"]),
+                        "name": prompt["name"],
+                        "type": "team",
+                        "team_access": team_shared.get(str(prompt["id"])),
+                    }
+                )
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving prompts: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify(list_prompts), 200)
+
+
+@prompts_ns.route("/get_single_prompt")
+class GetSinglePrompt(Resource):
+    @api.doc(params={"id": "ID of the prompt"}, description="Get a single prompt by ID")
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        prompt_id = request.args.get("id")
+        if not prompt_id:
+            return make_response(
+                jsonify({"success": False, "message": "ID is required"}), 400
+            )
+        try:
+            if is_composed_preset(prompt_id):
+                return make_response(
+                    jsonify({"content": compose_preset(prompt_id)}), 200
+                )
+            with db_readonly() as conn:
+                repo = PromptsRepository(conn)
+                prompt = repo.get_any(prompt_id, user)
+                if not prompt and team_access_for(conn, user, "prompt", prompt_id):
+                    # Team fallback: ownerless fetch only after a grant check.
+                    prompt = repo.get_for_rendering(prompt_id)
+            if not prompt:
+                return make_response(
+                    jsonify({"success": False, "message": "Prompt not found"}), 404
+                )
+        except Exception as err:
+            current_app.logger.error(f"Error retrieving prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"content": prompt["content"]}), 200)
+
+
+@prompts_ns.route("/delete_prompt")
+class DeletePrompt(Resource):
+    delete_prompt_model = api.model(
+        "DeletePromptModel",
+        {"id": fields.String(required=True, description="Prompt ID to delete")},
+    )
+
+    @api.expect(delete_prompt_model)
+    @api.doc(description="Delete a prompt by ID")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            with db_session() as conn:
+                repo = PromptsRepository(conn)
+                prompt = repo.get_any(data["id"], user)
+                if not prompt:
+                    return make_response(
+                        jsonify({"success": False, "message": "Prompt not found"}),
+                        404,
+                    )
+                repo.delete(str(prompt["id"]), user)
+        except Exception as err:
+            current_app.logger.error(f"Error deleting prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@prompts_ns.route("/update_prompt")
+class UpdatePrompt(Resource):
+    update_prompt_model = api.model(
+        "UpdatePromptModel",
+        {
+            "id": fields.String(required=True, description="Prompt ID to update"),
+            "name": fields.String(required=True, description="New name of the prompt"),
+            "content": fields.String(
+                required=True, description="New content of the prompt"
+            ),
+        },
+    )
+
+    @api.expect(update_prompt_model)
+    @api.doc(description="Update an existing prompt")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id", "name", "content"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            with db_session() as conn:
+                repo = PromptsRepository(conn)
+                prompt = repo.get_any(data["id"], user)
+                if prompt:
+                    repo.update(str(prompt["id"]), user, data["name"], data["content"])
+                else:
+                    # Team editor write path (viewer is read-only).
+                    access = team_access_for(conn, user, "prompt", data["id"])
+                    if access == "editor":
+                        result = repo.update_by_id(
+                            data["id"],
+                            data["name"],
+                            data["content"],
+                            expected_updated_at=data.get("expected_updated_at"),
+                        )
+                        if result is None:
+                            return make_response(
+                                jsonify(
+                                    {
+                                        "success": False,
+                                        "message": "Prompt was modified by someone else",
+                                        "code": "stale_write",
+                                    }
+                                ),
+                                409,
+                            )
+                    elif access == "viewer":
+                        return make_response(
+                            jsonify(
+                                {"success": False, "message": "Read-only: editor access required"}
+                            ),
+                            403,
+                        )
+                    else:
+                        return make_response(
+                            jsonify({"success": False, "message": "Prompt not found"}),
+                            404,
+                        )
+        except Exception as err:
+            current_app.logger.error(f"Error updating prompt: {err}", exc_info=True)
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(jsonify({"success": True}), 200)

docsgpt/api/user/reconciliation.py

@@ -0,0 +1,501 @@
+"""Reconciler tick: sweep stuck rows and escalate to terminal status + alert."""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Any, Dict, Iterator, Optional, TYPE_CHECKING
+
+from sqlalchemy import Connection, Engine
+
+from docsgpt.api.user.idempotency import MAX_TASK_ATTEMPTS
+from docsgpt.core.settings import settings
+from docsgpt.storage.db.engine import get_engine
+from docsgpt.storage.db.repositories.pending_tool_state import (
+    PendingToolStateRepository,
+)
+from docsgpt.storage.db.repositories.reconciliation import (
+    ReconciliationRepository,
+)
+from docsgpt.storage.db.repositories.stack_logs import StackLogsRepository
+
+if TYPE_CHECKING:
+    from docsgpt.storage.db.repositories.schedules import SchedulesRepository
+
+logger = logging.getLogger(__name__)
+
+
+MAX_MESSAGE_RECONCILE_ATTEMPTS = 3
+
+
+def zero_summary() -> Dict[str, int]:
+    """The counter keys every tick reports, all at zero.
+
+    Single source of truth so the early-return and the caller's error
+    fallback cannot drift from the sweeps that actually populate them.
+    """
+    return {
+        "messages_failed": 0,
+        "tool_calls_failed": 0,
+        "ingests_stalled": 0,
+        "idempotency_pending_failed": 0,
+        "schedule_runs_failed": 0,
+    }
+
+
+def run_reconciliation() -> Dict[str, Any]:
+    """Single tick of the reconciler. Five sweeps, FOR UPDATE SKIP LOCKED.
+
+    Stuck ``executed`` tool calls always flip to ``failed`` — operators
+    handle cleanup manually via the structured alert. The side effect is
+    assumed to have committed; no automated rollback is attempted.
+
+    Stuck ``task_dedup`` rows (lease expired AND attempts >= max)
+    promote to ``failed`` so a same-key retry can re-claim instead of
+    sitting in ``pending`` until 24 h TTL.
+    """
+    if not settings.POSTGRES_URI:
+        return {**zero_summary(), "skipped": "POSTGRES_URI not set"}
+
+    engine = get_engine()
+    summary: Dict[str, Any] = zero_summary()
+    # User-facing events to fan out once their DB writes have committed
+    # (publish-after-commit). Each item is
+    # ``(user_id, event_type, payload, scope)``.
+    events: list[tuple] = []
+
+    # Each sweep below commits in its own transaction, and the queued
+    # user-facing events are fanned out once, at the end. The ``finally`` is
+    # what makes a LATER sweep's failure non-destructive: without it, a
+    # transient DB error discards events whose writes already landed — and the
+    # loss is permanent, because every sweep filters out the rows it just made
+    # terminal (``find_and_lock_stalled_ingests`` looks only at
+    # ``status='active'``, ``find_and_lock_stuck_messages`` only at
+    # pending/streaming), so no later tick re-finds them to re-emit. The user
+    # is left with an upload toast spinning on "Training..." forever. Seen
+    # live on 2026-08-21 when an Azure DNS blip took Postgres unreachable
+    # mid-tick.
+    #
+    # Publish-after-commit is preserved by ``_sweep``: each sweep stages its
+    # own events and they only reach ``events`` once that sweep's transaction
+    # has committed. A sweep that raises PARTWAY THROUGH takes its staged
+    # events down with its rolled-back writes, which is the point — otherwise
+    # the ``finally`` would fan out terminal events for rows that are still
+    # active, and the next tick would re-find those rows and emit a duplicate.
+    try:
+        return _run_sweeps(engine, summary, events)
+    finally:
+        _publish_events(events)
+
+
+@contextmanager
+def _sweep(engine: Engine, events: list[tuple]) -> Iterator[tuple[Connection, list[tuple]]]:
+    """Open one sweep transaction, staging its events until it commits.
+
+    Yields ``(conn, staged)``. Append user-facing events to ``staged``, never
+    to ``events`` directly: ``staged`` is merged into ``events`` only after the
+    ``engine.begin()`` block exits cleanly, so a sweep that raises mid-loop
+    publishes nothing for the writes Postgres just rolled back.
+    """
+    staged: list[tuple] = []
+    with engine.begin() as conn:
+        yield conn, staged
+    events.extend(staged)
+
+
+def _run_sweeps(engine: Engine, summary: Dict[str, Any], events: list[tuple]) -> Dict[str, Any]:
+    """Run every reconciler sweep, appending user-facing events to ``events``.
+
+    Split out of :func:`run_reconciliation` so the caller can guarantee that
+    events from sweeps that already COMMITTED are published even when a later
+    sweep raises. Each sweep stages into its own list via :func:`_sweep`; see
+    the comment in :func:`run_reconciliation` for why staging matters.
+    """
+    with _sweep(engine, events) as (conn, staged):
+        repo = ReconciliationRepository(conn)
+        pt_repo = PendingToolStateRepository(conn)
+        for msg in repo.find_and_lock_stuck_messages():
+            new_count = repo.increment_message_reconcile_attempts(msg["id"])
+            if new_count >= MAX_MESSAGE_RECONCILE_ATTEMPTS:
+                repo.mark_message_failed(
+                    msg["id"],
+                    error=(
+                        "reconciler: stuck in pending/streaming for >5 min "
+                        f"after {new_count} attempts"
+                    ),
+                )
+                summary["messages_failed"] += 1
+                # Revoke any awaiting-approval prompt: the resumable state
+                # dies with the message, so the durable
+                # ``tool.approval.required`` envelope must be cleared or the
+                # UI toast lingers on reconnect. Only emit when a row was
+                # actually deleted so non-approval failures stay quiet.
+                user_id = msg.get("user_id")
+                conversation_id = msg.get("conversation_id")
+                if (
+                    user_id
+                    and conversation_id
+                    and pt_repo.delete_state(str(conversation_id), str(user_id))
+                ):
+                    # Mark the row as having had its approval revoked. A late
+                    # finalize is allowed to reclaim a reconciler-failed row
+                    # (see ``finalize_message``), but must not reclaim this
+                    # one: the resumable state is gone and the UI has already
+                    # been told the approval was cleared, so landing an answer
+                    # here would contradict what the user was shown.
+                    repo.mark_message_approval_cleared(msg["id"])
+                    staged.append(
+                        (
+                            str(user_id),
+                            "tool.approval.cleared",
+                            {
+                                "conversation_id": str(conversation_id),
+                                "message_id": str(msg["id"]),
+                                "reason": "failed",
+                            },
+                            {"kind": "conversation", "id": str(conversation_id)},
+                        )
+                    )
+                _emit_alert(
+                    conn,
+                    name="reconciler_message_failed",
+                    user_id=msg.get("user_id"),
+                    detail={
+                        "message_id": str(msg["id"]),
+                        "attempts": new_count,
+                    },
+                )
+
+    with _sweep(engine, events) as (conn, staged):
+        repo = ReconciliationRepository(conn)
+        for row in repo.find_and_lock_proposed_tool_calls():
+            repo.mark_tool_call_failed(
+                row["call_id"],
+                error=(
+                    "reconciler: never advanced past 'proposed' for >5 min. "
+                    "The row is written immediately before the tool runs, so "
+                    "either the tool is still executing or the stream was "
+                    "abandoned before it reported back; in the abandoned case "
+                    "no side effect was committed."
+                ),
+            )
+            summary["tool_calls_failed"] += 1
+            _emit_alert(
+                conn,
+                name="reconciler_tool_call_failed_proposed",
+                user_id=None,
+                detail={
+                    "call_id": row["call_id"],
+                    "tool_name": row.get("tool_name"),
+                },
+            )
+
+    with _sweep(engine, events) as (conn, staged):
+        repo = ReconciliationRepository(conn)
+        for row in repo.find_and_lock_executed_tool_calls():
+            repo.mark_tool_call_failed(
+                row["call_id"],
+                error=(
+                    "reconciler: executed but not confirmed within 15 min. "
+                    "The tool SUCCEEDED and its side effect is committed; "
+                    "confirmation only lands when the whole turn finalizes, "
+                    "so a long-running turn reaches this state while healthy. "
+                    "Check the parent message before assuming cleanup is "
+                    "needed."
+                ),
+            )
+            summary["tool_calls_failed"] += 1
+            _emit_alert(
+                conn,
+                name="reconciler_tool_call_failed_executed",
+                user_id=None,
+                detail={
+                    "call_id": row["call_id"],
+                    "tool_name": row.get("tool_name"),
+                    "action_name": row.get("action_name"),
+                },
+            )
+
+    # Q4: ingest checkpoints whose heartbeat has gone silent. Each is
+    # escalated to terminal ``status='stalled'`` and alerted once — no
+    # worker kill, no rollback of the partial embed. The 'stalled' flag
+    # ends the re-alert loop and drives the "indexing failed" badge the
+    # sources list derives from this row.
+    with _sweep(engine, events) as (conn, staged):
+        repo = ReconciliationRepository(conn)
+        for row in repo.find_and_lock_stalled_ingests():
+            summary["ingests_stalled"] += 1
+            _emit_alert(
+                conn,
+                name="reconciler_ingest_stalled",
+                user_id=row.get("user_id"),
+                detail={
+                    "source_id": str(row.get("source_id")),
+                    "embedded_chunks": row.get("embedded_chunks"),
+                    "total_chunks": row.get("total_chunks"),
+                    "last_updated": str(row.get("last_updated")),
+                },
+            )
+            repo.mark_ingest_stalled(str(row["source_id"]))
+            # Tell the upload toast the ingest is done-for. Without it the
+            # toast spins on "Training…" for the whole live session; only
+            # ``source.ingest.failed`` flips it to a terminal error.
+            user_id = row.get("user_id")
+            source_id = row.get("source_id")
+            if user_id and source_id:
+                staged.append(
+                    (
+                        str(user_id),
+                        "source.ingest.failed",
+                        {
+                            "source_id": str(source_id),
+                            "filename": row.get("source_name") or "",
+                            "operation": "upload",
+                            "error": "Indexing stopped after a stall.",
+                        },
+                        {"kind": "source", "id": str(source_id)},
+                    )
+                )
+
+    # Q5: idempotency rows whose lease expired with attempts exhausted.
+    # The wrapper's poison-loop guard normally finalises these, but if
+    # the wrapper itself died mid-task (worker SIGKILL, OOM during
+    # heartbeat) the row sits in ``pending`` blocking same-key retries
+    # via ``_lookup_completed`` returning None for the whole 24 h TTL.
+    # Promote to ``failed`` so a retry can re-claim and either resume
+    # or fail loudly.
+    with _sweep(engine, events) as (conn, staged):
+        repo = ReconciliationRepository(conn)
+        for row in repo.find_stuck_idempotency_pending(
+            max_attempts=MAX_TASK_ATTEMPTS,
+        ):
+            error_msg = (
+                "reconciler: idempotency lease expired with attempts "
+                f"({row['attempt_count']}) >= {MAX_TASK_ATTEMPTS}; "
+                "task abandoned"
+            )
+            repo.mark_idempotency_pending_failed(
+                row["idempotency_key"], error=error_msg,
+            )
+            summary["idempotency_pending_failed"] += 1
+            _emit_alert(
+                conn,
+                name="reconciler_idempotency_pending_failed",
+                user_id=None,
+                detail={
+                    "idempotency_key": row["idempotency_key"],
+                    "task_name": row.get("task_name"),
+                    "task_id": row.get("task_id"),
+                    "attempts": row.get("attempt_count"),
+                },
+            )
+
+    # Q6: scheduler runs stuck in 'running' past the soft-time-limit window.
+    from docsgpt.storage.db.repositories.schedule_runs import (
+        ScheduleRunsRepository,
+    )
+    from docsgpt.storage.db.repositories.schedules import SchedulesRepository
+    from docsgpt.core.settings import settings as _settings
+
+    stuck_age = max(
+        15, int(_settings.SCHEDULE_RUN_TIMEOUT // 60) + 5,
+    )
+    with _sweep(engine, events) as (conn, staged):
+        runs_repo = ScheduleRunsRepository(conn)
+        schedules_repo = SchedulesRepository(conn)
+        for run in runs_repo.list_stuck_running(age_minutes=stuck_age):
+            runs_repo.update(
+                run["id"],
+                {
+                    "status": "timeout",
+                    "finished_at": datetime.now(timezone.utc),
+                    "error_type": "timeout",
+                    "error": (
+                        "reconciler: schedule_run stuck in 'running' past "
+                        f"{stuck_age} min"
+                    ),
+                },
+            )
+            schedules_repo.bump_failure_count(str(run["schedule_id"]))
+            flipped = _terminal_flip_once_schedule(
+                schedules_repo, str(run["schedule_id"]),
+            )
+            summary["schedule_runs_failed"] += 1
+            staged.extend(
+                _schedule_terminal_events(
+                    run,
+                    error_type="timeout",
+                    error=(
+                        "reconciler: schedule_run stuck in 'running' past "
+                        f"{stuck_age} min"
+                    ),
+                    once_completed=flipped,
+                )
+            )
+            _emit_alert(
+                conn,
+                name="reconciler_schedule_run_timeout",
+                user_id=run.get("user_id"),
+                detail={
+                    "run_id": str(run["id"]),
+                    "schedule_id": str(run["schedule_id"]),
+                },
+            )
+
+    # Q7: scheduler runs orphaned in 'pending' — dispatcher committed but
+    # apply_async failed (broker outage / crash mid-dispatch).
+    with _sweep(engine, events) as (conn, staged):
+        runs_repo = ScheduleRunsRepository(conn)
+        schedules_repo = SchedulesRepository(conn)
+        for run in runs_repo.list_stuck_pending(age_minutes=stuck_age):
+            runs_repo.update(
+                run["id"],
+                {
+                    "status": "failed",
+                    "finished_at": datetime.now(timezone.utc),
+                    "error_type": "internal",
+                    "error": (
+                        "reconciler: schedule_run stuck in 'pending' past "
+                        f"{stuck_age} min (worker_never_started)"
+                    ),
+                },
+            )
+            schedules_repo.bump_failure_count(str(run["schedule_id"]))
+            flipped = _terminal_flip_once_schedule(
+                schedules_repo, str(run["schedule_id"]),
+            )
+            summary["schedule_runs_failed"] += 1
+            staged.extend(
+                _schedule_terminal_events(
+                    run,
+                    error_type="internal",
+                    error=(
+                        "reconciler: schedule_run stuck in 'pending' past "
+                        f"{stuck_age} min (worker_never_started)"
+                    ),
+                    once_completed=flipped,
+                )
+            )
+            _emit_alert(
+                conn,
+                name="reconciler_schedule_run_pending",
+                user_id=run.get("user_id"),
+                detail={
+                    "run_id": str(run["id"]),
+                    "schedule_id": str(run["schedule_id"]),
+                },
+            )
+
+    return summary
+
+
+def _terminal_flip_once_schedule(
+    schedules_repo: "SchedulesRepository", schedule_id: str,
+) -> bool:
+    """Flip a once-schedule to 'completed' after its run terminates.
+
+    Recurring schedules keep firing; once-schedules would otherwise read
+    'active forever' since next_run_at is already NULL. Returns ``True``
+    when the flip happened so the caller can publish a status event.
+    """
+    schedule = schedules_repo.get_internal(schedule_id)
+    if schedule is None or schedule.get("trigger_type") != "once":
+        return False
+    if schedule.get("status") in {"completed", "cancelled"}:
+        return False
+    schedules_repo.update_internal(
+        schedule_id, {"status": "completed", "next_run_at": None},
+    )
+    return True
+
+
+def _schedule_terminal_events(
+    run: Dict[str, Any],
+    *,
+    error_type: str,
+    error: str,
+    once_completed: bool,
+) -> list:
+    """Build the user-facing events for a reconciler-failed schedule run.
+
+    Always a ``schedule.run.failed`` (so a watching run-log updates live),
+    plus ``schedule.completed`` when a once-schedule flipped terminal — the
+    UI otherwise shows it 'active' with stale Edit/Cancel actions until a
+    manual refresh.
+    """
+    user_id = run.get("user_id")
+    schedule_id = run.get("schedule_id")
+    if not user_id or not schedule_id:
+        return []
+    scope = {"kind": "schedule", "id": str(schedule_id)}
+    out: list = [
+        (
+            str(user_id),
+            "schedule.run.failed",
+            {
+                "run_id": str(run["id"]),
+                "schedule_id": str(schedule_id),
+                "status": "failed",
+                "error_type": error_type,
+                "error": error,
+            },
+            scope,
+        )
+    ]
+    if once_completed:
+        out.append(
+            (
+                str(user_id),
+                "schedule.completed",
+                {"schedule_id": str(schedule_id), "status": "completed"},
+                scope,
+            )
+        )
+    return out
+
+
+def _publish_events(events: list) -> None:
+    """Fan out user-facing events after their DB writes have committed.
+
+    Each item is ``(user_id, event_type, payload, scope)``. Best-effort:
+    a publish miss is swallowed per event so one failure can't strand the
+    rest, and notifications never surface as a reconciler-task failure.
+    """
+    if not events:
+        return
+    from docsgpt.events.publisher import publish_user_event
+
+    for user_id, event_type, payload, scope in events:
+        try:
+            publish_user_event(user_id, event_type, payload, scope=scope)
+        except Exception:
+            logger.exception(
+                "reconciler: failed to publish %s for user=%s",
+                event_type,
+                user_id,
+            )
+
+
+def _emit_alert(
+    conn: Connection,
+    *,
+    name: str,
+    user_id: Optional[str],
+    detail: Dict[str, Any],
+) -> None:
+    """Structured ``logger.error`` plus a ``stack_logs`` row for operators."""
+    extra = {"alert": name, **detail}
+    logger.error("reconciler alert: %s", name, extra=extra)
+    try:
+        StackLogsRepository(conn).insert(
+            activity_id=str(uuid.uuid4()),
+            endpoint="reconciliation_worker",
+            level="ERROR",
+            user_id=user_id,
+            query=name,
+            stacks=[extra],
+        )
+    except Exception:
+        logger.exception("reconciler: failed to write stack_logs row for %s", name)

docsgpt/api/user/routes.py

@@ -0,0 +1,87 @@
+"""
+Main user API routes - registers all namespace modules.
+"""
+
+from flask import Blueprint
+
+from docsgpt.api import api
+from .agents import (
+    agents_folders_ns,
+    agents_guardrails_ns,
+    agents_ns,
+    agents_portability_ns,
+    agents_sharing_ns,
+    agents_webhooks_ns,
+)
+from .analytics import analytics_ns
+from .artifacts import artifacts_ns
+from .attachments import attachments_ns
+from .conversations import conversations_ns
+from .me import me_ns
+from .models import models_ns
+from .prompts import prompts_ns
+from .schedules import schedules_ns
+from .sharing import sharing_ns
+from .sources import (
+    sources_chunks_ns,
+    sources_ns,
+    sources_search_ns,
+    sources_upload_ns,
+)
+from .teams import teams_ns
+from .tools import tools_mcp_ns, tools_ns
+from .workflows import workflows_ns
+
+
+user = Blueprint("user", __name__)
+
+# Analytics
+api.add_namespace(analytics_ns)
+
+# Artifacts
+api.add_namespace(artifacts_ns)
+
+# Attachments
+api.add_namespace(attachments_ns)
+
+# Conversations
+api.add_namespace(conversations_ns)
+
+# Current user (identity + roles)
+api.add_namespace(me_ns)
+
+# Models
+api.add_namespace(models_ns)
+
+# Agents (main, sharing, webhooks, folders, import/export)
+api.add_namespace(agents_ns)
+api.add_namespace(agents_sharing_ns)
+api.add_namespace(agents_webhooks_ns)
+api.add_namespace(agents_folders_ns)
+api.add_namespace(agents_portability_ns)
+api.add_namespace(agents_guardrails_ns)
+
+# Prompts
+api.add_namespace(prompts_ns)
+
+# Schedules
+api.add_namespace(schedules_ns)
+
+# Sharing
+api.add_namespace(sharing_ns)
+
+# Sources (main, chunks, retrieval test, upload)
+api.add_namespace(sources_ns)
+api.add_namespace(sources_chunks_ns)
+api.add_namespace(sources_search_ns)
+api.add_namespace(sources_upload_ns)
+
+# Teams (CRUD, membership, resource-sharing grants)
+api.add_namespace(teams_ns)
+
+# Tools (main, MCP)
+api.add_namespace(tools_ns)
+api.add_namespace(tools_mcp_ns)
+
+# Workflows
+api.add_namespace(workflows_ns)

docsgpt/api/user/scheduler_dispatcher.py

@@ -0,0 +1,186 @@
+"""Schedule dispatcher: poll Postgres, claim due rows under FOR UPDATE SKIP LOCKED,
+advance next_run_at atomically with the run claim, then enqueue.
+
+Per-schedule IANA tz semantics (croniter+zoneinfo) outside Celery's app-wide tz,
+plus Postgres-native dedup avoid Redis visibility_timeout double-fires.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict, List, Optional
+
+from docsgpt.agents.scheduler_utils import next_cron_run
+from docsgpt.core.settings import settings
+from docsgpt.storage.db.engine import get_engine
+from docsgpt.storage.db.repositories.schedule_runs import (
+    ScheduleRunsRepository,
+)
+from docsgpt.storage.db.repositories.schedules import SchedulesRepository
+
+logger = logging.getLogger(__name__)
+
+
+def _normalize_dt(value: Any) -> Optional[datetime]:
+    """Accept a datetime / ISO string / None and return a tz-aware UTC dt."""
+    if value is None:
+        return None
+    if isinstance(value, datetime):
+        return value.astimezone(timezone.utc) if value.tzinfo else (
+            value.replace(tzinfo=timezone.utc)
+        )
+    if isinstance(value, str):
+        try:
+            parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except ValueError:
+            return None
+        return parsed.astimezone(timezone.utc) if parsed.tzinfo else (
+            parsed.replace(tzinfo=timezone.utc)
+        )
+    return None
+
+
+def _compute_next(
+    schedule: Dict[str, Any],
+    *,
+    after: datetime,
+) -> Optional[datetime]:
+    """Next next_run_at for a recurring schedule, or None when past end_at."""
+    cron = schedule.get("cron")
+    if not cron:
+        return None
+    end_at = _normalize_dt(schedule.get("end_at"))
+    candidate = next_cron_run(cron, schedule.get("timezone"), after=after)
+    if end_at is not None and candidate > end_at:
+        return None
+    return candidate
+
+
+def dispatch_due_runs() -> Dict[str, int]:
+    """One dispatcher tick; returns counts for schedule_syncs-style logging."""
+    if not settings.POSTGRES_URI:
+        return {"enqueued": 0, "skipped": 0, "advanced": 0}
+
+    from docsgpt.api.user.tasks import execute_scheduled_run
+
+    now = datetime.now(timezone.utc)
+    grace = timedelta(seconds=max(0, settings.SCHEDULE_MISFIRE_GRACE))
+    engine = get_engine()
+    counts = {"enqueued": 0, "skipped": 0, "advanced": 0}
+    enqueue_args: List[str] = []
+
+    with engine.begin() as conn:
+        schedules_repo = SchedulesRepository(conn)
+        runs_repo = ScheduleRunsRepository(conn)
+        for schedule in schedules_repo.list_due():
+            scheduled_for = _normalize_dt(schedule.get("next_run_at"))
+            if scheduled_for is None:
+                continue
+
+            trigger_type = schedule.get("trigger_type")
+            agent_id_raw = schedule.get("agent_id")
+            agent_id = str(agent_id_raw) if agent_id_raw else None
+
+            # Misfire grace applies to recurring only — once-tasks fire late, not vanish.
+            if (
+                trigger_type == "recurring"
+                and grace > timedelta(0)
+                and (now - scheduled_for) > grace
+            ):
+                runs_repo.record_skipped(
+                    str(schedule["id"]),
+                    schedule["user_id"],
+                    agent_id,
+                    scheduled_for,
+                    error_type="missed",
+                    error="misfire grace exceeded",
+                )
+                counts["skipped"] += 1
+                nxt = _compute_next(schedule, after=now)
+                if nxt is None:
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"status": "completed", "next_run_at": None,
+                         "last_run_at": now},
+                    )
+                else:
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"next_run_at": nxt, "last_run_at": now},
+                    )
+                counts["advanced"] += 1
+                continue
+
+            # Overlap guard: never enqueue while a previous run is active.
+            if runs_repo.has_active_run(str(schedule["id"])):
+                runs_repo.record_skipped(
+                    str(schedule["id"]),
+                    schedule["user_id"],
+                    agent_id,
+                    scheduled_for,
+                    error_type="overlap",
+                    error="previous run still active",
+                )
+                counts["skipped"] += 1
+                if trigger_type == "recurring":
+                    nxt = _compute_next(schedule, after=scheduled_for)
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"next_run_at": nxt, "last_run_at": now},
+                    )
+                else:
+                    # Once: null next_run_at so we don't re-pick; the in-flight
+                    # run will terminal-flip the schedule when it finishes.
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"next_run_at": None, "last_run_at": now},
+                    )
+                continue
+
+            # Dedup primitive: two racing dispatchers see exactly one row.
+            run = runs_repo.record_pending(
+                str(schedule["id"]),
+                schedule["user_id"],
+                agent_id,
+                scheduled_for,
+                trigger_source="cron",
+            )
+            if run is None:
+                counts["skipped"] += 1
+            else:
+                enqueue_args.append(str(run["id"]))
+                counts["enqueued"] += 1
+
+            # Advance: recurring picks next tick, once nulls next_run_at
+            # (worker terminal-flips status on completion).
+            if trigger_type == "recurring":
+                nxt = _compute_next(schedule, after=scheduled_for)
+                if nxt is None:
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"status": "completed", "next_run_at": None,
+                         "last_run_at": now},
+                    )
+                else:
+                    schedules_repo.update_internal(
+                        str(schedule["id"]),
+                        {"next_run_at": nxt, "last_run_at": now},
+                    )
+            else:
+                schedules_repo.update_internal(
+                    str(schedule["id"]),
+                    {"next_run_at": None, "last_run_at": now},
+                )
+            counts["advanced"] += 1
+
+    # Enqueue after commit so the worker sees the schedule_runs row on pick-up.
+    for run_id in enqueue_args:
+        try:
+            execute_scheduled_run.apply_async(args=[run_id], queue="docsgpt")
+        except Exception:
+            logger.exception(
+                "dispatcher: failed to enqueue execute_scheduled_run for %s",
+                run_id,
+            )
+    return counts

docsgpt/api/user/scheduler_worker.py

@@ -0,0 +1,450 @@
+"""Body of ``execute_scheduled_run`` — runs a single agent execution.
+
+Not a DURABLE_TASK: agent runs have side effects (messages, CRM writes)
+and blind auto-retry would double them. Failures after agent.gen starts
+are terminal and recorded; only the pre-start load is retry-safe.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+
+from celery.exceptions import SoftTimeLimitExceeded
+from sqlalchemy import text as sql_text
+
+from docsgpt.agents.headless_runner import run_agent_headless
+from docsgpt.core.settings import settings
+from docsgpt.events.publisher import publish_user_event
+from docsgpt.storage.db.base_repository import row_to_dict
+from docsgpt.storage.db.engine import get_engine
+from docsgpt.storage.db.repositories.conversations import (
+    ConversationsRepository,
+)
+from docsgpt.storage.db.repositories.schedule_runs import (
+    ScheduleRunsRepository,
+)
+from docsgpt.storage.db.repositories.schedules import SchedulesRepository
+from docsgpt.storage.db.repositories.token_usage import TokenUsageRepository
+
+logger = logging.getLogger(__name__)
+
+
+# Cap output verbatim in the run log; beyond the cap we keep the head and stamp output_truncated.
+_OUTPUT_CAP_CHARS = 24_000
+
+
+def _agent_config_for_schedule(schedule: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+    """Resolve the agent row (agent-bound) or build an ephemeral classic config.
+
+    For agentless schedules (``agent_id IS NULL``), the worker constructs an
+    in-memory agent shape carrying just enough fields for ``run_agent_headless``:
+    classic agent type, system-default retriever/chunks/prompt, no source, and
+    the optional ``model_id`` override. The runtime toolset is rebuilt by
+    ``ToolExecutor`` at fire time (current ``user_tools`` + non-disabled,
+    non-headless-excluded defaults), so a snapshot here would be dead code.
+    """
+    if schedule.get("agent_id"):
+        engine = get_engine()
+        with engine.connect() as conn:
+            row = conn.execute(
+                sql_text("SELECT * FROM agents WHERE id = CAST(:id AS uuid)"),
+                {"id": str(schedule["agent_id"])},
+            ).fetchone()
+        return row_to_dict(row) if row is not None else None
+    return _ephemeral_agent_for_agentless(schedule)
+
+
+def _ephemeral_agent_for_agentless(
+    schedule: Dict[str, Any],
+) -> Optional[Dict[str, Any]]:
+    """Build an agent-shaped config for a schedule with no parent agent."""
+    # ``agent_config["tools"]`` is intentionally omitted: ``run_agent_headless``
+    # never reads it. The runtime toolset is rebuilt by
+    # ``ToolExecutor._get_user_tools(owner)`` at fire time — same dereference
+    # the agent-bound path uses, so a tool added/disabled after creation is
+    # reflected. Headless mode there filters chat-only tools (``scheduler``).
+    user_id = schedule.get("user_id")
+    if not user_id:
+        return None
+    return {
+        "id": None,
+        "user_id": user_id,
+        "agent_type": "classic",
+        "retriever": "classic",
+        "chunks": 2,
+        "prompt_id": "default",
+        "source_id": None,
+        "default_model_id": schedule.get("model_id") or "",
+    }
+
+
+def _load_chat_history(schedule: Dict[str, Any]) -> list:
+    """Originating conversation history (one-time only; recurring has none)."""
+    origin = schedule.get("origin_conversation_id")
+    if not origin or schedule.get("trigger_type") != "once":
+        return []
+    user_id = schedule.get("user_id")
+    if not user_id:
+        return []
+    try:
+        engine = get_engine()
+        with engine.connect() as conn:
+            conv = ConversationsRepository(conn).get_any(str(origin), user_id)
+            if conv is None:
+                return []
+            messages = ConversationsRepository(conn).get_messages(str(conv["id"]))
+    except Exception:
+        logger.exception("scheduler: failed loading chat history")
+        return []
+    history: list = []
+    for msg in messages:
+        if msg.get("prompt") and msg.get("response"):
+            history.append({
+                "prompt": msg["prompt"],
+                "response": msg["response"],
+            })
+    return history
+
+
+def _publish_run_event(
+    event_type: str, run: Dict[str, Any], schedule: Dict[str, Any], **extra: Any
+) -> None:
+    """Best-effort SSE publish for a scheduler run state transition."""
+    user_id = run.get("user_id") or schedule.get("user_id")
+    if not user_id:
+        return
+    agent_id_raw = schedule.get("agent_id")
+    payload = {
+        "run_id": str(run["id"]),
+        "schedule_id": str(schedule["id"]),
+        "agent_id": str(agent_id_raw) if agent_id_raw else None,
+        "trigger_type": schedule.get("trigger_type"),
+        "status": run.get("status"),
+        **extra,
+    }
+    try:
+        publish_user_event(
+            user_id,
+            event_type,
+            payload,
+            scope={"kind": "schedule", "id": str(schedule["id"])},
+        )
+    except Exception:
+        logger.exception(
+            "scheduler: SSE publish failed event=%s run=%s",
+            event_type, run.get("id"),
+        )
+
+
+def _publish_message_appended(
+    user_id: str,
+    conversation_id: str,
+    message: Dict[str, Any],
+    schedule_id: str,
+    run_id: str,
+) -> None:
+    """SSE message-appended event for a one-time run's chat turn."""
+    try:
+        publish_user_event(
+            user_id,
+            "schedule.message.appended",
+            {
+                "conversation_id": str(conversation_id),
+                "message_id": str(message["id"]),
+                "schedule_id": str(schedule_id),
+                "run_id": str(run_id),
+                "position": int(message.get("position", 0)),
+            },
+            scope={"kind": "conversation", "id": str(conversation_id)},
+        )
+    except Exception:
+        logger.exception(
+            "scheduler: message.appended publish failed run=%s", run_id,
+        )
+
+
+def _append_one_time_turn(
+    schedule: Dict[str, Any],
+    run: Dict[str, Any],
+    outcome: Dict[str, Any],
+) -> Optional[Dict[str, Any]]:
+    """Insert an assistant turn in the originating conversation (once only)."""
+    origin = schedule.get("origin_conversation_id")
+    if not origin:
+        return None
+    engine = get_engine()
+    user_id = schedule.get("user_id")
+    metadata = {
+        "scheduled": True,
+        "schedule_id": str(schedule["id"]),
+        "run_id": str(run["id"]),
+        "scheduled_run_at": (
+            run.get("scheduled_for")
+            if isinstance(run.get("scheduled_for"), str)
+            else None
+        ),
+    }
+    with engine.begin() as conn:
+        conv = ConversationsRepository(conn).get_any(str(origin), user_id)
+        if conv is None:
+            return None
+        message = ConversationsRepository(conn).append_message(
+            str(conv["id"]),
+            {
+                "prompt": schedule.get("instruction") or "",
+                "response": outcome.get("answer") or "",
+                "thought": outcome.get("thought") or "",
+                "sources": outcome.get("sources") or [],
+                "tool_calls": outcome.get("tool_calls") or [],
+                "model_id": outcome.get("model_id"),
+                "metadata": metadata,
+            },
+        )
+    return message
+
+
+def execute_scheduled_run_body(run_id: str, celery_task_id: Optional[str]) -> Dict[str, Any]:
+    """Execute one scheduled run by id; returns a result dict for tracing."""
+    if not settings.POSTGRES_URI:
+        return {"status": "skipped", "reason": "POSTGRES_URI not set"}
+
+    engine = get_engine()
+
+    with engine.connect() as conn:
+        run = ScheduleRunsRepository(conn).get_internal(run_id)
+        if run is None:
+            return {"status": "skipped", "reason": "run not found"}
+        schedule = SchedulesRepository(conn).get_internal(str(run["schedule_id"]))
+    if schedule is None:
+        return {"status": "skipped", "reason": "schedule not found"}
+
+    # Refuse non-runnable terminal states; manual run-now bypasses.
+    if run.get("status") != "pending":
+        return {"status": "skipped", "reason": f"run status={run.get('status')}"}
+    if schedule.get("status") in {"cancelled", "completed"} and run.get(
+        "trigger_source"
+    ) != "manual":
+        with engine.begin() as conn:
+            ScheduleRunsRepository(conn).update(
+                run_id,
+                {
+                    "status": "skipped",
+                    "finished_at": datetime.now(timezone.utc),
+                    "error_type": "internal",
+                    "error": "schedule no longer active",
+                },
+            )
+        return {"status": "skipped", "reason": "schedule terminal"}
+
+    agent_config = _agent_config_for_schedule(schedule)
+    if agent_config is None:
+        with engine.begin() as conn:
+            updated = ScheduleRunsRepository(conn).update(
+                run_id,
+                {
+                    "status": "failed",
+                    "finished_at": datetime.now(timezone.utc),
+                    "error_type": "internal",
+                    "error": "agent missing",
+                },
+            )
+            SchedulesRepository(conn).bump_failure_count(str(schedule["id"]))
+        _publish_run_event("schedule.run.failed", updated or run, schedule,
+                           error="agent missing")
+        return {"status": "failed", "reason": "agent missing"}
+
+    with engine.begin() as conn:
+        if not ScheduleRunsRepository(conn).mark_running(run_id, celery_task_id):
+            return {"status": "skipped", "reason": "lost race to mark_running"}
+
+    started = datetime.now(timezone.utc)
+    instruction = schedule.get("instruction") or ""
+    allowlist = schedule.get("tool_allowlist") or []
+    chat_history = _load_chat_history(schedule)
+    outcome: Dict[str, Any]
+    error_type: Optional[str] = None
+    error_text: Optional[str] = None
+    timed_out = False
+    try:
+        outcome = run_agent_headless(
+            agent_config,
+            instruction,
+            tool_allowlist=allowlist,
+            model_id_override=schedule.get("model_id"),
+            endpoint="schedule",
+            conversation_id=schedule.get("origin_conversation_id"),
+            chat_history=chat_history,
+        )
+    except SoftTimeLimitExceeded:
+        timed_out = True
+        outcome = {"answer": "", "tool_calls": [], "sources": [], "thought": ""}
+        error_type = "timeout"
+        error_text = "run exceeded soft time limit"
+    except Exception as exc:
+        outcome = {"answer": "", "tool_calls": [], "sources": [], "thought": ""}
+        error_type = "agent_error"
+        error_text = str(exc)
+        logger.exception("scheduler: agent run failed run=%s", run_id)
+
+    finished = datetime.now(timezone.utc)
+
+    # Honour the outcome the runner reported: it classifies headless denials
+    # (``tool_not_allowed``) and mid-stream failures (``stream_error``) alike,
+    # and owns the message. Re-deriving the denial case here only duplicated
+    # the runner's rule while leaving a failed stream — which returns normally,
+    # with an empty answer — to fall through to "success".
+    if error_type is None and outcome.get("error_type"):
+        error_type = str(outcome["error_type"])
+        error_text = str(outcome.get("error") or "") or error_type
+
+    prompt_tokens = int(outcome.get("prompt_tokens", 0) or 0)
+    generated_tokens = int(outcome.get("generated_tokens", 0) or 0)
+    used_tokens = prompt_tokens + generated_tokens
+    if (
+        schedule.get("token_budget") is not None
+        and int(schedule["token_budget"]) > 0
+        and used_tokens > int(schedule["token_budget"])
+    ):
+        error_type = "budget_exceeded"
+        error_text = (
+            f"used {used_tokens} tokens exceeds budget "
+            f"{schedule['token_budget']}"
+        )
+
+    # Backstop for silent failures that raise nothing and emit no error event:
+    # a run that produced no answer, did no work and burned no completion
+    # tokens did not do its job, whatever the absence of an exception suggests.
+    # "Work" is deliberately broad — tool calls for a chat agent, completed
+    # nodes for a workflow — so a schedule whose whole purpose is a side effect
+    # (post a message, file a ticket) is never flagged for staying quiet.
+    if (
+        error_type is None
+        and not (outcome.get("answer") or "").strip()
+        and not (outcome.get("tool_calls") or [])
+        and not int(outcome.get("steps_completed") or 0)
+        and generated_tokens == 0
+    ):
+        error_type = "empty_output"
+        error_text = "run produced no answer, tool calls or completion tokens"
+
+    answer = outcome.get("answer") or ""
+    truncated = False
+    if len(answer) > _OUTPUT_CAP_CHARS:
+        answer = answer[:_OUTPUT_CAP_CHARS]
+        truncated = True
+
+    new_status = (
+        "timeout" if timed_out else ("failed" if error_type else "success")
+    )
+
+    with engine.begin() as conn:
+        update_fields: Dict[str, Any] = {
+            "status": new_status,
+            "started_at": started,
+            "finished_at": finished,
+            "output": answer or None,
+            "output_truncated": truncated,
+            "prompt_tokens": prompt_tokens,
+            "generated_tokens": generated_tokens,
+        }
+        if error_type:
+            update_fields["error_type"] = error_type
+            update_fields["error"] = error_text
+        updated_run = ScheduleRunsRepository(conn).update(run_id, update_fields)
+        if used_tokens > 0:
+            agent_id_raw = schedule.get("agent_id")
+            try:
+                TokenUsageRepository(conn).insert(
+                    user_id=schedule.get("user_id"),
+                    api_key=None,
+                    prompt_tokens=prompt_tokens,
+                    generated_tokens=generated_tokens,
+                    timestamp=finished,
+                    agent_id=str(agent_id_raw) if agent_id_raw else None,
+                    source="schedule",
+                    request_id=str(run_id),
+                    model_id=outcome.get("model_id"),
+                )
+            except Exception:
+                logger.exception(
+                    "scheduler: token_usage insert failed run=%s", run_id,
+                )
+        schedules_repo = SchedulesRepository(conn)
+        autopaused = False
+        if new_status == "success":
+            schedules_repo.reset_failure_count(str(schedule["id"]))
+        elif new_status in ("failed", "timeout"):
+            count = schedules_repo.bump_failure_count(str(schedule["id"]))
+            if (
+                settings.SCHEDULE_AUTOPAUSE_FAILURES > 0
+                and count >= settings.SCHEDULE_AUTOPAUSE_FAILURES
+                and schedule.get("trigger_type") == "recurring"
+            ):
+                autopaused = schedules_repo.autopause(str(schedule["id"]))
+        # Once: terminal-flip on cron-fired runs only; manual runs on a
+        # still-active once-schedule leave the future cadence intact.
+        if (
+            schedule.get("trigger_type") == "once"
+            and run.get("trigger_source") != "manual"
+            and schedule.get("status") == "active"
+        ):
+            schedules_repo.update_internal(
+                str(schedule["id"]),
+                {"status": "completed", "next_run_at": None},
+            )
+
+    appended: Optional[Dict[str, Any]] = None
+    if (
+        schedule.get("trigger_type") == "once"
+        and new_status == "success"
+        and schedule.get("origin_conversation_id")
+    ):
+        try:
+            appended = _append_one_time_turn(schedule, updated_run or run, outcome)
+        except Exception:
+            logger.exception(
+                "scheduler: append turn failed run=%s", run_id,
+            )
+        if appended is not None:
+            with engine.begin() as conn:
+                ScheduleRunsRepository(conn).update(
+                    run_id,
+                    {
+                        "conversation_id": str(appended["conversation_id"]),
+                        "message_id": str(appended["id"]),
+                    },
+                )
+            _publish_message_appended(
+                schedule.get("user_id"),
+                str(appended["conversation_id"]),
+                appended,
+                str(schedule["id"]),
+                run_id,
+            )
+
+    if new_status == "success":
+        _publish_run_event("schedule.run.completed", updated_run or run, schedule)
+    else:
+        _publish_run_event(
+            "schedule.run.failed",
+            updated_run or run,
+            schedule,
+            error_type=error_type,
+            error=error_text,
+        )
+
+    if autopaused:
+        _publish_run_event(
+            "schedule.autopaused",
+            updated_run or run,
+            schedule,
+            consecutive_failure_count=settings.SCHEDULE_AUTOPAUSE_FAILURES,
+        )
+
+    return {
+        "status": new_status,
+        "run_id": run_id,
+        "error_type": error_type,
+    }

docsgpt/api/user/schedules/__init__.py

@@ -0,0 +1,5 @@
+"""Schedules module."""
+
+from .routes import schedules_ns
+
+__all__ = ["schedules_ns"]

docsgpt/api/user/schedules/routes.py

@@ -0,0 +1,621 @@
+"""Schedules REST API (owner-scoped via request.decoded_token)."""
+
+from __future__ import annotations
+
+import functools
+import logging
+from datetime import datetime, timezone
+from typing import Any, Callable, Dict, Optional
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import Namespace, Resource, fields
+
+from docsgpt.agents.scheduler_utils import (
+    ScheduleValidationError,
+    clamp_once_horizon,
+    cron_interval_seconds,
+    next_cron_run,
+    parse_cron,
+    parse_run_at,
+    resolve_timezone,
+)
+from docsgpt.api import api
+from docsgpt.core.settings import settings
+from docsgpt.storage.db.base_repository import looks_like_uuid
+from docsgpt.storage.db.repositories.agents import AgentsRepository
+from docsgpt.storage.db.repositories.schedule_runs import (
+    ScheduleRunsRepository,
+)
+from docsgpt.storage.db.repositories.schedules import SchedulesRepository
+from docsgpt.storage.db.session import db_readonly, db_session
+
+
+logger = logging.getLogger(__name__)
+
+
+schedules_ns = Namespace(
+    "schedules", description="Agent schedule management", path="/api",
+)
+
+
+def _ok(data: Any, status: int = 200):
+    return make_response(jsonify(data), status)
+
+
+def _err(message: str, status: int = 400):
+    return make_response(jsonify({"success": False, "message": message}), status)
+
+
+def _safe_route(func: Callable) -> Callable:
+    """Decorator: log + mask exceptions that escape a route body as 500.
+
+    ``ScheduleValidationError`` messages are explicitly surfaced via
+    ``_err(str(exc))`` at each call site (deliberate, user-safe). This
+    decorator only fires for *unexpected* exceptions (DB driver errors,
+    NPEs, etc.) that escape the route body. The full trace is logged
+    server-side; the response body carries no internal detail.
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as exc:  # noqa: BLE001
+            try:
+                current_app.logger.exception(
+                    "unhandled exception in schedules route %s: %s",
+                    func.__qualname__, exc,
+                )
+            except RuntimeError:
+                # Out of Flask app context (rare in tests); use module logger.
+                logger.exception(
+                    "unhandled exception in schedules route %s: %s",
+                    func.__qualname__, exc,
+                )
+            return _err("internal error", 500)
+    return wrapper
+
+
+def _format_schedule(row: Dict[str, Any]) -> Dict[str, Any]:
+    """Render a schedule row for the API (id-as-string + ISO timestamps)."""
+    if not row:
+        return {}
+    out = dict(row)
+    for key in (
+        "id", "agent_id", "origin_conversation_id",
+    ):
+        if out.get(key) is not None:
+            out[key] = str(out[key])
+    out.pop("_id", None)  # drop dual-id legacy mirror
+    return out
+
+
+def _format_run(row: Dict[str, Any]) -> Dict[str, Any]:
+    """Render a schedule_run row for the API."""
+    if not row:
+        return {}
+    out = dict(row)
+    for key in (
+        "id", "schedule_id", "agent_id", "conversation_id", "message_id",
+    ):
+        if out.get(key) is not None:
+            out[key] = str(out[key])
+    out.pop("_id", None)
+    return out
+
+
+def _agent_owned(agent_id: str, user_id: str) -> Optional[Dict[str, Any]]:
+    if not looks_like_uuid(str(agent_id)):
+        return None
+    with db_readonly() as conn:
+        return AgentsRepository(conn).get_any(agent_id, user_id)
+
+
+def _user_id() -> Optional[str]:
+    decoded = getattr(request, "decoded_token", None)
+    if not decoded:
+        return None
+    return decoded.get("sub")
+
+
+def _publish_schedule_event(
+    user_id: str, event_type: str, schedule_id: str, *, status: str,
+) -> None:
+    """Best-effort SSE so other clients revoke a stale schedule state.
+
+    A resume/cancel must override an earlier ``schedule.autopaused``; that
+    envelope is durable and replays from the backlog on reconnect, so
+    without a newer event the schedule reads 'paused' even though it will
+    fire on the next tick.
+    """
+    try:
+        from docsgpt.events.publisher import publish_user_event
+
+        publish_user_event(
+            user_id,
+            event_type,
+            {"schedule_id": str(schedule_id), "status": status},
+            scope={"kind": "schedule", "id": str(schedule_id)},
+        )
+    except Exception:
+        logger.exception(
+            "schedules: publish %s failed for %s", event_type, schedule_id,
+        )
+
+
+@schedules_ns.route("/agents/<string:agent_id>/schedules")
+class AgentSchedules(Resource):
+    @api.doc(description="List schedules for an agent (recurring + one-time).")
+    @_safe_route
+    def get(self, agent_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        agent = _agent_owned(agent_id, user_id)
+        if agent is None:
+            return _err("agent not found", 404)
+        try:
+            with db_readonly() as conn:
+                rows = SchedulesRepository(conn).list_for_agent(
+                    str(agent["id"]), user_id,
+                )
+        except Exception as exc:
+            current_app.logger.error("list schedules failed: %s", exc, exc_info=True)
+            return _err("internal error", 500)
+        return _ok({"schedules": [_format_schedule(r) for r in rows]})
+
+    create_model = api.model(
+        "ScheduleCreate",
+        {
+            "instruction": fields.String(required=True),
+            "trigger_type": fields.String(
+                required=False,
+                description="'recurring' (default) or 'once'",
+            ),
+            "cron": fields.String(
+                required=False,
+                description="Required when trigger_type == 'recurring'",
+            ),
+            "run_at": fields.String(
+                required=False,
+                description="ISO 8601 — required when trigger_type == 'once'",
+            ),
+            "timezone": fields.String(required=False),
+            "name": fields.String(required=False),
+            "end_at": fields.String(required=False, description="ISO 8601"),
+            "tool_allowlist": fields.List(fields.String, required=False),
+            "model_id": fields.String(required=False),
+            "token_budget": fields.Integer(required=False),
+        },
+    )
+
+    @api.expect(create_model)
+    @api.doc(description="Create a schedule (recurring or one-time) for an agent.")
+    @_safe_route
+    def post(self, agent_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        agent = _agent_owned(agent_id, user_id)
+        if agent is None:
+            return _err("agent not found", 404)
+        data = request.get_json(silent=True) or {}
+        instruction = (data.get("instruction") or "").strip()
+        tz_name = (data.get("timezone") or "UTC").strip() or "UTC"
+        trigger_type = (data.get("trigger_type") or "recurring").strip().lower()
+        if trigger_type not in ("recurring", "once"):
+            return _err("trigger_type must be 'recurring' or 'once'")
+        if not instruction:
+            return _err("instruction is required")
+        try:
+            resolve_timezone(tz_name)
+        except ScheduleValidationError as exc:
+            return _err(str(exc))
+        token_budget = data.get("token_budget")
+        if token_budget is not None:
+            try:
+                token_budget = int(token_budget)
+                if token_budget < 0:
+                    raise ValueError
+            except (TypeError, ValueError):
+                return _err("token_budget must be a non-negative integer")
+        with db_readonly() as conn:
+            count = SchedulesRepository(conn).count_active_for_user(user_id)
+        if (
+            settings.SCHEDULE_MAX_PER_USER > 0
+            and count >= settings.SCHEDULE_MAX_PER_USER
+        ):
+            return _err("max schedules per user reached", 429)
+
+        if trigger_type == "once":
+            run_at_raw = (data.get("run_at") or "").strip()
+            if not run_at_raw:
+                return _err("run_at is required for trigger_type 'once'")
+            try:
+                fire = parse_run_at(run_at_raw, tz_name)
+                clamp_once_horizon(
+                    fire, settings.SCHEDULE_ONCE_MAX_HORIZON,
+                )
+            except ScheduleValidationError as exc:
+                return _err(str(exc))
+            try:
+                with db_session() as conn:
+                    created = SchedulesRepository(conn).create(
+                        user_id=user_id,
+                        agent_id=str(agent["id"]),
+                        trigger_type="once",
+                        instruction=instruction,
+                        run_at=fire,
+                        next_run_at=fire,
+                        timezone=tz_name,
+                        name=(data.get("name") or "").strip() or None,
+                        tool_allowlist=data.get("tool_allowlist") or [],
+                        model_id=(data.get("model_id") or None),
+                        token_budget=token_budget,
+                        created_via="ui",
+                    )
+            except Exception as exc:
+                current_app.logger.error(
+                    "create one-time schedule failed: %s", exc, exc_info=True,
+                )
+                return _err("internal error", 500)
+            return _ok({"schedule": _format_schedule(created)}, status=201)
+
+        cron = (data.get("cron") or "").strip()
+        if not cron:
+            return _err("cron is required")
+        try:
+            parse_cron(cron)
+        except ScheduleValidationError as exc:
+            return _err(str(exc))
+        min_interval = max(0, int(settings.SCHEDULE_MIN_INTERVAL))
+        if min_interval > 0:
+            try:
+                cadence = cron_interval_seconds(cron, tz_name)
+            except ScheduleValidationError as exc:
+                return _err(str(exc))
+            if cadence < min_interval:
+                return _err(
+                    "cadence below minimum interval "
+                    f"({cadence}s < {min_interval}s)",
+                )
+        end_at = None
+        if data.get("end_at"):
+            try:
+                end_at = datetime.fromisoformat(
+                    str(data["end_at"]).replace("Z", "+00:00"),
+                )
+            except ValueError:
+                return _err("invalid end_at")
+        try:
+            next_run = next_cron_run(cron, tz_name, after=datetime.now(timezone.utc))
+        except ScheduleValidationError as exc:
+            return _err(str(exc))
+        if end_at is not None and next_run > end_at:
+            return _err("end_at is before the first cron tick")
+        try:
+            with db_session() as conn:
+                created = SchedulesRepository(conn).create(
+                    user_id=user_id,
+                    agent_id=str(agent["id"]),
+                    trigger_type="recurring",
+                    instruction=instruction,
+                    cron=cron,
+                    timezone=tz_name,
+                    next_run_at=next_run,
+                    end_at=end_at,
+                    name=(data.get("name") or "").strip() or None,
+                    tool_allowlist=data.get("tool_allowlist") or [],
+                    model_id=(data.get("model_id") or None),
+                    token_budget=token_budget,
+                    created_via="ui",
+                )
+        except Exception as exc:
+            current_app.logger.error(
+                "create schedule failed: %s", exc, exc_info=True,
+            )
+            return _err("internal error", 500)
+        return _ok({"schedule": _format_schedule(created)}, status=201)
+
+
+@schedules_ns.route("/schedules/<string:schedule_id>")
+class ScheduleResource(Resource):
+    @api.doc(description="Get schedule by id.")
+    @_safe_route
+    def get(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        with db_readonly() as conn:
+            row = SchedulesRepository(conn).get(schedule_id, user_id)
+        if row is None:
+            return _err("schedule not found", 404)
+        return _ok({"schedule": _format_schedule(row)})
+
+    @api.doc(description="Edit a schedule's editable fields.")
+    @_safe_route
+    def put(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        data = request.get_json(silent=True) or {}
+        fields_in: Dict[str, Any] = {}
+        if "instruction" in data:
+            inst = (data["instruction"] or "").strip()
+            if not inst:
+                return _err("instruction must not be empty")
+            fields_in["instruction"] = inst
+        if "cron" in data:
+            cron = (data["cron"] or "").strip()
+            try:
+                parse_cron(cron)
+            except ScheduleValidationError as exc:
+                return _err(str(exc))
+            fields_in["cron"] = cron
+        if "timezone" in data:
+            tz_name = (data["timezone"] or "UTC").strip() or "UTC"
+            try:
+                resolve_timezone(tz_name)
+            except ScheduleValidationError as exc:
+                return _err(str(exc))
+            fields_in["timezone"] = tz_name
+        if "tool_allowlist" in data:
+            fields_in["tool_allowlist"] = data["tool_allowlist"] or []
+        if "name" in data:
+            fields_in["name"] = (data["name"] or "").strip() or None
+        if "model_id" in data:
+            fields_in["model_id"] = (data["model_id"] or None)
+        if "token_budget" in data:
+            tb = data["token_budget"]
+            if tb is not None:
+                try:
+                    tb = int(tb)
+                    if tb < 0:
+                        raise ValueError
+                except (TypeError, ValueError):
+                    return _err("token_budget must be a non-negative integer")
+            fields_in["token_budget"] = tb
+        if "end_at" in data:
+            if data["end_at"]:
+                try:
+                    fields_in["end_at"] = datetime.fromisoformat(
+                        str(data["end_at"]).replace("Z", "+00:00"),
+                    )
+                except ValueError:
+                    return _err("invalid end_at")
+            else:
+                fields_in["end_at"] = None
+        # Recompute next_run_at when cron/tz changes.
+        with db_session() as conn:
+            existing = SchedulesRepository(conn).get(schedule_id, user_id)
+            if existing is None:
+                return _err("schedule not found", 404)
+            if (
+                ("cron" in fields_in or "timezone" in fields_in)
+                and existing.get("trigger_type") == "recurring"
+            ):
+                cron_eff = fields_in.get("cron") or existing.get("cron")
+                tz_eff = fields_in.get("timezone") or existing.get("timezone")
+                if cron_eff:
+                    min_interval = max(0, int(settings.SCHEDULE_MIN_INTERVAL))
+                    if min_interval > 0:
+                        try:
+                            cadence = cron_interval_seconds(cron_eff, tz_eff)
+                        except ScheduleValidationError as exc:
+                            return _err(str(exc))
+                        if cadence < min_interval:
+                            return _err(
+                                "cadence below minimum interval "
+                                f"({cadence}s < {min_interval}s)",
+                            )
+                    try:
+                        fields_in["next_run_at"] = next_cron_run(
+                            cron_eff, tz_eff, after=datetime.now(timezone.utc),
+                        )
+                    except ScheduleValidationError as exc:
+                        return _err(str(exc))
+            updated = SchedulesRepository(conn).update(
+                schedule_id, user_id, fields_in,
+            )
+        return _ok({"schedule": _format_schedule(updated or {})})
+
+    @api.doc(description="Pause / resume a schedule.")
+    @_safe_route
+    def patch(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        data = request.get_json(silent=True) or {}
+        action = (data.get("action") or "").lower().strip()
+        if action not in {"pause", "resume"}:
+            return _err("action must be 'pause' or 'resume'")
+        with db_session() as conn:
+            existing = SchedulesRepository(conn).get(schedule_id, user_id)
+            if existing is None:
+                return _err("schedule not found", 404)
+            if existing.get("status") in ("cancelled", "completed"):
+                return _err("schedule is terminal", 409)
+            if action == "pause":
+                fields_in: Dict[str, Any] = {"status": "paused", "next_run_at": None}
+            else:
+                # Resume: recurring recomputes from now; once honours run_at if still future.
+                fields_in = {"status": "active"}
+                if existing.get("trigger_type") == "recurring":
+                    try:
+                        fields_in["next_run_at"] = next_cron_run(
+                            existing["cron"],
+                            existing["timezone"],
+                            after=datetime.now(timezone.utc),
+                        )
+                    except ScheduleValidationError as exc:
+                        return _err(str(exc))
+                else:
+                    new_run_at = data.get("run_at")
+                    if new_run_at:
+                        try:
+                            run_at_dt = datetime.fromisoformat(
+                                str(new_run_at).replace("Z", "+00:00"),
+                            )
+                        except ValueError:
+                            return _err("invalid run_at")
+                        if run_at_dt <= datetime.now(timezone.utc):
+                            return _err(
+                                "run_at must be in the future to resume", 409,
+                            )
+                        fields_in["next_run_at"] = run_at_dt
+                        fields_in["run_at"] = run_at_dt
+                    else:
+                        run_at = existing.get("run_at")
+                        if run_at:
+                            if isinstance(run_at, str):
+                                try:
+                                    run_at_dt = datetime.fromisoformat(
+                                        run_at.replace("Z", "+00:00"),
+                                    )
+                                except ValueError:
+                                    return _err("schedule run_at is invalid")
+                            else:
+                                run_at_dt = run_at
+                            if run_at_dt <= datetime.now(timezone.utc):
+                                return _err(
+                                    "the once schedule has elapsed; recreate "
+                                    "it or supply a new run_at",
+                                    409,
+                                )
+                            fields_in["next_run_at"] = run_at_dt
+            updated = SchedulesRepository(conn).update(
+                schedule_id, user_id, fields_in,
+            )
+            if action == "resume":
+                SchedulesRepository(conn).reset_failure_count(schedule_id)
+        if action == "resume" and updated:
+            _publish_schedule_event(
+                user_id, "schedule.resumed", schedule_id, status="active",
+            )
+        return _ok({"schedule": _format_schedule(updated or {})})
+
+    @api.doc(description="Cancel / delete a schedule.")
+    @_safe_route
+    def delete(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        with db_session() as conn:
+            ok = SchedulesRepository(conn).delete(schedule_id, user_id)
+        if not ok:
+            return _err("schedule not found", 404)
+        _publish_schedule_event(
+            user_id, "schedule.cancelled", schedule_id, status="cancelled",
+        )
+        return _ok({"success": True})
+
+
+@schedules_ns.route("/schedules/<string:schedule_id>/run")
+class ScheduleRunNow(Resource):
+    @api.doc(description="Run a schedule immediately (trigger_source='manual').")
+    @_safe_route
+    def post(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        # FOR UPDATE serializes concurrent Run-Now POSTs (timestamp-unique
+        # scheduled_for values would otherwise sneak past the unique index).
+        with db_session() as conn:
+            schedule = SchedulesRepository(conn).get_for_update(
+                schedule_id, user_id,
+            )
+            if schedule is None:
+                return _err("schedule not found", 404)
+            if schedule.get("status") == "cancelled":
+                return _err("schedule is cancelled", 409)
+            if ScheduleRunsRepository(conn).has_active_run(schedule_id):
+                return _err("a run is already in flight", 409)
+            scheduled_for = datetime.now(timezone.utc)
+            agent_id_raw = schedule.get("agent_id")
+            run = ScheduleRunsRepository(conn).record_pending(
+                schedule_id,
+                user_id,
+                str(agent_id_raw) if agent_id_raw else None,
+                scheduled_for,
+                trigger_source="manual",
+            )
+        if run is None:
+            return _err("could not claim run (concurrent dispatch)", 409)
+        # Import inside the handler to avoid a circular tasks <-> routes import.
+        try:
+            from docsgpt.api.user.tasks import execute_scheduled_run
+            execute_scheduled_run.apply_async(args=[str(run["id"])], queue="docsgpt")
+        except Exception as exc:
+            current_app.logger.error(
+                "run-now enqueue failed: %s", exc, exc_info=True,
+            )
+            return _err("enqueue failed", 500)
+        return _ok({"run": _format_run(run)}, status=202)
+
+
+@schedules_ns.route("/schedules/<string:schedule_id>/runs")
+class ScheduleRunList(Resource):
+    @api.doc(
+        description="Paginated run log for a schedule.",
+        params={"limit": "Page size (default 50)", "offset": "Page offset"},
+    )
+    @_safe_route
+    def get(self, schedule_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id):
+            return _err("invalid schedule id", 400)
+        try:
+            limit = max(1, min(int(request.args.get("limit", 50)), 200))
+        except (TypeError, ValueError):
+            limit = 50
+        try:
+            offset = max(0, int(request.args.get("offset", 0)))
+        except (TypeError, ValueError):
+            offset = 0
+        with db_readonly() as conn:
+            schedule = SchedulesRepository(conn).get(schedule_id, user_id)
+            if schedule is None:
+                return _err("schedule not found", 404)
+            rows = ScheduleRunsRepository(conn).list_runs(
+                schedule_id, user_id, limit=limit, offset=offset,
+            )
+        return _ok(
+            {
+                "runs": [_format_run(r) for r in rows],
+                "limit": limit,
+                "offset": offset,
+            }
+        )
+
+
+@schedules_ns.route("/schedules/<string:schedule_id>/runs/<string:run_id>")
+class ScheduleRunDetail(Resource):
+    @api.doc(description="Full output / error for a single run.")
+    @_safe_route
+    def get(self, schedule_id, run_id):
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        if not looks_like_uuid(schedule_id) or not looks_like_uuid(run_id):
+            return _err("invalid id", 400)
+        with db_readonly() as conn:
+            schedule = SchedulesRepository(conn).get(schedule_id, user_id)
+            if schedule is None:
+                return _err("schedule not found", 404)
+            run = ScheduleRunsRepository(conn).get(run_id, user_id)
+            if run is None or str(run.get("schedule_id")) != str(
+                schedule["id"]
+            ):
+                return _err("run not found", 404)
+        return _ok({"run": _format_run(run)})

docsgpt/api/user/sharing/__init__.py

@@ -0,0 +1,5 @@
+"""Sharing module."""
+
+from .routes import sharing_ns
+
+__all__ = ["sharing_ns"]

docsgpt/api/user/sharing/routes.py

@@ -0,0 +1,393 @@
+"""Conversation sharing routes."""
+
+import logging
+import uuid
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, inputs, Namespace, Resource
+from sqlalchemy import text as _sql_text
+
+from docsgpt.api import api
+from docsgpt.storage.db.base_repository import looks_like_uuid
+from docsgpt.storage.db.repositories.agents import AgentsRepository
+from docsgpt.storage.db.repositories.attachments import AttachmentsRepository
+from docsgpt.storage.db.repositories.conversations import ConversationsRepository
+from docsgpt.storage.db.repositories.shared_conversations import (
+    SharedConversationsRepository,
+)
+from docsgpt.storage.db.session import db_readonly, db_session
+from docsgpt.utils import check_required_fields
+
+logger = logging.getLogger(__name__)
+
+
+sharing_ns = Namespace(
+    "sharing", description="Conversation sharing operations", path="/api"
+)
+
+
+def _resolve_prompt_pg_id(conn, prompt_id_raw, user_id):
+    """Translate an incoming prompt id (UUID or legacy Mongo ObjectId) to a PG UUID.
+
+    Scoped by ``user_id`` so a caller can't link another user's prompt
+    into their share record. Returns ``None`` for sentinel values
+    (``"default"``) or unresolved ids.
+    """
+    if not prompt_id_raw or prompt_id_raw == "default":
+        return None
+    value = str(prompt_id_raw)
+    # Already UUID — trust it but still require ownership. A shape-gate
+    # (rather than a loose ``len == 36 and '-' in value`` check) keeps
+    # non-UUID input out of ``CAST(:pid AS uuid)``; the cast would raise
+    # and poison the readonly transaction otherwise.
+    if looks_like_uuid(value):
+        row = conn.execute(
+            _sql_text(
+                "SELECT id FROM prompts WHERE id = CAST(:pid AS uuid) "
+                "AND user_id = :uid"
+            ),
+            {"pid": value, "uid": user_id},
+        ).fetchone()
+        return str(row[0]) if row else None
+    # Legacy Mongo ObjectId fallback.
+    row = conn.execute(
+        _sql_text(
+            "SELECT id FROM prompts WHERE legacy_mongo_id = :pid "
+            "AND user_id = :uid"
+        ),
+        {"pid": value, "uid": user_id},
+    ).fetchone()
+    return str(row[0]) if row else None
+
+
+def _resolve_source_pg_id(conn, source_raw, user=None):
+    """Translate a source id (UUID or legacy Mongo ObjectId) to a PG UUID.
+
+    ``source_raw`` is client-supplied and was resolved with no ownership
+    predicate, so a caller could bake any tenant's source id into the agent
+    this share creates — and ``/api/search`` then searched it. ``user`` is
+    required to authorize; without it nothing resolves.
+    """
+    if not source_raw:
+        return None
+    if not user:
+        logger.warning("Refusing to resolve a share source with no principal.")
+        return None
+    value = str(source_raw)
+    # See ``_resolve_prompt_pg_id`` for the shape-gate rationale.
+    if looks_like_uuid(value):
+        row = conn.execute(
+            _sql_text(
+                "SELECT id FROM sources WHERE id = CAST(:sid AS uuid)"
+            ),
+            {"sid": value},
+        ).fetchone()
+        return _authorized_source(conn, row, user)
+    row = conn.execute(
+        _sql_text("SELECT id FROM sources WHERE legacy_mongo_id = :sid"),
+        {"sid": value},
+    ).fetchone()
+    return _authorized_source(conn, row, user)
+
+
+def _authorized_source(conn, row, user):
+    """Return the resolved source id only if ``user`` may reference it."""
+    if not row:
+        return None
+    resolved = str(row[0])
+    from docsgpt.api.user.team_sharing import can_access
+
+    if not can_access(conn, "source", resolved, user):
+        logger.warning(
+            "Refusing to attach source %s to a share: %s has no access.",
+            resolved, user,
+        )
+        return None
+    return resolved
+
+
+def _find_reusable_share_agent(
+    conn, user_id, *, prompt_pg_id, chunks, source_pg_id, retriever,
+):
+    """Find an existing share-as-agent key row matching these parameters.
+
+    Mirrors the legacy Mongo ``agents_collection.find_one`` pre-existence
+    check. Used to reuse an api key across repeated shares of the same
+    conversation with the same prompt/chunks/source/retriever.
+    """
+    clauses = ["user_id = :uid", "key IS NOT NULL"]
+    params: dict = {"uid": user_id}
+    if prompt_pg_id is None:
+        clauses.append("prompt_id IS NULL")
+    else:
+        clauses.append("prompt_id = CAST(:pid AS uuid)")
+        params["pid"] = prompt_pg_id
+    if chunks is None:
+        clauses.append("chunks IS NULL")
+    else:
+        clauses.append("chunks = :chunks")
+        params["chunks"] = int(chunks)
+    if source_pg_id is None:
+        clauses.append("source_id IS NULL")
+    else:
+        clauses.append("source_id = CAST(:sid AS uuid)")
+        params["sid"] = source_pg_id
+    if retriever is None:
+        clauses.append("retriever IS NULL")
+    else:
+        clauses.append("retriever = :retr")
+        params["retr"] = retriever
+    sql = (
+        "SELECT * FROM agents WHERE "
+        + " AND ".join(clauses)
+        + " LIMIT 1"
+    )
+    row = conn.execute(_sql_text(sql), params).fetchone()
+    if row is None:
+        return None
+    mapping = dict(row._mapping)
+    mapping["id"] = str(mapping["id"]) if mapping.get("id") else None
+    return mapping
+
+
+@sharing_ns.route("/share")
+class ShareConversation(Resource):
+    share_conversation_model = api.model(
+        "ShareConversationModel",
+        {
+            "conversation_id": fields.String(
+                required=True, description="Conversation ID"
+            ),
+            "user": fields.String(description="User ID (optional)"),
+            "prompt_id": fields.String(description="Prompt ID (optional)"),
+            "chunks": fields.Integer(description="Chunks count (optional)"),
+        },
+    )
+
+    @api.expect(share_conversation_model)
+    @api.doc(description="Share a conversation")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["conversation_id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        is_promptable = request.args.get("isPromptable", type=inputs.boolean)
+        if is_promptable is None:
+            return make_response(
+                jsonify({"success": False, "message": "isPromptable is required"}), 400
+            )
+        conversation_id = data["conversation_id"]
+
+        try:
+            with db_session() as conn:
+                conv_repo = ConversationsRepository(conn)
+                shared_repo = SharedConversationsRepository(conn)
+                agents_repo = AgentsRepository(conn)
+
+                conversation = conv_repo.get_any(conversation_id, user)
+                if conversation is None:
+                    return make_response(
+                        jsonify(
+                            {
+                                "status": "error",
+                                "message": "Conversation does not exist",
+                            }
+                        ),
+                        404,
+                    )
+                conv_pg_id = str(conversation["id"])
+                current_n_queries = conv_repo.message_count(conv_pg_id)
+
+                if is_promptable:
+                    prompt_id_raw = data.get("prompt_id", "default")
+                    chunks_raw = data.get("chunks", "2")
+                    try:
+                        chunks_int = int(chunks_raw) if chunks_raw not in (None, "") else None
+                    except (TypeError, ValueError):
+                        chunks_int = None
+
+                    prompt_pg_id = _resolve_prompt_pg_id(conn, prompt_id_raw, user)
+                    source_pg_id = _resolve_source_pg_id(
+                        conn, data.get("source"), user
+                    )
+                    retriever = data.get("retriever")
+
+                    reusable = _find_reusable_share_agent(
+                        conn, user,
+                        prompt_pg_id=prompt_pg_id,
+                        chunks=chunks_int,
+                        source_pg_id=source_pg_id,
+                        retriever=retriever,
+                    )
+                    if reusable:
+                        api_uuid = reusable.get("key")
+                    else:
+                        api_uuid = str(uuid.uuid4())
+                        name = (conversation.get("name") or "") + "(shared)"
+                        agents_repo.create(
+                            user,
+                            name,
+                            "published",
+                            key=api_uuid,
+                            retriever=retriever,
+                            chunks=chunks_int,
+                            prompt_id=prompt_pg_id,
+                            source_id=source_pg_id,
+                        )
+
+                    share = shared_repo.get_or_create(
+                        conv_pg_id,
+                        user,
+                        is_promptable=True,
+                        first_n_queries=current_n_queries,
+                        api_key=api_uuid,
+                        prompt_id=prompt_pg_id,
+                        chunks=chunks_int,
+                    )
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": True,
+                                "identifier": str(share["uuid"]),
+                            }
+                        ),
+                        201 if reusable is None else 200,
+                    )
+
+                # Non-promptable share path.
+                share = shared_repo.get_or_create(
+                    conv_pg_id,
+                    user,
+                    is_promptable=False,
+                    first_n_queries=current_n_queries,
+                    api_key=None,
+                )
+                return make_response(
+                    jsonify(
+                        {
+                            "success": True,
+                            "identifier": str(share["uuid"]),
+                        }
+                    ),
+                    201,
+                )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error sharing conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)
+
+
+@sharing_ns.route("/shared_conversation/<string:identifier>")
+class GetPubliclySharedConversations(Resource):
+    @api.doc(description="Get publicly shared conversations by identifier")
+    def get(self, identifier: str):
+        try:
+            with db_readonly() as conn:
+                shared_repo = SharedConversationsRepository(conn)
+                conv_repo = ConversationsRepository(conn)
+                attach_repo = AttachmentsRepository(conn)
+
+                shared = shared_repo.find_by_uuid(identifier)
+                if not shared or not shared.get("conversation_id"):
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "error": "might have broken url or the conversation does not exist",
+                            }
+                        ),
+                        404,
+                    )
+                conv_pg_id = str(shared["conversation_id"])
+                owner_user = shared.get("user_id")
+
+                conversation = conv_repo.get_owned(conv_pg_id, owner_user) if owner_user else None
+                if conversation is None:
+                    # Fall back to any-user lookup in case shared row's
+                    # user_id is missing — still keyed by PG UUID.
+                    row = conn.execute(
+                        _sql_text(
+                            "SELECT * FROM conversations WHERE id = CAST(:id AS uuid)"
+                        ),
+                        {"id": conv_pg_id},
+                    ).fetchone()
+                    if row is None:
+                        return make_response(
+                            jsonify(
+                                {
+                                    "success": False,
+                                    "error": "might have broken url or the conversation does not exist",
+                                }
+                            ),
+                            404,
+                        )
+                    conversation = dict(row._mapping)
+
+                messages = conv_repo.get_messages(conv_pg_id)
+                first_n = shared.get("first_n_queries") or 0
+                conversation_queries = []
+                for msg in messages[:first_n]:
+                    query = {
+                        "prompt": msg.get("prompt"),
+                        "response": msg.get("response"),
+                        "thought": msg.get("thought"),
+                        "sources": msg.get("sources") or [],
+                        "tool_calls": msg.get("tool_calls") or [],
+                        "timestamp": (
+                            msg["timestamp"].isoformat()
+                            if hasattr(msg.get("timestamp"), "isoformat")
+                            else msg.get("timestamp")
+                        ),
+                        "feedback": msg.get("feedback"),
+                    }
+                    attachments = msg.get("attachments") or []
+                    if attachments:
+                        attachment_details = []
+                        for attachment_id in attachments:
+                            try:
+                                attachment = attach_repo.get_any(
+                                    str(attachment_id), owner_user,
+                                ) if owner_user else None
+                                if attachment:
+                                    attachment_details.append(
+                                        {
+                                            "id": str(attachment["id"]),
+                                            "fileName": attachment.get(
+                                                "filename", "Unknown file"
+                                            ),
+                                        }
+                                    )
+                            except Exception as e:
+                                current_app.logger.error(
+                                    f"Error retrieving attachment {attachment_id}: {e}",
+                                    exc_info=True,
+                                )
+                        query["attachments"] = attachment_details
+                    conversation_queries.append(query)
+
+                created = conversation.get("created_at") or conversation.get("date")
+                date_iso = (
+                    created.isoformat()
+                    if hasattr(created, "isoformat")
+                    else (str(created) if created is not None else None)
+                )
+                res = {
+                    "success": True,
+                    "queries": conversation_queries,
+                    "title": conversation.get("name"),
+                    "timestamp": date_iso,
+                }
+                if shared.get("is_promptable") and shared.get("api_key"):
+                    res["api_key"] = shared["api_key"]
+                return make_response(jsonify(res), 200)
+        except Exception as err:
+            current_app.logger.error(
+                f"Error getting shared conversation: {err}", exc_info=True
+            )
+            return make_response(jsonify({"success": False}), 400)

docsgpt/api/user/sources/__init__.py

@@ -0,0 +1,13 @@
+"""Sources module."""
+
+from .chunks import sources_chunks_ns
+from .retrieval_test import sources_search_ns
+from .routes import sources_ns
+from .upload import sources_upload_ns
+
+__all__ = [
+    "sources_ns",
+    "sources_chunks_ns",
+    "sources_search_ns",
+    "sources_upload_ns",
+]

docsgpt/api/user/sources/chunks.py

@@ -0,0 +1,311 @@
+"""Source document management chunk management."""
+
+from flask import current_app, jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+
+from docsgpt.api import api
+from docsgpt.api.user.base import get_vector_store
+from docsgpt.api.user.team_sharing import effective_write_owner
+from docsgpt.storage.db.repositories.sources import SourcesRepository
+from docsgpt.storage.db.session import db_readonly
+from docsgpt.utils import check_required_fields, num_tokens_from_string
+
+sources_chunks_ns = Namespace(
+    "sources", description="Source document management operations", path="/api"
+)
+
+
+def _resolve_source(doc_id: str, user: str):
+    """Resolve a source (UUID or legacy ObjectId) for the caller.
+
+    Returns the row dict (with PG UUID in ``id``) or ``None`` if missing.
+    """
+    with db_readonly() as conn:
+        return SourcesRepository(conn).get_any(doc_id, user)
+
+
+def _resolve_source_for_write(doc_id: str, user: str):
+    """Resolve a source the caller may WRITE chunks on.
+
+    Returns the row dict when ``user`` owns the source, or when they hold a
+    team ``editor`` grant (adding/removing/editing documents is editor-allowed
+    — the vector partition is keyed by source_id, owner-agnostic). Returns
+    ``None`` for viewer-only / no access. Source deletion stays owner-only and
+    is handled elsewhere.
+    """
+    with db_readonly() as conn:
+        doc = SourcesRepository(conn).get_any(doc_id, user)
+        if doc is not None:
+            return doc
+        owner = effective_write_owner(conn, "source", doc_id, user)
+        if not owner:
+            return None
+        return SourcesRepository(conn).get_any(doc_id, owner)
+
+
+@sources_chunks_ns.route("/get_chunks")
+class GetChunks(Resource):
+    @api.doc(
+        description="Retrieves chunks from a document, optionally filtered by file path and search term",
+        params={
+            "id": "The document ID",
+            "page": "Page number for pagination",
+            "per_page": "Number of chunks per page",
+            "path": "Optional: Filter chunks by relative file path",
+            "search": "Optional: Search term to filter chunks by title or content",
+        },
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        doc_id = request.args.get("id")
+        page = int(request.args.get("page", 1))
+        per_page = int(request.args.get("per_page", 10))
+        path = request.args.get("path")
+        search_term = request.args.get("search", "").strip().lower()
+
+        if not doc_id:
+            return make_response(jsonify({"error": "Invalid doc_id"}), 400)
+        try:
+            doc = _resolve_source(doc_id, user)
+        except Exception as e:
+            current_app.logger.error(f"Error resolving source: {e}", exc_info=True)
+            return make_response(jsonify({"error": "Invalid doc_id"}), 400)
+        if not doc:
+            return make_response(
+                jsonify({"error": "Document not found or access denied"}), 404
+            )
+        resolved_id = str(doc["id"])
+        try:
+            store = get_vector_store(resolved_id)
+            chunks = store.get_chunks()
+
+            filtered_chunks = []
+            for chunk in chunks:
+                metadata = chunk.get("metadata", {})
+
+                if path:
+                    chunk_source = metadata.get("source", "")
+                    chunk_file_path = metadata.get("file_path", "")
+                    source_match = chunk_source and chunk_source.endswith(path)
+                    file_path_match = chunk_file_path and chunk_file_path.endswith(path)
+
+                    if not (source_match or file_path_match):
+                        continue
+                if search_term:
+                    text_match = search_term in chunk.get("text", "").lower()
+                    title_match = search_term in metadata.get("title", "").lower()
+
+                    if not (text_match or title_match):
+                        continue
+                filtered_chunks.append(chunk)
+            chunks = filtered_chunks
+
+            total_chunks = len(chunks)
+            start = (page - 1) * per_page
+            end = start + per_page
+            paginated_chunks = chunks[start:end]
+
+            return make_response(
+                jsonify(
+                    {
+                        "page": page,
+                        "per_page": per_page,
+                        "total": total_chunks,
+                        "chunks": paginated_chunks,
+                        "path": path if path else None,
+                        "search": search_term if search_term else None,
+                    }
+                ),
+                200,
+            )
+        except Exception as e:
+            current_app.logger.error(f"Error getting chunks: {e}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)
+
+
+@sources_chunks_ns.route("/add_chunk")
+class AddChunk(Resource):
+    @api.expect(
+        api.model(
+            "AddChunkModel",
+            {
+                "id": fields.String(required=True, description="Document ID"),
+                "text": fields.String(required=True, description="Text of the chunk"),
+                "metadata": fields.Raw(
+                    required=False,
+                    description="Metadata associated with the chunk",
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Adds a new chunk to the document",
+    )
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id", "text"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        doc_id = data.get("id")
+        text = data.get("text")
+        metadata = data.get("metadata", {})
+        token_count = num_tokens_from_string(text)
+        metadata["token_count"] = token_count
+
+        try:
+            doc = _resolve_source_for_write(doc_id, user)
+        except Exception as e:
+            current_app.logger.error(f"Error resolving source: {e}", exc_info=True)
+            return make_response(jsonify({"error": "Invalid doc_id"}), 400)
+        if not doc:
+            return make_response(jsonify({"error": "Source not accessible"}), 403)
+        try:
+            store = get_vector_store(str(doc["id"]))
+            chunk_id = store.add_chunk(text, metadata)
+            return make_response(
+                jsonify({"message": "Chunk added successfully", "chunk_id": chunk_id}),
+                201,
+            )
+        except Exception as e:
+            current_app.logger.error(f"Error adding chunk: {e}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)
+
+
+@sources_chunks_ns.route("/delete_chunk")
+class DeleteChunk(Resource):
+    @api.doc(
+        description="Deletes a specific chunk from the document.",
+        params={"id": "The document ID", "chunk_id": "The ID of the chunk to delete"},
+    )
+    def delete(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        doc_id = request.args.get("id")
+        chunk_id = request.args.get("chunk_id")
+
+        try:
+            doc = _resolve_source_for_write(doc_id, user)
+        except Exception as e:
+            current_app.logger.error(f"Error resolving source: {e}", exc_info=True)
+            return make_response(jsonify({"error": "Invalid doc_id"}), 400)
+        if not doc:
+            return make_response(jsonify({"error": "Source not accessible"}), 403)
+        try:
+            store = get_vector_store(str(doc["id"]))
+            deleted = store.delete_chunk(chunk_id)
+            if deleted:
+                return make_response(
+                    jsonify({"message": "Chunk deleted successfully"}), 200
+                )
+            else:
+                return make_response(
+                    jsonify({"message": "Chunk not found or could not be deleted"}),
+                    404,
+                )
+        except Exception as e:
+            current_app.logger.error(f"Error deleting chunk: {e}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)
+
+
+@sources_chunks_ns.route("/update_chunk")
+class UpdateChunk(Resource):
+    @api.expect(
+        api.model(
+            "UpdateChunkModel",
+            {
+                "id": fields.String(required=True, description="Document ID"),
+                "chunk_id": fields.String(
+                    required=True, description="Chunk ID to update"
+                ),
+                "text": fields.String(
+                    required=False, description="New text of the chunk"
+                ),
+                "metadata": fields.Raw(
+                    required=False,
+                    description="Updated metadata associated with the chunk",
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Updates an existing chunk in the document.",
+    )
+    def put(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+        required_fields = ["id", "chunk_id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        doc_id = data.get("id")
+        chunk_id = data.get("chunk_id")
+        text = data.get("text")
+        metadata = data.get("metadata")
+
+        if text is not None:
+            token_count = num_tokens_from_string(text)
+            if metadata is None:
+                metadata = {}
+            metadata["token_count"] = token_count
+        try:
+            doc = _resolve_source_for_write(doc_id, user)
+        except Exception as e:
+            current_app.logger.error(f"Error resolving source: {e}", exc_info=True)
+            return make_response(jsonify({"error": "Invalid doc_id"}), 400)
+        if not doc:
+            return make_response(jsonify({"error": "Source not accessible"}), 403)
+        try:
+            store = get_vector_store(str(doc["id"]))
+
+            chunks = store.get_chunks()
+            existing_chunk = next((c for c in chunks if c["doc_id"] == chunk_id), None)
+            if not existing_chunk:
+                return make_response(jsonify({"error": "Chunk not found"}), 404)
+            new_text = text if text is not None else existing_chunk["text"]
+
+            if metadata is not None:
+                new_metadata = existing_chunk["metadata"].copy()
+                new_metadata.update(metadata)
+            else:
+                new_metadata = existing_chunk["metadata"].copy()
+            if text is not None:
+                new_metadata["token_count"] = num_tokens_from_string(new_text)
+            try:
+                new_chunk_id = store.add_chunk(new_text, new_metadata)
+
+                deleted = store.delete_chunk(chunk_id)
+                if not deleted:
+                    current_app.logger.warning(
+                        f"Failed to delete old chunk {chunk_id}, but new chunk {new_chunk_id} was created"
+                    )
+                return make_response(
+                    jsonify(
+                        {
+                            "message": "Chunk updated successfully",
+                            "chunk_id": new_chunk_id,
+                            "original_chunk_id": chunk_id,
+                        }
+                    ),
+                    200,
+                )
+            except Exception as add_error:
+                current_app.logger.error(f"Failed to add updated chunk: {add_error}")
+                return make_response(
+                    jsonify({"error": "Failed to update chunk - addition failed"}), 500
+                )
+        except Exception as e:
+            current_app.logger.error(f"Error updating chunk: {e}", exc_info=True)
+            return make_response(jsonify({"success": False}), 500)

docsgpt/api/user/sources/retrieval_test.py

@@ -0,0 +1,254 @@
+"""Retrieval test — preview the chunks a query actually retrieves from a source.
+
+The chunk browser's ``search`` (``/api/get_chunks``) is a substring filter over
+stored text; it says nothing about what a RAG query would retrieve. This
+endpoint runs the *production* retrieval path against a single source so a user
+can see the real ranked chunks for a query, and try retrieval settings without
+saving them.
+
+It deliberately builds a ``Dispatcher`` — the same object the answer pipeline
+uses — so retriever selection, per-source ``chunks`` / ``score_threshold``, the
+prescreen stage and the token budget all behave exactly as they do at answer
+time, rather than drifting from them.
+"""
+
+import logging
+import math
+import time
+
+from flask import jsonify, make_response, request
+from flask_restx import fields, Namespace, Resource
+from pydantic import ValidationError
+
+from docsgpt.api import api
+from docsgpt.api.user.sources.routes import _resolve_readable_source
+from docsgpt.core.model_utils import get_default_model_id
+from docsgpt.retriever.dispatcher import Dispatcher
+from docsgpt.retriever.retriever_creator import RetrieverCreator
+from docsgpt.storage.db.session import db_readonly
+from docsgpt.storage.db.source_config import RetrievalConfig, SourceConfig
+from docsgpt.utils import num_tokens_from_string
+
+logger = logging.getLogger(__name__)
+
+sources_search_ns = Namespace(
+    "sources", description="Source retrieval testing", path="/api"
+)
+
+# A retrieval query is a search query, not a document.
+MAX_QUERY_LENGTH = 2000
+
+# The answer pipeline's default; keeps the preview's budgeting identical to it.
+DOC_TOKEN_LIMIT = 50000
+
+# Cost-attribution tag for any LLM call the preview makes (prescreen only —
+# with no chat history the rephrase side-call is skipped entirely).
+USAGE_SOURCE = "retrieval_test"
+
+# Ceiling on the prescreen LLM calls a single test may trigger.
+MAX_PRESCREEN_BATCHES = 20
+
+
+@sources_search_ns.route("/sources/<string:source_id>/search")
+class SourceSearch(Resource):
+    search_model = api.model(
+        "SourceSearchModel",
+        {
+            "query": fields.String(
+                required=True, description="The query to retrieve for"
+            ),
+            "retrieval": fields.Raw(
+                required=False,
+                description="Ad-hoc retrieval config to test with. Omit to use "
+                "the source's saved config. Never persisted.",
+            ),
+        },
+    )
+
+    @api.expect(search_model)
+    @api.doc(
+        description="Run the real retrieval pipeline against one source and "
+        "return the ranked chunks it produces, optionally under an ad-hoc "
+        "retrieval config. Read-only: nothing is saved."
+    )
+    def post(self, source_id):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+
+        body = request.get_json(silent=True)
+        if not isinstance(body, dict):
+            return make_response(
+                jsonify({"success": False, "message": "Invalid request body"}), 400
+            )
+        query = (body.get("query") or "").strip()
+        if not query:
+            return make_response(
+                jsonify({"success": False, "message": "Query is required"}), 400
+            )
+        if len(query) > MAX_QUERY_LENGTH:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": f"Query must be at most {MAX_QUERY_LENGTH} characters",
+                    }
+                ),
+                400,
+            )
+
+        try:
+            # Read access = owner or any team grant (viewer included), matching
+            # the other source read endpoints (wiki pages, graph).
+            with db_readonly() as conn:
+                doc = _resolve_readable_source(conn, source_id, user)
+        except Exception as e:
+            # An unresolvable id yields None (→ 404); reaching here means the
+            # lookup itself failed, which is ours, not the caller's.
+            logger.error(f"Error resolving source: {e}", exc_info=True)
+            return make_response(
+                jsonify({"success": False, "message": "Could not resolve source"}), 500
+            )
+        if not doc:
+            return make_response(
+                jsonify(
+                    {"success": False, "message": "Source not found or access denied"}
+                ),
+                404,
+            )
+        resolved_id = str(doc["id"])
+
+        # A supplied config is validated exactly as strictly as a saved one (D7
+        # strict-on-write), with the same static message so validation internals
+        # aren't echoed back. Absent → the source's saved config.
+        saved = SourceConfig.parse(doc.get("config")).retrieval
+        if body.get("retrieval") is not None:
+            try:
+                retrieval = RetrievalConfig.model_validate(body["retrieval"])
+            except ValidationError:
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "message": "Invalid retrieval config: one or more "
+                            "fields failed validation.",
+                        }
+                    ),
+                    400,
+                )
+        else:
+            retrieval = saved
+
+        # "Ad-hoc" means the caller is testing something the source is NOT
+        # already configured to do. The client always sends the config it has on
+        # screen — including an untouched one — so compare by value rather than
+        # trusting the body's presence, or a source whose saved config is
+        # expensive could never be tested at all (see the ceiling below).
+        ad_hoc = retrieval != saved
+
+        # RetrievalConfig.retriever is a free string, so an unknown key would
+        # only blow up inside RetrieverCreator — a caller's typo must be a 400,
+        # not a 500.
+        if retrieval.retriever not in RetrieverCreator.retrievers:
+            return make_response(
+                jsonify(
+                    {
+                        "success": False,
+                        "message": f"Unknown retriever '{retrieval.retriever}'.",
+                    }
+                ),
+                400,
+            )
+
+        # Prescreen screens candidate_k candidates in batches of batch_size, one
+        # synchronous LLM call each, so candidate_k=500 / batch_size=1 is 500
+        # provider calls in one request; RetrievalConfig bounds each field but
+        # not their ratio. Only an ad-hoc config is capped: a config already
+        # saved on the source runs at answer time anyway, so refusing to test it
+        # would defeat the endpoint.
+        if ad_hoc:
+            ps = retrieval.prescreen_config()
+            if ps is not None:
+                batches = math.ceil(ps.candidate_k / ps.batch_size)
+                if batches > MAX_PRESCREEN_BATCHES:
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "message": (
+                                    f"This prescreen config needs {batches} LLM "
+                                    f"calls to test (limit "
+                                    f"{MAX_PRESCREEN_BATCHES}). Raise batch_size "
+                                    "or lower candidate_k."
+                                ),
+                            }
+                        ),
+                        400,
+                    )
+
+        # A test has no chat behind it, so no model was requested. Resolve the
+        # same instance default the answer path falls back to: with a bogus id
+        # the prescreen stage would call the provider, fail, and silently keep
+        # every candidate. On an instance with no models configured this is
+        # None — leave the retriever's own default in place rather than
+        # forwarding it.
+        dispatcher_kwargs = {}
+        default_model_id = get_default_model_id()
+        if default_model_id:
+            dispatcher_kwargs["model_id"] = default_model_id
+
+        try:
+            started = time.monotonic()
+            # Dispatcher directly, NOT build_dispatcher: the
+            # PER_SOURCE_RETRIEVAL_ENABLED kill-switch would fall back to a
+            # stock classic retriever and ignore the config under test.
+            retriever = Dispatcher(
+                source={"active_docs": [resolved_id], "question": query},
+                chat_history=[],  # no history ⇒ no rephrase side-call
+                chunks=retrieval.chunks,
+                doc_token_limit=DOC_TOKEN_LIMIT,
+                decoded_token=decoded_token,
+                sources=[{"id": resolved_id, "retrieval": retrieval}],
+                include_scores=True,
+                usage_source=USAGE_SOURCE,
+                **dispatcher_kwargs,
+            )
+            docs = retriever.search(query) or []
+            latency_ms = int((time.monotonic() - started) * 1000)
+        except Exception as e:
+            logger.error(f"Retrieval test failed: {e}", exc_info=True)
+            return make_response(
+                jsonify({"success": False, "message": "Retrieval failed"}), 500
+            )
+
+        chunks = [
+            {
+                "rank": idx,
+                "text": d.get("text", ""),
+                "title": d.get("title"),
+                "filename": d.get("filename"),
+                "source": d.get("source"),
+                "tokens": num_tokens_from_string(d.get("text", "")),
+                # None for retrievers/stores that produce no comparable score
+                # (graphrag's PPR ranking, stores without a score seam).
+                "score": d.get("score"),
+                "score_kind": d.get("score_kind"),
+            }
+            for idx, d in enumerate(docs, start=1)
+        ]
+
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "query": query,
+                    "retriever": retrieval.retriever,
+                    "retrieval": retrieval.model_dump(),
+                    "total": len(chunks),
+                    "latency_ms": latency_ms,
+                    "chunks": chunks,
+                }
+            ),
+            200,
+        )

docsgpt/api/user/tasks.py

@@ -0,0 +1,860 @@
+import logging
+from datetime import timedelta
+from typing import Dict, Optional
+
+from sqlalchemy.exc import DataError
+
+from docsgpt.api.user.idempotency import with_idempotency
+from docsgpt.celery_init import celery
+from docsgpt.parser.file.base_parser import DocumentParseError
+from docsgpt.worker import (
+    AttachmentRejectedError,
+    agent_webhook_worker,
+    attachment_worker,
+    ingest_worker,
+    mcp_oauth,
+    parse_document_worker,
+    reembed_wiki_page_worker,
+    remote_worker,
+    sync,
+    sync_worker,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# Shared decorator config for long-running, side-effecting tasks. ``acks_late``
+# is also the celeryconfig default but stays explicit here so each task's
+# durability story is grep-able next to the body. Combined with
+# ``autoretry_for=(Exception,)`` and a bounded ``max_retries`` so a poison
+# message can't loop forever.
+#
+# ``retry_backoff`` is the factor, NOT a boolean toggle: celery's
+# ``add_autoretry_behaviour`` OVERWRITES ``retry_kwargs["countdown"]`` whenever
+# it is truthy (celery/app/autoretry.py). With the old ``retry_backoff=True``
+# the factor was ``int(max(1.0, True)) == 1``, so the three waits were jittered
+# 0-1 s, 0-2 s and 0-4 s — a whole retry envelope of at most 7 seconds. A
+# 3.5-minute network blip (2026-08-21) therefore exhausted every attempt and
+# published a terminal ``source.ingest.failed`` / ``attachment.failed``.
+#
+# celery applies FULL JITTER (``retry_jitter`` defaults to True), so each wait
+# is drawn uniformly from ``[0, factor * 2**retries]`` — the ceilings are NOT
+# the waits, and the envelope is a distribution, not a guarantee. Factor 60
+# gives ceilings 60/120/240 s: a median envelope of ~210 s, which covers a
+# 60 s blip ~98% of the time and a 120 s blip ~85%. Factor 30 would have
+# covered the 210 s incident 0% of the time, because its 210 s nominal
+# maximum was reachable only by drawing all three waits at their ceiling.
+# Raising ``max_retries`` would buy more headroom but is deliberately not
+# done here: attempt 6 would newly reach the ``MAX_TASK_ATTEMPTS=5``
+# poison-loop guard in ``idempotency.py``, which is a separate behaviour
+# change from widening the waits.
+#
+# Jitter is deliberately left ON. These nine task types share their
+# dependencies (Postgres, object storage, the embedding provider), so an
+# outage fails them all at once; ``retry_jitter=False`` would wake every
+# in-flight task at the same instant and stampede the service that just
+# recovered.
+#
+# ``max_retries`` is passed at the top level rather than inside
+# ``retry_kwargs``: celery captures that dict BY REFERENCE and writes
+# ``retry_kwargs["countdown"]`` into it on every retry, so one shared dict
+# spread across all the decorators below would let concurrent retries race
+# and would permanently mutate this module constant.
+#
+# ``dont_autoretry_for`` is the necessary counterweight to that widened
+# envelope. ``autoretry_for=(Exception,)`` retries EVERYTHING, but a
+# ``DocumentParseError`` (unparseable, empty, or image-only file) fails
+# identically on every attempt — with factor 60 it now burns a median 3.5 min
+# and up to ~7 before reaching a terminal state, during which anything polling
+# ``/api/task_status`` (the wiki-convert and GraphRAG-enable modals) just keeps
+# reporting "pending". It is the DEFAULT here so a new durable task cannot
+# forget it; tasks needing a wider set pass their own via ``durable_task()``.
+DURABLE_TASK = dict(
+    bind=True,
+    acks_late=True,
+    autoretry_for=(Exception,),
+    dont_autoretry_for=(DocumentParseError,),
+    max_retries=3,
+    retry_backoff=60,
+)
+
+
+def durable_task(**overrides) -> Dict:
+    """Return ``DURABLE_TASK`` with per-task overrides applied.
+
+    Needed because ``@celery.task(**DURABLE_TASK, dont_autoretry_for=...)``
+    would be a duplicate keyword argument.
+
+    Args:
+        **overrides: Task options replacing the shared defaults.
+
+    Returns:
+        The merged options dict, for ``@celery.task(**durable_task(...))``.
+    """
+    return {**DURABLE_TASK, **overrides}
+
+
+# operation tag for the poison-path source.ingest.failed event, per task.
+_INGEST_POISON_OPERATION = {
+    "ingest": "upload",
+    "ingest_remote": "upload",
+    "ingest_connector_task": "upload",
+    "reingest_source_task": "reingest",
+}
+
+
+def _emit_ingest_poison_event(task_name, bound):
+    """Publish a terminal ``source.ingest.failed`` when the poison-guard trips.
+
+    The guard returns before the worker runs, so the worker's own failed
+    event never fires — without this the upload toast spins on "training".
+    """
+    user = bound.get("user")
+    source_id = bound.get("source_id")
+    if not user or not source_id:
+        return
+    from docsgpt.events.publisher import publish_user_event
+
+    publish_user_event(
+        user,
+        "source.ingest.failed",
+        {
+            "source_id": str(source_id),
+            "filename": bound.get("filename") or "",
+            "operation": _INGEST_POISON_OPERATION.get(task_name, "upload"),
+            "error": "Ingestion stopped after repeated failures.",
+        },
+        scope={"kind": "source", "id": str(source_id)},
+    )
+
+
+# ``dont_autoretry_for=(DocumentParseError,)`` now comes from ``DURABLE_TASK``:
+# a file that cannot be converted to text fails identically on every attempt,
+# so retrying only multiplies the log noise before the same failure — go
+# straight to the poison/failure path.
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="ingest", on_poison=_emit_ingest_poison_event)
+def ingest(
+    self,
+    directory,
+    formats,
+    job_name,
+    user,
+    file_path,
+    filename,
+    file_name_map=None,
+    config=None,
+    idempotency_key=None,
+    source_id=None,
+):
+    resp = ingest_worker(
+        self,
+        directory,
+        formats,
+        job_name,
+        file_path,
+        filename,
+        user,
+        file_name_map=file_name_map,
+        config=config,
+        idempotency_key=idempotency_key,
+        source_id=source_id,
+    )
+    return resp
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="ingest_remote", on_poison=_emit_ingest_poison_event)
+def ingest_remote(
+    self, source_data, job_name, user, loader,
+    config=None, idempotency_key=None, source_id=None,
+):
+    resp = remote_worker(
+        self, source_data, job_name, user, loader,
+        config=config,
+        idempotency_key=idempotency_key,
+        source_id=source_id,
+    )
+    return resp
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(
+    task_name="reingest_source_task", on_poison=_emit_ingest_poison_event,
+)
+def reingest_source_task(self, source_id, user, idempotency_key=None):
+    from docsgpt.worker import reingest_source_worker
+
+    resp = reingest_source_worker(self, source_id, user)
+    return resp
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="reembed_wiki_page")
+def reembed_wiki_page(
+    self, source_id, path, content_hash, user, idempotency_key=None,
+):
+    resp = reembed_wiki_page_worker(self, source_id, path, content_hash, user)
+    return resp
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="convert_source_to_wiki")
+def convert_source_to_wiki(self, source_id, user, idempotency_key=None):
+    from docsgpt.worker import convert_source_to_wiki_worker
+
+    resp = convert_source_to_wiki_worker(self, source_id, user)
+    return resp
+
+
+def _emit_graph_poison_event(task_name, bound):
+    """Publish a terminal ``graph.extract.failed`` when the poison-guard trips.
+
+    The guard returns before the worker runs, so the worker's own failed event
+    never fires — without this the build UI spins forever.
+    """
+    user = bound.get("user")
+    source_id = bound.get("source_id")
+    if not user or not source_id:
+        return
+    from docsgpt.events.publisher import publish_user_event
+
+    publish_user_event(
+        user,
+        "graph.extract.failed",
+        {
+            "source_id": str(source_id),
+            "error": "Graph extraction stopped after repeated failures.",
+        },
+        scope={"kind": "source", "id": str(source_id)},
+    )
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="extract_graph", on_poison=_emit_graph_poison_event)
+def extract_graph(self, source_id, user, idempotency_key=None):
+    from docsgpt.worker import extract_graph_worker
+
+    resp = extract_graph_worker(self, source_id, user)
+    return resp
+
+
+# Beat-driven dispatch tasks default to ``acks_late=False``: a SIGKILL
+# of a beat tick is harmless to redeliver only if the dispatch itself is
+# idempotent. We keep these early-ACK so the broker doesn't replay a
+# dispatch that already enqueued downstream work.
+@celery.task(bind=True, acks_late=False)
+def schedule_syncs(self, frequency):
+    resp = sync_worker(self, frequency)
+    return resp
+
+
+@celery.task(bind=True)
+def sync_source(
+    self,
+    source_data,
+    job_name,
+    user,
+    loader,
+    sync_frequency,
+    retriever,
+    doc_id,
+):
+    resp = sync(
+        self,
+        source_data,
+        job_name,
+        user,
+        loader,
+        sync_frequency,
+        retriever,
+        doc_id,
+    )
+    return resp
+
+
+def _emit_attachment_poison_event(task_name, bound):
+    """Publish a terminal ``attachment.failed`` when the poison-guard trips.
+
+    Mirrors ``_emit_ingest_poison_event``: the guard returns before the
+    worker runs, so ``attachment_worker``'s own events never fire and the
+    upload toast would otherwise spin on "processing" forever. Also writes
+    the failure row the worker never got to write, so the poisoned upload
+    stays visible to a DB scan and not just to whoever saw the toast.
+    """
+    user = bound.get("user")
+    file_info = bound.get("file_info") or {}
+    attachment_id = file_info.get("attachment_id")
+    if not user or not attachment_id:
+        return
+    from docsgpt.events.publisher import publish_user_event
+    from docsgpt.worker import record_attachment_failure
+
+    record_attachment_failure(
+        user, file_info, "Attachment processing stopped after repeated failures."
+    )
+    publish_user_event(
+        user,
+        "attachment.failed",
+        {
+            "attachment_id": str(attachment_id),
+            "filename": file_info.get("filename") or "",
+            "error": "Attachment processing stopped after repeated failures.",
+        },
+        scope={"kind": "attachment", "id": str(attachment_id)},
+    )
+
+
+# ``dont_autoretry_for``: a DataError (poison payload, e.g. NUL bytes or an
+# over-long value), an AttachmentRejectedError (zip bomb) or a
+# DocumentParseError (the file cannot be converted to text at all) is
+# deterministic — retrying re-fails identically and multiplies log noise, so it
+# goes straight to the failure path.
+@celery.task(
+    **durable_task(
+        dont_autoretry_for=(DataError, AttachmentRejectedError, DocumentParseError),
+    )
+)
+@with_idempotency(
+    task_name="store_attachment", on_poison=_emit_attachment_poison_event,
+)
+def store_attachment(self, file_info, user, idempotency_key=None):
+    resp = attachment_worker(self, file_info, user)
+    return resp
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(task_name="process_agent_webhook")
+def process_agent_webhook(self, agent_id, payload, idempotency_key=None):
+    resp = agent_webhook_worker(self, agent_id, payload)
+    return resp
+
+
+# Seconds the hard (SIGKILL) limit trails the soft limit, giving the soft handler
+# room to unwind and return a terminal result before the worker is killed.
+_PARSE_HARD_LIMIT_GRACE = 30
+
+
+def parse_timeout_for_size(size_bytes: Optional[int]) -> float:
+    """Seconds to allow one ``parse_document``: floored at the base timeout, scaled by size, capped.
+
+    Args:
+        size_bytes: Byte size of the document, or None when unknown (base timeout).
+
+    Returns:
+        The parse window in seconds.
+    """
+    from docsgpt.core.settings import settings
+
+    base = float(getattr(settings, "DOCUMENT_PARSE_TIMEOUT", 120) or 120)
+    per_mib = float(getattr(settings, "DOCUMENT_PARSE_TIMEOUT_PER_MB", 0) or 0)
+    ceiling = float(getattr(settings, "DOCUMENT_PARSE_TIMEOUT_MAX", base) or base)
+    size = float(size_bytes) if isinstance(size_bytes, (int, float)) else 0.0
+    scaled = base + per_mib * max(size, 0.0) / (1024 * 1024)
+    return min(ceiling, max(base, scaled))
+
+
+def parse_task_time_limits(timeout: float) -> Dict[str, int]:
+    """Per-call Celery limits for a ``parse_document`` the caller awaits for ``timeout`` seconds.
+
+    The task's import-time ``soft_time_limit`` is bound to the BASE timeout, so a caller
+    awaiting a longer, size-scaled window must raise the per-call limits to match or the
+    worker self-terminates the parse first.
+
+    Args:
+        timeout: The awaited parse window in seconds.
+
+    Returns:
+        ``apply_async`` kwargs carrying the soft and hard time limits.
+    """
+    soft = max(1, int(timeout))
+    return {"soft_time_limit": soft, "time_limit": soft + _PARSE_HARD_LIMIT_GRACE}
+
+
+# Not DURABLE: the read_document tool awaits this synchronously with a timeout, so a
+# blind autoretry would double-parse and the caller would already have degraded. The
+# task is routed to the dedicated ``parsing`` queue (celeryconfig task_routes) so a
+# parse enqueued from inside a Celery worker (headless/scheduled agent) is served by a
+# separate parsing worker and never self-deadlocks the awaiting worker.
+@celery.task(bind=True, acks_late=False, autoretry_for=())
+def parse_document(self, artifact_id, parent, user_id, options=None):
+    """Parse an input artifact on the parsing queue; self-terminate at the soft time limit."""
+    from celery.exceptions import SoftTimeLimitExceeded
+
+    try:
+        return parse_document_worker(self, artifact_id, parent, user_id, options or {})
+    except SoftTimeLimitExceeded:
+        # A pathological/malicious document must not pin a parsing-worker slot past the
+        # window the caller already abandoned. Return the worker's clean error shape so
+        # the slot frees and the Redis result backend still gets a terminal result.
+        # ``request.timelimit`` is (hard, soft) and carries the caller's PER-CALL limit,
+        # so prefer it over the import-time default when reporting the window.
+        limit = (getattr(self.request, "timelimit", None) or (None, None))[1]
+        if limit is None:
+            limit = getattr(self, "soft_time_limit", None)
+        suffix = f" after {int(limit)}s" if limit else ""
+        return {"status": "error", "error": f"document parsing timed out{suffix}."}
+
+
+# Bind the soft limit to DOCUMENT_PARSE_TIMEOUT (the floor of the window callers await) so
+# the prefork worker self-terminates a runaway parse instead of pinning the slot; the hard
+# limit is the SIGKILL backstop if the soft handler can't unwind in time. Callers awaiting a
+# size-scaled window override both per call via ``parse_task_time_limits``.
+try:
+    from docsgpt.core.settings import settings as _parse_settings
+    parse_document.soft_time_limit = int(_parse_settings.DOCUMENT_PARSE_TIMEOUT)
+    parse_document.time_limit = parse_document.soft_time_limit + _PARSE_HARD_LIMIT_GRACE
+except Exception:
+    pass
+
+
+@celery.task(**DURABLE_TASK)
+@with_idempotency(
+    task_name="ingest_connector_task", on_poison=_emit_ingest_poison_event,
+)
+def ingest_connector_task(
+    self,
+    job_name,
+    user,
+    source_type,
+    session_token=None,
+    file_ids=None,
+    folder_ids=None,
+    recursive=True,
+    retriever="classic",
+    operation_mode="upload",
+    doc_id=None,
+    sync_frequency="never",
+    config=None,
+    idempotency_key=None,
+    source_id=None,
+):
+    from docsgpt.worker import ingest_connector
+
+    resp = ingest_connector(
+        self,
+        job_name,
+        user,
+        source_type,
+        session_token=session_token,
+        file_ids=file_ids,
+        folder_ids=folder_ids,
+        recursive=recursive,
+        retriever=retriever,
+        operation_mode=operation_mode,
+        doc_id=doc_id,
+        sync_frequency=sync_frequency,
+        config=config,
+        idempotency_key=idempotency_key,
+        source_id=source_id,
+    )
+    return resp
+
+
+@celery.task(bind=True, acks_late=False)
+def dispatch_scheduled_runs(self):
+    """Beat-driven scheduler poller (body in scheduler_dispatcher)."""
+    from docsgpt.api.user.scheduler_dispatcher import dispatch_due_runs
+
+    return dispatch_due_runs()
+
+
+@celery.task(
+    bind=True,
+    acks_late=True,
+    # Not DURABLE_TASK: agent runs have side effects; blind retry would double them.
+    autoretry_for=(),
+    max_retries=0,
+)
+def execute_scheduled_run(self, run_id):
+    """Execute one scheduled run; soft-time-limit honors SCHEDULE_RUN_TIMEOUT."""
+    from docsgpt.api.user.scheduler_worker import execute_scheduled_run_body
+
+    return execute_scheduled_run_body(run_id, getattr(self.request, "id", None))
+
+
+# Bind runtime soft-time-limit so the prefork worker can raise mid-agent.
+try:
+    from docsgpt.core.settings import settings as _scheduler_settings
+    execute_scheduled_run.soft_time_limit = max(
+        30, int(_scheduler_settings.SCHEDULE_RUN_TIMEOUT),
+    )
+    execute_scheduled_run.time_limit = (
+        execute_scheduled_run.soft_time_limit + 60
+    )
+except Exception:
+    pass
+
+
+@celery.task(bind=True, acks_late=False)
+def cleanup_schedule_runs(self):
+    """Trim ``schedule_runs`` per ``SCHEDULE_RUN_OUTPUT_RETENTION_DAYS``."""
+    from docsgpt.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {"deleted": 0, "skipped": "POSTGRES_URI not set"}
+
+    from docsgpt.storage.db.engine import get_engine
+    from docsgpt.storage.db.repositories.schedule_runs import (
+        ScheduleRunsRepository,
+    )
+
+    ttl_days = settings.SCHEDULE_RUN_OUTPUT_RETENTION_DAYS
+    engine = get_engine()
+    with engine.begin() as conn:
+        deleted = ScheduleRunsRepository(conn).cleanup_older_than(ttl_days)
+    return {"deleted": deleted, "ttl_days": ttl_days}
+
+
+@celery.task(bind=True, acks_late=False)
+def reap_sandbox_sessions(self):
+    """Close sandbox sessions idle past their TTL in this worker process.
+
+    The SandboxManager registry is per-process, so this reaps only sessions
+    bound in THIS worker; the API processes reap their own opportunistically on
+    ``open``. Artifacts are persisted eagerly, so reaping only closes idle
+    kernels and never loses a user-facing artifact.
+    """
+    try:
+        from docsgpt.sandbox.sandbox_creator import SandboxCreator
+
+        reaped = SandboxCreator.get_manager().reap_expired()
+    except Exception:  # noqa: BLE001 - housekeeping must never crash the beat loop
+        logging.getLogger(__name__).exception("reap_sandbox_sessions failed")
+        return {"reaped": 0, "error": True}
+    return {"reaped": len(reaped)}
+
+
+@celery.task(bind=True, acks_late=False)
+def reap_stale_workflow_runs(self):
+    """Fail workflow runs stranded in ``running`` past the stale deadline.
+
+    A run row is pre-created as ``running`` and finalized when its generator
+    finishes; a client disconnect or worker crash can leave it ``running``
+    forever. This closes those rows out so the UI/API stop showing a run that
+    will never complete.
+    """
+    from datetime import datetime, timezone
+
+    from docsgpt.core.settings import settings
+    from docsgpt.storage.db.engine import get_engine
+    from docsgpt.storage.db.repositories.workflow_runs import WorkflowRunsRepository
+
+    try:
+        stale_seconds = max(60, int(settings.WORKFLOW_RUN_STALE_SECONDS))
+        cutoff = datetime.now(timezone.utc) - timedelta(seconds=stale_seconds)
+        engine = get_engine()
+        with engine.begin() as conn:
+            reaped = WorkflowRunsRepository(conn).mark_stale_running_failed(cutoff)
+    except Exception:  # noqa: BLE001 - housekeeping must never crash the beat loop
+        logging.getLogger(__name__).exception("reap_stale_workflow_runs failed")
+        return {"reaped": 0, "error": True}
+    return {"reaped": reaped}
+
+
+@celery.on_after_configure.connect
+def setup_periodic_tasks(sender, **kwargs):
+    from docsgpt.core.settings import settings
+
+    sender.add_periodic_task(
+        timedelta(days=1),
+        schedule_syncs.s("daily"),
+    )
+    sender.add_periodic_task(
+        timedelta(weeks=1),
+        schedule_syncs.s("weekly"),
+    )
+    sender.add_periodic_task(
+        timedelta(days=30),
+        schedule_syncs.s("monthly"),
+    )
+    # Replaces Mongo's TTL index on pending_tool_state.expires_at.
+    sender.add_periodic_task(
+        timedelta(seconds=60),
+        cleanup_pending_tool_state.s(),
+        name="cleanup-pending-tool-state",
+    )
+    # Pure housekeeping for ``task_dedup`` / ``webhook_dedup`` — the
+    # upsert paths already handle stale rows, so cadence only bounds
+    # table size. Hourly is plenty for typical traffic.
+    sender.add_periodic_task(
+        timedelta(hours=1),
+        cleanup_idempotency_dedup.s(),
+        name="cleanup-idempotency-dedup",
+    )
+    sender.add_periodic_task(
+        timedelta(seconds=30),
+        reconciliation_task.s(),
+        name="reconciliation",
+    )
+    sender.add_periodic_task(
+        timedelta(hours=7),
+        version_check_task.s(),
+        name="version-check",
+    )
+    # Bound ``message_events`` growth — every streamed SSE chunk writes
+    # one row, so retained chats accumulate hundreds of rows per
+    # message. Reconnect-replay is only meaningful for streams the user
+    # could plausibly still be waiting on, so 14 days is generous.
+    sender.add_periodic_task(
+        timedelta(hours=24),
+        cleanup_message_events.s(),
+        name="cleanup-message-events",
+    )
+    sender.add_periodic_task(
+        timedelta(hours=24),
+        cleanup_guardrail_events.s(),
+        name="cleanup-guardrail-events",
+    )
+    sender.add_periodic_task(
+        timedelta(hours=24),
+        cleanup_orphan_memories.s(),
+        name="cleanup-orphan-memories",
+    )
+    # Scheduler dispatcher and run-log trim.
+    sender.add_periodic_task(
+        timedelta(seconds=max(15, settings.SCHEDULE_DISPATCHER_INTERVAL)),
+        dispatch_scheduled_runs.s(),
+        name="dispatch-scheduled-runs",
+    )
+    sender.add_periodic_task(
+        timedelta(hours=24),
+        cleanup_schedule_runs.s(),
+        name="cleanup-schedule-runs",
+    )
+    # Close idle-past-TTL sandbox sessions roughly every minute. The on-open
+    # opportunistic reap still runs in the API processes; this covers worker
+    # processes (and quiet periods where no new session is opened).
+    sender.add_periodic_task(
+        timedelta(seconds=60),
+        reap_sandbox_sessions.s(),
+        name="reap-sandbox-sessions",
+    )
+    # Fail workflow runs stranded in ``running`` (client disconnect / crash) so
+    # they don't linger forever. Every few minutes is plenty; the cutoff is hours.
+    sender.add_periodic_task(
+        timedelta(seconds=300),
+        reap_stale_workflow_runs.s(),
+        name="reap-stale-workflow-runs",
+    )
+
+
+# Bound time limits so a hung OAuth discovery (user never finishes the
+# consent flow, upstream never redirects) self-terminates instead of
+# stranding the ``mcp.oauth.awaiting_redirect`` envelope forever. The
+# soft limit raises inside ``mcp_oauth``'s ``try`` so it publishes a
+# terminal ``mcp.oauth.failed``; the hard limit is the prefork backstop.
+# Generous so a human actively clicking through OAuth isn't cut off.
+@celery.task(bind=True, soft_time_limit=600, time_limit=660)
+def mcp_oauth_task(self, config, user):
+    resp = mcp_oauth(self, config, user)
+    return resp
+
+
+@celery.task(bind=True, acks_late=False)
+def cleanup_pending_tool_state(self):
+    """Revert stale ``resuming`` rows, then delete TTL-expired rows."""
+    from docsgpt.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {"deleted": 0, "reverted": 0, "skipped": "POSTGRES_URI not set"}
+
+    from docsgpt.storage.db.engine import get_engine
+    from docsgpt.storage.db.repositories.pending_tool_state import (
+        PendingToolStateRepository,
+    )
+
+    engine = get_engine()
+    with engine.begin() as conn:
+        repo = PendingToolStateRepository(conn)
+        reverted = repo.revert_stale_resuming(grace_seconds=600)
+        cleared = repo.cleanup_expired()
+
+    # Reaping the resumable state retires any awaiting-approval prompt
+    # tied to it. Without a clearing event the durable
+    # ``tool.approval.required`` envelope replays on reconnect and the UI
+    # toast lingers for a conversation that can no longer be resumed.
+    from docsgpt.events.publisher import publish_user_event
+
+    for row in cleared:
+        user_id = row.get("user_id")
+        conversation_id = row.get("conversation_id")
+        if not user_id or not conversation_id:
+            continue
+        agent_config = row.get("agent_config") or {}
+        reserved_message_id = (
+            agent_config.get("reserved_message_id")
+            if isinstance(agent_config, dict)
+            else None
+        )
+        if reserved_message_id:
+            try:
+                from docsgpt.api.answer.services.conversation_service import (
+                    ConversationService,
+                    TERMINATED_RESPONSE_PLACEHOLDER,
+                )
+
+                ConversationService().finalize_message(
+                    str(reserved_message_id),
+                    TERMINATED_RESPONSE_PLACEHOLDER,
+                    status="failed",
+                    error=TimeoutError("Tool continuation expired before resume"),
+                )
+            except Exception:
+                logger.exception(
+                    "Failed to retire expired continuation message %s",
+                    reserved_message_id,
+                )
+        publish_user_event(
+            str(user_id),
+            "tool.approval.cleared",
+            {"conversation_id": str(conversation_id), "reason": "expired"},
+            scope={"kind": "conversation", "id": str(conversation_id)},
+        )
+    return {"deleted": len(cleared), "reverted": reverted}
+
+
+@celery.task(bind=True, acks_late=False)
+def cleanup_idempotency_dedup(self):
+    """Delete TTL-expired rows from ``task_dedup`` and ``webhook_dedup``.
+
+    Pure housekeeping — the upsert paths already ignore stale rows
+    (TTL-aware ``ON CONFLICT DO UPDATE``), so this only bounds table
+    growth and keeps SELECT planning tight on large deployments.
+    """
+    from docsgpt.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {
+            "task_dedup_deleted": 0,
+            "webhook_dedup_deleted": 0,
+            "skipped": "POSTGRES_URI not set",
+        }
+
+    from docsgpt.storage.db.engine import get_engine
+    from docsgpt.storage.db.repositories.idempotency import (
+        IdempotencyRepository,
+    )
+
+    engine = get_engine()
+    with engine.begin() as conn:
+        return IdempotencyRepository(conn).cleanup_expired()
+
+
+@celery.task(bind=True, acks_late=False)
+def reconciliation_task(self):
+    """Sweep stuck durability rows and escalate them to terminal status + alert.
+
+    Never retries: the task is on a 30 s beat, so the next tick IS the retry,
+    and ``acks_late=False`` keeps a failing run from being redelivered. It only
+    guards itself so a transient connectivity error emits one WARNING instead
+    of a full traceback on the same ERROR channel ``_emit_alert`` uses for real
+    reconciler findings — at 30 s cadence a multi-minute outage would otherwise
+    bury genuine alerts under dozens of identical stack traces.
+    """
+    from docsgpt.api.user.reconciliation import run_reconciliation, zero_summary
+
+    try:
+        return run_reconciliation()
+    except Exception:  # noqa: BLE001 - housekeeping must never crash the beat loop
+        # ``exception`` not ``warning``: without the traceback a programming
+        # error in a sweep logs one context-free line every 30 s forever and
+        # there is no way to locate it. Celery still records the tick as
+        # SUCCESS because the exception is swallowed here, so this log is the
+        # only channel that carries the stack.
+        logger.exception("reconciliation_task failed; the next beat retries")
+        # Keys come from ``zero_summary`` so a failed tick reports the same
+        # shape as a successful one; hand-writing them drifted immediately.
+        return {**zero_summary(), "error": True}
+
+
+@celery.task(bind=True, acks_late=False)
+def cleanup_message_events(self):
+    """Delete ``message_events`` rows older than the retention window.
+
+    Streamed answer responses write one journal row per SSE yield,
+    so unbounded growth would dominate Postgres for any retained-
+    conversations deployment. The reconnect-replay path only needs
+    rows for in-flight streams; 14 days covers paused/tool-action
+    flows comfortably.
+    """
+    from docsgpt.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {"deleted": 0, "skipped": "POSTGRES_URI not set"}
+
+    from docsgpt.storage.db.engine import get_engine
+    from docsgpt.storage.db.repositories.message_events import (
+        MessageEventsRepository,
+    )
+
+    from docsgpt.storage.db.repositories.conversations import (
+        ConversationsRepository,
+    )
+
+    ttl_days = settings.MESSAGE_EVENTS_RETENTION_DAYS
+    engine = get_engine()
+    with engine.begin() as conn:
+        deleted = MessageEventsRepository(conn).cleanup_older_than(ttl_days)
+        # Supersede tombstones ride the same beat: both are per-stream
+        # bookkeeping with the same retention story.
+        tombstones = ConversationsRepository(conn).cleanup_superseded_older_than(
+            ttl_days
+        )
+    return {"deleted": deleted, "superseded_deleted": tombstones, "ttl_days": ttl_days}
+
+
+@celery.task(bind=True, acks_late=False)
+def cleanup_guardrail_events(self):
+    """Delete ``guardrail_events`` rows older than the retention window.
+
+    The journal has no natural bound: every triggered control on every turn
+    writes a row, and the table carries scanned text when the operator opted
+    into storing it, so it should not be kept indefinitely.
+    """
+    from docsgpt.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {"deleted": 0, "skipped": "POSTGRES_URI not set"}
+
+    from docsgpt.storage.db.engine import get_engine
+    from docsgpt.storage.db.repositories.guardrail_events import (
+        GuardrailEventsRepository,
+    )
+
+    ttl_days = settings.GUARDRAILS_EVENTS_RETENTION_DAYS
+    engine = get_engine()
+    with engine.begin() as conn:
+        deleted = GuardrailEventsRepository(conn).purge_older_than(ttl_days)
+    return {"deleted": deleted, "ttl_days": ttl_days}
+
+
+@celery.task(bind=True, acks_late=False)
+def cleanup_orphan_memories(self):
+    """Sweep orphan memories left by the 0009 FK-to-trigger orphan window.
+
+    A ``memories`` INSERT for a real ``tool_id`` racing a ``user_tools``
+    DELETE leaves a permanent orphan the dropped FK would have rejected.
+    Default-tool synthetic ids are preserved (legitimate built-in data).
+    """
+    from docsgpt.core.settings import settings
+    if not settings.POSTGRES_URI:
+        return {"deleted": 0, "skipped": "POSTGRES_URI not set"}
+
+    from docsgpt.agents.default_tools import default_tool_ids
+    from docsgpt.storage.db.engine import get_engine
+    from docsgpt.storage.db.repositories.memories import MemoriesRepository
+
+    keep_tool_ids = list(default_tool_ids().values())
+    engine = get_engine()
+    with engine.begin() as conn:
+        deleted = MemoriesRepository(conn).delete_orphans(keep_tool_ids)
+    return {"deleted": deleted}
+
+
+@celery.task(bind=True, acks_late=False)
+def version_check_task(self):
+    """Periodic anonymous version check.
+
+    Complements the ``worker_ready`` boot trigger so long-running
+    deployments (>6h cache TTL) still refresh advisories. ``run_check``
+    is fail-silent and coordinates across replicas via Redis lock +
+    cache (see ``docsgpt.updates.version_check``).
+    """
+    from docsgpt.updates.version_check import run_check
+    run_check()

docsgpt/api/user/team_authz.py

@@ -0,0 +1,109 @@
+"""Team-scoped authorization — the SECOND authz plane beside global RBAC.
+
+Mirrors ``docsgpt/api/user/authz.py`` (the global admin/user plane) but for
+per-team roles. The two planes never mix: a global ``admin`` is a superuser over
+all teams (short-circuit below), but a ``team_admin`` is NOT a global admin.
+
+Key differences from the global resolver, all deliberate:
+- Team roles are resolved PER ROUTE (lazy), not eagerly in the app.py chokepoint
+  — most requests never touch a team route, so we don't pay a team query on the
+  universal path.
+- Team roles are NEVER read from the JWT (same reason as global RBAC: simple_jwt
+  / session_jwt are self-mintable).
+- ``team_id`` is read ONLY from ``request.view_args`` (the URL path), NEVER the
+  request body — a body-supplied team_id would be a trivial escalation vector.
+- Resolution FAILS CLOSED: a DB error while reading membership denies access
+  (the inverse of the global resolver, which fails open — here, failing open
+  would grant team access during an outage).
+"""
+
+from __future__ import annotations
+
+import logging
+from functools import wraps
+
+from flask import jsonify, make_response, request
+
+from docsgpt.api.user.authz import ROLE_ADMIN, has_role
+from docsgpt.storage.db.repositories.team_members import (
+    ROLE_TEAM_ADMIN,
+    ROLE_TEAM_MEMBER,
+    TeamMembersRepository,
+)
+from docsgpt.storage.db.session import db_readonly
+
+logger = logging.getLogger(__name__)
+
+
+def team_role_for(token: dict | None, team_id: str | None) -> str | None:
+    """Strongest team role the principal holds in ``team_id``, or None.
+
+    Fails closed to None on a DB error so a membership-read outage denies team
+    access rather than granting it.
+    """
+    if not token or not team_id:
+        return None
+    sub = token.get("sub")
+    if not sub:
+        return None
+    try:
+        with db_readonly() as conn:
+            return TeamMembersRepository(conn).role_for(sub, team_id)
+    except Exception:
+        logger.error(
+            "team_role_for: team_members read failed for sub=%s team=%s",
+            sub,
+            team_id,
+            exc_info=True,
+        )
+        return None
+
+
+def has_team_role(token: dict | None, team_id: str | None, name: str) -> bool:
+    """True if the principal satisfies ``name`` for ``team_id``.
+
+    A global ``admin`` satisfies any team role (superuser over all teams).
+    Otherwise ``team_admin`` implies ``team_member`` (admin ⊇ member).
+    """
+    if has_role(token, ROLE_ADMIN):
+        return True
+    role = team_role_for(token, team_id)
+    if role is None:
+        return False
+    if name == ROLE_TEAM_MEMBER:
+        return True
+    return role == name
+
+
+def require_team_role(name: str):
+    """Decorator factory: 401 when unauthenticated, 403 when lacking ``name``.
+
+    Reads ``team_id`` from the route kwargs (URL path) ONLY — never the body.
+    Fails closed: a missing token, missing team_id, or membership-read error
+    never passes through.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            token = getattr(request, "decoded_token", None)
+            if not token:
+                return make_response(
+                    jsonify({"success": False, "message": "Authentication required"}), 401
+                )
+            team_id = kwargs.get("team_id")
+            if not team_id:
+                return make_response(
+                    jsonify({"success": False, "message": "Team not specified"}), 400
+                )
+            if not has_team_role(token, team_id, name):
+                return make_response(jsonify({"success": False, "message": "Forbidden"}), 403)
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+team_admin_required = require_team_role(ROLE_TEAM_ADMIN)
+team_member_required = require_team_role(ROLE_TEAM_MEMBER)

docsgpt/api/user/team_sharing.py

@@ -0,0 +1,124 @@
+"""Cross-cutting helpers for team resource sharing.
+
+Centralises the polymorphic dispatch over the four shareable resource types so
+the grant endpoint and the per-entity list/get/update paths share one source of
+truth for "does this user own resource R?" and "what can this user see/edit via
+their teams?". Every shareable repo exposes the same ``get_any(id, user_id)``
+ownership accessor, which is what makes the dispatch uniform.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from sqlalchemy import Connection
+
+from docsgpt.storage.db.base_repository import looks_like_uuid
+from docsgpt.storage.db.repositories.agents import AgentsRepository
+from docsgpt.storage.db.repositories.prompts import PromptsRepository
+from docsgpt.storage.db.repositories.sources import SourcesRepository
+from docsgpt.storage.db.repositories.team_resource_grants import (
+    TeamResourceGrantsRepository,
+)
+from docsgpt.storage.db.repositories.team_scope import TeamScopeRepository
+from docsgpt.storage.db.repositories.user_tools import UserToolsRepository
+
+RESOURCE_TYPES = ("agent", "source", "prompt", "tool")
+
+_REPO_FOR_TYPE = {
+    "agent": AgentsRepository,
+    "source": SourcesRepository,
+    "prompt": PromptsRepository,
+    "tool": UserToolsRepository,
+}
+
+
+def is_valid_resource_type(resource_type: str) -> bool:
+    return resource_type in _REPO_FOR_TYPE
+
+
+def owns_resource(
+    conn: Connection, resource_type: str, resource_id: str, user_id: str
+) -> bool:
+    """True if ``user_id`` is the OWNER of the resource.
+
+    Dispatches to the correct repo by ``resource_type`` — this is the guard that
+    stops a caller registering a grant with a mismatched ``resource_type`` /
+    ``resource_id`` (the polymorphic grant table has no FK to catch it). Owners
+    only — team-granted ``editor`` access does NOT make you an owner.
+    """
+    repo_cls = _REPO_FOR_TYPE.get(resource_type)
+    if repo_cls is None:
+        return False
+    return repo_cls(conn).get_any(resource_id, user_id) is not None
+
+
+def team_access_for(
+    conn: Connection, user_id: str, resource_type: str, resource_id: str
+) -> Optional[str]:
+    """Strongest team access (``editor``/``viewer``) ``user_id`` has on a resource.
+
+    None when no team grant reaches the user (they may still be the owner — that
+    is a separate dual-key check at the repo).
+    """
+    return TeamScopeRepository(conn).effective_access(user_id, resource_type, resource_id)
+
+
+def visible_ids_via_teams(
+    conn: Connection, user_id: str, resource_type: str
+) -> set[str]:
+    """The id set of ``resource_type`` shared to any team ``user_id`` belongs to."""
+    return TeamScopeRepository(conn).visible_resource_ids(user_id, resource_type)
+
+
+def visible_with_access(
+    conn: Connection, user_id: str, resource_type: str
+) -> dict[str, str]:
+    """Map ``resource_id -> strongest access`` shared to the user's teams."""
+    return TeamScopeRepository(conn).visible_with_access(user_id, resource_type)
+
+
+def effective_write_owner(
+    conn: Connection, resource_type: str, resource_id: str, user_id: str
+) -> Optional[str]:
+    """The owner id to write a resource AS, or None if the caller can't write.
+
+    Returns ``user_id`` when the caller owns the resource, or the resource's
+    real ``owner_id`` when the caller holds a team ``editor`` grant — so callers
+    can pass the result straight to the existing owner-scoped ``update(id, owner,
+    ...)`` repo methods (which match on ``WHERE id AND user_id = :owner``) without
+    a separate ownerless write path. None means viewer-only or no access → the
+    route should answer 403/404. Delete is never authorized here — owner-only.
+    """
+    if owns_resource(conn, resource_type, resource_id, user_id):
+        return user_id
+    # Past the ownership check, only canonical-UUID resources can carry a team
+    # grant; a legacy/non-UUID id can't, and casting it would poison the txn.
+    if not looks_like_uuid(resource_id):
+        return None
+    grants = TeamResourceGrantsRepository(conn).list_for_resource(
+        resource_type, resource_id
+    )
+    if not grants:
+        return None
+    if TeamScopeRepository(conn).can_write(user_id, resource_type, resource_id):
+        # All grant rows carry the same denormalised owner_id.
+        return grants[0].get("owner_id")
+    return None
+
+
+def can_access(
+    conn: Connection, resource_type: str, resource_id: str, user_id: str
+) -> bool:
+    """True if ``user_id`` owns the resource OR has any team grant on it (read).
+
+    This is the write-path gate for *referencing* a resource (e.g. attaching a
+    ``source_id`` to an agent): you may reference what you own or what a team has
+    shared with you directly. Transitive access *through* a shared agent is a
+    separate, run-time concept and is intentionally NOT gated here.
+    """
+    if not resource_id:
+        return True
+    if owns_resource(conn, resource_type, resource_id, user_id):
+        return True
+    return TeamScopeRepository(conn).can_read(user_id, resource_type, resource_id)

docsgpt/api/user/teams/__init__.py

@@ -0,0 +1,5 @@
+"""Teams API namespace: team CRUD, membership, and resource-sharing grants."""
+
+from .routes import teams_ns
+
+__all__ = ["teams_ns"]

docsgpt/api/user/teams/routes.py

@@ -0,0 +1,636 @@
+"""Team management API: CRUD, membership, and resource-sharing grants.
+
+Authorization model (two planes, see ``team_authz.py``):
+- Any authenticated user may create a team (self-serve); the creator becomes a
+  ``team_admin`` in the same transaction.
+- Team detail / member list / grant list require team membership.
+- Member management and team edit require ``team_admin``.
+- Team deletion and owner transfer are owner-only (a global ``admin`` overrides).
+- Sharing a resource requires the caller to OWN it (dispatched by resource_type)
+  and be a member of the target team. Sharing is additive visibility — the
+  resource's owner is never changed.
+
+``team_id`` always comes from the URL path (never the body) — enforced by
+``require_team_role`` and by reading ``team_id`` as a route kwarg here.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import uuid
+
+from flask import jsonify, make_response, request
+from flask_restx import Namespace, Resource
+
+from docsgpt.api.user.authz import ROLE_ADMIN, has_role
+from docsgpt.api.user.team_authz import (
+    has_team_role,
+    team_admin_required,
+    team_member_required,
+)
+from docsgpt.api.user.team_sharing import is_valid_resource_type, owns_resource
+from docsgpt.events.publisher import publish_user_event
+from docsgpt.storage.db.base_repository import looks_like_uuid
+from docsgpt.storage.db.repositories.agents import AgentsRepository
+from docsgpt.storage.db.repositories.auth_events import AuthEventsRepository
+from docsgpt.storage.db.repositories.prompts import PromptsRepository
+from docsgpt.storage.db.repositories.sources import SourcesRepository
+from docsgpt.storage.db.repositories.team_members import (
+    ROLE_TEAM_ADMIN,
+    ROLE_TEAM_MEMBER,
+    TeamMembersRepository,
+)
+from docsgpt.storage.db.repositories.team_resource_grants import (
+    TeamResourceGrantsRepository,
+)
+from docsgpt.storage.db.repositories.teams import TeamsRepository
+from docsgpt.storage.db.repositories.user_tools import UserToolsRepository
+from docsgpt.storage.db.repositories.users import UsersRepository
+from docsgpt.storage.db.session import db_readonly, db_session
+
+logger = logging.getLogger(__name__)
+
+teams_ns = Namespace("teams", description="Team management and resource sharing", path="/api")
+
+_VALID_TEAM_ROLES = (ROLE_TEAM_ADMIN, ROLE_TEAM_MEMBER)
+_VALID_ACCESS_LEVELS = ("viewer", "editor")
+
+
+def _current_user() -> str | None:
+    token = getattr(request, "decoded_token", None)
+    return token.get("sub") if isinstance(token, dict) else None
+
+
+def _audit(conn, actor: str | None, event: str, **metadata) -> None:
+    """Append a team management event to the audit trail (best-effort).
+
+    Runs inside the action's transaction so the audit row commits atomically
+    with the change. Never raises into the request path — an audit failure must
+    not fail the operation.
+    """
+    try:
+        # SAVEPOINT: a failed audit insert poisons the surrounding txn, so
+        # nest it — on failure only the audit rolls back, not the action.
+        with conn.begin_nested():
+            AuthEventsRepository(conn).insert(
+                user_id=actor or "unknown",
+                event=event,
+                ip=request.remote_addr,
+                user_agent=request.headers.get("User-Agent"),
+                metadata={k: v for k, v in metadata.items() if v is not None},
+            )
+    except Exception:
+        logger.warning("team audit insert failed for event=%s", event, exc_info=True)
+
+
+def _slugify(name: str) -> str:
+    base = re.sub(r"[^a-z0-9]+", "-", (name or "").lower()).strip("-")
+    return base or "team"
+
+
+def _unique_slug(repo: TeamsRepository, base: str) -> str:
+    """A slug not yet taken: ``base`` if free, else ``base-<4hex>`` until unique."""
+    if not repo.slug_exists(base):
+        return base
+    for _ in range(8):
+        candidate = f"{base}-{uuid.uuid4().hex[:4]}"
+        if not repo.slug_exists(candidate):
+            return candidate
+    return f"{base}-{uuid.uuid4().hex}"
+
+
+# --- Notifications (best-effort, fire-and-forget; never fail the request) ----
+# Emitted AFTER the mutation commits, over their own read connection, so a
+# name-lookup or Redis hiccup can never roll back the share/membership change.
+
+
+def _resource_display_name(conn, resource_type: str, resource_id: str) -> str | None:
+    """Best-effort human label for a shared resource (ownerless fetch)."""
+    try:
+        if resource_type == "agent":
+            row = AgentsRepository(conn).get_by_id(resource_id)
+        elif resource_type == "source":
+            row = SourcesRepository(conn).get_by_id(resource_id)
+        elif resource_type == "prompt":
+            row = PromptsRepository(conn).get_for_rendering(resource_id)
+        elif resource_type == "tool":
+            row = UserToolsRepository(conn).get_by_id(resource_id)
+        else:
+            return None
+        if not row:
+            return None
+        return row.get("custom_name") or row.get("display_name") or row.get("name")
+    except Exception:
+        logger.warning("resource name resolve failed (%s)", resource_type, exc_info=True)
+        return None
+
+
+def _notify_member_added(
+    team_id: str, new_user: str | None, role: str, actor: str | None
+) -> None:
+    """Toast the new member that they were added to a team."""
+    if not new_user or new_user == actor:
+        return
+    try:
+        with db_readonly() as conn:
+            team = TeamsRepository(conn).get(team_id)
+        publish_user_event(
+            new_user,
+            "team.member_added",
+            {
+                "team_id": str(team_id),
+                "team_name": team.get("name") if team else None,
+                "role": role,
+                "added_by": actor,
+            },
+            scope={"kind": "team", "id": str(team_id)},
+        )
+    except Exception:
+        logger.warning("member_added notify failed", exc_info=True)
+
+
+def _notify_resource_shared(
+    actor: str | None,
+    team_id: str,
+    resource_type: str,
+    resource_id: str,
+    access_level: str,
+    target_user_id: str | None,
+) -> None:
+    """Toast the recipient(s) of a share.
+
+    Per-member share notifies just that member; whole-team share notifies every
+    current member except the sharer.
+    """
+    try:
+        with db_readonly() as conn:
+            resource_name = _resource_display_name(conn, resource_type, resource_id)
+            team = TeamsRepository(conn).get(team_id)
+            if target_user_id:
+                recipients = [target_user_id]
+            else:
+                members = TeamMembersRepository(conn).list_members(team_id)
+                recipients = list({m["user_id"] for m in members})
+        payload = {
+            "resource_type": resource_type,
+            "resource_id": str(resource_id),
+            "resource_name": resource_name,
+            "access_level": access_level,
+            "team_id": str(team_id),
+            "team_name": team.get("name") if team else None,
+            "shared_by": actor,
+        }
+        for recipient in recipients:
+            if recipient and recipient != actor:
+                publish_user_event(
+                    recipient,
+                    "resource.shared",
+                    payload,
+                    scope={"kind": "resource", "id": str(resource_id)},
+                )
+    except Exception:
+        logger.warning("resource_shared notify failed", exc_info=True)
+
+
+@teams_ns.route("/teams")
+class Teams(Resource):
+    def get(self):
+        """List the teams the caller belongs to, each annotated with their role."""
+        user = _current_user()
+        if not user:
+            return {"success": False}, 401
+        try:
+            with db_readonly() as conn:
+                teams = TeamsRepository(conn).list_for_user(user)
+            return make_response(jsonify({"success": True, "teams": teams}), 200)
+        except Exception as err:
+            logger.error("List teams failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    def post(self):
+        """Create a team (self-serve). The creator becomes its first team_admin."""
+        user = _current_user()
+        if not user:
+            return {"success": False}, 401
+        data = request.get_json(silent=True) or {}
+        name = (data.get("name") or "").strip()
+        if not name:
+            return {"success": False, "message": "name required"}, 400
+        description = data.get("description")
+        try:
+            with db_session() as conn:
+                teams = TeamsRepository(conn)
+                slug = _unique_slug(teams, _slugify(name))
+                team = teams.create(name, slug, owner_id=user, description=description)
+                TeamMembersRepository(conn).add_member(
+                    team["id"], user, role=ROLE_TEAM_ADMIN, source="manual", granted_by=user
+                )
+                _audit(conn, user, "team.create", team_id=team["id"], name=name)
+            team["member_role"] = ROLE_TEAM_ADMIN
+            return make_response(jsonify({"success": True, "team": team}), 201)
+        except Exception as err:
+            logger.error("Create team failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/teams/<string:team_id>")
+class Team(Resource):
+    @team_member_required
+    def get(self, team_id):
+        """Team detail with members and the caller's role. Requires membership."""
+        user = _current_user()
+        try:
+            with db_readonly() as conn:
+                team = TeamsRepository(conn).get(team_id)
+                if not team:
+                    return {"success": False, "message": "Not found"}, 404
+                members = TeamMembersRepository(conn)
+                team["members"] = members.list_members(team_id)
+                team["member_role"] = members.role_for(user, team_id)
+            return make_response(jsonify({"success": True, "team": team}), 200)
+        except Exception as err:
+            logger.error("Get team failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @team_admin_required
+    def put(self, team_id):
+        """Update team name/description. Requires team_admin."""
+        data = request.get_json(silent=True) or {}
+        fields = {k: data[k] for k in ("name", "description") if k in data}
+        if not fields:
+            return {"success": False, "message": "nothing to update"}, 400
+        try:
+            with db_session() as conn:
+                updated = TeamsRepository(conn).update(team_id, fields)
+            return make_response(jsonify({"success": bool(updated)}), 200)
+        except Exception as err:
+            logger.error("Update team failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    def delete(self, team_id):
+        """Delete the team. Owner-only (a global admin overrides)."""
+        user = _current_user()
+        if not user:
+            return {"success": False}, 401
+        try:
+            with db_session() as conn:
+                team = TeamsRepository(conn).get(team_id)
+                if not team:
+                    return {"success": False, "message": "Not found"}, 404
+                token = getattr(request, "decoded_token", None)
+                if team["owner_id"] != user and not has_role(token, ROLE_ADMIN):
+                    return {"success": False, "message": "Only the team owner can delete"}, 403
+                TeamsRepository(conn).delete(team_id)
+                _audit(conn, user, "team.delete", team_id=team_id)
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            logger.error("Delete team failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/teams/<string:team_id>/members")
+class TeamMembers(Resource):
+    @team_member_required
+    def get(self, team_id):
+        """List members. Requires membership."""
+        try:
+            with db_readonly() as conn:
+                members = TeamMembersRepository(conn).list_members(team_id)
+            return make_response(jsonify({"success": True, "members": members}), 200)
+        except Exception as err:
+            logger.error("List members failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @team_admin_required
+    def post(self, team_id):
+        """Add a member by email (preferred) or raw user_id. Requires team_admin."""
+        data = request.get_json(silent=True) or {}
+        new_user = (data.get("user_id") or "").strip()
+        email = (data.get("email") or "").strip()
+        role = data.get("role", ROLE_TEAM_MEMBER)
+        if role not in _VALID_TEAM_ROLES:
+            return {"success": False, "message": "invalid role"}, 400
+        if not new_user and not email:
+            return {"success": False, "message": "email or user_id required"}, 400
+        try:
+            with db_session() as conn:
+                # Resolve an email to its sub (the user must have logged in at
+                # least once for their email to be on file).
+                if not new_user and email:
+                    user_row = UsersRepository(conn).find_by_email(email)
+                    if not user_row:
+                        return {
+                            "success": False,
+                            "message": "No user found with that email (they must sign in once first)",
+                        }, 404
+                    new_user = user_row["user_id"]
+                TeamMembersRepository(conn).set_manual_role(
+                    team_id, new_user, role, granted_by=_current_user()
+                )
+                _audit(
+                    conn,
+                    _current_user(),
+                    "team.member_add",
+                    team_id=team_id,
+                    target_user=new_user,
+                    role=role,
+                )
+            # Post-commit, best-effort: tell the new member they were added.
+            _notify_member_added(team_id, new_user, role, _current_user())
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            logger.error("Add member failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/teams/<string:team_id>/members/<string:member_id>")
+class TeamMember(Resource):
+    @team_admin_required
+    def put(self, team_id, member_id):
+        """Change a member's role. Requires team_admin. Guards the last admin."""
+        data = request.get_json(silent=True) or {}
+        role = data.get("role")
+        if role not in _VALID_TEAM_ROLES:
+            return {"success": False, "message": "invalid role"}, 400
+        try:
+            with db_session() as conn:
+                members = TeamMembersRepository(conn)
+                if role == ROLE_TEAM_MEMBER and self._would_orphan_admins(
+                    members, team_id, member_id
+                ):
+                    return {
+                        "success": False,
+                        "message": "Cannot demote the last team admin",
+                    }, 409
+                members.set_manual_role(team_id, member_id, role, granted_by=_current_user())
+                _audit(
+                    conn,
+                    _current_user(),
+                    "team.member_role",
+                    team_id=team_id,
+                    target_user=member_id,
+                    role=role,
+                )
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            logger.error("Update member role failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @team_member_required
+    def delete(self, team_id, member_id):
+        """Remove a member. team_admin removes anyone; a member may remove self
+        (leave). Guards the last admin."""
+        user = _current_user()
+        token = getattr(request, "decoded_token", None)
+        is_self = member_id == user
+        if not is_self and not has_team_role(token, team_id, ROLE_TEAM_ADMIN):
+            return {"success": False, "message": "Forbidden"}, 403
+        try:
+            with db_session() as conn:
+                members = TeamMembersRepository(conn)
+                if self._would_orphan_admins(members, team_id, member_id):
+                    return {
+                        "success": False,
+                        "message": "Cannot remove the last team admin",
+                    }, 409
+                members.remove_member(team_id, member_id)
+                _audit(
+                    conn,
+                    user,
+                    "team.member_remove",
+                    team_id=team_id,
+                    target_user=member_id,
+                    self_leave=is_self,
+                )
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            logger.error("Remove member failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @staticmethod
+    def _would_orphan_admins(
+        members: TeamMembersRepository, team_id: str, member_id: str
+    ) -> bool:
+        """True if removing/demoting ``member_id`` leaves the team with no admin.
+
+        Locks the admin rows (FOR UPDATE) so concurrent demote/remove calls
+        serialize — preventing two simultaneous removals of distinct admins from
+        both passing the guard and orphaning the team.
+        """
+        admins = members.lock_admins(team_id)
+        if member_id not in admins:
+            return False
+        return len(admins) <= 1
+
+
+@teams_ns.route("/teams/<string:team_id>/grants")
+class TeamGrants(Resource):
+    @team_member_required
+    def get(self, team_id):
+        """List resources shared with this team. Requires membership."""
+        resource_type = request.args.get("resource_type")
+        try:
+            with db_readonly() as conn:
+                grants = TeamResourceGrantsRepository(conn).list_for_team(
+                    team_id, resource_type
+                )
+            return make_response(jsonify({"success": True, "grants": grants}), 200)
+        except Exception as err:
+            logger.error("List grants failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @team_member_required
+    def post(self, team_id):
+        """Share a resource the caller OWNS with this team (additive visibility)."""
+        user = _current_user()
+        data = request.get_json(silent=True) or {}
+        resource_type = data.get("resource_type")
+        resource_id = data.get("resource_id")
+        access_level = data.get("access_level", "viewer")
+        # None → share with the whole team; a sub → share with that one member.
+        target_user_id = (data.get("target_user_id") or "").strip() or None
+        if (
+            not is_valid_resource_type(resource_type)
+            or not resource_id
+            or not looks_like_uuid(resource_id)
+        ):
+            return {"success": False, "message": "invalid resource"}, 400
+        if access_level not in _VALID_ACCESS_LEVELS:
+            return {"success": False, "message": "invalid access_level"}, 400
+        try:
+            with db_session() as conn:
+                # Ownership is the security boundary: dispatch by resource_type so
+                # a mismatched type/id can't register a bogus grant.
+                if not owns_resource(conn, resource_type, resource_id, user):
+                    return {"success": False, "message": "Not the resource owner"}, 403
+                # A per-member share target must actually be a member of the team.
+                if target_user_id and not TeamMembersRepository(conn).is_member(
+                    target_user_id, team_id
+                ):
+                    return {"success": False, "message": "Target is not a team member"}, 400
+                grant = TeamResourceGrantsRepository(conn).grant(
+                    team_id,
+                    resource_type,
+                    resource_id,
+                    owner_id=user,
+                    granted_by=user,
+                    access_level=access_level,
+                    target_user_id=target_user_id,
+                )
+                _audit(
+                    conn,
+                    user,
+                    "team.share",
+                    team_id=team_id,
+                    resource_type=resource_type,
+                    resource_id=resource_id,
+                    access_level=access_level,
+                    target_user_id=target_user_id,
+                )
+            # Post-commit, best-effort: tell the recipient(s) it was shared.
+            _notify_resource_shared(
+                user, team_id, resource_type, resource_id, access_level, target_user_id
+            )
+            return make_response(jsonify({"success": True, "grant": grant}), 201)
+        except Exception as err:
+            logger.error("Share resource failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+    @team_member_required
+    def delete(self, team_id):
+        """Unshare a resource. Allowed for the resource owner or a team_admin.
+
+        Identifiers come from query params (some proxies strip DELETE bodies),
+        with a JSON-body fallback for older clients.
+        """
+        user = _current_user()
+        token = getattr(request, "decoded_token", None)
+        data = request.get_json(silent=True) or {}
+        resource_type = request.args.get("resource_type") or data.get("resource_type")
+        resource_id = request.args.get("resource_id") or data.get("resource_id")
+        # Which grant to remove: whole-team (None) or a specific member's.
+        target_user_id = (
+            request.args.get("target_user_id") or data.get("target_user_id") or ""
+        ).strip() or None
+        if (
+            not is_valid_resource_type(resource_type)
+            or not resource_id
+            or not looks_like_uuid(resource_id)
+        ):
+            return {"success": False, "message": "invalid resource"}, 400
+        try:
+            with db_session() as conn:
+                is_owner = owns_resource(conn, resource_type, resource_id, user)
+                if not is_owner and not has_team_role(token, team_id, ROLE_TEAM_ADMIN):
+                    return {"success": False, "message": "Forbidden"}, 403
+                revoked = TeamResourceGrantsRepository(conn).revoke(
+                    team_id, resource_type, resource_id, target_user_id=target_user_id
+                )
+                if revoked:
+                    _audit(
+                        conn,
+                        user,
+                        "team.unshare",
+                        team_id=team_id,
+                        resource_type=resource_type,
+                        resource_id=resource_id,
+                        target_user_id=target_user_id,
+                    )
+            return make_response(jsonify({"success": bool(revoked)}), 200)
+        except Exception as err:
+            logger.error("Unshare resource failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/teams/<string:team_id>/transfer_owner")
+class TeamOwnerTransfer(Resource):
+    def post(self, team_id):
+        """Transfer team ownership. Owner-only (global admin overrides).
+
+        The new owner must already be a member; they are promoted to team_admin.
+        """
+        user = _current_user()
+        if not user:
+            return {"success": False}, 401
+        data = request.get_json(silent=True) or {}
+        new_owner = (data.get("user_id") or "").strip()
+        if not new_owner:
+            return {"success": False, "message": "user_id required"}, 400
+        token = getattr(request, "decoded_token", None)
+        try:
+            with db_session() as conn:
+                teams = TeamsRepository(conn)
+                team = teams.get(team_id)
+                if not team:
+                    return {"success": False, "message": "Not found"}, 404
+                if team["owner_id"] != user and not has_role(token, ROLE_ADMIN):
+                    return {"success": False, "message": "Only the team owner can transfer"}, 403
+                members = TeamMembersRepository(conn)
+                if not members.is_member(new_owner, team_id):
+                    return {"success": False, "message": "New owner must be a member"}, 400
+                members.set_manual_role(team_id, new_owner, ROLE_TEAM_ADMIN, granted_by=user)
+                teams.reassign_owner(team_id, new_owner)
+                _audit(
+                    conn,
+                    user,
+                    "team.transfer_owner",
+                    team_id=team_id,
+                    new_owner=new_owner,
+                )
+            return make_response(jsonify({"success": True}), 200)
+        except Exception as err:
+            logger.error("Transfer owner failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/resource_shares")
+class ResourceShares(Resource):
+    def get(self):
+        """List the teams a resource the caller OWNS is shared with.
+
+        Powers the share dialog (show current shares + unshare). Owner-only so a
+        non-owner can't enumerate a resource's sharing graph.
+        """
+        user = _current_user()
+        if not user:
+            return {"success": False}, 401
+        resource_type = request.args.get("resource_type")
+        resource_id = request.args.get("resource_id")
+        if (
+            not is_valid_resource_type(resource_type)
+            or not resource_id
+            or not looks_like_uuid(resource_id)
+        ):
+            return {"success": False, "message": "invalid resource"}, 400
+        try:
+            with db_readonly() as conn:
+                if not owns_resource(conn, resource_type, resource_id, user):
+                    return {"success": False, "message": "Not the resource owner"}, 403
+                shares = TeamResourceGrantsRepository(conn).list_for_resource(
+                    resource_type, resource_id
+                )
+            return make_response(jsonify({"success": True, "shares": shares}), 200)
+        except Exception as err:
+            logger.error("List resource shares failed: %s", err, exc_info=True)
+            return {"success": False}, 400
+
+
+@teams_ns.route("/admin/teams")
+class AllTeams(Resource):
+    method_decorators = []
+
+    def get(self):
+        """Global-admin oversight: every team with member counts."""
+        token = getattr(request, "decoded_token", None)
+        if not token:
+            return {"success": False}, 401
+        if not has_role(token, ROLE_ADMIN):
+            return {"success": False, "message": "Forbidden"}, 403
+        try:
+            with db_readonly() as conn:
+                teams = TeamsRepository(conn).list_all()
+            return make_response(jsonify({"success": True, "teams": teams}), 200)
+        except Exception as err:
+            logger.error("List all teams failed: %s", err, exc_info=True)
+            return {"success": False}, 400

docsgpt/api/user/tools/__init__.py

@@ -0,0 +1,6 @@
+"""Tools module."""
+
+from .mcp import tools_mcp_ns
+from .routes import tools_ns
+
+__all__ = ["tools_ns", "tools_mcp_ns"]

docsgpt/api/user/tools/mcp.py

@@ -0,0 +1,516 @@
+"""Tool management MCP server integration."""
+
+from urllib.parse import urlencode, urlparse
+
+from flask import current_app, jsonify, make_response, redirect, request
+from flask_restx import Namespace, Resource, fields
+
+from docsgpt.agents.tools.mcp_tool import MCPOAuthManager, MCPTool
+from docsgpt.api import api
+from docsgpt.api.user.tools.routes import transform_actions
+from docsgpt.cache import get_redis_instance
+from docsgpt.core.url_validation import SSRFError, validate_url
+from docsgpt.security.encryption import decrypt_credentials, encrypt_credentials
+from docsgpt.storage.db.repositories.connector_sessions import (
+    ConnectorSessionsRepository,
+)
+from docsgpt.storage.db.repositories.user_tools import UserToolsRepository
+from docsgpt.storage.db.session import db_readonly, db_session
+from docsgpt.utils import check_required_fields
+
+tools_mcp_ns = Namespace("tools", description="Tool management operations", path="/api")
+
+_ALLOWED_TRANSPORTS = {"auto", "sse", "http"}
+
+
+def _sanitize_mcp_transport(config):
+    """Normalise and validate the transport_type field.
+
+    Strips ``command`` / ``args`` keys that are only valid for local STDIO
+    transports and returns the cleaned transport type string.
+    """
+    transport_type = (config.get("transport_type") or "auto").lower()
+    if transport_type not in _ALLOWED_TRANSPORTS:
+        raise ValueError(f"Unsupported transport_type: {transport_type}")
+    config.pop("command", None)
+    config.pop("args", None)
+    config["transport_type"] = transport_type
+    return transport_type
+
+
+def _extract_auth_credentials(config):
+    """Build an ``auth_credentials`` dict from the raw MCP config."""
+    auth_credentials = {}
+    auth_type = config.get("auth_type", "none")
+
+    if auth_type == "api_key":
+        if config.get("api_key"):
+            auth_credentials["api_key"] = config["api_key"]
+        if config.get("api_key_header"):
+            auth_credentials["api_key_header"] = config["api_key_header"]
+    elif auth_type == "bearer":
+        if config.get("bearer_token"):
+            auth_credentials["bearer_token"] = config["bearer_token"]
+    elif auth_type == "basic":
+        if config.get("username"):
+            auth_credentials["username"] = config["username"]
+        if config.get("password"):
+            auth_credentials["password"] = config["password"]
+
+    return auth_credentials
+
+
+def _validate_mcp_server_url(config: dict) -> None:
+    """Validate the server_url in an MCP config to prevent SSRF.
+
+    Raises:
+        ValueError: If the URL is missing or points to a blocked address.
+    """
+    server_url = (config.get("server_url") or "").strip()
+    if not server_url:
+        raise ValueError("server_url is required")
+    try:
+        validate_url(server_url)
+    except SSRFError as exc:
+        raise ValueError(f"Invalid server URL: {exc}") from exc
+
+
+@tools_mcp_ns.route("/mcp_server/test")
+class TestMCPServerConfig(Resource):
+    @api.expect(
+        api.model(
+            "MCPServerTestModel",
+            {
+                "config": fields.Raw(
+                    required=True, description="MCP server configuration to test"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Test MCP server connection with provided configuration")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+
+        required_fields = ["config"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            config = data["config"]
+            try:
+                _sanitize_mcp_transport(config)
+            except ValueError:
+                return make_response(
+                    jsonify({"success": False, "error": "Unsupported transport_type"}),
+                    400,
+                )
+
+            _validate_mcp_server_url(config)
+
+            auth_credentials = _extract_auth_credentials(config)
+            test_config = config.copy()
+            test_config["auth_credentials"] = auth_credentials
+
+            mcp_tool = MCPTool(config=test_config, user_id=user)
+            result = mcp_tool.test_connection()
+
+            if result.get("requires_oauth"):
+                safe_result = {
+                    k: v
+                    for k, v in result.items()
+                    if k in ("success", "requires_oauth", "auth_url")
+                }
+                return make_response(jsonify(safe_result), 200)
+
+            if not result.get("success"):
+                current_app.logger.error(
+                    f"MCP connection test failed: {result.get('message')}"
+                )
+                return make_response(
+                    jsonify(
+                        {
+                            "success": False,
+                            "message": "Connection test failed",
+                            "tools_count": 0,
+                        }
+                    ),
+                    200,
+                )
+
+            safe_result = {
+                "success": True,
+                "message": result.get("message", "Connection successful"),
+                "tools_count": result.get("tools_count", 0),
+                "tools": result.get("tools", []),
+            }
+            return make_response(jsonify(safe_result), 200)
+        except ValueError as e:
+            current_app.logger.warning(f"Invalid MCP server test request: {e}")
+            return make_response(
+                jsonify({"success": False, "error": "Invalid MCP server configuration"}),
+                400,
+            )
+        except Exception as e:
+            current_app.logger.error(f"Error testing MCP server: {e}", exc_info=True)
+            return make_response(
+                jsonify({"success": False, "error": "Connection test failed"}),
+                500,
+            )
+
+
+@tools_mcp_ns.route("/mcp_server/save")
+class MCPServerSave(Resource):
+    @api.expect(
+        api.model(
+            "MCPServerSaveModel",
+            {
+                "id": fields.String(
+                    required=False, description="Tool ID for updates (optional)"
+                ),
+                "displayName": fields.String(
+                    required=True, description="Display name for the MCP server"
+                ),
+                "config": fields.Raw(
+                    required=True, description="MCP server configuration"
+                ),
+                "status": fields.Boolean(
+                    required=False, default=True, description="Tool status"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Create or update MCP server with automatic tool discovery")
+    def post(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        data = request.get_json()
+
+        required_fields = ["displayName", "config"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+        try:
+            config = data["config"]
+            try:
+                _sanitize_mcp_transport(config)
+            except ValueError:
+                return make_response(
+                    jsonify({"success": False, "error": "Unsupported transport_type"}),
+                    400,
+                )
+
+            _validate_mcp_server_url(config)
+
+            auth_credentials = _extract_auth_credentials(config)
+            auth_type = config.get("auth_type", "none")
+            mcp_config = config.copy()
+            mcp_config["auth_credentials"] = auth_credentials
+
+            if auth_type == "oauth":
+                if not config.get("oauth_task_id"):
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "error": "Connection not authorized. Please complete the OAuth authorization first.",
+                            }
+                        ),
+                        400,
+                    )
+                redis_client = get_redis_instance()
+                manager = MCPOAuthManager(redis_client)
+                result = manager.get_oauth_status(
+                    config["oauth_task_id"], user
+                )
+                if not result.get("status") == "completed":
+                    return make_response(
+                        jsonify(
+                            {
+                                "success": False,
+                                "error": "OAuth failed or not completed. Please try authorizing again.",
+                            }
+                        ),
+                        400,
+                    )
+                actions_metadata = result.get("tools", [])
+            elif auth_type == "none" or auth_credentials:
+                mcp_tool = MCPTool(config=mcp_config, user_id=user)
+                mcp_tool.discover_tools()
+                actions_metadata = mcp_tool.get_actions_metadata()
+            else:
+                raise Exception(
+                    "No valid credentials provided for the selected authentication type"
+                )
+            storage_config = config.copy()
+
+            tool_id = data.get("id")
+            existing_doc = None
+            existing_encrypted = None
+            if tool_id:
+                with db_readonly() as conn:
+                    repo = UserToolsRepository(conn)
+                    existing_doc = repo.get_any(tool_id, user)
+                if existing_doc and existing_doc.get("name") == "mcp_tool":
+                    existing_encrypted = (existing_doc.get("config") or {}).get(
+                        "encrypted_credentials"
+                    )
+                else:
+                    existing_doc = None
+
+            if auth_credentials:
+                if existing_encrypted:
+                    existing_secrets = decrypt_credentials(existing_encrypted, user)
+                    existing_secrets.update(auth_credentials)
+                    auth_credentials = existing_secrets
+                storage_config["encrypted_credentials"] = encrypt_credentials(
+                    auth_credentials, user
+                )
+            elif existing_encrypted:
+                storage_config["encrypted_credentials"] = existing_encrypted
+
+            for field in [
+                "api_key",
+                "bearer_token",
+                "username",
+                "password",
+                "api_key_header",
+                "redirect_uri",
+            ]:
+                storage_config.pop(field, None)
+            transformed_actions = transform_actions(actions_metadata)
+
+            display_name = data["displayName"]
+            description = f"MCP Server: {storage_config.get('server_url', 'Unknown')}"
+            status_bool = bool(data.get("status", True))
+
+            with db_session() as conn:
+                repo = UserToolsRepository(conn)
+                if existing_doc:
+                    repo.update(
+                        str(existing_doc["id"]), user,
+                        {
+                            "display_name": display_name,
+                            "custom_name": display_name,
+                            "description": description,
+                            "config": storage_config,
+                            "actions": transformed_actions,
+                            "status": status_bool,
+                        },
+                    )
+                    saved_id = str(existing_doc["id"])
+                    response_data = {
+                        "success": True,
+                        "id": saved_id,
+                        "message": f"MCP server updated successfully! Discovered {len(transformed_actions)} tools.",
+                        "tools_count": len(transformed_actions),
+                    }
+                else:
+                    # Fall back to find_by_user_and_name — the original
+                    # dual-write path also ran an existence check before
+                    # deciding between insert and update.
+                    existing_by_name = repo.find_by_user_and_name(user, "mcp_tool")
+                    if tool_id is None and existing_by_name and (
+                        (existing_by_name.get("config") or {}).get("server_url")
+                        == storage_config.get("server_url")
+                    ):
+                        repo.update(
+                            str(existing_by_name["id"]), user,
+                            {
+                                "display_name": display_name,
+                                "custom_name": display_name,
+                                "description": description,
+                                "config": storage_config,
+                                "actions": transformed_actions,
+                                "status": status_bool,
+                            },
+                        )
+                        saved_id = str(existing_by_name["id"])
+                        response_data = {
+                            "success": True,
+                            "id": saved_id,
+                            "message": f"MCP server updated successfully! Discovered {len(transformed_actions)} tools.",
+                            "tools_count": len(transformed_actions),
+                        }
+                    else:
+                        created = repo.create(
+                            user, "mcp_tool",
+                            config=storage_config,
+                            custom_name=display_name,
+                            display_name=display_name,
+                            description=description,
+                            config_requirements={},
+                            actions=transformed_actions,
+                            status=status_bool,
+                        )
+                        saved_id = str(created["id"])
+                        response_data = {
+                            "success": True,
+                            "id": saved_id,
+                            "message": f"MCP server created successfully! Discovered {len(transformed_actions)} tools.",
+                            "tools_count": len(transformed_actions),
+                        }
+                    if tool_id and existing_doc is None:
+                        # Client requested update on a non-existent tool id.
+                        return make_response(
+                            jsonify(
+                                {
+                                    "success": False,
+                                    "error": "Tool not found or access denied",
+                                }
+                            ),
+                            404,
+                        )
+            return make_response(jsonify(response_data), 200)
+        except ValueError as e:
+            current_app.logger.warning(f"Invalid MCP server save request: {e}")
+            return make_response(
+                jsonify({"success": False, "error": "Invalid MCP server configuration"}),
+                400,
+            )
+        except Exception as e:
+            current_app.logger.error(f"Error saving MCP server: {e}", exc_info=True)
+            return make_response(
+                jsonify({"success": False, "error": "Failed to save MCP server"}),
+                500,
+            )
+
+
+@tools_mcp_ns.route("/mcp_server/callback")
+class MCPOAuthCallback(Resource):
+    @api.expect(
+        api.model(
+            "MCPServerCallbackModel",
+            {
+                "code": fields.String(required=True, description="Authorization code"),
+                "state": fields.String(required=True, description="State parameter"),
+                "error": fields.String(
+                    required=False, description="Error message (if any)"
+                ),
+            },
+        )
+    )
+    @api.doc(
+        description="Handle OAuth callback by providing the authorization code and state"
+    )
+    def get(self):
+        code = request.args.get("code")
+        state = request.args.get("state")
+        error = request.args.get("error")
+
+        if error:
+            params = {
+                "status": "error",
+                "message": f"OAuth error: {error}. Please try again and make sure to grant all requested permissions, including offline access.",
+                "provider": "mcp_tool",
+            }
+            return redirect(f"/api/connectors/callback-status?{urlencode(params)}")
+        if not code or not state:
+            return redirect(
+                "/api/connectors/callback-status?status=error&message=Authorization+code+or+state+not+provided.+Please+complete+the+authorization+process+and+make+sure+to+grant+offline+access.&provider=mcp_tool"
+            )
+        try:
+            redis_client = get_redis_instance()
+            if not redis_client:
+                return redirect(
+                    "/api/connectors/callback-status?status=error&message=Internal+server+error:+Redis+not+available.&provider=mcp_tool"
+                )
+            manager = MCPOAuthManager(redis_client)
+            success = manager.handle_oauth_callback(state, code, error)
+            if success:
+                return redirect(
+                    "/api/connectors/callback-status?status=success&message=Authorization+code+received+successfully.+You+can+close+this+window.&provider=mcp_tool"
+                )
+            else:
+                return redirect(
+                    "/api/connectors/callback-status?status=error&message=OAuth+callback+failed.&provider=mcp_tool"
+                )
+        except Exception as e:
+            current_app.logger.error(
+                f"Error handling MCP OAuth callback: {str(e)}", exc_info=True
+            )
+            return redirect(
+                "/api/connectors/callback-status?status=error&message=Internal+server+error.&provider=mcp_tool"
+            )
+
+
+@tools_mcp_ns.route("/mcp_server/auth_status")
+class MCPAuthStatus(Resource):
+    @api.doc(
+        description="Batch check auth status for all MCP tools. "
+        "Lightweight DB-only check — no network calls to MCP servers."
+    )
+    def get(self):
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        try:
+            with db_readonly() as conn:
+                tools_repo = UserToolsRepository(conn)
+                sessions_repo = ConnectorSessionsRepository(conn)
+                all_tools = tools_repo.list_for_user(user)
+                mcp_tools = [t for t in all_tools if t.get("name") == "mcp_tool"]
+                if not mcp_tools:
+                    return make_response(
+                        jsonify({"success": True, "statuses": {}}), 200
+                    )
+
+                oauth_server_urls: dict = {}
+                statuses: dict = {}
+                for tool in mcp_tools:
+                    tool_id = str(tool["id"])
+                    config = tool.get("config") or {}
+                    auth_type = config.get("auth_type", "none")
+                    if auth_type == "oauth":
+                        server_url = config.get("server_url", "")
+                        if server_url:
+                            parsed = urlparse(server_url)
+                            base_url = f"{parsed.scheme}://{parsed.netloc}"
+                            oauth_server_urls[tool_id] = base_url
+                        else:
+                            statuses[tool_id] = "needs_auth"
+                    else:
+                        statuses[tool_id] = "configured"
+
+                if oauth_server_urls:
+                    # Look up a session per distinct base URL. MCP sessions
+                    # are stored with ``provider = "mcp:<server_url>"``
+                    # and the URL in ``server_url``; reuse the repo's
+                    # per-URL accessor rather than an ad-hoc $in query.
+                    url_has_tokens: dict = {}
+                    for base_url in set(oauth_server_urls.values()):
+                        session = sessions_repo.get_by_user_and_server_url(
+                            user, base_url,
+                        )
+                        tokens = (
+                            (session or {}).get("session_data", {}) or {}
+                        ).get("tokens", {}) or {}
+                        # MCP code also stashes tokens into token_info on
+                        # the row; consider either present as "connected".
+                        token_info = (session or {}).get("token_info") or {}
+                        url_has_tokens[base_url] = bool(
+                            tokens.get("access_token")
+                            or token_info.get("access_token")
+                        )
+
+                    for tool_id, base_url in oauth_server_urls.items():
+                        if url_has_tokens.get(base_url):
+                            statuses[tool_id] = "connected"
+                        else:
+                            statuses[tool_id] = "needs_auth"
+
+            return make_response(jsonify({"success": True, "statuses": statuses}), 200)
+        except Exception as e:
+            current_app.logger.error(
+                "Error checking MCP auth status: %s", e, exc_info=True
+            )
+            return make_response(
+                jsonify({"success": False, "error": "Failed to check auth status"}),
+                500,
+            )

docsgpt/api/user/utils.py

@@ -0,0 +1,75 @@
+"""Centralized utilities for API routes.
+
+Post-Mongo-cutover slim: the old Mongo-shaped helpers (``validate_object_id``,
+``check_resource_ownership``, ``paginated_response``, ``serialize_object_id``,
+``safe_db_operation``, ``validate_enum``, ``extract_sort_params``) have been
+removed — they carried ``bson`` / ``pymongo`` imports and had zero callers.
+"""
+
+from functools import wraps
+from typing import Callable, Optional
+
+from flask import (
+    Response,
+    jsonify,
+    make_response,
+    request,
+)
+
+
+def get_user_id() -> Optional[str]:
+    """Extract user ID from decoded JWT token, or None if unauthenticated."""
+    decoded_token = getattr(request, "decoded_token", None)
+    return decoded_token.get("sub") if decoded_token else None
+
+
+def require_auth(func: Callable) -> Callable:
+    """Decorator to require authentication. Returns 401 when absent."""
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        user_id = get_user_id()
+        if not user_id:
+            return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401)
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def success_response(
+    data=None, message: Optional[str] = None, status: int = 200
+) -> Response:
+    """Shape a successful JSON response."""
+    body = {"success": True}
+    if data is not None:
+        body["data"] = data
+    if message is not None:
+        body["message"] = message
+    return make_response(jsonify(body), status)
+
+
+def error_response(message: str, status: int = 400, **kwargs) -> Response:
+    """Shape an error JSON response; any kwargs are merged into the body."""
+    body = {"success": False, "error": message, **kwargs}
+    return make_response(jsonify(body), status)
+
+
+def require_fields(required: list) -> Callable:
+    """Decorator: return 400 if any listed field is missing/falsy in the JSON body."""
+
+    def decorator(func: Callable) -> Callable:
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            data = request.get_json()
+            if not data:
+                return error_response("Request body required")
+            missing = [field for field in required if not data.get(field)]
+            if missing:
+                return error_response(
+                    f"Missing required fields: {', '.join(missing)}"
+                )
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator

docsgpt/api/user/workflows/__init__.py

@@ -0,0 +1,3 @@
+from .routes import workflows_ns
+
+__all__ = ["workflows_ns"]

docsgpt/api/user/workflows/routes.py

@@ -0,0 +1,593 @@
+"""Workflow management routes."""
+
+from typing import Any, Dict, List, Optional, Set
+
+from flask import current_app, request
+from flask_restx import Namespace, Resource
+
+from docsgpt.agents.workflows.cel_evaluator import (
+    CelEvaluationError,
+    validate_cel_expression,
+)
+from docsgpt.storage.db.base_repository import looks_like_uuid
+from docsgpt.storage.db.repositories.workflow_edges import WorkflowEdgesRepository
+from docsgpt.storage.db.repositories.workflow_nodes import WorkflowNodesRepository
+from docsgpt.storage.db.repositories.workflows import WorkflowsRepository
+from docsgpt.storage.db.session import db_readonly, db_session
+from docsgpt.core.json_schema_utils import (
+    JsonSchemaValidationError,
+    normalize_json_schema_payload,
+)
+from docsgpt.core.model_utils import get_model_capabilities
+from docsgpt.api.user.utils import (
+    error_response,
+    get_user_id,
+    require_auth,
+    require_fields,
+    success_response,
+)
+
+workflows_ns = Namespace("workflows", path="/api")
+
+
+def _workflow_error_response(message: str, err: Exception):
+    current_app.logger.error(f"{message}: {err}", exc_info=True)
+    return error_response(message)
+
+
+def _resolve_workflow(repo: WorkflowsRepository, workflow_id: str, user_id: str):
+    """Resolve a workflow by UUID or legacy Mongo id, scoped to user."""
+    if not workflow_id:
+        return None
+    if looks_like_uuid(workflow_id):
+        row = repo.get(workflow_id, user_id)
+        if row is not None:
+            return row
+    return repo.get_by_legacy_id(workflow_id, user_id)
+
+
+def _write_graph(
+    conn,
+    pg_workflow_id: str,
+    graph_version: int,
+    nodes_data: List[Dict],
+    edges_data: List[Dict],
+) -> List[Dict]:
+    """Bulk-create nodes + edges for one graph version. Uses ON CONFLICT upsert.
+
+    Edges arrive with source/target as user-provided node-id strings. We
+    insert nodes first, capture their ``node_id → UUID`` map, then
+    translate edges before insertion. Edges referencing missing nodes are
+    dropped with a warning.
+    """
+    nodes_repo = WorkflowNodesRepository(conn)
+    edges_repo = WorkflowEdgesRepository(conn)
+
+    if nodes_data:
+        created_nodes = nodes_repo.bulk_create(
+            pg_workflow_id, graph_version,
+            [
+                {
+                    "node_id": n["id"],
+                    "node_type": n["type"],
+                    "title": n.get("title", ""),
+                    "description": n.get("description", ""),
+                    "position": n.get("position", {"x": 0, "y": 0}),
+                    "config": n.get("data", {}),
+                }
+                for n in nodes_data
+            ],
+        )
+        node_uuid_by_str = {n["node_id"]: n["id"] for n in created_nodes}
+    else:
+        created_nodes = []
+        node_uuid_by_str = {}
+
+    if edges_data:
+        translated_edges: List[Dict] = []
+        for e in edges_data:
+            src = e.get("source")
+            tgt = e.get("target")
+            from_uuid = node_uuid_by_str.get(src)
+            to_uuid = node_uuid_by_str.get(tgt)
+            if not from_uuid or not to_uuid:
+                current_app.logger.warning(
+                    "Workflow graph write: dropping edge %s; node refs unresolved "
+                    "(source=%s, target=%s)",
+                    e.get("id"), src, tgt,
+                )
+                continue
+            translated_edges.append({
+                "edge_id": e["id"],
+                "from_node_id": from_uuid,
+                "to_node_id": to_uuid,
+                "source_handle": e.get("sourceHandle"),
+                "target_handle": e.get("targetHandle"),
+            })
+        if translated_edges:
+            edges_repo.bulk_create(
+                pg_workflow_id, graph_version, translated_edges,
+            )
+
+    return created_nodes
+
+
+def serialize_workflow(w: Dict) -> Dict:
+    """Serialize workflow row to API response format."""
+    created_at = w.get("created_at")
+    updated_at = w.get("updated_at")
+    return {
+        "id": str(w["id"]),
+        "name": w.get("name"),
+        "description": w.get("description"),
+        "created_at": created_at.isoformat() if hasattr(created_at, "isoformat") else created_at,
+        "updated_at": updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at,
+    }
+
+
+def serialize_node(n: Dict) -> Dict:
+    """Serialize workflow node row to API response format."""
+    return {
+        "id": n["node_id"],
+        "type": n["node_type"],
+        "title": n.get("title"),
+        "description": n.get("description"),
+        "position": n.get("position"),
+        "data": n.get("config", {}) or {},
+    }
+
+
+def serialize_edge(e: Dict) -> Dict:
+    """Serialize workflow edge row to API response format."""
+    return {
+        "id": e["edge_id"],
+        "source": e.get("source_id"),
+        "target": e.get("target_id"),
+        "sourceHandle": e.get("source_handle"),
+        "targetHandle": e.get("target_handle"),
+    }
+
+
+def get_workflow_graph_version(workflow: Dict) -> int:
+    """Get current graph version with fallback."""
+    raw_version = workflow.get("current_graph_version", 1)
+    try:
+        version = int(raw_version)
+        return version if version > 0 else 1
+    except (ValueError, TypeError):
+        return 1
+
+
+def validate_json_schema_payload(
+    json_schema: Any,
+) -> tuple[Optional[Dict[str, Any]], Optional[str]]:
+    """Validate and normalize optional JSON schema payload for structured output."""
+    if json_schema is None:
+        return None, None
+    try:
+        return normalize_json_schema_payload(json_schema), None
+    except JsonSchemaValidationError as exc:
+        return None, str(exc)
+
+
+def normalize_agent_node_json_schemas(nodes: List[Dict]) -> List[Dict]:
+    """Normalize agent-node JSON schema payloads before persistence."""
+    normalized_nodes: List[Dict] = []
+    for node in nodes:
+        if not isinstance(node, dict):
+            normalized_nodes.append(node)
+            continue
+
+        normalized_node = dict(node)
+        if normalized_node.get("type") != "agent":
+            normalized_nodes.append(normalized_node)
+            continue
+
+        raw_config = normalized_node.get("data")
+        if not isinstance(raw_config, dict) or "json_schema" not in raw_config:
+            normalized_nodes.append(normalized_node)
+            continue
+
+        normalized_config = dict(raw_config)
+        try:
+            normalized_config["json_schema"] = normalize_json_schema_payload(
+                raw_config.get("json_schema")
+            )
+        except JsonSchemaValidationError:
+            # Validation runs before normalization; keep original on unexpected shape.
+            normalized_config["json_schema"] = raw_config.get("json_schema")
+        normalized_node["data"] = normalized_config
+        normalized_nodes.append(normalized_node)
+
+    return normalized_nodes
+
+
+def validate_workflow_structure(
+    nodes: List[Dict], edges: List[Dict], user_id: str | None = None
+) -> List[str]:
+    """Validate workflow graph structure.
+
+    ``user_id`` is required so per-user BYOM custom-model UUIDs resolve
+    when checking each agent node's structured-output capability.
+    """
+    errors = []
+
+    if not nodes:
+        errors.append("Workflow must have at least one node")
+        return errors
+
+    start_nodes = [n for n in nodes if n.get("type") == "start"]
+    if len(start_nodes) != 1:
+        errors.append("Workflow must have exactly one start node")
+
+    end_nodes = [n for n in nodes if n.get("type") == "end"]
+    if not end_nodes:
+        errors.append("Workflow must have at least one end node")
+
+    node_ids = {n.get("id") for n in nodes}
+    node_map = {n.get("id"): n for n in nodes}
+    end_ids = {n.get("id") for n in end_nodes}
+
+    for edge in edges:
+        source_id = edge.get("source")
+        target_id = edge.get("target")
+        if source_id not in node_ids:
+            errors.append(f"Edge references non-existent source: {source_id}")
+        if target_id not in node_ids:
+            errors.append(f"Edge references non-existent target: {target_id}")
+
+    if start_nodes:
+        start_id = start_nodes[0].get("id")
+        if not any(e.get("source") == start_id for e in edges):
+            errors.append("Start node must have at least one outgoing edge")
+
+    condition_nodes = [n for n in nodes if n.get("type") == "condition"]
+    for cnode in condition_nodes:
+        cnode_id = cnode.get("id")
+        cnode_title = cnode.get("title", cnode_id)
+        outgoing = [e for e in edges if e.get("source") == cnode_id]
+        if len(outgoing) < 2:
+            errors.append(
+                f"Condition node '{cnode_title}' must have at least 2 outgoing edges"
+            )
+        node_data = cnode.get("data", {}) or {}
+        cases = node_data.get("cases", [])
+        if not isinstance(cases, list):
+            cases = []
+        if not cases or not any(
+            isinstance(c, dict) and str(c.get("expression", "")).strip() for c in cases
+        ):
+            errors.append(
+                f"Condition node '{cnode_title}' must have at least one case with an expression"
+            )
+
+        case_handles: Set[str] = set()
+        duplicate_case_handles: Set[str] = set()
+        for case in cases:
+            if not isinstance(case, dict):
+                continue
+            raw_handle = case.get("sourceHandle", "")
+            handle = raw_handle.strip() if isinstance(raw_handle, str) else ""
+            if not handle:
+                errors.append(
+                    f"Condition node '{cnode_title}' has a case without a branch handle"
+                )
+                continue
+            if handle in case_handles:
+                duplicate_case_handles.add(handle)
+            case_handles.add(handle)
+
+        for handle in duplicate_case_handles:
+            errors.append(
+                f"Condition node '{cnode_title}' has duplicate case handle '{handle}'"
+            )
+
+        outgoing_by_handle: Dict[str, List[Dict]] = {}
+        for out_edge in outgoing:
+            raw_handle = out_edge.get("sourceHandle", "")
+            handle = raw_handle.strip() if isinstance(raw_handle, str) else ""
+            outgoing_by_handle.setdefault(handle, []).append(out_edge)
+
+        for handle, handle_edges in outgoing_by_handle.items():
+            if not handle:
+                errors.append(
+                    f"Condition node '{cnode_title}' has an outgoing edge without sourceHandle"
+                )
+                continue
+            if handle != "else" and handle not in case_handles:
+                errors.append(
+                    f"Condition node '{cnode_title}' has a connection from unknown branch '{handle}'"
+                )
+            if len(handle_edges) > 1:
+                errors.append(
+                    f"Condition node '{cnode_title}' has multiple outgoing edges from branch '{handle}'"
+                )
+
+        if "else" not in outgoing_by_handle:
+            errors.append(f"Condition node '{cnode_title}' must have an 'else' branch")
+
+        for case in cases:
+            if not isinstance(case, dict):
+                continue
+            raw_handle = case.get("sourceHandle", "")
+            handle = raw_handle.strip() if isinstance(raw_handle, str) else ""
+            if not handle:
+                continue
+
+            raw_expression = case.get("expression", "")
+            has_expression = isinstance(raw_expression, str) and bool(
+                raw_expression.strip()
+            )
+            has_outgoing = bool(outgoing_by_handle.get(handle))
+            if has_expression and not has_outgoing:
+                errors.append(
+                    f"Condition node '{cnode_title}' case '{handle}' has an expression but no outgoing edge"
+                )
+            if not has_expression and has_outgoing:
+                errors.append(
+                    f"Condition node '{cnode_title}' case '{handle}' has an outgoing edge but no expression"
+                )
+            if has_expression:
+                try:
+                    validate_cel_expression(raw_expression)
+                except CelEvaluationError as exc:
+                    errors.append(
+                        f"Condition node '{cnode_title}' case '{handle}' "
+                        f"has an invalid expression: {exc}"
+                    )
+
+        for handle, handle_edges in outgoing_by_handle.items():
+            if not handle:
+                continue
+            for out_edge in handle_edges:
+                target = out_edge.get("target")
+                if target and not _can_reach_end(target, edges, node_map, end_ids):
+                    errors.append(
+                        f"Branch '{handle}' of condition '{cnode_title}' "
+                        f"must eventually reach an end node"
+                    )
+
+    # Set State nodes were validated nowhere. A node whose CEL does not compile
+    # saves and publishes clean, then aborts the run on first execution — the
+    # user only finds out as a failed answer, with no pointer to the node.
+    state_nodes = [n for n in nodes if n.get("type") == "state"]
+    for snode in state_nodes:
+        snode_title = snode.get("title", snode.get("id"))
+        node_data = snode.get("data", {}) or {}
+        # The builder writes state config under ``data.config`` but other
+        # payloads keep it flat on ``data``; the engine reads
+        # ``node.config.get("config", node.config)`` for exactly this reason,
+        # so accept both rather than silently validating nothing.
+        nested = node_data.get("config")
+        source = nested if isinstance(nested, dict) else node_data
+        operations = source.get("operations", [])
+        if not isinstance(operations, list):
+            continue
+        for index, operation in enumerate(operations):
+            if not isinstance(operation, dict):
+                continue
+            raw_expression = operation.get("expression", "")
+            target_variable = operation.get("target_variable", "")
+            has_expression = isinstance(raw_expression, str) and bool(
+                raw_expression.strip()
+            )
+            has_target = isinstance(target_variable, str) and bool(
+                target_variable.strip()
+            )
+            # The engine silently skips an operation missing either half
+            # (workflow_engine._execute_state_node), so downstream nodes read a
+            # variable that never gets set. Surface it at save time instead.
+            if has_expression and not has_target:
+                errors.append(
+                    f"Set State node '{snode_title}' operation {index + 1} "
+                    f"has an expression but no target variable"
+                )
+            if has_target and not has_expression:
+                errors.append(
+                    f"Set State node '{snode_title}' operation {index + 1} "
+                    f"has a target variable but no expression"
+                )
+            if has_expression:
+                try:
+                    validate_cel_expression(raw_expression)
+                except CelEvaluationError as exc:
+                    errors.append(
+                        f"Set State node '{snode_title}' operation {index + 1} "
+                        f"has an invalid expression: {exc}"
+                    )
+
+    agent_nodes = [n for n in nodes if n.get("type") == "agent"]
+    for agent_node in agent_nodes:
+        agent_title = agent_node.get("title", agent_node.get("id", "unknown"))
+        raw_config = agent_node.get("data", {}) or {}
+        if not isinstance(raw_config, dict):
+            errors.append(f"Agent node '{agent_title}' has invalid configuration")
+            continue
+        normalized_schema, schema_error = validate_json_schema_payload(
+            raw_config.get("json_schema")
+        )
+        has_json_schema = normalized_schema is not None
+
+        model_id = raw_config.get("model_id")
+        if has_json_schema and isinstance(model_id, str) and model_id.strip():
+            capabilities = get_model_capabilities(model_id.strip(), user_id=user_id)
+            if capabilities and not capabilities.get("supports_structured_output", False):
+                errors.append(
+                    f"Agent node '{agent_title}' selected model does not support structured output"
+                )
+        if schema_error:
+            errors.append(f"Agent node '{agent_title}' JSON schema {schema_error}")
+
+    code_nodes = [n for n in nodes if n.get("type") == "code"]
+    for code_node in code_nodes:
+        code_title = code_node.get("title", code_node.get("id", "unknown"))
+        raw_config = code_node.get("data", {}) or {}
+        if not isinstance(raw_config, dict):
+            errors.append(f"Code node '{code_title}' has invalid configuration")
+            continue
+        if not str(raw_config.get("code", "")).strip():
+            errors.append(f"Code node '{code_title}' must have code to execute")
+        _, schema_error = validate_json_schema_payload(raw_config.get("json_schema"))
+        if schema_error:
+            errors.append(f"Code node '{code_title}' JSON schema {schema_error}")
+
+    for node in nodes:
+        if not node.get("id"):
+            errors.append("All nodes must have an id")
+        if not node.get("type"):
+            errors.append(f"Node {node.get('id', 'unknown')} must have a type")
+
+    return errors
+
+
+def _can_reach_end(
+    node_id: str, edges: List[Dict], node_map: Dict, end_ids: set, visited: set = None
+) -> bool:
+    if visited is None:
+        visited = set()
+    if node_id in end_ids:
+        return True
+    if node_id in visited or node_id not in node_map:
+        return False
+    visited.add(node_id)
+    outgoing = [e.get("target") for e in edges if e.get("source") == node_id]
+    return any(_can_reach_end(t, edges, node_map, end_ids, visited) for t in outgoing if t)
+
+
+@workflows_ns.route("/workflows")
+class WorkflowList(Resource):
+
+    @require_auth
+    @require_fields(["name"])
+    def post(self):
+        """Create a new workflow with nodes and edges."""
+        user_id = get_user_id()
+        data = request.get_json()
+
+        name = data.get("name", "").strip()
+        description = data.get("description", "")
+        nodes_data = data.get("nodes", [])
+        edges_data = data.get("edges", [])
+
+        validation_errors = validate_workflow_structure(
+            nodes_data, edges_data, user_id=user_id
+        )
+        if validation_errors:
+            return error_response(
+                "Workflow validation failed", errors=validation_errors
+            )
+        nodes_data = normalize_agent_node_json_schemas(nodes_data)
+
+        try:
+            with db_session() as conn:
+                repo = WorkflowsRepository(conn)
+                workflow = repo.create(user_id, name, description=description)
+                pg_workflow_id = str(workflow["id"])
+                _write_graph(conn, pg_workflow_id, 1, nodes_data, edges_data)
+        except Exception as err:
+            return _workflow_error_response("Failed to create workflow", err)
+
+        return success_response({"id": pg_workflow_id}, 201)
+
+
+@workflows_ns.route("/workflows/<string:workflow_id>")
+class WorkflowDetail(Resource):
+
+    @require_auth
+    def get(self, workflow_id: str):
+        """Get workflow details with nodes and edges."""
+        user_id = get_user_id()
+        try:
+            with db_readonly() as conn:
+                repo = WorkflowsRepository(conn)
+                workflow = _resolve_workflow(repo, workflow_id, user_id)
+                if workflow is None:
+                    return error_response("Workflow not found", 404)
+                pg_workflow_id = str(workflow["id"])
+                graph_version = get_workflow_graph_version(workflow)
+                nodes = WorkflowNodesRepository(conn).find_by_version(
+                    pg_workflow_id, graph_version,
+                )
+                edges = WorkflowEdgesRepository(conn).find_by_version(
+                    pg_workflow_id, graph_version,
+                )
+        except Exception as err:
+            return _workflow_error_response("Failed to fetch workflow", err)
+
+        return success_response(
+            {
+                "workflow": serialize_workflow(workflow),
+                "nodes": [serialize_node(n) for n in nodes],
+                "edges": [serialize_edge(e) for e in edges],
+            }
+        )
+
+    @require_auth
+    @require_fields(["name"])
+    def put(self, workflow_id: str):
+        """Update workflow and replace nodes/edges."""
+        user_id = get_user_id()
+        data = request.get_json()
+        name = data.get("name", "").strip()
+        description = data.get("description", "")
+        nodes_data = data.get("nodes", [])
+        edges_data = data.get("edges", [])
+
+        validation_errors = validate_workflow_structure(
+            nodes_data, edges_data, user_id=user_id
+        )
+        if validation_errors:
+            return error_response(
+                "Workflow validation failed", errors=validation_errors
+            )
+        nodes_data = normalize_agent_node_json_schemas(nodes_data)
+
+        try:
+            with db_session() as conn:
+                repo = WorkflowsRepository(conn)
+                workflow = _resolve_workflow(repo, workflow_id, user_id)
+                if workflow is None:
+                    return error_response("Workflow not found", 404)
+                pg_workflow_id = str(workflow["id"])
+                current_graph_version = get_workflow_graph_version(workflow)
+                next_graph_version = current_graph_version + 1
+
+                _write_graph(
+                    conn, pg_workflow_id, next_graph_version,
+                    nodes_data, edges_data,
+                )
+                repo.update(
+                    pg_workflow_id, user_id,
+                    {
+                        "name": name,
+                        "description": description,
+                        "current_graph_version": next_graph_version,
+                    },
+                )
+                WorkflowNodesRepository(conn).delete_other_versions(
+                    pg_workflow_id, next_graph_version,
+                )
+                WorkflowEdgesRepository(conn).delete_other_versions(
+                    pg_workflow_id, next_graph_version,
+                )
+        except Exception as err:
+            return _workflow_error_response("Failed to update workflow", err)
+
+        return success_response()
+
+    @require_auth
+    def delete(self, workflow_id: str):
+        """Delete workflow and its graph."""
+        user_id = get_user_id()
+        try:
+            with db_session() as conn:
+                repo = WorkflowsRepository(conn)
+                workflow = _resolve_workflow(repo, workflow_id, user_id)
+                if workflow is None:
+                    return error_response("Workflow not found", 404)
+                # ON DELETE CASCADE on workflow_nodes/edges cleans children.
+                repo.delete(str(workflow["id"]), user_id)
+        except Exception as err:
+            return _workflow_error_response("Failed to delete workflow", err)
+
+        return success_response()