Merge pull request #2812 from arc53-machine/feat/personal-access-tokens

feat: personal access tokens (scoped API tokens for CLI and CI/CD)

docsgpt/api/admin/routes.py

@@ -23,6 +23,9 @@ from docsgpt.api.user.authz import ROLE_ADMIN, admin_required
 from docsgpt.storage.db.repositories.admin_stats import AdminStatsRepository
 from docsgpt.storage.db.repositories.auth_events import AuthEventsRepository
 from docsgpt.storage.db.repositories.device_audit_log import DeviceAuditLogRepository
+from docsgpt.storage.db.repositories.personal_access_tokens import (
+    PersonalAccessTokensRepository,
+)
 from docsgpt.storage.db.repositories.token_usage import TokenUsageRepository
 from docsgpt.storage.db.repositories.user_roles import UserRolesRepository
 from docsgpt.storage.db.repositories.users import UsersRepository
@@ -246,12 +249,30 @@ class AdminUserSessionsResource(Resource):
         """Force-logout: revoke the user's live OIDC sessions (best-effort)."""
         ok = denylist.deny_user(user_id)
         with db_session() as conn:
+            # A forced logout that left API credentials alive would not be one.
+            revoked_token_ids = PersonalAccessTokensRepository(conn).revoke_all_for_user(
+                user_id, reason="admin_sessions_revoked"
+            )
+            # One pat_revoked event per token, like every other revocation path.
+            for token_id in revoked_token_ids:
+                AuthEventsRepository(conn).insert(
+                    user_id,
+                    "pat_revoked",
+                    ip=request.remote_addr,
+                    user_agent=request.headers.get("User-Agent"),
+                    metadata={"token_id": token_id, "by": _actor(), "via": "admin_sessions_revoked"},
+                )
             AuthEventsRepository(conn).insert(
                 user_id,
                 "admin_sessions_revoked",
                 ip=request.remote_addr,
                 user_agent=request.headers.get("User-Agent"),
-                metadata={"by": _actor(), "via": "admin_api", "persisted": ok},
+                metadata={
+                    "by": _actor(),
+                    "via": "admin_api",
+                    "persisted": ok,
+                    "personal_access_tokens_revoked": len(revoked_token_ids),
+                },
             )
         return make_response(jsonify({"success": True, "revoked": ok}), 200)
 

docsgpt/api/asgi_auth.py

@@ -9,34 +9,63 @@ from __future__ import annotations
 
 import uuid
 from contextvars import Token
-from typing import Optional, Tuple
+from typing import Optional, Sequence, Tuple, Union
 
 import anyio
 from starlette.requests import Request
 from starlette.responses import JSONResponse
 
 from docsgpt.api.oidc.denylist import is_denied as oidc_session_denied
+from docsgpt.api.pat.tokens import is_pat
 from docsgpt.auth import handle_auth
 from docsgpt.core import log_context
 from docsgpt.core.settings import settings
 
 
-async def authenticate(request: Request) -> Tuple[Optional[dict], Optional[JSONResponse]]:
+async def authenticate(
+    request: Request, *, pat_scope: Union[str, Sequence[str], None] = None
+) -> Tuple[Optional[dict], Optional[JSONResponse]]:
     """Decode the caller's JWT the way Flask's ``authenticate_request`` does.
 
     Args:
         request: The incoming Starlette request.
+        pat_scope: Scope (or any-of scopes) a personal access token needs for this route. Left
+            unset, the route rejects PATs outright (deny by default, matching
+            the Flask rule table in ``docsgpt/api/pat/rules.py``). A token with
+            a resource filter is always rejected.
 
     Returns:
         tuple: ``(claims, None)`` for an authenticated caller, ``(None, None)``
         when no token was sent (the route decides whether that is allowed), or
         ``(None, response)`` carrying the 401 to return.
     """
-    decoded = handle_auth(request)
+    # A personal access token resolves against Postgres; keep that sync read off the event loop.
+    decoded = await anyio.to_thread.run_sync(handle_auth, request)
     if not decoded:
         return None, None
     if "error" in decoded:
         return None, JSONResponse(decoded, status_code=401)
+    if is_pat(decoded):
+        # A PAT lookup already excludes revoked tokens and deactivated users,
+        # so the session denylist below does not apply to it.
+        accepted = (pat_scope,) if isinstance(pat_scope, str) else tuple(pat_scope or ())
+        if not set(accepted).intersection(decoded.get("scopes") or []):
+            return None, JSONResponse(
+                {"success": False, "message": "Token lacks the required scope", "error": "insufficient_scope"},
+                status_code=403,
+            )
+        if decoded.get("resource_filter"):
+            # These routes sit outside the Flask rule table and cannot tie what
+            # they serve to an allowlist, so a restricted token is kept out.
+            return None, JSONResponse(
+                {
+                    "success": False,
+                    "message": "This endpoint is not available to a resource-restricted token",
+                    "error": "resource_not_allowed",
+                },
+                status_code=403,
+            )
+        return decoded, None
     # The denylist is a sync Redis read; keep it off the event loop.
     if settings.AUTH_TYPE == "oidc" and await anyio.to_thread.run_sync(oidc_session_denied, decoded):
         return None, JSONResponse(

docsgpt/api/async_sse.py

@@ -25,6 +25,7 @@ from starlette.responses import Response
 from starlette.routing import Route
 
 from docsgpt.api.asgi_auth import authenticate, bind_log_context, json_error
+from docsgpt.api.pat.rules import MESSAGE_REPLAY_SCOPES
 from docsgpt.api.asgi_stream import sse_response
 from docsgpt.core.settings import settings
 from docsgpt.storage.db.session import db_readonly
@@ -94,7 +95,8 @@ async def stream_message_events(request: Request) -> Response:
     """
     # Same JWT decoder and OIDC revocation check as the Flask routes. With
     # AUTH_TYPE unset the caller resolves to ``{"sub": "local"}``.
-    decoded, error = await authenticate(request)
+    # Same scopes as its Flask sibling GET /api/messages/<id>/tail.
+    decoded, error = await authenticate(request, pat_scope=MESSAGE_REPLAY_SCOPES)
     if error is not None:
         return error
     user_id = decoded.get("sub") if isinstance(decoded, dict) else None

docsgpt/api/pat/routes.py

@@ -0,0 +1,313 @@
+"""Personal access token management.
+
+``/api/user/tokens`` lets a signed-in user list, create and revoke their own
+tokens; ``/api/admin/...`` lets an admin inspect and revoke anyone's. None of
+these routes accept a PAT (see ``docsgpt/api/pat/rules.py``), so a leaked
+token can neither mint a replacement nor widen itself.
+"""
+
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timezone
+
+from flask import jsonify, make_response, request
+from flask_restx import Namespace, Resource
+from sqlalchemy.exc import IntegrityError
+
+from docsgpt.api.pat.tokens import (
+    FILTERABLE_FAMILIES,
+    SCOPES,
+    auth_type_supports_pats,
+    generate_token,
+    is_pat,
+    normalize_resource_filter,
+    normalize_scopes,
+    renewal_lifetime_days,
+    resolve_expiry,
+)
+from docsgpt.api.user.authz import admin_required
+from docsgpt.core.settings import settings
+from docsgpt.storage.db.repositories.auth_events import AuthEventsRepository
+from docsgpt.storage.db.repositories.personal_access_tokens import (
+    PersonalAccessTokensRepository,
+)
+from docsgpt.storage.db.session import db_readonly, db_session
+
+pat_ns = Namespace("tokens", description="Personal access tokens", path="/api")
+
+_MAX_NAME_LENGTH = 100
+
+
+def _error(message: str, status: int):
+    return make_response(jsonify({"success": False, "message": message}), status)
+
+
+def _session_user_id():
+    """The caller's id, or ``None`` for anonymous and PAT callers alike."""
+    decoded = getattr(request, "decoded_token", None)
+    if not decoded or is_pat(decoded):
+        return None
+    return decoded.get("sub")
+
+
+def _valid_uuid(value: str) -> bool:
+    """Canonical form only: ``uuid.UUID`` also accepts ``urn:uuid:…`` and braces, which Postgres does not."""
+    try:
+        return str(uuid.UUID(str(value))) == str(value).lower()
+    except (ValueError, AttributeError, TypeError):
+        return False
+
+
+def _is_expired(expires_at) -> bool:
+    if not expires_at:
+        return False
+    try:
+        moment = expires_at if isinstance(expires_at, datetime) else datetime.fromisoformat(str(expires_at))
+    except ValueError:
+        return False
+    if moment.tzinfo is None:
+        moment = moment.replace(tzinfo=timezone.utc)
+    return moment <= datetime.now(timezone.utc)
+
+
+def serialize_token(row: dict) -> dict:
+    # The row keeps status 'active' until someone revokes it; report what is true for a caller.
+    status = "expired" if row["status"] == "active" and _is_expired(row.get("expires_at")) else row["status"]
+    return {
+        "id": str(row["id"]),
+        "name": row["name"],
+        "token_prefix": row["token_prefix"],
+        "scopes": list(row.get("scopes") or []),
+        "resource_filter": row.get("resource_filter") or {},
+        "status": status,
+        "expires_at": row.get("expires_at"),
+        "last_used_at": row.get("last_used_at"),
+        "last_used_ip": row.get("last_used_ip"),
+        "created_at": row.get("created_at"),
+        "regenerated_at": row.get("regenerated_at"),
+        "revoked_at": row.get("revoked_at"),
+    }
+
+
+def _policy() -> dict:
+    return {
+        "enabled": auth_type_supports_pats(),
+        "default_lifetime_days": settings.PAT_DEFAULT_LIFETIME_DAYS,
+        "max_lifetime_days": settings.PAT_MAX_LIFETIME_DAYS,
+        "allow_non_expiring": settings.PAT_ALLOW_NON_EXPIRING,
+        "max_per_user": settings.PAT_MAX_PER_USER,
+        "filterable_families": list(FILTERABLE_FAMILIES),
+    }
+
+
+@pat_ns.route("/user/tokens")
+class PersonalAccessTokens(Resource):
+    def get(self):
+        """List the caller's tokens with the scope catalog and the server's token policy."""
+        user_id = _session_user_id()
+        if not user_id:
+            return _error("Authentication required", 401)
+        with db_readonly() as conn:
+            rows = PersonalAccessTokensRepository(conn).list_for_user(user_id)
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "tokens": [serialize_token(r) for r in rows],
+                    "scopes": [{"name": k, "description": v} for k, v in SCOPES.items()],
+                    "policy": _policy(),
+                }
+            ),
+            200,
+        )
+
+    def post(self):
+        """Create a token. The plaintext ``token`` is returned here and never again."""
+        user_id = _session_user_id()
+        if not user_id:
+            return _error("Authentication required", 401)
+        if not auth_type_supports_pats():
+            return _error("Personal access tokens are not available on this server", 403)
+
+        body = request.get_json(silent=True)
+        if body is None:
+            body = {}
+        if not isinstance(body, dict):
+            return _error("Request body must be a JSON object", 400)
+        name = body.get("name")
+        if not isinstance(name, str) or not name.strip():
+            return _error("name is required", 400)
+        name = name.strip()
+        if len(name) > _MAX_NAME_LENGTH:
+            return _error(f"name must be at most {_MAX_NAME_LENGTH} characters", 400)
+        try:
+            scopes = normalize_scopes(body.get("scopes"))
+            resource_filter = normalize_resource_filter(body.get("resource_filter"), scopes)
+            expires_at = resolve_expiry(body.get("expires_in_days"))
+        except ValueError as exc:
+            return _error(str(exc), 400)
+
+        token, token_hash, token_prefix = generate_token()
+        try:
+            with db_session() as conn:
+                repo = PersonalAccessTokensRepository(conn)
+                # Serialise this user's creates so concurrent requests cannot both pass the cap check.
+                repo.lock_user(user_id)
+                if repo.count_active(user_id) >= settings.PAT_MAX_PER_USER:
+                    return _error(
+                        f"Token limit reached ({settings.PAT_MAX_PER_USER}); revoke one first", 409
+                    )
+                repo.retire_expired_name(user_id, name)
+                if repo.name_in_use(user_id, name):
+                    return _error("A token with this name already exists", 409)
+                row = repo.create(
+                    user_id,
+                    name,
+                    token_hash=token_hash,
+                    token_prefix=token_prefix,
+                    scopes=scopes,
+                    resource_filter=resource_filter,
+                    expires_at=expires_at,
+                )
+                AuthEventsRepository(conn).insert(
+                    user_id,
+                    "pat_created",
+                    ip=request.remote_addr,
+                    user_agent=request.headers.get("User-Agent"),
+                    metadata={
+                        "token_id": str(row["id"]),
+                        "name": name,
+                        "scopes": scopes,
+                        "resource_filter": resource_filter,
+                        "expires_at": row.get("expires_at"),
+                    },
+                )
+        except IntegrityError:
+            # Lost a race against a concurrent create with the same name.
+            return _error("A token with this name already exists", 409)
+        return make_response(
+            jsonify({"success": True, "token": token, "personal_access_token": serialize_token(row)}),
+            201,
+        )
+
+
+@pat_ns.route("/user/tokens/<string:token_id>")
+class PersonalAccessToken(Resource):
+    def delete(self, token_id):
+        """Revoke one of the caller's tokens. Takes effect on the next request."""
+        user_id = _session_user_id()
+        if not user_id:
+            return _error("Authentication required", 401)
+        if not _valid_uuid(token_id):
+            return _error("Token not found", 404)
+        with db_session() as conn:
+            revoked = PersonalAccessTokensRepository(conn).revoke(token_id, user_id)
+            if revoked:
+                AuthEventsRepository(conn).insert(
+                    user_id,
+                    "pat_revoked",
+                    ip=request.remote_addr,
+                    user_agent=request.headers.get("User-Agent"),
+                    metadata={"token_id": token_id, "by": user_id},
+                )
+        if not revoked:
+            return _error("Token not found", 404)
+        return make_response(jsonify({"success": True}), 200)
+
+
+@pat_ns.route("/user/tokens/<string:token_id>/regenerate")
+class PersonalAccessTokenRegenerate(Resource):
+    def post(self, token_id):
+        """Issue a new secret for a token and reset its expiry.
+
+        Name, scopes and restrictions stay; the old secret stops working at
+        once. ``expires_in_days`` is optional and defaults to the lifetime the
+        token was last issued with. An expired token can be renewed this way; a
+        revoked one cannot. The plaintext ``token`` is returned here and never again.
+        """
+        user_id = _session_user_id()
+        if not user_id:
+            return _error("Authentication required", 401)
+        if not auth_type_supports_pats():
+            return _error("Personal access tokens are not available on this server", 403)
+        if not _valid_uuid(token_id):
+            return _error("Token not found", 404)
+        body = request.get_json(silent=True)
+        if body is None:
+            body = {}
+        if not isinstance(body, dict):
+            return _error("Request body must be a JSON object", 400)
+
+        token, token_hash, token_prefix = generate_token()
+        with db_session() as conn:
+            repo = PersonalAccessTokensRepository(conn)
+            current = repo.get(token_id, user_id)
+            if not current or current["status"] != "active":
+                return _error("Token not found", 404)
+            requested = body.get("expires_in_days")
+            if requested is None:
+                requested = renewal_lifetime_days(current)
+            try:
+                expires_at = resolve_expiry(requested)
+            except ValueError as exc:
+                return _error(str(exc), 400)
+            row = repo.regenerate(
+                token_id, user_id, token_hash=token_hash, token_prefix=token_prefix, expires_at=expires_at
+            )
+            if not row:
+                # Revoked between the read and the write.
+                return _error("Token not found", 404)
+            AuthEventsRepository(conn).insert(
+                user_id,
+                "pat_regenerated",
+                ip=request.remote_addr,
+                user_agent=request.headers.get("User-Agent"),
+                metadata={
+                    "token_id": token_id,
+                    "name": row["name"],
+                    "expires_at": row.get("expires_at"),
+                    "previous_expires_at": current.get("expires_at"),
+                },
+            )
+        return make_response(
+            jsonify({"success": True, "token": token, "personal_access_token": serialize_token(row)}),
+            200,
+        )
+
+
+@pat_ns.route("/admin/users/<string:user_id>/tokens")
+class AdminUserTokens(Resource):
+    @admin_required
+    def get(self, user_id):
+        """List a user's tokens, revoked ones included."""
+        with db_readonly() as conn:
+            rows = PersonalAccessTokensRepository(conn).list_for_user(user_id, include_revoked=True)
+        return make_response(
+            jsonify({"success": True, "tokens": [serialize_token(r) for r in rows]}), 200
+        )
+
+
+@pat_ns.route("/admin/tokens/<string:token_id>")
+class AdminToken(Resource):
+    @admin_required
+    def delete(self, token_id):
+        """Revoke any user's token."""
+        if not _valid_uuid(token_id):
+            return _error("Token not found", 404)
+        actor = (getattr(request, "decoded_token", None) or {}).get("sub")
+        with db_session() as conn:
+            repo = PersonalAccessTokensRepository(conn)
+            row = repo.get(token_id)
+            revoked = bool(row) and repo.revoke(token_id, reason="admin_revoked")
+            if revoked:
+                AuthEventsRepository(conn).insert(
+                    row["user_id"],
+                    "pat_revoked",
+                    ip=request.remote_addr,
+                    user_agent=request.headers.get("User-Agent"),
+                    metadata={"token_id": token_id, "by": actor, "via": "admin_api"},
+                )
+        if not revoked:
+            return _error("Token not found", 404)
+        return make_response(jsonify({"success": True}), 200)

docsgpt/api/pat/rules.py

@@ -0,0 +1,536 @@
+"""What a personal access token may call: the scope and resource rule table.
+
+Authorization for PATs is central and deny by default. ``RULES`` maps a Flask
+route (its rule string and method) to the scope it needs; a PAT request to a
+route that is not listed is refused, so a new endpoint is unreachable by token
+until someone classifies it here. ``tests/api/test_pat_rules.py`` fails when a
+registered route is in neither ``RULES`` nor ``DENIED``.
+
+A token may also carry a resource filter (``{"agents": [ids]}``). For a
+restricted family the rule must be able to prove the request stays inside the
+allowlist: it names where the id travels (``ids``), or declares that the route
+filters its own listing (``listing``), or delegates to the route
+(``in_route``). Anything else, creation included, is refused. ``refs`` cover
+ids of *other* families a route accepts (an agent update naming a source), and
+``blocked_by`` closes routes whose rows hang off a family the rule cannot see
+(a schedule belongs to an agent).
+
+Session (JWT) callers never pass through here.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import dataclass
+from typing import Any, Callable, Iterable, Optional
+
+from docsgpt.api.pat.tokens import is_pat
+
+VIEW, QUERY, JSON, FORM, BODY = "view", "query", "json", "form", "body"
+
+Locator = tuple[str, str]
+
+
+@dataclass(frozen=True)
+class Rule:
+    """Requirement for one route+method. ``scopes`` is any-of; empty means any valid token."""
+
+    scopes: tuple[str, ...] = ()
+    family: Optional[str] = None
+    ids: tuple[Locator, ...] = ()
+    refs: tuple[tuple[str, Locator], ...] = ()
+    listing: bool = False
+    open: bool = False
+    in_route: bool = False
+    blocked_by: tuple[str, ...] = ()
+    check: Optional[Callable[[Any, dict, Optional[str]], Optional[str]]] = None
+
+
+def _rule(scope: Optional[str] = None, *ids: Locator, any_of: tuple[str, ...] = (), **kwargs) -> Rule:
+    scopes = any_of or ((scope,) if scope else ())
+    family = kwargs.pop("family", None)
+    if family is None and scope:
+        family = scope.partition(":")[0]
+    return Rule(scopes=scopes, family=family, ids=tuple(ids), **kwargs)
+
+
+#: Scopes that admit a token to message replay. Shared by the Flask tail route
+#: below and its ASGI sibling GET /api/messages/<id>/events (docsgpt/api/async_sse.py),
+#: which sits outside this table.
+MESSAGE_REPLAY_SCOPES = ("conversations:read", "chat:run")
+
+_ALL_FAMILIES = ("agents", "sources", "prompts", "tools", "workflows")
+_WORKFLOW_CONTENT_FAMILIES = ("sources", "tools", "prompts")
+# A route reached through an agent id can prove the agent; nothing else about it.
+_NON_AGENT_FAMILIES = ("sources", "prompts", "tools", "workflows")
+
+# Ids of other families that agent create/update accept in their JSON-or-form body.
+_AGENT_BODY_REFS = (
+    ("sources", (BODY, "source")),
+    ("sources", (BODY, "sources")),
+    ("prompts", (BODY, "prompt_id")),
+    ("tools", (BODY, "tools")),
+    ("workflows", (BODY, "workflow")),
+)
+
+
+def _conversation_agent_id(conversation_id: str, user_id: Optional[str]) -> tuple[bool, str]:
+    """``(found, agent_id)`` for a conversation the user can reach; ``agent_id`` is "" when it has none."""
+    from docsgpt.storage.db.repositories.conversations import ConversationsRepository
+    from docsgpt.storage.db.session import db_readonly
+
+    if not user_id:
+        return False, ""
+    try:
+        with db_readonly() as conn:
+            row = ConversationsRepository(conn).get_any(str(conversation_id), user_id)
+    except Exception:
+        return False, ""
+    if not row:
+        return False, ""
+    return True, str(row.get("agent_id") or "")
+
+
+def _chat_check(request, resource_filter: dict, user_id: Optional[str]) -> Optional[str]:
+    """Keep a restricted token's chat traffic inside its allowlists.
+
+    An agent brings its own sources, prompt and tools, which this table cannot
+    see, so a restricted token must name an allowed agent. The one exception is
+    a token restricted on sources only, which may chat against allowed sources
+    directly. Everything that could swap in another agent or another set of
+    resources is refused: an agent ``api_key``, an inline workflow, and a
+    ``conversation_id`` that belongs to a different agent (the server would
+    otherwise continue, append to, or resume tool calls of that conversation).
+
+    Chat executes tools: an agent's own, or the user's defaults when there is
+    no agent. Neither can be held to a tools allowlist from here, so a token
+    restricted on tools cannot chat at all.
+    """
+    body = _json_body(request)
+    if "tools" in resource_filter:
+        return "A token restricted to specific tools cannot use chat endpoints"
+    if body.get("api_key"):
+        return "A restricted token cannot chat with an agent API key; pass agent_id"
+    if body.get("workflow"):
+        # An inline workflow graph (builder preview) can reference any resource.
+        return "A restricted token cannot run an inline workflow"
+    agent_ids = _as_ids(body.get("agent_id"))
+    if "agents" in resource_filter:
+        if len(agent_ids) != 1:
+            return "This token is restricted to specific agents; pass agent_id"
+        # the agent id itself is verified through ``refs``
+    elif set(resource_filter) - {"sources"}:
+        return "Restrict this token to specific agents to use chat endpoints"
+    elif agent_ids:
+        return "This token is restricted to specific sources and cannot run agents"
+    conversation_id = body.get("conversation_id")
+    if conversation_id:
+        found, conversation_agent = _conversation_agent_id(conversation_id, user_id)
+        expected = agent_ids[0] if agent_ids else ""
+        if not found or _canonical(conversation_agent) != _canonical(expected):
+            return "This conversation does not belong to the agent this token may use"
+    return None
+
+
+def _agent_body_check(request, resource_filter: dict, user_id: Optional[str]) -> Optional[str]:
+    """A workflow pulls in its own sources, tools and prompts, which a reference check cannot see.
+
+    A token restricted on any of those may attach a workflow to an agent only
+    when it is also restricted on workflows, so the workflow is one its owner
+    chose (``refs`` then verifies the id).
+    """
+    if "workflows" in resource_filter or not set(resource_filter) & {"sources", "tools", "prompts"}:
+        return None
+    if _read(request, (BODY, "workflow")):
+        return "A token restricted to specific sources, tools or prompts cannot attach a workflow to an agent"
+    return None
+
+
+_CHAT = dict(
+    family=None,
+    refs=(
+        ("agents", (JSON, "agent_id")),
+        ("sources", (JSON, "active_docs")),
+        ("prompts", (JSON, "prompt_id")),
+        ("workflows", (JSON, "workflow_id")),
+    ),
+    check=_chat_check,
+)
+
+RULES: dict[tuple[str, str], Rule] = {
+    # Identity and public metadata: any valid token.
+    ("/api/user/me", "GET"): _rule(open=True),
+    ("/api/health", "GET"): _rule(open=True),
+    ("/api/config", "GET"): _rule(open=True),
+    # Agents
+    ("/api/get_agent", "GET"): _rule("agents:read", (QUERY, "id")),
+    ("/api/get_agents", "GET"): _rule("agents:read", listing=True),
+    ("/api/pinned_agents", "GET"): _rule("agents:read"),
+    ("/api/shared_agents", "GET"): _rule("agents:read"),
+    ("/api/template_agents", "GET"): _rule("agents:read", open=True),
+    ("/api/export_agent", "GET"): _rule("agents:read", (QUERY, "id")),
+    ("/api/guardrails/catalog", "GET"): _rule("agents:read", open=True),
+    ("/api/guardrails/events", "GET"): _rule("agents:read", (QUERY, "agent_id")),
+    ("/api/guardrails/summary", "GET"): _rule("agents:read", (QUERY, "agent_id")),
+    ("/api/agents/folders/", "GET"): _rule("agents:read", open=True),
+    ("/api/agents/folders/<string:folder_id>", "GET"): _rule("agents:read"),
+    ("/api/create_agent", "POST"): _rule("agents:write", refs=_AGENT_BODY_REFS, check=_agent_body_check),
+    ("/api/update_agent/<string:agent_id>", "PUT"): _rule(
+        "agents:write", (VIEW, "agent_id"), refs=_AGENT_BODY_REFS, check=_agent_body_check
+    ),
+    ("/api/delete_agent", "DELETE"): _rule("agents:write", (QUERY, "id")),
+    ("/api/adopt_agent", "POST"): _rule("agents:write"),
+    ("/api/pin_agent", "POST"): _rule("agents:write", (QUERY, "id")),
+    ("/api/remove_shared_agent", "DELETE"): _rule("agents:write", (QUERY, "id")),
+    ("/api/share_agent", "PUT"): _rule("agents:write", (JSON, "id")),
+    ("/api/import_agent/plan", "POST"): _rule("agents:write", in_route=True),
+    ("/api/import_agent", "POST"): _rule("agents:write", in_route=True),
+    ("/api/agents/folders/", "POST"): _rule("agents:write"),
+    ("/api/agents/folders/<string:folder_id>", "PUT"): _rule("agents:write"),
+    ("/api/agents/folders/<string:folder_id>", "DELETE"): _rule("agents:write"),
+    ("/api/agents/folders/move_agent", "POST"): _rule("agents:write", (JSON, "agent_id")),
+    ("/api/agents/folders/bulk_move", "POST"): _rule("agents:write", (JSON, "agent_ids")),
+    ("/api/regenerate_agent_key/<string:agent_id>", "POST"): _rule("agents:keys", (VIEW, "agent_id")),
+    ("/api/agent_webhook", "GET"): _rule("agents:keys", (QUERY, "id")),
+    # Schedules hang off an agent, and a schedule runs that agent with a free-form
+    # instruction and stores the output. The agent id proves the agent and nothing
+    # else, so tokens restricted on any other family are kept out; routes that
+    # carry only a schedule id prove nothing and are closed to every restricted token.
+    ("/api/agents/<string:agent_id>/schedules", "GET"): _rule(
+        "schedules:read", refs=(("agents", (VIEW, "agent_id")),), blocked_by=_NON_AGENT_FAMILIES
+    ),
+    ("/api/agents/<string:agent_id>/schedules", "POST"): _rule(
+        "schedules:write", refs=(("agents", (VIEW, "agent_id")),), blocked_by=_NON_AGENT_FAMILIES
+    ),
+    ("/api/schedules/<string:schedule_id>", "GET"): _rule("schedules:read", blocked_by=_ALL_FAMILIES),
+    ("/api/schedules/<string:schedule_id>/runs", "GET"): _rule("schedules:read", blocked_by=_ALL_FAMILIES),
+    ("/api/schedules/<string:schedule_id>/runs/<string:run_id>", "GET"): _rule(
+        "schedules:read", blocked_by=_ALL_FAMILIES
+    ),
+    ("/api/schedules/<string:schedule_id>", "PUT"): _rule("schedules:write", blocked_by=_ALL_FAMILIES),
+    ("/api/schedules/<string:schedule_id>", "PATCH"): _rule("schedules:write", blocked_by=_ALL_FAMILIES),
+    ("/api/schedules/<string:schedule_id>", "DELETE"): _rule("schedules:write", blocked_by=_ALL_FAMILIES),
+    ("/api/schedules/<string:schedule_id>/run", "POST"): _rule("schedules:write", blocked_by=_ALL_FAMILIES),
+    # Sources
+    ("/api/sources", "GET"): _rule("sources:read", listing=True),
+    # Counted and paged in SQL, so it cannot be narrowed here; restricted tokens use /api/sources.
+    ("/api/sources/paginated", "GET"): _rule("sources:read"),
+    ("/api/directory_structure", "GET"): _rule("sources:read", (QUERY, "id")),
+    ("/api/get_chunks", "GET"): _rule("sources:read", (QUERY, "id")),
+    ("/api/sources/<string:source_id>/wiki/pages", "GET"): _rule("sources:read", (VIEW, "source_id")),
+    ("/api/sources/<string:source_id>/wiki/page", "GET"): _rule("sources:read", (VIEW, "source_id")),
+    ("/api/sources/<string:source_id>/graph", "GET"): _rule("sources:read", (VIEW, "source_id")),
+    ("/api/sources/<string:source_id>/graph/node/<string:node_id>", "GET"): _rule(
+        "sources:read", (VIEW, "source_id")
+    ),
+    # Ingestion and attachment extraction both report through this poll.
+    ("/api/task_status", "GET"): _rule(any_of=("sources:read", "sources:write", "chat:run"), open=True),
+    ("/api/upload", "POST"): _rule("sources:write"),
+    ("/api/remote", "POST"): _rule("sources:write"),
+    ("/api/sources/wiki", "POST"): _rule("sources:write"),
+    ("/api/delete_old", "GET"): _rule("sources:write", (QUERY, "source_id")),
+    ("/api/manage_sync", "POST"): _rule("sources:write", (JSON, "source_id")),
+    ("/api/sync_source", "POST"): _rule("sources:write", (JSON, "source_id")),
+    ("/api/sources/reingest", "POST"): _rule("sources:write", (JSON, "source_id")),
+    ("/api/manage_source_files", "POST"): _rule("sources:write", (FORM, "source_id")),
+    ("/api/sources/<string:source_id>/config", "PATCH"): _rule("sources:write", (VIEW, "source_id")),
+    ("/api/sources/<string:source_id>/wiki/page", "PUT"): _rule("sources:write", (VIEW, "source_id")),
+    ("/api/sources/<string:source_id>/wiki/convert", "POST"): _rule("sources:write", (VIEW, "source_id")),
+    ("/api/sources/<string:source_id>/graphrag/enable", "POST"): _rule(
+        "sources:write", (VIEW, "source_id")
+    ),
+    ("/api/add_chunk", "POST"): _rule("sources:write", (JSON, "id")),
+    ("/api/update_chunk", "PUT"): _rule("sources:write", (JSON, "id")),
+    ("/api/delete_chunk", "DELETE"): _rule("sources:write", (QUERY, "id")),
+    # Prompts
+    ("/api/get_prompts", "GET"): _rule("prompts:read", listing=True),
+    ("/api/get_single_prompt", "GET"): _rule("prompts:read", (QUERY, "id")),
+    ("/api/create_prompt", "POST"): _rule("prompts:write"),
+    ("/api/update_prompt", "POST"): _rule("prompts:write", (JSON, "id")),
+    ("/api/delete_prompt", "POST"): _rule("prompts:write", (JSON, "id")),
+    # Tools
+    ("/api/available_tools", "GET"): _rule("tools:read", open=True),
+    ("/api/get_tools", "GET"): _rule("tools:read", listing=True),
+    ("/api/create_tool", "POST"): _rule("tools:write"),
+    ("/api/parse_spec", "POST"): _rule("tools:write", open=True),
+    ("/api/update_tool", "POST"): _rule("tools:write", (JSON, "id")),
+    ("/api/update_tool_config", "POST"): _rule("tools:write", (JSON, "id")),
+    ("/api/update_tool_actions", "POST"): _rule("tools:write", (JSON, "id")),
+    ("/api/update_tool_status", "POST"): _rule("tools:write", (JSON, "id")),
+    ("/api/delete_tool", "POST"): _rule("tools:write", (JSON, "id")),
+    ("/api/mcp_server/test", "POST"): _rule("tools:write", open=True),
+    ("/api/mcp_server/save", "POST"): _rule("tools:write", (JSON, "id")),
+    # Models
+    ("/api/models", "GET"): _rule(any_of=("models:read", "chat:run"), open=True),
+    ("/api/user/models", "GET"): _rule("models:read"),
+    ("/api/user/models/<string:model_id>", "GET"): _rule("models:read"),
+    ("/api/user/models", "POST"): _rule("models:write"),
+    ("/api/user/models/<string:model_id>", "PATCH"): _rule("models:write"),
+    ("/api/user/models/<string:model_id>", "DELETE"): _rule("models:write"),
+    ("/api/user/models/test", "POST"): _rule("models:write"),
+    ("/api/user/models/<string:model_id>/test", "POST"): _rule("models:write"),
+    # Workflows
+    # A workflow graph names sources, tools and prompts inside its nodes, out of reach of ``refs``.
+    ("/api/workflows", "POST"): _rule("workflows:write", blocked_by=_WORKFLOW_CONTENT_FAMILIES),
+    ("/api/workflows/<string:workflow_id>", "GET"): _rule("workflows:read", (VIEW, "workflow_id")),
+    ("/api/workflows/<string:workflow_id>", "PUT"): _rule(
+        "workflows:write", (VIEW, "workflow_id"), blocked_by=_WORKFLOW_CONTENT_FAMILIES
+    ),
+    ("/api/workflows/<string:workflow_id>", "DELETE"): _rule("workflows:write", (VIEW, "workflow_id")),
+    # Conversations and analytics span every agent and carry cited source text and tool
+    # output, so they are closed to every restricted token.
+    ("/api/get_conversations", "GET"): _rule("conversations:read", blocked_by=_ALL_FAMILIES),
+    ("/api/search_conversations", "GET"): _rule("conversations:read", blocked_by=_ALL_FAMILIES),
+    ("/api/get_single_conversation", "GET"): _rule("conversations:read", blocked_by=_ALL_FAMILIES),
+    # A message cannot be tied to an allowlist from here, so any restricted token is kept out.
+    ("/api/messages/<string:message_id>/tail", "GET"): _rule(
+        any_of=MESSAGE_REPLAY_SCOPES, family=None, blocked_by=_ALL_FAMILIES
+    ),
+    ("/api/delete_conversation", "POST"): _rule("conversations:write", blocked_by=_ALL_FAMILIES),
+    ("/api/delete_all_conversations", "GET"): _rule("conversations:write", blocked_by=_ALL_FAMILIES),
+    ("/api/update_conversation_name", "POST"): _rule("conversations:write", blocked_by=_ALL_FAMILIES),
+    ("/api/feedback", "POST"): _rule("conversations:write", blocked_by=_ALL_FAMILIES),
+    ("/api/get_message_analytics", "POST"): _rule("analytics:read", blocked_by=_ALL_FAMILIES),
+    ("/api/get_token_analytics", "POST"): _rule("analytics:read", blocked_by=_ALL_FAMILIES),
+    ("/api/get_feedback_analytics", "POST"): _rule("analytics:read", blocked_by=_ALL_FAMILIES),
+    ("/api/get_tool_analytics", "POST"): _rule("analytics:read", blocked_by=_ALL_FAMILIES),
+    ("/api/get_schedule_analytics", "POST"): _rule("analytics:read", blocked_by=_ALL_FAMILIES),
+    ("/api/get_user_logs", "POST"): _rule("analytics:read", blocked_by=_ALL_FAMILIES),
+    # Teams (read only)
+    ("/api/teams", "GET"): _rule("teams:read"),
+    ("/api/teams/<string:team_id>", "GET"): _rule("teams:read"),
+    ("/api/teams/<string:team_id>/members", "GET"): _rule("teams:read"),
+    ("/api/teams/<string:team_id>/grants", "GET"): _rule("teams:read"),
+    ("/api/resource_shares", "GET"): _rule("teams:read"),
+    # Chat
+    ("/api/answer", "POST"): _rule("chat:run", **_CHAT),
+    ("/stream", "POST"): _rule("chat:run", **_CHAT),
+    ("/api/search", "POST"): _rule("chat:run", **_CHAT),
+    ("/api/store_attachment", "POST"): _rule("chat:run", family=None),
+    ("/api/sources/<string:source_id>/search", "POST"): _rule(
+        "chat:run", family=None, refs=(("sources", (VIEW, "source_id")),)
+    ),
+}
+
+#: Routes a PAT may never call, by exact rule string ("*" = every method) or prefix.
+#: Token management, admin, login flows and interactive OAuth handshakes need a
+#: signed-in session; the rest have no scope yet. Listing them keeps the
+#: classification test honest: a new route must land here or in ``RULES``.
+DENIED: dict[str, tuple[str, ...]] = {
+    "/": ("*",),
+    "/api/user/tokens": ("*",),
+    "/api/user/tokens/<string:token_id>": ("*",),
+    "/api/user/tokens/<string:token_id>/regenerate": ("*",),
+    "/api/generate_token": ("*",),
+    "/api/combine": ("*",),
+    "/api/download": ("*",),
+    "/api/upload_index": ("*",),
+    "/api/share": ("*",),
+    "/api/shared_agent": ("*",),
+    "/api/shared_conversation/<string:identifier>": ("*",),
+    "/api/webhooks/agents/<string:webhook_token>": ("*",),
+    "/api/images/<string:agent_id>/<string:capability>": ("*",),
+    "/api/mcp_server/callback": ("*",),
+    "/api/mcp_server/auth_status": ("*",),
+    "/api/artifact/<artifact_id>": ("*",),
+    "/api/artifacts": ("*",),
+    "/api/artifacts/<artifact_id>": ("*",),
+    "/api/artifacts/<artifact_id>/restore": ("*",),
+    "/api/artifacts/<artifact_id>/versions/<int:version>": ("*",),
+    "/api/stt": ("*",),
+    "/api/stt/live/start": ("*",),
+    "/api/stt/live/chunk": ("*",),
+    "/api/stt/live/finish": ("*",),
+    "/api/tts": ("*",),
+    "/api/teams": ("POST",),
+    "/api/teams/<string:team_id>": ("PUT", "DELETE"),
+    "/api/teams/<string:team_id>/members": ("POST",),
+    "/api/teams/<string:team_id>/members/<string:member_id>": ("*",),
+    "/api/teams/<string:team_id>/grants": ("POST", "DELETE"),
+    "/api/teams/<string:team_id>/transfer_owner": ("*",),
+    "/swagger.json": ("*",),
+}
+DENIED_PREFIXES = (
+    "/api/admin/",
+    "/api/auth/oidc/",
+    "/api/connectors/",
+    "/api/devices",
+    "/scim/",
+    "/static/",
+    "/swaggerui/",
+    "/v1/",
+)
+
+
+def is_denied(rule: str, method: str) -> bool:
+    if rule.startswith(DENIED_PREFIXES):
+        return True
+    methods = DENIED.get(rule)
+    return bool(methods) and ("*" in methods or method in methods)
+
+
+def _json_body(request) -> dict:
+    body = request.get_json(silent=True)
+    return body if isinstance(body, dict) else {}
+
+
+def _as_ids(value: Any) -> list[str]:
+    """Flatten whatever a route accepts as ids: a string, a JSON-encoded or plain list, or an ``{id}`` dict."""
+    if value is None or value == "":
+        return []
+    if isinstance(value, dict):
+        return _as_ids(value.get("id") or value.get("_id") or value.get("workflow_id"))
+    if isinstance(value, (list, tuple)):
+        out: list[str] = []
+        for item in value:
+            out.extend(_as_ids(item))
+        return out
+    text = str(value).strip()
+    if text[:1] in "[{":
+        try:
+            return _as_ids(json.loads(text))
+        except ValueError:
+            return [text]
+    return [text]
+
+
+def _read(request, locator: Locator) -> list[str]:
+    where, key = locator
+    if where == VIEW:
+        return _as_ids((request.view_args or {}).get(key))
+    if where == QUERY:
+        return _as_ids(request.args.get(key))
+    if where == JSON:
+        return _as_ids(_json_body(request).get(key))
+    if where == FORM:
+        return _as_ids(request.form.get(key))
+    # BODY: routes that accept JSON or a multipart form interchangeably.
+    if request.is_json:
+        return _as_ids(_json_body(request).get(key))
+    return _as_ids(request.form.get(key))
+
+
+def _canonical(value: str) -> str:
+    try:
+        return str(uuid.UUID(value))
+    except (ValueError, AttributeError, TypeError):
+        return value
+
+
+def _all_allowed(ids: Iterable[str], allowed: Iterable[str]) -> bool:
+    allowlist = {_canonical(a) for a in allowed}
+    return all(_canonical(i) in allowlist for i in ids)
+
+
+def authorize(request, decoded_token: dict) -> Optional[tuple[dict, int]]:
+    """Check a PAT request against the table. ``None`` allows; otherwise ``(body, status)``."""
+    url_rule = getattr(request, "url_rule", None)
+    if url_rule is None:
+        # Routing failed (unknown path or wrong method): no view will run, so
+        # let Flask answer 404/405 instead of masking it with a 403.
+        return None
+    rule = RULES.get((url_rule.rule, request.method))
+    if rule is None:
+        return (
+            {
+                "success": False,
+                "error": "not_available_to_tokens",
+                "message": "This endpoint cannot be called with a personal access token",
+            },
+            403,
+        )
+    granted = set(decoded_token.get("scopes") or [])
+    if rule.scopes and not granted.intersection(rule.scopes):
+        return (
+            {
+                "success": False,
+                "error": "insufficient_scope",
+                "message": f"Token lacks the required scope: {' or '.join(rule.scopes)}",
+                "required_scope": rule.scopes[0],
+            },
+            403,
+        )
+    resource_filter = decoded_token.get("resource_filter") or {}
+    if not resource_filter:
+        return None
+    reason = _check_resources(request, rule, resource_filter, decoded_token.get("sub"))
+    if reason is None:
+        return None
+    return ({"success": False, "error": "resource_not_allowed", "message": reason}, 403)
+
+
+def _check_resources(request, rule: Rule, resource_filter: dict, user_id: Optional[str] = None) -> Optional[str]:
+    for family in rule.blocked_by:
+        if family in resource_filter:
+            return f"This endpoint is not available to a token restricted to specific {family}"
+    if rule.check is not None:
+        reason = rule.check(request, resource_filter, user_id)
+        if reason:
+            return reason
+    for family, locator in rule.refs:
+        if family not in resource_filter:
+            continue
+        ids = _read(request, locator)
+        if ids and not _all_allowed(ids, resource_filter[family]):
+            return f"Token is not allowed to use one of the referenced {family}"
+    family = rule.family
+    if family is None or family not in resource_filter or rule.open or rule.listing or rule.in_route:
+        return None
+    ids = [i for locator in rule.ids for i in _read(request, locator)]
+    if not ids:
+        return f"This token is restricted to specific {family} and cannot use this endpoint"
+    if not _all_allowed(ids, resource_filter[family]):
+        return f"Token is not allowed to access this resource ({family})"
+    return None
+
+
+def allowed_ids(request, family: str) -> Optional[set[str]]:
+    """The caller's allowlist for ``family``, or ``None`` when unrestricted (or not a PAT).
+
+    Used by listing routes (``listing=True``) and by ``in_route`` handlers.
+    """
+    decoded = getattr(request, "decoded_token", None)
+    if not is_pat(decoded):
+        return None
+    ids = (decoded.get("resource_filter") or {}).get(family)
+    if ids is None:
+        return None
+    return {_canonical(str(i)) for i in ids}
+
+
+def filter_listing(request, family: str, items: list, key: str = "id") -> list:
+    """Drop rows outside the caller's allowlist. Rows without a UUID id (built-in presets) are kept."""
+    allowed = allowed_ids(request, family)
+    if allowed is None:
+        return items
+    kept = []
+    for item in items:
+        value = str(item.get(key, ""))
+        if not _is_uuid(value) or _canonical(value) in allowed:
+            kept.append(item)
+    return kept
+
+
+def _is_uuid(value: str) -> bool:
+    try:
+        uuid.UUID(value)
+    except (ValueError, AttributeError, TypeError):
+        return False
+    return True
+
+
+def may_see_agent_keys(request) -> bool:
+    """False for a token without ``agents:keys``: it must not receive a plaintext agent API key.
+
+    Create, first publish and adopt all mint a key and used to return it, which
+    handed a deploy token a secret that outlives the token's own revocation.
+    """
+    decoded = getattr(request, "decoded_token", None)
+    if not is_pat(decoded):
+        return True
+    return "agents:keys" in (decoded.get("scopes") or [])
+
+
+def mask_agent_key(key: Optional[str]) -> str:
+    return f"{key[:4]}...{key[-4:]}" if key else ""

docsgpt/api/pat/tokens.py

@@ -0,0 +1,264 @@
+"""Personal access tokens: format, scope catalog and the per-request verifier.
+
+A PAT is ``dgpt_pat_`` + 32 random bytes (urlsafe). Only its SHA-256 is stored
+(``personal_access_tokens.token_hash``), the same shape as device session
+tokens. Scopes and the resource filter are always read from the database row;
+nothing about a token's authority is encoded in the credential itself.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import secrets
+import uuid
+from datetime import datetime, timedelta, timezone
+from typing import Any, Optional
+
+from docsgpt.core.settings import settings
+from docsgpt.storage.db.repositories.personal_access_tokens import (
+    PersonalAccessTokensRepository,
+)
+from docsgpt.storage.db.session import db_readonly, db_session
+
+logger = logging.getLogger(__name__)
+
+TOKEN_PREFIX = "dgpt_pat_"
+# Characters of the secret kept in ``token_prefix`` so users can tell tokens apart.
+_DISPLAY_CHARS = 6
+
+AUTH_METHOD_PAT = "pat"
+
+#: Every grantable scope with the description shown in the UI and docs.
+SCOPES: dict[str, str] = {
+    "agents:read": "View agents, folders, guardrail events and export agent definitions",
+    "agents:write": "Create, update, delete, share and import (apply) agents and folders",
+    "agents:keys": "Regenerate agent API keys and read incoming webhook URLs",
+    "sources:read": "View sources, their files, chunks and ingestion task status",
+    "sources:write": "Upload, ingest, sync, edit and delete sources and chunks",
+    "prompts:read": "View prompts",
+    "prompts:write": "Create, update and delete prompts",
+    "tools:read": "View configured tools",
+    "tools:write": "Create, update and delete tools and MCP servers",
+    "models:read": "View available and custom models",
+    "models:write": "Create, update, test and delete custom models",
+    "workflows:read": "View workflows",
+    "workflows:write": "Create, update and delete workflows",
+    "schedules:read": "View agent schedules and their runs",
+    "schedules:write": "Create, update, run and delete agent schedules",
+    "conversations:read": "View conversations and messages",
+    "conversations:write": "Rename, delete and give feedback on conversations",
+    "analytics:read": "View usage analytics and logs",
+    "teams:read": "View teams, members and resource shares",
+    "chat:run": "Ask agents and search sources (answer, stream, search); used for benchmarking",
+}
+
+#: Resource families whose tokens can be narrowed to specific ids.
+FILTERABLE_FAMILIES = ("agents", "sources", "prompts", "tools", "workflows")
+_MAX_FILTER_IDS = 200
+
+
+def auth_type_supports_pats() -> bool:
+    """PATs bind to a stable user id, which simple_jwt/session_jwt don't have."""
+    return bool(settings.PAT_ENABLED) and settings.AUTH_TYPE in (None, "oidc")
+
+
+def generate_token() -> tuple[str, str, str]:
+    """Mint a token. Returns ``(plaintext, sha256_hex, display_prefix)``."""
+    secret = secrets.token_urlsafe(32)
+    token = TOKEN_PREFIX + secret
+    return token, hash_token(token), TOKEN_PREFIX + secret[:_DISPLAY_CHARS]
+
+
+def hash_token(token: str) -> str:
+    return hashlib.sha256(token.encode("utf-8")).hexdigest()
+
+
+def looks_like_pat(value: Optional[str]) -> bool:
+    return bool(value) and value.startswith(TOKEN_PREFIX)
+
+
+def redact(value: Optional[str]) -> str:
+    """Log-safe form of a credential: the display prefix only."""
+    if not value:
+        return ""
+    if looks_like_pat(value):
+        return value[: len(TOKEN_PREFIX) + _DISPLAY_CHARS] + "…"
+    return value[:4] + "…"
+
+
+def expand_scopes(scopes) -> set[str]:
+    """Granted scopes plus what they imply (``x:write`` includes ``x:read``)."""
+    granted = set(scopes or [])
+    for scope in list(granted):
+        family, _, action = scope.partition(":")
+        if action == "write" and f"{family}:read" in SCOPES:
+            granted.add(f"{family}:read")
+    return granted
+
+
+def normalize_scopes(raw: Any) -> list[str]:
+    """Validate a requested scope list. Raises ``ValueError`` with a user-facing message."""
+    if not isinstance(raw, list) or not raw:
+        raise ValueError("scopes must be a non-empty list")
+    unknown = sorted({s for s in raw if not isinstance(s, str) or s not in SCOPES}, key=str)
+    if unknown:
+        raise ValueError(f"Unknown scopes: {', '.join(map(str, unknown))}")
+    return sorted(set(raw))
+
+
+def normalize_resource_filter(raw: Any, scopes: list[str]) -> dict[str, list[str]]:
+    """Validate ``{"<family>": ["<uuid>", ...]}``. Raises ``ValueError`` with a user-facing message.
+
+    A family may only be restricted when the token holds a scope in it;
+    otherwise the restriction would be dead weight that reads as protection.
+    """
+    if raw in (None, {}):
+        return {}
+    if not isinstance(raw, dict):
+        raise ValueError("resource_filter must be an object")
+    families = {s.partition(":")[0] for s in scopes}
+    # chat:run acts on agents and sources, so both may be restricted alongside it.
+    if "chat" in families:
+        families.update({"agents", "sources"})
+    if "tools" in raw and "chat:run" in scopes:
+        # Chat executes tools (an agent's own, or the user's defaults), which cannot be held to an allowlist.
+        raise ValueError("resource_filter.tools cannot be combined with the chat:run scope")
+    out: dict[str, list[str]] = {}
+    for family, ids in raw.items():
+        if family not in FILTERABLE_FAMILIES:
+            raise ValueError(
+                f"resource_filter supports only: {', '.join(FILTERABLE_FAMILIES)}"
+            )
+        if family not in families:
+            raise ValueError(f"resource_filter.{family} needs a {family} scope on the token")
+        if not isinstance(ids, list) or not ids:
+            raise ValueError(f"resource_filter.{family} must be a non-empty list of ids")
+        if len(ids) > _MAX_FILTER_IDS:
+            raise ValueError(f"resource_filter.{family} allows at most {_MAX_FILTER_IDS} ids")
+        normalized = []
+        for value in ids:
+            try:
+                normalized.append(str(uuid.UUID(str(value))))
+            except (ValueError, AttributeError, TypeError):
+                raise ValueError(f"resource_filter.{family} contains an invalid id: {value!r}")
+        out[family] = sorted(set(normalized))
+    return out
+
+
+def resolve_expiry(expires_in_days: Any) -> Optional[datetime]:
+    """Map the requested lifetime to ``expires_at``. Raises ``ValueError`` with a user-facing message.
+
+    ``None`` means "use the default"; ``0`` asks for a non-expiring token,
+    which only an operator setting can allow.
+    """
+    if expires_in_days is None:
+        days = settings.PAT_DEFAULT_LIFETIME_DAYS
+    elif isinstance(expires_in_days, bool) or not isinstance(expires_in_days, int):
+        raise ValueError("expires_in_days must be an integer")
+    elif expires_in_days == 0:
+        if not settings.PAT_ALLOW_NON_EXPIRING:
+            raise ValueError("Non-expiring tokens are disabled on this server")
+        return None
+    elif expires_in_days < 0:
+        raise ValueError("expires_in_days must be positive")
+    else:
+        days = expires_in_days
+    if days > settings.PAT_MAX_LIFETIME_DAYS:
+        raise ValueError(f"expires_in_days must not exceed {settings.PAT_MAX_LIFETIME_DAYS}")
+    return datetime.now(timezone.utc) + timedelta(days=days)
+
+
+def _parse_moment(value: Any) -> Optional[datetime]:
+    if not value:
+        return None
+    try:
+        moment = value if isinstance(value, datetime) else datetime.fromisoformat(str(value))
+    except ValueError:
+        return None
+    return moment if moment.tzinfo else moment.replace(tzinfo=timezone.utc)
+
+
+def renewal_lifetime_days(row: dict) -> Optional[int]:
+    """The lifetime a token was last issued with, for renewing it on the same terms.
+
+    ``0`` for a non-expiring token, ``None`` when it cannot be derived (the
+    caller then falls back to the default). The result is clamped to today's
+    maximum, since the policy may have tightened since the token was issued.
+    """
+    issued = _parse_moment(row.get("regenerated_at")) or _parse_moment(row.get("created_at"))
+    expires = _parse_moment(row.get("expires_at"))
+    if expires is None:
+        return 0 if settings.PAT_ALLOW_NON_EXPIRING else None
+    if issued is None:
+        return None
+    days = round((expires - issued).total_seconds() / 86400)
+    return max(1, min(days, settings.PAT_MAX_LIFETIME_DAYS))
+
+
+def _client_ip(request) -> Optional[str]:
+    # Flask exposes remote_addr; Starlette exposes client.host.
+    ip = getattr(request, "remote_addr", None)
+    if ip:
+        return ip
+    client = getattr(request, "client", None)
+    return getattr(client, "host", None)
+
+
+_TOUCH_INTERVAL_SECONDS = 60
+
+
+def _usage_is_stale(last_used_at: Any) -> bool:
+    """True when ``last_used_at`` is old enough to be worth a write transaction."""
+    if not last_used_at:
+        return True
+    try:
+        seen = last_used_at if isinstance(last_used_at, datetime) else datetime.fromisoformat(str(last_used_at))
+    except ValueError:
+        return True
+    if seen.tzinfo is None:
+        seen = seen.replace(tzinfo=timezone.utc)
+    return (datetime.now(timezone.utc) - seen).total_seconds() >= _TOUCH_INTERVAL_SECONDS
+
+
+_INVALID = {"message": "Authentication error: invalid token", "error": "invalid_token"}
+
+
+def authenticate_pat(token: str, request) -> dict:
+    """Resolve a PAT into the claims dict the rest of the app reads.
+
+    Fails closed: an unknown, revoked or expired token, a disabled feature or
+    a database error all yield the same ``invalid_token`` error.
+    """
+    if not auth_type_supports_pats():
+        return dict(_INVALID)
+    try:
+        with db_readonly() as conn:
+            row = PersonalAccessTokensRepository(conn).find_active_by_hash(hash_token(token))
+    except Exception:
+        logger.error("PAT lookup failed for %s", redact(token), exc_info=True)
+        return dict(_INVALID)
+    if not row:
+        logger.warning("Rejected personal access token %s", redact(token))
+        return dict(_INVALID)
+    if _usage_is_stale(row.get("last_used_at")):
+        try:
+            with db_session() as conn:
+                PersonalAccessTokensRepository(conn).touch_last_used(
+                    str(row["id"]), _client_ip(request), min_interval_seconds=_TOUCH_INTERVAL_SECONDS
+                )
+        except Exception:
+            # Usage telemetry must never fail a request.
+            logger.debug("PAT last-used update failed", exc_info=True)
+    return {
+        "sub": row["user_id"],
+        "auth_method": AUTH_METHOD_PAT,
+        "pat_id": str(row["id"]),
+        "pat_name": row["name"],
+        "scopes": sorted(expand_scopes(row.get("scopes"))),
+        "resource_filter": row.get("resource_filter") or {},
+    }
+
+
+def is_pat(decoded_token: Optional[dict]) -> bool:
+    return bool(decoded_token) and decoded_token.get("auth_method") == AUTH_METHOD_PAT

docsgpt/api/user/agents/portability.py

@@ -36,6 +36,7 @@ from docsgpt.agents.default_tools import (
     synthesized_tool_name_for_id,
 )
 from docsgpt.api import api
+from docsgpt.api.pat.rules import allowed_ids
 from docsgpt.core.model_utils import validate_model_id
 from docsgpt.core.url_validation import SSRFError, validate_url
 from docsgpt.security.safe_url import UnsafeUserUrlError, validate_user_base_url
@@ -1717,6 +1718,32 @@ def _read_import_payload(req):
     return raw.decode("utf-8", "replace"), {}
 
 
+def _restricted_token_denial(conn, user: str, doc: dict) -> Optional[str]:
+    """Why a resource-restricted personal access token may not import ``doc``, if it may not.
+
+    An import resolves sources, prompts and tools by name and may create them,
+    so a token restricted on any of those families cannot be held to its
+    allowlist here. A token restricted to specific agents may update exactly
+    those; it can never create one.
+    """
+    for family in ("sources", "prompts", "tools", "workflows"):
+        if allowed_ids(request, family) is not None:
+            return f"A token restricted to specific {family} cannot import agents"
+    allowed_agents = allowed_ids(request, "agents")
+    if allowed_agents is None:
+        return None
+    target = _resolve_target(conn, user, doc.get("metadata") or {})
+    if target["action"] != "update" or target["agent_id"] not in allowed_agents:
+        return "This token is restricted to specific agents and may only update those"
+    return None
+
+
+def _token_denied_response(reason: str):
+    return make_response(
+        jsonify({"success": False, "error": "resource_not_allowed", "message": reason}), 403
+    )
+
+
 @agents_portability_ns.route("/export_agent")
 class ExportAgent(Resource):
     @api.doc(params={"id": "Agent ID"}, description="Export an agent as YAML")
@@ -1763,6 +1790,8 @@ class ImportAgentPlan(Resource):
             return make_response(jsonify({"success": False, "message": str(exc)}), 400)
         try:
             with db_readonly() as conn:
+                if reason := _restricted_token_denial(conn, user, doc):
+                    return _token_denied_response(reason)
                 plan = plan_import(conn, user, doc)
         except Exception:
             current_app.logger.error("Agent import plan failed", exc_info=True)
@@ -1786,6 +1815,8 @@ class ImportAgent(Resource):
             return make_response(jsonify({"success": False, "message": str(exc)}), 400)
         try:
             with db_session() as conn:
+                if reason := _restricted_token_denial(conn, user, doc):
+                    return _token_denied_response(reason)
                 result = apply_import(conn, user, doc, resolution)
         except AgentImportError as exc:
             # Apply-time rejection of the user's document (e.g. the workflow

docsgpt/api/user/agents/routes.py

@@ -9,6 +9,7 @@ from flask_restx import fields, Namespace, Resource
 from pydantic import ValidationError as PydanticValidationError
 
 from docsgpt.api import api
+from docsgpt.api.pat.rules import filter_listing, mask_agent_key, may_see_agent_keys
 from docsgpt.guardrails.config import AgentConfig
 from docsgpt.api.user.base import (
     copy_agent_image_for_user,
@@ -498,7 +499,7 @@ class GetAgents(Resource):
         except Exception as err:
             current_app.logger.error(f"Error retrieving agents: {err}", exc_info=True)
             return make_response(jsonify({"success": False}), 400)
-        return make_response(jsonify(list_agents), 200)
+        return make_response(jsonify(filter_listing(request, "agents", list_agents)), 200)
 
 
 @agents_ns.route("/create_agent")
@@ -762,7 +763,9 @@ class CreateAgent(Resource):
         except Exception as err:
             current_app.logger.error(f"Error creating agent: {err}", exc_info=True)
             return make_response(jsonify({"success": False}), 400)
-        return make_response(jsonify({"id": new_id, "key": key}), 201)
+        # A token without agents:keys never receives the plaintext agent key.
+        visible_key = key if may_see_agent_keys(request) else mask_agent_key(key)
+        return make_response(jsonify({"id": new_id, "key": visible_key}), 201)
 
 
 @agents_ns.route("/update_agent/<string:agent_id>")
@@ -1306,7 +1309,11 @@ class UpdateAgent(Resource):
             "message": "Agent updated successfully",
         }
         if newly_generated_key:
-            response_data["key"] = newly_generated_key
+            response_data["key"] = (
+                newly_generated_key
+                if may_see_agent_keys(request)
+                else mask_agent_key(newly_generated_key)
+            )
         return make_response(jsonify(response_data), 200)
 
 
@@ -1810,7 +1817,9 @@ class AdoptAgent(Resource):
                 )
 
             response_agent = _format_agent_output(new_agent, include_key_masked=False)
-            response_agent["key"] = new_key
+            response_agent["key"] = (
+                new_key if may_see_agent_keys(request) else mask_agent_key(new_key)
+            )
             return make_response(
                 jsonify({"success": True, "agent": response_agent}), 200
             )

docsgpt/api/user/me/routes.py

@@ -12,6 +12,8 @@ from __future__ import annotations
 from flask import jsonify, make_response, request
 from flask_restx import Namespace, Resource
 
+from docsgpt.api.pat.tokens import is_pat
+
 me_ns = Namespace("me", description="Current user identity and roles", path="/api")
 
 
@@ -31,4 +33,13 @@ class MeResource(Resource):
             value = decoded_token.get(field)
             if value:
                 body[field] = value
+        if is_pat(decoded_token):
+            # Lets a CLI or pipeline confirm what its token is allowed to do.
+            body["auth_method"] = "pat"
+            body["token"] = {
+                "id": decoded_token.get("pat_id"),
+                "name": decoded_token.get("pat_name"),
+                "scopes": decoded_token.get("scopes") or [],
+                "resource_filter": decoded_token.get("resource_filter") or {},
+            }
         return make_response(jsonify(body), 200)

docsgpt/api/user/prompts/routes.py

@@ -5,6 +5,7 @@ from flask import current_app, jsonify, make_response, request
 from flask_restx import fields, Namespace, Resource
 
 from docsgpt.api import api
+from docsgpt.api.pat.rules import filter_listing
 from docsgpt.api.user.team_sharing import team_access_for, visible_with_access
 from docsgpt.storage.db.repositories.prompts import PromptsRepository
 from docsgpt.prompts.composer import compose_preset, is_composed_preset
@@ -91,7 +92,8 @@ class GetPrompts(Resource):
         except Exception as err:
             current_app.logger.error(f"Error retrieving prompts: {err}", exc_info=True)
             return make_response(jsonify({"success": False}), 400)
-        return make_response(jsonify(list_prompts), 200)
+        # Presets (default/creative/strict) have no row id and stay visible to a restricted token.
+        return make_response(jsonify(filter_listing(request, "prompts", list_prompts)), 200)
 
 
 @prompts_ns.route("/get_single_prompt")

docsgpt/api/user/sources/routes.py

@@ -10,6 +10,7 @@ from pydantic import ValidationError
 
 from docsgpt.agents.tools.path_utils import validate_tool_path
 from docsgpt.api import api
+from docsgpt.api.pat.rules import filter_listing
 from docsgpt.api.user.tasks import (
     convert_source_to_wiki,
     extract_graph,
@@ -130,7 +131,7 @@ class CombinedJson(Resource):
         except Exception as err:
             current_app.logger.error(f"Error retrieving sources: {err}", exc_info=True)
             return make_response(jsonify({"success": False}), 400)
-        return make_response(jsonify(data), 200)
+        return make_response(jsonify(filter_listing(request, "sources", data)), 200)
 
 
 @sources_ns.route("/sources/paginated")

docsgpt/api/user/tools/routes.py

@@ -16,6 +16,7 @@ from docsgpt.agents.default_tools import (
 from docsgpt.agents.tools.spec_parser import parse_spec
 from docsgpt.agents.tools.tool_manager import ToolManager
 from docsgpt.api import api
+from docsgpt.api.pat.rules import filter_listing
 from docsgpt.api.user.artifacts.authz import Principal, authorize_artifact
 from docsgpt.api.user.team_sharing import effective_write_owner, visible_with_access
 from docsgpt.core.settings import settings
@@ -294,6 +295,9 @@ class GetTools(Resource):
                     builtin_copy.get("name") in WORKFLOW_ONLY_BUILTINS
                 )
                 user_tools.append(builtin_copy)
+            # A resource-restricted token sees only its allowed tools. Default
+            # and builtin rows have ids too, so they follow the same allowlist.
+            user_tools = filter_listing(request, "tools", user_tools)
         except Exception as err:
             current_app.logger.error(f"Error getting user tools: {err}", exc_info=True)
             return make_response(jsonify({"success": False}), 400)