Connections own their credentials

Migration 0038 moves every stored secret (OAuth tokens, MCP OAuth
tokens and client registrations, API keys) into the connection's
encrypted envelope, links API-key tools to one connection per distinct
credential, allows several accounts per provider, and adds
credential_mode to sources and tools. OAuth MCP tools keep resolving
each member's own token, as they did before.

docsgpt.connectors.service is now the only reader of OAuth tokens:
get_valid_token_info refreshes under a row lock and persists rotated
refresh tokens, and a revoked grant flags the connection, pauses its
sources and notifies the owner. Loaders build from a connection
(BaseConnectorLoader.from_connection), so scheduled sync covers Drive,
SharePoint and Confluence sources with no browser. S3 and Reddit keys
stay on the connection instead of in remote_data.

New endpoints: POST /api/connections, /setup, /reconnect,
/picker-token, /claim, DELETE /api/connections/<id>, per-action
permissions and MCP refresh-tools. Upload, file listing, sync and
validate-session take a connection_id; session tokens keep working for
this release. The tool executor reads credentials from the resolved
connection (owner or member mode) and pauses on a Connect card when a
connection needs signing in. docsgpt connectors reencrypt rewrites
stored credentials after a key rotation.

docsgpt/api/user/tools/mcp.py

@@ -75,6 +75,43 @@ def _validate_mcp_server_url(config: dict) -> None:
         raise ValueError(f"Invalid server URL: {exc}") from exc
 
 
+def _mcp_connection(user, config, auth_type, auth_credentials, display_name):
+    """The connection an MCP tool runs with; created on first save.
+
+    OAuth servers already have one (the sign-in stored its tokens there).
+    Key, bearer and basic auth store their secret on a connection; servers
+    with no auth get a credential-less connection so they still appear on
+    the Connectors page. Returns None when a multi-user install runs on the
+    default encryption key, which keeps the legacy per-tool secret.
+    """
+    from docsgpt.connectors import catalog, service
+
+    base = catalog.base_url(config.get("server_url"))
+    if not base:
+        return None
+    if auth_type == "oauth":
+        with db_readonly() as conn:
+            row = service._mcp_row(conn, user, base, None)
+        return str(row["id"]) if row else None
+    definition = catalog.get_definition("custom_mcp")
+    host = base.split("://")[-1]
+    try:
+        with db_session() as conn:
+            if auth_credentials:
+                row, _ = service.create_api_key_connection(
+                    conn, user, definition, auth_credentials, server_url=base, display_name=display_name,
+                )
+            else:
+                repo = ConnectorSessionsRepository(conn)
+                row = repo.find_account(user, "custom_mcp", server_url=base, account_label=host) or repo.create(
+                    user, "custom_mcp", connector_key="custom_mcp", auth_kind="none",
+                    display_name=display_name, account_label=host, server_url=base,
+                )
+    except service.EncryptionKeyNotConfigured:
+        return None
+    return str(row["id"]) if row else None
+
+
 @tools_mcp_ns.route("/mcp_server/test")
 class TestMCPServerConfig(Resource):
     @api.expect(
@@ -283,9 +320,19 @@ class MCPServerSave(Resource):
                 "redirect_uri",
             ]:
                 storage_config.pop(field, None)
-            transformed_actions = transform_actions(actions_metadata)
+            from docsgpt.connectors.permissions import apply_default_permissions
+
+            transformed_actions = apply_default_permissions(
+                "mcp_tool", transform_actions(actions_metadata),
+            )
 
             display_name = data["displayName"]
+            connection_id = _mcp_connection(
+                user, storage_config, auth_type, auth_credentials, display_name,
+            ) or (str(existing_doc["connection_id"]) if existing_doc and existing_doc.get("connection_id") else None)
+            if connection_id and auth_type != "oauth":
+                # The secret lives on the connection only.
+                storage_config.pop("encrypted_credentials", None)
             description = f"MCP Server: {storage_config.get('server_url', 'Unknown')}"
             status_bool = bool(data.get("status", True))
 
@@ -301,6 +348,7 @@ class MCPServerSave(Resource):
                             "config": storage_config,
                             "actions": transformed_actions,
                             "status": status_bool,
+                            "connection_id": connection_id,
                         },
                     )
                     saved_id = str(existing_doc["id"])
@@ -328,6 +376,7 @@ class MCPServerSave(Resource):
                                 "config": storage_config,
                                 "actions": transformed_actions,
                                 "status": status_bool,
+                                "connection_id": connection_id,
                             },
                         )
                         saved_id = str(existing_by_name["id"])
@@ -347,6 +396,7 @@ class MCPServerSave(Resource):
                             config_requirements={},
                             actions=transformed_actions,
                             status=status_bool,
+                            connection_id=connection_id,
                         )
                         saved_id = str(created["id"])
                         response_data = {
@@ -461,50 +511,34 @@ class MCPAuthStatus(Resource):
                         jsonify({"success": True, "statuses": {}}), 200
                     )
 
-                oauth_server_urls: dict = {}
+                from docsgpt.connectors import service
+
+                # Read from connection status alone: status checks never
+                # decrypt credentials.
                 statuses: dict = {}
                 for tool in mcp_tools:
                     tool_id = str(tool["id"])
                     config = tool.get("config") or {}
                     auth_type = config.get("auth_type", "none")
-                    if auth_type == "oauth":
-                        server_url = config.get("server_url", "")
-                        if server_url:
-                            parsed = urlparse(server_url)
-                            base_url = f"{parsed.scheme}://{parsed.netloc}"
-                            oauth_server_urls[tool_id] = base_url
+                    row = None
+                    if tool.get("connection_id"):
+                        row = sessions_repo.get(str(tool["connection_id"]))
+                    elif auth_type == "oauth" and config.get("server_url"):
+                        parsed = urlparse(config["server_url"])
+                        row = sessions_repo.get_by_user_provider(
+                            user, service.mcp_provider(f"{parsed.scheme}://{parsed.netloc}"),
+                        )
+                    if row is not None:
+                        connected = service.normalize_status(row) == service.STATUS_CONNECTED
+                        if auth_type == "oauth" or not connected:
+                            statuses[tool_id] = "connected" if connected else "needs_auth"
                         else:
-                            statuses[tool_id] = "needs_auth"
+                            statuses[tool_id] = "configured"
+                    elif auth_type == "oauth":
+                        statuses[tool_id] = "needs_auth"
                     else:
                         statuses[tool_id] = "configured"
 
-                if oauth_server_urls:
-                    # Look up a session per distinct base URL. MCP sessions
-                    # are stored with ``provider = "mcp:<server_url>"``
-                    # and the URL in ``server_url``; reuse the repo's
-                    # per-URL accessor rather than an ad-hoc $in query.
-                    url_has_tokens: dict = {}
-                    for base_url in set(oauth_server_urls.values()):
-                        session = sessions_repo.get_by_user_and_server_url(
-                            user, base_url,
-                        )
-                        tokens = (
-                            (session or {}).get("session_data", {}) or {}
-                        ).get("tokens", {}) or {}
-                        # MCP code also stashes tokens into token_info on
-                        # the row; consider either present as "connected".
-                        token_info = (session or {}).get("token_info") or {}
-                        url_has_tokens[base_url] = bool(
-                            tokens.get("access_token")
-                            or token_info.get("access_token")
-                        )
-
-                    for tool_id, base_url in oauth_server_urls.items():
-                        if url_has_tokens.get(base_url):
-                            statuses[tool_id] = "connected"
-                        else:
-                            statuses[tool_id] = "needs_auth"
-
             return make_response(jsonify({"success": True, "statuses": statuses}), 200)
         except Exception as e:
             current_app.logger.error(

docsgpt/api/user/tools/routes.py

@@ -271,6 +271,10 @@ class GetTools(Resource):
                 ):
                     tool_copy["config"]["has_encrypted_credentials"] = True
                     tool_copy["config"].pop("encrypted_credentials", None)
+                if tool_copy.get("connection_id"):
+                    # The secret lives on the connection; the form must not
+                    # ask for it again.
+                    tool_copy.setdefault("config", {})["has_encrypted_credentials"] = True
                 tool_copy["ownership"] = ownership
                 return tool_copy
 
@@ -383,6 +387,11 @@ class CreateTool(Resource):
                 f"Error getting tool actions: {err}", exc_info=True
             )
             return make_response(jsonify({"success": False}), 400)
+        definition = definition_for_tool(data["name"])
+        if definition is not None:
+            connected = _create_connected_tool(user, data, definition, tool_instance)
+            if connected is not None:
+                return connected
         try:
             config_requirements = tool_instance.get_config_requirements()
             if config_requirements:
@@ -422,6 +431,74 @@ class CreateTool(Resource):
         return make_response(jsonify({"id": new_id}), 200)
 
 
+def _create_connected_tool(user, data, definition, tool_instance):
+    """Create a service tool whose secret lives on a connection, not the tool.
+
+    Uses ``connection_id`` when given, otherwise stores the pasted secret on
+    a connection (reusing an identical one). Returns None to fall back to the
+    legacy path when a multi-user install still runs on the default key.
+    """
+    from docsgpt.connectors import catalog as connector_catalog
+    from docsgpt.connectors import service as connection_service
+    from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository
+
+    config_requirements = tool_instance.get_config_requirements()
+    public, secrets = connection_service.split_secrets(data.get("config") or {}, config_requirements)
+    connection_id = data.get("connection_id")
+    if not connection_id:
+        validation_errors = _validate_config(data.get("config") or {}, config_requirements)
+        if validation_errors:
+            return make_response(
+                jsonify({"success": False, "message": "Validation failed", "errors": validation_errors}), 400,
+            )
+    try:
+        with db_session() as conn:
+            if connection_id:
+                connection = ConnectorSessionsRepository(conn).get_for_user(str(connection_id), user)
+                if connection is None or connector_catalog.connector_key_for_row(connection) != definition.key:
+                    return make_response(jsonify({"success": False, "message": "Connection not found"}), 404)
+            else:
+                connection, _ = connection_service.create_api_key_connection(conn, user, definition, secrets)
+            created = connection_service.create_tool_for_connection(
+                conn,
+                user,
+                connection,
+                template=data["name"],
+                display_name=data.get("customName") or data.get("displayName") or definition.name,
+                config=public,
+                status=bool(data.get("status", True)),
+            )
+    except connection_service.EncryptionKeyNotConfigured:
+        return None
+    except ValueError as err:
+        return make_response(jsonify({"success": False, "message": str(err)}), 400)
+    return make_response(jsonify({"id": str(created["id"]), "connection_id": str(connection["id"])}), 200)
+
+
+def _update_connection_secrets(conn, user, tool_doc, config, config_requirements):
+    """Write changed secrets onto the tool's connection; return the tool's public config.
+
+    Returns None when the caller does not own the connection (an editor on a
+    team share may change actions, never the owner's credentials).
+    """
+    from docsgpt.connectors import service as connection_service
+    from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository
+
+    public, secrets = connection_service.split_secrets(config or {}, config_requirements)
+    if secrets:
+        connection = ConnectorSessionsRepository(conn).get_for_user(str(tool_doc["connection_id"]), user)
+        if connection is None:
+            return None
+        stored = connection_service.read_secrets(connection)
+        credentials = {**(stored.get("credentials") or {}), **secrets}
+        connection_service.write_secrets(
+            conn, connection, {**stored, "credentials": credentials},
+            status=connection_service.STATUS_CONNECTED, last_error=None,
+        )
+        connection_service.resume_sources(conn, str(connection["id"]))
+    return public
+
+
 @tools_ns.route("/update_tool")
 class UpdateTool(Resource):
     @api.expect(
@@ -528,6 +605,17 @@ class UpdateTool(Resource):
                     )
                     existing_config = tool_doc.get("config", {}) or {}
                     has_existing_secrets = "encrypted_credentials" in existing_config
+                    if tool_doc.get("connection_id"):
+                        has_existing_secrets = True
+                        new_config = _update_connection_secrets(
+                            conn, user, tool_doc, data["config"], config_requirements,
+                        )
+                        if new_config is None:
+                            return make_response(
+                                jsonify({"success": False, "message": "Only the owner can change the credentials"}),
+                                403,
+                            )
+                        data = {**data, "config": new_config}
 
                     if config_requirements:
                         validation_errors = _validate_config(