Connections own their credentials

Migration 0038 moves every stored secret (OAuth tokens, MCP OAuth
tokens and client registrations, API keys) into the connection's
encrypted envelope, links API-key tools to one connection per distinct
credential, allows several accounts per provider, and adds
credential_mode to sources and tools. OAuth MCP tools keep resolving
each member's own token, as they did before.

docsgpt.connectors.service is now the only reader of OAuth tokens:
get_valid_token_info refreshes under a row lock and persists rotated
refresh tokens, and a revoked grant flags the connection, pauses its
sources and notifies the owner. Loaders build from a connection
(BaseConnectorLoader.from_connection), so scheduled sync covers Drive,
SharePoint and Confluence sources with no browser. S3 and Reddit keys
stay on the connection instead of in remote_data.

New endpoints: POST /api/connections, /setup, /reconnect,
/picker-token, /claim, DELETE /api/connections/<id>, per-action
permissions and MCP refresh-tools. Upload, file listing, sync and
validate-session take a connection_id; session tokens keep working for
this release. The tool executor reads credentials from the resolved
connection (owner or member mode) and pauses on a Connect card when a
connection needs signing in. docsgpt connectors reencrypt rewrites
stored credentials after a key rotation.

docsgpt/api/user/sources/routes.py

@@ -231,6 +231,63 @@ class PaginatedSources(Resource):
             return make_response(jsonify({"success": False}), 400)
 
 
+def delete_source(user: str, doc: dict) -> bool:
+    """Delete a source's index, stored files and row. Returns whether it worked.
+
+    Args:
+        user: The owner deleting it.
+        doc: The source row, already authorised for ``user``.
+    """
+    storage = StorageCreator.get_storage()
+    resolved_id = str(doc["id"])
+    source_id = resolved_id
+
+    try:
+        if settings.VECTOR_STORE == "faiss":
+            index_path = f"indexes/{resolved_id}"
+            # index.pkl is the legacy sidecar; index.json the current one.
+            # Older sources have only the former, so clear whichever exist.
+            for index_file in ("index.faiss", "index.json", "index.pkl"):
+                if storage.file_exists(f"{index_path}/{index_file}"):
+                    storage.delete_file(f"{index_path}/{index_file}")
+        else:
+            vectorstore = VectorCreator.create_vectorstore(
+                settings.VECTOR_STORE, source_id=source_id
+            )
+            vectorstore.delete_index()
+        if "file_path" in doc and doc["file_path"]:
+            file_path = doc["file_path"]
+            if storage.is_directory(file_path):
+                files = storage.list_files(file_path)
+                for f in files:
+                    storage.delete_file(f)
+            else:
+                storage.delete_file(file_path)
+    except FileNotFoundError:
+        pass
+    except Exception as err:
+        current_app.logger.error(
+            f"Error deleting files and indexes: {err}", exc_info=True
+        )
+        return False
+    try:
+        with db_session() as conn:
+            SourcesRepository(conn).delete(resolved_id, user)
+            record_event(
+                conn,
+                "source.deleted",
+                actor=user,
+                source_id=resolved_id,
+                name=doc.get("name"),
+            )
+    except Exception as err:
+        current_app.logger.error(
+            f"Error deleting source row: {err}", exc_info=True
+        )
+        return False
+    return True
+
+
 @sources_ns.route("/delete_old")
 class DeleteOldIndexes(Resource):
     @api.doc(
@@ -255,51 +312,7 @@ class DeleteOldIndexes(Resource):
             return make_response(jsonify({"success": False}), 400)
         if not doc:
             return make_response(jsonify({"status": "not found"}), 404)
-        storage = StorageCreator.get_storage()
-        resolved_id = str(doc["id"])
-
-        try:
-            if settings.VECTOR_STORE == "faiss":
-                index_path = f"indexes/{resolved_id}"
-                # index.pkl is the legacy sidecar; index.json the current one.
-                # Older sources have only the former, so clear whichever exist.
-                for index_file in ("index.faiss", "index.json", "index.pkl"):
-                    if storage.file_exists(f"{index_path}/{index_file}"):
-                        storage.delete_file(f"{index_path}/{index_file}")
-            else:
-                vectorstore = VectorCreator.create_vectorstore(
-                    settings.VECTOR_STORE, source_id=resolved_id
-                )
-                vectorstore.delete_index()
-            if "file_path" in doc and doc["file_path"]:
-                file_path = doc["file_path"]
-                if storage.is_directory(file_path):
-                    files = storage.list_files(file_path)
-                    for f in files:
-                        storage.delete_file(f)
-                else:
-                    storage.delete_file(file_path)
-        except FileNotFoundError:
-            pass
-        except Exception as err:
-            current_app.logger.error(
-                f"Error deleting files and indexes: {err}", exc_info=True
-            )
-            return make_response(jsonify({"success": False}), 400)
-        try:
-            with db_session() as conn:
-                SourcesRepository(conn).delete(resolved_id, user)
-                record_event(
-                    conn,
-                    "source.deleted",
-                    actor=user,
-                    source_id=resolved_id,
-                    name=doc.get("name"),
-                )
-        except Exception as err:
-            current_app.logger.error(
-                f"Error deleting source row: {err}", exc_info=True
-            )
+        if not delete_source(user, doc):
             return make_response(jsonify({"success": False}), 400)
         return make_response(jsonify({"success": True}), 200)
 

docsgpt/api/user/sources/upload.py

@@ -28,7 +28,6 @@ from docsgpt.security.zip_archive import (
 )
 from docsgpt.storage.db.repositories.connector_sessions import (
     ConnectorSessionsRepository,
-    owns_connector_session,
 )
 from docsgpt.storage.db.repositories.idempotency import IdempotencyRepository
 from docsgpt.storage.db.repositories.sources import SourcesRepository
@@ -451,6 +450,43 @@ class UploadFile(Resource):
         return make_response(jsonify(response_payload), 200)
 
 
+def _remote_credentials(user, source, config):
+    """Split an S3 / Reddit request into loader config and the connection holding its keys.
+
+    A request naming a ``connection_id`` uses that connection's stored keys.
+    A request carrying keys (the form before connections) stores them on a
+    connection, so they are entered once and never land in
+    ``sources.remote_data``. When a multi-user install still runs on the
+    public default encryption key, the keys stay with the source as before.
+
+    Returns:
+        ``(source_data, connection_id, error_response)``.
+    """
+    from docsgpt.connectors import catalog, service
+
+    definition = catalog.get_definition(source)
+    credential_keys = {f.key for f in definition.credential_fields}
+    public = {k: v for k, v in config.items() if k not in credential_keys and k != "connection_id"}
+    connection_id = config.get("connection_id")
+    if connection_id:
+        with db_readonly() as conn:
+            row = ConnectorSessionsRepository(conn).get_for_user(str(connection_id), user)
+        if row is None or catalog.connector_key_for_row(row) != source:
+            return None, None, make_response(
+                jsonify({"success": False, "error": "Invalid or unauthorized connection"}), 401,
+            )
+        return public, str(row["id"]), None
+    provided = {k: config[k] for k in credential_keys if config.get(k) not in (None, "")}
+    if not provided:
+        return config, None, None
+    try:
+        with db_session() as conn:
+            row, _ = service.create_api_key_connection(conn, user, definition, provided)
+    except (service.EncryptionKeyNotConfigured, ValueError):
+        return config, None, None
+    return public, str(row["id"]), None
+
+
 @sources_upload_ns.route("/remote")
 class UploadRemote(Resource):
     @api.expect(
@@ -520,32 +556,35 @@ class UploadRemote(Resource):
         try:
             config = json.loads(data["data"])
             source_data = None
+            connection_id = None
 
             if data["source"] == "github":
                 source_data = config.get("repo_url")
             elif data["source"] in ["crawler", "url", "sitemap"]:
                 source_data = config.get("url")
-            elif data["source"] == "reddit":
-                source_data = config
-            elif data["source"] == "s3":
-                source_data = config
+            elif data["source"] in ("reddit", "s3"):
+                source_data, connection_id, error = _remote_credentials(user, data["source"], config)
+                if error is not None:
+                    if scoped_key:
+                        _release_claim(scoped_key)
+                    return error
             elif data["source"] in ConnectorCreator.get_supported_connectors():
-                session_token = config.get("session_token")
-                if not session_token:
+                if not (config.get("connection_id") or config.get("session_token")):
                     if scoped_key:
                         _release_claim(scoped_key)
                     return make_response(
                         jsonify(
                             {
                                 "success": False,
-                                "error": f"Missing session_token in {data['source']} configuration",
+                                "error": f"Missing connection_id in {data['source']} configuration",
                             }
                         ),
                         400,
                     )
-                with db_readonly() as conn:
-                    connector_session = ConnectorSessionsRepository(conn).get_by_session_token(session_token)
-                if not owns_connector_session(connector_session, user, data["source"]):
+                from docsgpt.connectors import service as connection_service
+
+                connector_session = connection_service.resolve_request_connection(user, data["source"], config)
+                if connector_session is None:
                     if scoped_key:
                         _release_claim(scoped_key)
                     return make_response(
@@ -576,7 +615,7 @@ class UploadRemote(Resource):
                         "job_name": data["name"],
                         "user": user,
                         "source_type": data["source"],
-                        "session_token": session_token,
+                        "connection_id": str(connector_session["id"]),
                         "file_ids": file_ids,
                         "folder_ids": folder_ids,
                         "recursive": config.get("recursive", False),
@@ -611,6 +650,7 @@ class UploadRemote(Resource):
             remote_kwargs = {
                 "kwargs": {
                     "source_data": source_data,
+                    "connection_id": connection_id,
                     "job_name": data["name"],
                     "user": user,
                     "loader": data["source"],

docsgpt/api/user/tasks.py

@@ -79,6 +79,12 @@ DURABLE_TASK = dict(
 )
 
 
+def _connection_unavailable():
+    from docsgpt.connectors.service import ConnectionUnavailable
+
+    return ConnectionUnavailable
+
+
 def durable_task(**overrides) -> Dict:
     """Return ``DURABLE_TASK`` with per-task overrides applied.
 
@@ -167,13 +173,16 @@ def ingest(
 @with_idempotency(task_name="ingest_remote", on_poison=_emit_ingest_poison_event)
 def ingest_remote(
     self, source_data, job_name, user, loader,
-    config=None, idempotency_key=None, source_id=None,
+    config=None, idempotency_key=None, source_id=None, connection_id=None,
+    sync_frequency="never",
 ):
     resp = remote_worker(
         self, source_data, job_name, user, loader,
+        sync_frequency=sync_frequency,
         config=config,
         idempotency_key=idempotency_key,
         source_id=source_id,
+        connection_id=connection_id,
     )
     return resp
 
@@ -249,6 +258,15 @@ def schedule_syncs(self, frequency):
     return resp
 
 
+@celery.task(bind=True, acks_late=True, autoretry_for=(Exception,), max_retries=3, retry_backoff=60,
+             dont_autoretry_for=(_connection_unavailable(),))
+def sync_connector_source(self, source_id):
+    """Re-sync one connector source from its connection, with no browser involved."""
+    from docsgpt.worker import sync_connector_source as run
+
+    return run(self, source_id)
+
+
 @celery.task(bind=True)
 def sync_source(
     self,
@@ -408,7 +426,9 @@ except Exception:
     pass
 
 
-@celery.task(**DURABLE_TASK)
+# A revoked or disconnected connection will not heal by retrying; the
+# service has already paused the source and told its owner to reconnect.
+@celery.task(**durable_task(dont_autoretry_for=(DocumentParseError, _connection_unavailable())))
 @with_idempotency(
     task_name="ingest_connector_task", on_poison=_emit_ingest_poison_event,
 )
@@ -428,6 +448,7 @@ def ingest_connector_task(
     config=None,
     idempotency_key=None,
     source_id=None,
+    connection_id=None,
 ):
     from docsgpt.worker import ingest_connector
 
@@ -437,6 +458,7 @@ def ingest_connector_task(
         user,
         source_type,
         session_token=session_token,
+        connection_id=connection_id,
         file_ids=file_ids,
         folder_ids=folder_ids,
         recursive=recursive,

docsgpt/api/user/tools/mcp.py

@@ -75,6 +75,43 @@ def _validate_mcp_server_url(config: dict) -> None:
         raise ValueError(f"Invalid server URL: {exc}") from exc
 
 
+def _mcp_connection(user, config, auth_type, auth_credentials, display_name):
+    """The connection an MCP tool runs with; created on first save.
+
+    OAuth servers already have one (the sign-in stored its tokens there).
+    Key, bearer and basic auth store their secret on a connection; servers
+    with no auth get a credential-less connection so they still appear on
+    the Connectors page. Returns None when a multi-user install runs on the
+    default encryption key, which keeps the legacy per-tool secret.
+    """
+    from docsgpt.connectors import catalog, service
+
+    base = catalog.base_url(config.get("server_url"))
+    if not base:
+        return None
+    if auth_type == "oauth":
+        with db_readonly() as conn:
+            row = service._mcp_row(conn, user, base, None)
+        return str(row["id"]) if row else None
+    definition = catalog.get_definition("custom_mcp")
+    host = base.split("://")[-1]
+    try:
+        with db_session() as conn:
+            if auth_credentials:
+                row, _ = service.create_api_key_connection(
+                    conn, user, definition, auth_credentials, server_url=base, display_name=display_name,
+                )
+            else:
+                repo = ConnectorSessionsRepository(conn)
+                row = repo.find_account(user, "custom_mcp", server_url=base, account_label=host) or repo.create(
+                    user, "custom_mcp", connector_key="custom_mcp", auth_kind="none",
+                    display_name=display_name, account_label=host, server_url=base,
+                )
+    except service.EncryptionKeyNotConfigured:
+        return None
+    return str(row["id"]) if row else None
+
+
 @tools_mcp_ns.route("/mcp_server/test")
 class TestMCPServerConfig(Resource):
     @api.expect(
@@ -283,9 +320,19 @@ class MCPServerSave(Resource):
                 "redirect_uri",
             ]:
                 storage_config.pop(field, None)
-            transformed_actions = transform_actions(actions_metadata)
+            from docsgpt.connectors.permissions import apply_default_permissions
+
+            transformed_actions = apply_default_permissions(
+                "mcp_tool", transform_actions(actions_metadata),
+            )
 
             display_name = data["displayName"]
+            connection_id = _mcp_connection(
+                user, storage_config, auth_type, auth_credentials, display_name,
+            ) or (str(existing_doc["connection_id"]) if existing_doc and existing_doc.get("connection_id") else None)
+            if connection_id and auth_type != "oauth":
+                # The secret lives on the connection only.
+                storage_config.pop("encrypted_credentials", None)
             description = f"MCP Server: {storage_config.get('server_url', 'Unknown')}"
             status_bool = bool(data.get("status", True))
 
@@ -301,6 +348,7 @@ class MCPServerSave(Resource):
                             "config": storage_config,
                             "actions": transformed_actions,
                             "status": status_bool,
+                            "connection_id": connection_id,
                         },
                     )
                     saved_id = str(existing_doc["id"])
@@ -328,6 +376,7 @@ class MCPServerSave(Resource):
                                 "config": storage_config,
                                 "actions": transformed_actions,
                                 "status": status_bool,
+                                "connection_id": connection_id,
                             },
                         )
                         saved_id = str(existing_by_name["id"])
@@ -347,6 +396,7 @@ class MCPServerSave(Resource):
                             config_requirements={},
                             actions=transformed_actions,
                             status=status_bool,
+                            connection_id=connection_id,
                         )
                         saved_id = str(created["id"])
                         response_data = {
@@ -461,50 +511,34 @@ class MCPAuthStatus(Resource):
                         jsonify({"success": True, "statuses": {}}), 200
                     )
 
-                oauth_server_urls: dict = {}
+                from docsgpt.connectors import service
+
+                # Read from connection status alone: status checks never
+                # decrypt credentials.
                 statuses: dict = {}
                 for tool in mcp_tools:
                     tool_id = str(tool["id"])
                     config = tool.get("config") or {}
                     auth_type = config.get("auth_type", "none")
-                    if auth_type == "oauth":
-                        server_url = config.get("server_url", "")
-                        if server_url:
-                            parsed = urlparse(server_url)
-                            base_url = f"{parsed.scheme}://{parsed.netloc}"
-                            oauth_server_urls[tool_id] = base_url
+                    row = None
+                    if tool.get("connection_id"):
+                        row = sessions_repo.get(str(tool["connection_id"]))
+                    elif auth_type == "oauth" and config.get("server_url"):
+                        parsed = urlparse(config["server_url"])
+                        row = sessions_repo.get_by_user_provider(
+                            user, service.mcp_provider(f"{parsed.scheme}://{parsed.netloc}"),
+                        )
+                    if row is not None:
+                        connected = service.normalize_status(row) == service.STATUS_CONNECTED
+                        if auth_type == "oauth" or not connected:
+                            statuses[tool_id] = "connected" if connected else "needs_auth"
                         else:
-                            statuses[tool_id] = "needs_auth"
+                            statuses[tool_id] = "configured"
+                    elif auth_type == "oauth":
+                        statuses[tool_id] = "needs_auth"
                     else:
                         statuses[tool_id] = "configured"
 
-                if oauth_server_urls:
-                    # Look up a session per distinct base URL. MCP sessions
-                    # are stored with ``provider = "mcp:<server_url>"``
-                    # and the URL in ``server_url``; reuse the repo's
-                    # per-URL accessor rather than an ad-hoc $in query.
-                    url_has_tokens: dict = {}
-                    for base_url in set(oauth_server_urls.values()):
-                        session = sessions_repo.get_by_user_and_server_url(
-                            user, base_url,
-                        )
-                        tokens = (
-                            (session or {}).get("session_data", {}) or {}
-                        ).get("tokens", {}) or {}
-                        # MCP code also stashes tokens into token_info on
-                        # the row; consider either present as "connected".
-                        token_info = (session or {}).get("token_info") or {}
-                        url_has_tokens[base_url] = bool(
-                            tokens.get("access_token")
-                            or token_info.get("access_token")
-                        )
-
-                    for tool_id, base_url in oauth_server_urls.items():
-                        if url_has_tokens.get(base_url):
-                            statuses[tool_id] = "connected"
-                        else:
-                            statuses[tool_id] = "needs_auth"
-
             return make_response(jsonify({"success": True, "statuses": statuses}), 200)
         except Exception as e:
             current_app.logger.error(

docsgpt/api/user/tools/routes.py

@@ -271,6 +271,10 @@ class GetTools(Resource):
                 ):
                     tool_copy["config"]["has_encrypted_credentials"] = True
                     tool_copy["config"].pop("encrypted_credentials", None)
+                if tool_copy.get("connection_id"):
+                    # The secret lives on the connection; the form must not
+                    # ask for it again.
+                    tool_copy.setdefault("config", {})["has_encrypted_credentials"] = True
                 tool_copy["ownership"] = ownership
                 return tool_copy
 
@@ -383,6 +387,11 @@ class CreateTool(Resource):
                 f"Error getting tool actions: {err}", exc_info=True
             )
             return make_response(jsonify({"success": False}), 400)
+        definition = definition_for_tool(data["name"])
+        if definition is not None:
+            connected = _create_connected_tool(user, data, definition, tool_instance)
+            if connected is not None:
+                return connected
         try:
             config_requirements = tool_instance.get_config_requirements()
             if config_requirements:
@@ -422,6 +431,74 @@ class CreateTool(Resource):
         return make_response(jsonify({"id": new_id}), 200)
 
 
+def _create_connected_tool(user, data, definition, tool_instance):
+    """Create a service tool whose secret lives on a connection, not the tool.
+
+    Uses ``connection_id`` when given, otherwise stores the pasted secret on
+    a connection (reusing an identical one). Returns None to fall back to the
+    legacy path when a multi-user install still runs on the default key.
+    """
+    from docsgpt.connectors import catalog as connector_catalog
+    from docsgpt.connectors import service as connection_service
+    from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository
+
+    config_requirements = tool_instance.get_config_requirements()
+    public, secrets = connection_service.split_secrets(data.get("config") or {}, config_requirements)
+    connection_id = data.get("connection_id")
+    if not connection_id:
+        validation_errors = _validate_config(data.get("config") or {}, config_requirements)
+        if validation_errors:
+            return make_response(
+                jsonify({"success": False, "message": "Validation failed", "errors": validation_errors}), 400,
+            )
+    try:
+        with db_session() as conn:
+            if connection_id:
+                connection = ConnectorSessionsRepository(conn).get_for_user(str(connection_id), user)
+                if connection is None or connector_catalog.connector_key_for_row(connection) != definition.key:
+                    return make_response(jsonify({"success": False, "message": "Connection not found"}), 404)
+            else:
+                connection, _ = connection_service.create_api_key_connection(conn, user, definition, secrets)
+            created = connection_service.create_tool_for_connection(
+                conn,
+                user,
+                connection,
+                template=data["name"],
+                display_name=data.get("customName") or data.get("displayName") or definition.name,
+                config=public,
+                status=bool(data.get("status", True)),
+            )
+    except connection_service.EncryptionKeyNotConfigured:
+        return None
+    except ValueError as err:
+        return make_response(jsonify({"success": False, "message": str(err)}), 400)
+    return make_response(jsonify({"id": str(created["id"]), "connection_id": str(connection["id"])}), 200)
+
+
+def _update_connection_secrets(conn, user, tool_doc, config, config_requirements):
+    """Write changed secrets onto the tool's connection; return the tool's public config.
+
+    Returns None when the caller does not own the connection (an editor on a
+    team share may change actions, never the owner's credentials).
+    """
+    from docsgpt.connectors import service as connection_service
+    from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository
+
+    public, secrets = connection_service.split_secrets(config or {}, config_requirements)
+    if secrets:
+        connection = ConnectorSessionsRepository(conn).get_for_user(str(tool_doc["connection_id"]), user)
+        if connection is None:
+            return None
+        stored = connection_service.read_secrets(connection)
+        credentials = {**(stored.get("credentials") or {}), **secrets}
+        connection_service.write_secrets(
+            conn, connection, {**stored, "credentials": credentials},
+            status=connection_service.STATUS_CONNECTED, last_error=None,
+        )
+        connection_service.resume_sources(conn, str(connection["id"]))
+    return public
+
+
 @tools_ns.route("/update_tool")
 class UpdateTool(Resource):
     @api.expect(
@@ -528,6 +605,17 @@ class UpdateTool(Resource):
                     )
                     existing_config = tool_doc.get("config", {}) or {}
                     has_existing_secrets = "encrypted_credentials" in existing_config
+                    if tool_doc.get("connection_id"):
+                        has_existing_secrets = True
+                        new_config = _update_connection_secrets(
+                            conn, user, tool_doc, data["config"], config_requirements,
+                        )
+                        if new_config is None:
+                            return make_response(
+                                jsonify({"success": False, "message": "Only the owner can change the credentials"}),
+                                403,
+                            )
+                        data = {**data, "config": new_config}
 
                     if config_requirements:
                         validation_errors = _validate_config(