Connections own their credentials

Migration 0038 moves every stored secret (OAuth tokens, MCP OAuth
tokens and client registrations, API keys) into the connection's
encrypted envelope, links API-key tools to one connection per distinct
credential, allows several accounts per provider, and adds
credential_mode to sources and tools. OAuth MCP tools keep resolving
each member's own token, as they did before.

docsgpt.connectors.service is now the only reader of OAuth tokens:
get_valid_token_info refreshes under a row lock and persists rotated
refresh tokens, and a revoked grant flags the connection, pauses its
sources and notifies the owner. Loaders build from a connection
(BaseConnectorLoader.from_connection), so scheduled sync covers Drive,
SharePoint and Confluence sources with no browser. S3 and Reddit keys
stay on the connection instead of in remote_data.

New endpoints: POST /api/connections, /setup, /reconnect,
/picker-token, /claim, DELETE /api/connections/<id>, per-action
permissions and MCP refresh-tools. Upload, file listing, sync and
validate-session take a connection_id; session tokens keep working for
this release. The tool executor reads credentials from the resolved
connection (owner or member mode) and pauses on a Connect card when a
connection needs signing in. docsgpt connectors reencrypt rewrites
stored credentials after a key rotation.

docsgpt/api/user/sources/routes.py

@@ -231,6 +231,63 @@ class PaginatedSources(Resource):
             return make_response(jsonify({"success": False}), 400)
 
 
+def delete_source(user: str, doc: dict) -> bool:
+    """Delete a source's index, stored files and row. Returns whether it worked.
+
+    Args:
+        user: The owner deleting it.
+        doc: The source row, already authorised for ``user``.
+    """
+    storage = StorageCreator.get_storage()
+    resolved_id = str(doc["id"])
+    source_id = resolved_id
+
+    try:
+        if settings.VECTOR_STORE == "faiss":
+            index_path = f"indexes/{resolved_id}"
+            # index.pkl is the legacy sidecar; index.json the current one.
+            # Older sources have only the former, so clear whichever exist.
+            for index_file in ("index.faiss", "index.json", "index.pkl"):
+                if storage.file_exists(f"{index_path}/{index_file}"):
+                    storage.delete_file(f"{index_path}/{index_file}")
+        else:
+            vectorstore = VectorCreator.create_vectorstore(
+                settings.VECTOR_STORE, source_id=source_id
+            )
+            vectorstore.delete_index()
+        if "file_path" in doc and doc["file_path"]:
+            file_path = doc["file_path"]
+            if storage.is_directory(file_path):
+                files = storage.list_files(file_path)
+                for f in files:
+                    storage.delete_file(f)
+            else:
+                storage.delete_file(file_path)
+    except FileNotFoundError:
+        pass
+    except Exception as err:
+        current_app.logger.error(
+            f"Error deleting files and indexes: {err}", exc_info=True
+        )
+        return False
+    try:
+        with db_session() as conn:
+            SourcesRepository(conn).delete(resolved_id, user)
+            record_event(
+                conn,
+                "source.deleted",
+                actor=user,
+                source_id=resolved_id,
+                name=doc.get("name"),
+            )
+    except Exception as err:
+        current_app.logger.error(
+            f"Error deleting source row: {err}", exc_info=True
+        )
+        return False
+    return True
+
+
 @sources_ns.route("/delete_old")
 class DeleteOldIndexes(Resource):
     @api.doc(
@@ -255,51 +312,7 @@ class DeleteOldIndexes(Resource):
             return make_response(jsonify({"success": False}), 400)
         if not doc:
             return make_response(jsonify({"status": "not found"}), 404)
-        storage = StorageCreator.get_storage()
-        resolved_id = str(doc["id"])
-
-        try:
-            if settings.VECTOR_STORE == "faiss":
-                index_path = f"indexes/{resolved_id}"
-                # index.pkl is the legacy sidecar; index.json the current one.
-                # Older sources have only the former, so clear whichever exist.
-                for index_file in ("index.faiss", "index.json", "index.pkl"):
-                    if storage.file_exists(f"{index_path}/{index_file}"):
-                        storage.delete_file(f"{index_path}/{index_file}")
-            else:
-                vectorstore = VectorCreator.create_vectorstore(
-                    settings.VECTOR_STORE, source_id=resolved_id
-                )
-                vectorstore.delete_index()
-            if "file_path" in doc and doc["file_path"]:
-                file_path = doc["file_path"]
-                if storage.is_directory(file_path):
-                    files = storage.list_files(file_path)
-                    for f in files:
-                        storage.delete_file(f)
-                else:
-                    storage.delete_file(file_path)
-        except FileNotFoundError:
-            pass
-        except Exception as err:
-            current_app.logger.error(
-                f"Error deleting files and indexes: {err}", exc_info=True
-            )
-            return make_response(jsonify({"success": False}), 400)
-        try:
-            with db_session() as conn:
-                SourcesRepository(conn).delete(resolved_id, user)
-                record_event(
-                    conn,
-                    "source.deleted",
-                    actor=user,
-                    source_id=resolved_id,
-                    name=doc.get("name"),
-                )
-        except Exception as err:
-            current_app.logger.error(
-                f"Error deleting source row: {err}", exc_info=True
-            )
+        if not delete_source(user, doc):
             return make_response(jsonify({"success": False}), 400)
         return make_response(jsonify({"success": True}), 200)
 

docsgpt/api/user/sources/upload.py

@@ -28,7 +28,6 @@ from docsgpt.security.zip_archive import (
 )
 from docsgpt.storage.db.repositories.connector_sessions import (
     ConnectorSessionsRepository,
-    owns_connector_session,
 )
 from docsgpt.storage.db.repositories.idempotency import IdempotencyRepository
 from docsgpt.storage.db.repositories.sources import SourcesRepository
@@ -451,6 +450,43 @@ class UploadFile(Resource):
         return make_response(jsonify(response_payload), 200)
 
 
+def _remote_credentials(user, source, config):
+    """Split an S3 / Reddit request into loader config and the connection holding its keys.
+
+    A request naming a ``connection_id`` uses that connection's stored keys.
+    A request carrying keys (the form before connections) stores them on a
+    connection, so they are entered once and never land in
+    ``sources.remote_data``. When a multi-user install still runs on the
+    public default encryption key, the keys stay with the source as before.
+
+    Returns:
+        ``(source_data, connection_id, error_response)``.
+    """
+    from docsgpt.connectors import catalog, service
+
+    definition = catalog.get_definition(source)
+    credential_keys = {f.key for f in definition.credential_fields}
+    public = {k: v for k, v in config.items() if k not in credential_keys and k != "connection_id"}
+    connection_id = config.get("connection_id")
+    if connection_id:
+        with db_readonly() as conn:
+            row = ConnectorSessionsRepository(conn).get_for_user(str(connection_id), user)
+        if row is None or catalog.connector_key_for_row(row) != source:
+            return None, None, make_response(
+                jsonify({"success": False, "error": "Invalid or unauthorized connection"}), 401,
+            )
+        return public, str(row["id"]), None
+    provided = {k: config[k] for k in credential_keys if config.get(k) not in (None, "")}
+    if not provided:
+        return config, None, None
+    try:
+        with db_session() as conn:
+            row, _ = service.create_api_key_connection(conn, user, definition, provided)
+    except (service.EncryptionKeyNotConfigured, ValueError):
+        return config, None, None
+    return public, str(row["id"]), None
+
+
 @sources_upload_ns.route("/remote")
 class UploadRemote(Resource):
     @api.expect(
@@ -520,32 +556,35 @@ class UploadRemote(Resource):
         try:
             config = json.loads(data["data"])
             source_data = None
+            connection_id = None
 
             if data["source"] == "github":
                 source_data = config.get("repo_url")
             elif data["source"] in ["crawler", "url", "sitemap"]:
                 source_data = config.get("url")
-            elif data["source"] == "reddit":
-                source_data = config
-            elif data["source"] == "s3":
-                source_data = config
+            elif data["source"] in ("reddit", "s3"):
+                source_data, connection_id, error = _remote_credentials(user, data["source"], config)
+                if error is not None:
+                    if scoped_key:
+                        _release_claim(scoped_key)
+                    return error
             elif data["source"] in ConnectorCreator.get_supported_connectors():
-                session_token = config.get("session_token")
-                if not session_token:
+                if not (config.get("connection_id") or config.get("session_token")):
                     if scoped_key:
                         _release_claim(scoped_key)
                     return make_response(
                         jsonify(
                             {
                                 "success": False,
-                                "error": f"Missing session_token in {data['source']} configuration",
+                                "error": f"Missing connection_id in {data['source']} configuration",
                             }
                         ),
                         400,
                     )
-                with db_readonly() as conn:
-                    connector_session = ConnectorSessionsRepository(conn).get_by_session_token(session_token)
-                if not owns_connector_session(connector_session, user, data["source"]):
+                from docsgpt.connectors import service as connection_service
+
+                connector_session = connection_service.resolve_request_connection(user, data["source"], config)
+                if connector_session is None:
                     if scoped_key:
                         _release_claim(scoped_key)
                     return make_response(
@@ -576,7 +615,7 @@ class UploadRemote(Resource):
                         "job_name": data["name"],
                         "user": user,
                         "source_type": data["source"],
-                        "session_token": session_token,
+                        "connection_id": str(connector_session["id"]),
                         "file_ids": file_ids,
                         "folder_ids": folder_ids,
                         "recursive": config.get("recursive", False),
@@ -611,6 +650,7 @@ class UploadRemote(Resource):
             remote_kwargs = {
                 "kwargs": {
                     "source_data": source_data,
+                    "connection_id": connection_id,
                     "job_name": data["name"],
                     "user": user,
                     "loader": data["source"],