Connections own their credentials

Migration 0038 moves every stored secret (OAuth tokens, MCP OAuth
tokens and client registrations, API keys) into the connection's
encrypted envelope, links API-key tools to one connection per distinct
credential, allows several accounts per provider, and adds
credential_mode to sources and tools. OAuth MCP tools keep resolving
each member's own token, as they did before.

docsgpt.connectors.service is now the only reader of OAuth tokens:
get_valid_token_info refreshes under a row lock and persists rotated
refresh tokens, and a revoked grant flags the connection, pauses its
sources and notifies the owner. Loaders build from a connection
(BaseConnectorLoader.from_connection), so scheduled sync covers Drive,
SharePoint and Confluence sources with no browser. S3 and Reddit keys
stay on the connection instead of in remote_data.

New endpoints: POST /api/connections, /setup, /reconnect,
/picker-token, /claim, DELETE /api/connections/<id>, per-action
permissions and MCP refresh-tools. Upload, file listing, sync and
validate-session take a connection_id; session tokens keep working for
this release. The tool executor reads credentials from the resolved
connection (owner or member mode) and pauses on a Connect card when a
connection needs signing in. docsgpt connectors reencrypt rewrites
stored credentials after a key rotation.

docsgpt/parser/connectors/base.py

@@ -6,7 +6,7 @@ interface for external knowledge base connectors.
 """
 
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 from docsgpt.parser.schema.base import Document
 
@@ -88,18 +88,55 @@ class BaseConnectorLoader(ABC):
     Abstract base class for connector loaders.
     
     Defines the minimal interface that all connector loader
-    implementations must follow.
+    implementations must follow. A loader reads its OAuth tokens through
+    ``docsgpt.connectors.service`` from the connection it was built for,
+    either directly (``connection_id``, what background sync uses) or through
+    a legacy browser ``session_token`` that names the connection.
     """
-    
+
+    connection_id: Optional[str] = None
+
     @abstractmethod
-    def __init__(self, session_token: str):
+    def __init__(self, session_token: Optional[str] = None, *, connection_id: Optional[str] = None):
         """
         Initialize the connector loader.
         
         Args:
-            session_token: Authentication session token
+            session_token: Legacy browser session token naming the connection.
+            connection_id: The connection to read tokens from.
         """
         pass
+
+    @classmethod
+    def from_connection(cls, connection_id: str) -> "BaseConnectorLoader":
+        """Build a loader that reads its tokens from ``connection_id``."""
+        return cls(connection_id=connection_id)
+
+    def _load_token_info(
+        self, session_token: Optional[str], connection_id: Optional[str],
+    ) -> Tuple[str, Dict[str, Any]]:
+        """Resolve the connection and return ``(connection_id, token_info)``.
+
+        Raises:
+            ValueError: The connection is missing or needs reconnecting.
+        """
+        from docsgpt.connectors import service
+
+        resolved = connection_id or service.connection_id_for_session_token(session_token)
+        self.connection_id = resolved
+        return resolved, service.get_valid_token_info(resolved)
+
+    def _refresh_rejected_token(self, access_token: Optional[str]) -> Dict[str, Any]:
+        """Token info after the provider answered 401 to ``access_token``.
+
+        Refreshes under the connection's row lock and persists the rotated
+        refresh token, or returns the token another worker already renewed.
+        """
+        from docsgpt.connectors import service
+
+        if not self.connection_id:
+            raise ValueError("Loader has no connection to refresh")
+        return service.get_valid_token_info(self.connection_id, rejected_access_token=access_token)
     
     @abstractmethod
     def load_data(self, inputs: Dict[str, Any]) -> List[Document]:

docsgpt/parser/connectors/confluence/auth.py

@@ -6,7 +6,6 @@ from urllib.parse import urlencode
 import requests
 
 from docsgpt.core.settings import settings
-from docsgpt.parser.connectors._auth_utils import session_token_fingerprint
 from docsgpt.parser.connectors.base import BaseConnectorAuth
 
 logger = logging.getLogger(__name__)
@@ -152,32 +151,6 @@ class ConfluenceAuth(BaseConnectorAuth):
         except Exception:
             return True
 
-    def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]:
-        from docsgpt.storage.db.repositories.connector_sessions import (
-            ConnectorSessionsRepository,
-        )
-        from docsgpt.storage.db.session import db_readonly
-
-        with db_readonly() as conn:
-            session = ConnectorSessionsRepository(conn).get_by_session_token(
-                session_token
-            )
-        if not session:
-            raise ValueError(
-                f"Invalid session token ({session_token_fingerprint(session_token)})"
-            )
-
-        token_info = session.get("token_info")
-        if not token_info:
-            raise ValueError("Session missing token information")
-
-        required = ["access_token", "refresh_token", "cloud_id"]
-        missing = [f for f in required if not token_info.get(f)]
-        if missing:
-            raise ValueError(f"Missing required token fields: {missing}")
-
-        return token_info
-
     def sanitize_token_info(
         self, token_info: Dict[str, Any], **extra_fields
     ) -> Dict[str, Any]:

docsgpt/parser/connectors/confluence/loader.py

@@ -44,12 +44,8 @@ def _retry_on_auth_failure(func):
                     "Auth failure in %s, refreshing token and retrying", func.__name__
                 )
                 try:
-                    new_token_info = self.auth.refresh_access_token(self.refresh_token)
+                    new_token_info = self._refresh_rejected_token(self.access_token)
                     self.access_token = new_token_info["access_token"]
-                    self.refresh_token = new_token_info.get(
-                        "refresh_token", self.refresh_token
-                    )
-                    self._persist_refreshed_tokens(new_token_info)
                 except Exception as refresh_err:
                     raise ValueError(
                         f"Authentication failed and could not be refreshed: {refresh_err}"
@@ -62,13 +58,15 @@ def _retry_on_auth_failure(func):
 
 class ConfluenceLoader(BaseConnectorLoader):
 
-    def __init__(self, session_token: str):
+    def __init__(self, session_token: Optional[str] = None, *, connection_id: Optional[str] = None):
         self.auth = ConfluenceAuth()
         self.session_token = session_token
 
-        token_info = self.auth.get_token_info_from_session(session_token)
+        _, token_info = self._load_token_info(session_token, connection_id)
+        missing = [f for f in ("access_token", "cloud_id") if not token_info.get(f)]
+        if missing:
+            raise ValueError(f"Missing required token fields: {missing}")
         self.access_token = token_info["access_token"]
-        self.refresh_token = token_info["refresh_token"]
         self.cloud_id = token_info["cloud_id"]
 
         self.base_url = API_V2.format(cloud_id=self.cloud_id)
@@ -81,22 +79,6 @@ class ConfluenceLoader(BaseConnectorLoader):
             "Accept": "application/json",
         }
 
-    def _persist_refreshed_tokens(self, token_info: Dict[str, Any]) -> None:
-        try:
-            from docsgpt.storage.db.repositories.connector_sessions import (
-                ConnectorSessionsRepository,
-            )
-            from docsgpt.storage.db.session import db_session
-
-            sanitized = self.auth.sanitize_token_info(token_info)
-            with db_session() as conn:
-                repo = ConnectorSessionsRepository(conn)
-                session = repo.get_by_session_token(self.session_token)
-                if session:
-                    repo.update(str(session["id"]), {"token_info": sanitized})
-        except Exception as e:
-            logger.warning("Failed to persist refreshed tokens: %s", e)
-
     @_retry_on_auth_failure
     def load_data(self, inputs: Dict[str, Any]) -> List[Document]:
         folder_id = inputs.get("folder_id")

docsgpt/parser/connectors/google_drive/auth.py

@@ -8,7 +8,6 @@ from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 
 from docsgpt.core.settings import settings
-from docsgpt.parser.connectors._auth_utils import session_token_fingerprint
 from docsgpt.parser.connectors.base import BaseConnectorAuth
 
 
@@ -213,39 +212,6 @@ class GoogleDriveAuth(BaseConnectorAuth):
 
         return True
     
-    def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]:
-        try:
-            from docsgpt.storage.db.repositories.connector_sessions import (
-                ConnectorSessionsRepository,
-            )
-            from docsgpt.storage.db.session import db_readonly
-
-            with db_readonly() as conn:
-                session = ConnectorSessionsRepository(conn).get_by_session_token(
-                    session_token
-                )
-            if not session:
-                raise ValueError(
-                    f"Invalid session token ({session_token_fingerprint(session_token)})"
-                )
-
-            token_info = session.get("token_info")
-            if not token_info:
-                raise ValueError("Session missing token information")
-
-            required_fields = ["access_token", "refresh_token"]
-            missing_fields = [field for field in required_fields if field not in token_info or not token_info.get(field)]
-            if missing_fields:
-                raise ValueError(f"Missing required token fields: {missing_fields}")
-
-            if 'token_uri' not in token_info:
-                token_info['token_uri'] = 'https://oauth2.googleapis.com/token'
-
-            return token_info
-
-        except Exception as e:
-            raise ValueError(f"Failed to retrieve Google Drive token information: {str(e)}")
-
     def validate_credentials(self, credentials: Credentials) -> bool:
         """
         Validate Google Drive credentials by making a test API call.

docsgpt/parser/connectors/google_drive/loader.py

@@ -48,11 +48,13 @@ class GoogleDriveLoader(BaseConnectorLoader):
         'application/vnd.google-apps.spreadsheet': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
     }
 
-    def __init__(self, session_token: str):
+    def __init__(self, session_token: Optional[str] = None, *, connection_id: Optional[str] = None):
         self.auth = GoogleDriveAuth()
         self.session_token = session_token
 
-        token_info = self.auth.get_token_info_from_session(session_token)
+        _, token_info = self._load_token_info(session_token, connection_id)
+        # Google refresh tokens do not rotate, so the credentials object may
+        # renew the access token in memory for the rest of this run.
         self.credentials = self.auth.create_credentials_from_token_info(token_info)
 
         try:

docsgpt/parser/connectors/share_point/auth.py

@@ -5,7 +5,6 @@ from typing import Optional, Dict, Any
 from msal import ConfidentialClientApplication
 
 from docsgpt.core.settings import settings
-from docsgpt.parser.connectors._auth_utils import session_token_fingerprint
 from docsgpt.parser.connectors.base import BaseConnectorAuth
 
 logger = logging.getLogger(__name__)
@@ -76,41 +75,6 @@ class SharePointAuth(BaseConnectorAuth):
 
         return self.map_token_response(result)
 
-    def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]:
-        try:
-            from docsgpt.storage.db.repositories.connector_sessions import (
-                ConnectorSessionsRepository,
-            )
-            from docsgpt.storage.db.session import db_readonly
-
-            with db_readonly() as conn:
-                session = ConnectorSessionsRepository(conn).get_by_session_token(
-                    session_token
-                )
-
-            if not session:
-                raise ValueError(
-                    f"Invalid session token ({session_token_fingerprint(session_token)})"
-                )
-
-            token_info = session.get("token_info")
-            if not token_info:
-                raise ValueError("Session missing token information")
-
-            required_fields = ["access_token", "refresh_token"]
-            missing_fields = [field for field in required_fields if field not in token_info or not token_info.get(field)]
-            if missing_fields:
-                raise ValueError(f"Missing required token fields: {missing_fields}")
-
-            if 'token_uri' not in token_info:
-                token_info['token_uri'] = f"https://login.microsoftonline.com/{settings.MICROSOFT_TENANT_ID}/oauth2/v2.0/token"
-
-            return token_info
-
-        except Exception as e:
-            logger.error("Failed to retrieve token from session: %s", e)
-            raise ValueError(f"Failed to retrieve SharePoint token information: {str(e)}")
-
     def is_token_expired(self, token_info: Dict[str, Any]) -> bool:
         if not token_info:
             return True

docsgpt/parser/connectors/share_point/loader.py

@@ -26,8 +26,7 @@ def _retry_on_auth_failure(func):
             if e.response is not None and e.response.status_code in (401, 403):
                 logging.info(f"Auth failure in {func.__name__}, refreshing token and retrying")
                 try:
-                    new_token_info = self.auth.refresh_access_token(self.refresh_token)
-                    self.access_token = new_token_info.get('access_token')
+                    self._apply_token_info(self._refresh_rejected_token(self.access_token))
                 except Exception as refresh_error:
                     raise ValueError(
                         f"Authentication failed and could not be refreshed: {refresh_error}"
@@ -63,13 +62,12 @@ class SharePointLoader(BaseConnectorLoader):
 
     GRAPH_API_BASE = "https://graph.microsoft.com/v1.0"
 
-    def __init__(self, session_token: str):
+    def __init__(self, session_token: Optional[str] = None, *, connection_id: Optional[str] = None):
         self.auth = SharePointAuth()
         self.session_token = session_token
 
-        token_info = self.auth.get_token_info_from_session(session_token)
-        self.access_token = token_info.get('access_token')
-        self.refresh_token = token_info.get('refresh_token')
+        _, token_info = self._load_token_info(session_token, connection_id)
+        self._apply_token_info(token_info)
         self.allows_shared_content = token_info.get('allows_shared_content', False)
 
         if not self.access_token:
@@ -77,6 +75,10 @@ class SharePointLoader(BaseConnectorLoader):
 
         self.next_page_token = None
 
+    def _apply_token_info(self, token_info: Dict[str, Any]) -> None:
+        self.access_token = token_info.get('access_token')
+        self.expiry = token_info.get('expiry')
+
     def _get_headers(self) -> Dict[str, str]:
         return {
             'Authorization': f'Bearer {self.access_token}',
@@ -87,12 +89,14 @@ class SharePointLoader(BaseConnectorLoader):
         if not self.access_token:
             raise ValueError("No access token available")
 
-        token_info = {'access_token': self.access_token, 'expiry': None}
-        if self.auth.is_token_expired(token_info):
+        # The connection service refreshes under a row lock and stores the
+        # rotated refresh token; refreshing here would spend it and lose it.
+        if self.auth.is_token_expired({'access_token': self.access_token, 'expiry': self.expiry}):
             logging.info("Token expired, attempting refresh")
             try:
-                new_token_info = self.auth.refresh_access_token(self.refresh_token)
-                self.access_token = new_token_info.get('access_token')
+                from docsgpt.connectors import service
+
+                self._apply_token_info(service.get_valid_token_info(self.connection_id))
             except Exception:
                 raise ValueError("Failed to refresh access token")