From 5321a9438865d3bb736b6157724b477ceb1253ad Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:30:48 +0100 Subject: [PATCH 001/162] Add the connector catalog and read-only connections API connector_sessions already holds one row per signed-in account or MCP server. Migration 0038 names each row (connector_key, auth_kind, display_name, account_label) and lets sources and tools point at the row they use through connection_id, backfilling the links it can prove. docsgpt/connectors/catalog.py declares every service DocsGPT connects to, with the server settings it needs and what a connection provides. New endpoints: GET /api/connectors/catalog, GET /api/connections, GET /api/connections/ and POST /api/connections//disconnect. /api/available_tools now groups tools into built_in, service and custom, and source listings carry connectionId. --- docsgpt/alembic/versions/0038_connections.py | 136 ++++++ docsgpt/api/connector/connections.py | 106 +++++ docsgpt/api/pat/rules.py | 1 + docsgpt/api/user/sources/routes.py | 8 + docsgpt/api/user/tools/routes.py | 14 + docsgpt/app.py | 4 + docsgpt/connectors/__init__.py | 6 + docsgpt/connectors/catalog.py | 423 ++++++++++++++++++ docsgpt/connectors/permissions.py | 83 ++++ docsgpt/connectors/service.py | 264 +++++++++++ docsgpt/storage/db/models.py | 13 + .../db/repositories/connector_sessions.py | 78 +++- tests/connectors/__init__.py | 0 tests/connectors/test_catalog.py | 116 +++++ tests/connectors/test_permissions.py | 47 ++ tests/connectors/test_service_read.py | 263 +++++++++++ tests/storage/db/test_migration_0038.py | 154 +++++++ 17 files changed, 1714 insertions(+), 2 deletions(-) create mode 100644 docsgpt/alembic/versions/0038_connections.py create mode 100644 docsgpt/api/connector/connections.py create mode 100644 docsgpt/connectors/__init__.py create mode 100644 docsgpt/connectors/catalog.py create mode 100644 docsgpt/connectors/permissions.py create mode 100644 docsgpt/connectors/service.py create mode 100644 tests/connectors/__init__.py create mode 100644 tests/connectors/test_catalog.py create mode 100644 tests/connectors/test_permissions.py create mode 100644 tests/connectors/test_service_read.py create mode 100644 tests/storage/db/test_migration_0038.py diff --git a/docsgpt/alembic/versions/0038_connections.py b/docsgpt/alembic/versions/0038_connections.py new file mode 100644 index 00000000..3ce6aa43 --- /dev/null +++ b/docsgpt/alembic/versions/0038_connections.py @@ -0,0 +1,136 @@ +"""0038 connections — connector_sessions becomes the connections table. + +``connector_sessions`` already holds one row per signed-in account (OAuth +ingest providers) or per MCP server. This migration names what each row is +and lets sources and tools point at the row they use: + +* ``connector_key`` is the catalog entry (``google_drive``, ``custom_mcp``, + ``telegram``), ``auth_kind`` how the row signs in, ``display_name`` and + ``account_label`` what the Connectors page shows. +* ``sources.connection_id`` and ``user_tools.connection_id`` link the + resources a connection feeds. ``ON DELETE SET NULL`` keeps a source's + indexed content when its connection is removed. + +Backfill (idempotent, only fills NULLs): + +1. ``connector_key``, ``auth_kind``, ``display_name`` from ``provider``. +2. ``account_label`` from ``user_email`` for OAuth rows. +3. ``sources.connection_id`` for ``connector:file`` sources, matched to the + owner's only row for ``remote_data->>'provider'``. +4. ``user_tools.connection_id`` for OAuth MCP tools, matched to the owner's + row for the tool's server base URL. + +Revision ID: 0038_connections +Revises: 0037_request_traces +""" + +from typing import Sequence, Union + +from alembic import op + + +revision: str = "0038_connections" +down_revision: Union[str, None] = "0037_request_traces" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +_OAUTH_PROVIDERS = { + "google_drive": "Google Drive", + "share_point": "SharePoint", + "confluence": "Confluence", +} + + +def upgrade() -> None: + op.execute( + """ + ALTER TABLE connector_sessions + ADD COLUMN IF NOT EXISTS connector_key TEXT, + ADD COLUMN IF NOT EXISTS display_name TEXT, + ADD COLUMN IF NOT EXISTS account_label TEXT, + ADD COLUMN IF NOT EXISTS auth_kind TEXT, + ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now(); + """ + ) + op.execute( + "ALTER TABLE sources ADD COLUMN IF NOT EXISTS connection_id UUID " + "REFERENCES connector_sessions(id) ON DELETE SET NULL;" + ) + op.execute( + "ALTER TABLE user_tools ADD COLUMN IF NOT EXISTS connection_id UUID " + "REFERENCES connector_sessions(id) ON DELETE SET NULL;" + ) + op.execute("CREATE INDEX IF NOT EXISTS sources_connection_idx ON sources (connection_id);") + op.execute("CREATE INDEX IF NOT EXISTS user_tools_connection_idx ON user_tools (connection_id);") + + # 1 + 2: name the existing rows. + for provider, name in _OAUTH_PROVIDERS.items(): + op.execute( + f""" + UPDATE connector_sessions SET + connector_key = COALESCE(connector_key, '{provider}'), + auth_kind = COALESCE(auth_kind, 'oauth'), + display_name = COALESCE(display_name, '{name}'), + account_label = COALESCE(account_label, user_email) + WHERE provider = '{provider}'; + """ + ) + op.execute( + """ + UPDATE connector_sessions SET + connector_key = COALESCE(connector_key, 'custom_mcp'), + auth_kind = COALESCE(auth_kind, 'mcp_oauth'), + display_name = COALESCE( + display_name, + regexp_replace(COALESCE(server_url, substr(provider, 5)), '^https?://', '') + ) + WHERE provider LIKE 'mcp:%'; + """ + ) + + # 3: connector sources point at the owner's only session for the provider. + op.execute( + """ + UPDATE sources s SET connection_id = cs.id + FROM connector_sessions cs + WHERE s.connection_id IS NULL + AND s.type = 'connector:file' + AND cs.user_id = s.user_id + AND cs.provider = s.remote_data->>'provider' + AND ( + SELECT count(*) FROM connector_sessions c2 + WHERE c2.user_id = s.user_id AND c2.provider = cs.provider + ) = 1; + """ + ) + + # 4: OAuth MCP tools point at the owner's session for the server's base URL. + op.execute( + """ + UPDATE user_tools t SET connection_id = cs.id + FROM connector_sessions cs + WHERE t.connection_id IS NULL + AND t.name = 'mcp_tool' + AND t.config->>'auth_type' = 'oauth' + AND cs.user_id = t.user_id + AND cs.provider = 'mcp:' || substring(t.config->>'server_url' from '^(https?://[^/]+)'); + """ + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS user_tools_connection_idx;") + op.execute("DROP INDEX IF EXISTS sources_connection_idx;") + op.execute("ALTER TABLE user_tools DROP COLUMN IF EXISTS connection_id;") + op.execute("ALTER TABLE sources DROP COLUMN IF EXISTS connection_id;") + op.execute( + """ + ALTER TABLE connector_sessions + DROP COLUMN IF EXISTS updated_at, + DROP COLUMN IF EXISTS auth_kind, + DROP COLUMN IF EXISTS account_label, + DROP COLUMN IF EXISTS display_name, + DROP COLUMN IF EXISTS connector_key; + """ + ) diff --git a/docsgpt/api/connector/connections.py b/docsgpt/api/connector/connections.py new file mode 100644 index 00000000..46cb9edc --- /dev/null +++ b/docsgpt/api/connector/connections.py @@ -0,0 +1,106 @@ +"""Connectors catalog and connections API. + +``/api/connectors/catalog`` lists every service DocsGPT can connect to, with +whether the server is set up for it and the caller's connection summary. +``/api/connections`` lists and manages the caller's connections. Responses +never include tokens or secrets. +""" + +from __future__ import annotations + +from flask import current_app, jsonify, make_response, request +from flask_restx import Namespace, Resource + +from docsgpt.api import api +from docsgpt.api.user.authz import ROLE_ADMIN, has_role +from docsgpt.connectors import service +from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository +from docsgpt.storage.db.session import db_readonly, db_session + +connections_ns = Namespace("connections", description="Connectors and connections", path="/api") +api.add_namespace(connections_ns) + + +def _user_id() -> str | None: + token = getattr(request, "decoded_token", None) + return token.get("sub") if isinstance(token, dict) else None + + +def _unauthorized(): + return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401) + + +def _not_found(): + return make_response(jsonify({"success": False, "error": "Connection not found"}), 404) + + +@connections_ns.route("/connectors/catalog") +class ConnectorCatalog(Resource): + @api.doc(description="Every connector with its availability and the caller's connection summary") + def get(self): + user_id = _user_id() + if not user_id: + return _unauthorized() + try: + with db_readonly() as conn: + entries = service.catalog_for_user( + conn, user_id, is_admin=has_role(request.decoded_token, ROLE_ADMIN), + ) + except Exception as err: + current_app.logger.error(f"Error building connector catalog: {err}", exc_info=True) + return make_response(jsonify({"success": False, "error": "Failed to load connectors"}), 500) + return make_response(jsonify({"success": True, "connectors": entries}), 200) + + +@connections_ns.route("/connections") +class ConnectionsList(Resource): + @api.doc(description="The caller's connections with status and linked resource counts") + def get(self): + user_id = _user_id() + if not user_id: + return _unauthorized() + try: + with db_readonly() as conn: + connections = service.list_connections(conn, user_id) + except Exception as err: + current_app.logger.error(f"Error listing connections: {err}", exc_info=True) + return make_response(jsonify({"success": False, "error": "Failed to load connections"}), 500) + return make_response(jsonify({"success": True, "connections": connections}), 200) + + +@connections_ns.route("/connections/") +class ConnectionDetail(Resource): + @api.doc(description="One connection with the sources it syncs and the tools it provides") + def get(self, connection_id: str): + user_id = _user_id() + if not user_id: + return _unauthorized() + try: + with db_readonly() as conn: + row = ConnectorSessionsRepository(conn).get_for_user(connection_id, user_id) + if row is None: + return _not_found() + detail = service.connection_detail(conn, row) + except Exception as err: + current_app.logger.error(f"Error loading connection: {err}", exc_info=True) + return make_response(jsonify({"success": False, "error": "Failed to load connection"}), 500) + return make_response(jsonify({"success": True, "connection": detail}), 200) + + +@connections_ns.route("/connections//disconnect") +class ConnectionDisconnect(Resource): + @api.doc(description="Delete a connection's stored credentials; its sources and tools stay") + def post(self, connection_id: str): + user_id = _user_id() + if not user_id: + return _unauthorized() + try: + with db_session() as conn: + row = ConnectorSessionsRepository(conn).get_for_user(connection_id, user_id) + if row is None: + return _not_found() + connection = service.disconnect(conn, row) + except Exception as err: + current_app.logger.error(f"Error disconnecting connection: {err}", exc_info=True) + return make_response(jsonify({"success": False, "error": "Failed to disconnect"}), 500) + return make_response(jsonify({"success": True, "connection": connection}), 200) diff --git a/docsgpt/api/pat/rules.py b/docsgpt/api/pat/rules.py index c1d7683d..b1ebe9e4 100644 --- a/docsgpt/api/pat/rules.py +++ b/docsgpt/api/pat/rules.py @@ -357,6 +357,7 @@ DENIED_PREFIXES = ( "/api/admin/", "/api/auth/oidc/", "/api/connectors/", + "/api/connections", "/api/devices", "/scim/", "/static/", diff --git a/docsgpt/api/user/sources/routes.py b/docsgpt/api/user/sources/routes.py index 6826c9e9..7ca3b339 100644 --- a/docsgpt/api/user/sources/routes.py +++ b/docsgpt/api/user/sources/routes.py @@ -67,6 +67,12 @@ def _get_provider_from_remote_data(remote_data): return None +def _connection_id(row: dict) -> str | None: + """The connection a source syncs from, as a string id, or None.""" + value = row.get("connection_id") + return str(value) if value else None + + @sources_ns.route("/sources") class CombinedJson(Resource): @api.doc(description="Provide JSON file with combined available indexes") @@ -117,6 +123,7 @@ class CombinedJson(Resource): "config": SourceConfig.parse(index.get("config")).model_dump(), "ownership": ownership, "team_access": team_access, + "connectionId": _connection_id(index), } for index in indexes: @@ -207,6 +214,7 @@ class PaginatedSources(Resource): "team_access": ( None if owned else team_shared.get(str(doc["id"])) ), + "connectionId": _connection_id(doc), } ) response = { diff --git a/docsgpt/api/user/tools/routes.py b/docsgpt/api/user/tools/routes.py index ddc56916..c4464b68 100644 --- a/docsgpt/api/user/tools/routes.py +++ b/docsgpt/api/user/tools/routes.py @@ -19,6 +19,7 @@ from docsgpt.api import api from docsgpt.api.pat.rules import filter_listing from docsgpt.api.user.artifacts.authz import Principal, authorize_artifact from docsgpt.api.user.team_sharing import effective_write_owner, visible_with_access +from docsgpt.connectors.catalog import definition_for_tool from docsgpt.core.settings import settings from docsgpt.core.url_validation import SSRFError, validate_url from docsgpt.security.encryption import decrypt_credentials, encrypt_credentials @@ -186,6 +187,10 @@ def transform_actions(actions_metadata): tools_ns = Namespace("tools", description="Tool management operations", path="/api") +# Tools the Connectors page adds through "Add custom connector" rather than +# the Add Tool modal. +_CUSTOM_CONNECTOR_TOOLS = {"mcp_tool": "custom_mcp", "api_tool": "custom_openapi"} + @tools_ns.route("/available_tools") class AvailableTools(Resource): @@ -202,6 +207,13 @@ class AvailableTools(Resource): description = lines[1].strip() if len(lines) > 1 else "" config_req = tool_instance.get_config_requirements() actions = tool_instance.get_actions_metadata() + definition = definition_for_tool(tool_name) + if definition is not None: + group, connector_key = "service", definition.key + elif tool_name in _CUSTOM_CONNECTOR_TOOLS: + group, connector_key = "custom", _CUSTOM_CONNECTOR_TOOLS[tool_name] + else: + group, connector_key = "built_in", None tools_metadata.append( { "name": tool_name, @@ -209,6 +221,8 @@ class AvailableTools(Resource): "description": description, "configRequirements": config_req, "actions": actions, + "group": group, + "connector_key": connector_key, } ) except Exception as err: diff --git a/docsgpt/app.py b/docsgpt/app.py index 88e2d5cd..5a45ab31 100644 --- a/docsgpt/app.py +++ b/docsgpt/app.py @@ -29,6 +29,7 @@ from docsgpt.api.scim import scim_bp # noqa: E402 from docsgpt.api.user.authz import ROLE_USER, resolve_roles # noqa: E402 from docsgpt.api.user.routes import user # noqa: E402 from docsgpt.api.connector.routes import connector # noqa: E402 +from docsgpt.api.connector import connections as _connections # noqa: E402,F401 from docsgpt.api.v1 import v1_bp # noqa: E402 from docsgpt.celery_init import celery # noqa: E402 from docsgpt.core.secret_key import resolve_jwt_secret_key # noqa: E402 @@ -209,6 +210,9 @@ def get_config(): "hybrid_available": settings.VECTOR_STORE == "pgvector", "tts_available": TTSCreator.is_enabled(settings.TTS_PROVIDER), "stt_available": STTCreator.is_enabled(settings.STT_PROVIDER), + # Lets a frontend built before the Connectors page run against this + # backend, and a new frontend hide the page against an older one. + "connectors_enabled": True, } if settings.AUTH_TYPE == "oidc": response["oidc"] = { diff --git a/docsgpt/connectors/__init__.py b/docsgpt/connectors/__init__.py new file mode 100644 index 00000000..fc58c5c0 --- /dev/null +++ b/docsgpt/connectors/__init__.py @@ -0,0 +1,6 @@ +"""Connectors: one place to connect accounts that feed Sources and Tools. + +``catalog`` declares the services, ``service`` manages connections (the rows +of ``connector_sessions``), ``permissions`` classifies tool actions as read +or write. +""" diff --git a/docsgpt/connectors/catalog.py b/docsgpt/connectors/catalog.py new file mode 100644 index 00000000..dcc4e39a --- /dev/null +++ b/docsgpt/connectors/catalog.py @@ -0,0 +1,423 @@ +"""The connector catalog: every service DocsGPT can connect to. + +A connector is something the user connects once (an OAuth sign-in, an API key +or an MCP server). Each connection it produces can feed Sources (content +synced into DocsGPT) and Tools (actions an agent can take). The definitions +here are declarative: they say how a connector signs in, which server +settings it needs, what it can sync and which tools it creates, so the API +and the frontend never hard-code a list of services. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Optional +from urllib.parse import urlparse + +import yaml + +from docsgpt.core.settings import settings + +CATEGORIES = ("files", "knowledge", "dev", "messaging", "database", "search", "custom") +AUTH_KINDS = ("oauth", "mcp_oauth", "api_key", "none", "mcp") +PUBLISHERS = ("built_in", "preset", "custom") + + +@dataclass(frozen=True) +class CredentialField: + """One field the user fills in to connect an ``api_key`` connector. + + Attributes: + key: Name the credential is stored and passed to the tool or loader + under, e.g. ``token`` or ``aws_access_key_id``. + label: English label; the frontend shows the translated + ``settings.connectors.fields.`` when it has one. + secret: Masked in the form and never returned by the API. + required: Whether connecting fails without it. + """ + + key: str + label: str + secret: bool = True + required: bool = True + + def to_dict(self) -> dict: + return {"key": self.key, "label": self.label, "secret": self.secret, "required": self.required} + + +@dataclass(frozen=True) +class ConnectorDefinition: + """How one connector signs in and what a connection to it provides. + + Attributes: + key: Catalog key, stored on each connection as ``connector_key``. + name: Display name. + description: One line shown on the catalog card. + icon: Frontend asset key (``assets/connectors/.svg``). + category: One of :data:`CATEGORIES`. + auth_kind: ``oauth`` (server-side OAuth app), ``mcp_oauth`` (MCP + OAuth with dynamic client registration), ``api_key`` (fields the + user pastes), ``none`` or ``mcp`` (a custom MCP server whose auth + the user picks). + capabilities: Any of ``sync``, ``read``, ``write``. + credential_fields: Fields asked for by ``api_key`` connectors. + required_settings: Server settings that must be set for the + connector to be usable, e.g. ``GOOGLE_CLIENT_ID``. + sync_ingestor: Ingest loader a synced source uses (``google_drive``, + ``s3``), or None when the connector cannot sync. + default_sync_frequency: Sync frequency preselected in the wizard. + setup_fields: Per-source fields asked when choosing what to sync + (an S3 bucket, Reddit search queries). + tool_templates: ``user_tools`` names created on connect. + setup: What the wizard does after sign-in: ``tools`` is ``auto`` + (created and enabled), ``ask`` or ``off``; ``sync`` likewise. + mcp_url: MCP endpoint for presets. + publisher: ``built_in``, ``preset`` or ``custom``. + docs_url: Setup guide for admins. + oauth_scopes: Scopes an MCP preset requests. + """ + + key: str + name: str + description: str + icon: str + category: str + auth_kind: str + capabilities: tuple[str, ...] = () + credential_fields: tuple[CredentialField, ...] = () + required_settings: tuple[str, ...] = () + sync_ingestor: Optional[str] = None + default_sync_frequency: str = "weekly" + setup_fields: tuple[CredentialField, ...] = () + tool_templates: tuple[str, ...] = () + setup: dict = field(default_factory=lambda: {"tools": "auto", "sync": "ask"}) + mcp_url: Optional[str] = None + publisher: str = "built_in" + docs_url: Optional[str] = None + oauth_scopes: tuple[str, ...] = () + + @property + def missing_settings(self) -> list[str]: + """Server settings this connector needs that are not set.""" + return [name for name in self.required_settings if not getattr(settings, name, None)] + + @property + def configured(self) -> bool: + """Whether every required server setting is set.""" + return not self.missing_settings + + @property + def mcp_base_url(self) -> Optional[str]: + """``scheme://host`` of the preset's MCP endpoint (the MCP session key).""" + return base_url(self.mcp_url) if self.mcp_url else None + + def to_dict(self) -> dict: + """Serialise the parts the frontend needs (never server secrets).""" + return { + "key": self.key, + "name": self.name, + "description": self.description, + "icon": self.icon, + "category": self.category, + "auth_kind": self.auth_kind, + "capabilities": list(self.capabilities), + "credential_fields": [f.to_dict() for f in self.credential_fields], + "setup_fields": [f.to_dict() for f in self.setup_fields], + "sync_ingestor": self.sync_ingestor, + "default_sync_frequency": self.default_sync_frequency, + "tool_templates": list(self.tool_templates), + "setup": dict(self.setup), + "mcp_url": self.mcp_url, + "publisher": self.publisher, + "docs_url": self.docs_url, + "oauth_scopes": list(self.oauth_scopes), + } + + +def base_url(url: Optional[str]) -> str: + """``scheme://netloc`` of ``url``, the key MCP sessions are stored under.""" + parsed = urlparse(url or "") + if not parsed.scheme or not parsed.netloc: + return "" + return f"{parsed.scheme}://{parsed.netloc}" + + +_DOCS = "https://docs.docsgpt.cloud/Guides/Connectors" + +_BUILT_IN: tuple[ConnectorDefinition, ...] = ( + ConnectorDefinition( + key="google_drive", + name="Google Drive", + description="Sync Docs, Sheets and PDFs into Sources.", + icon="drive", + category="files", + auth_kind="oauth", + capabilities=("sync",), + required_settings=("GOOGLE_CLIENT_ID", "GOOGLE_CLIENT_SECRET"), + sync_ingestor="google_drive", + setup={"tools": "off", "sync": "ask"}, + docs_url=f"{_DOCS}#google-drive", + ), + ConnectorDefinition( + key="share_point", + name="SharePoint", + description="Sync files from SharePoint sites and OneDrive into Sources.", + icon="sharepoint", + category="files", + auth_kind="oauth", + capabilities=("sync",), + required_settings=("MICROSOFT_CLIENT_ID", "MICROSOFT_CLIENT_SECRET"), + sync_ingestor="share_point", + setup={"tools": "off", "sync": "ask"}, + docs_url=f"{_DOCS}#sharepoint-and-onedrive", + ), + ConnectorDefinition( + key="confluence", + name="Confluence", + description="Sync Confluence spaces and pages into Sources.", + icon="confluence", + category="knowledge", + auth_kind="oauth", + capabilities=("sync",), + required_settings=("CONFLUENCE_CLIENT_ID", "CONFLUENCE_CLIENT_SECRET"), + sync_ingestor="confluence", + setup={"tools": "off", "sync": "ask"}, + docs_url=f"{_DOCS}#confluence", + ), + ConnectorDefinition( + key="s3", + name="Amazon S3", + description="Sync documents from an S3 bucket into Sources.", + icon="s3", + category="files", + auth_kind="api_key", + capabilities=("sync",), + credential_fields=( + CredentialField("aws_access_key_id", "Access key ID", secret=False), + CredentialField("aws_secret_access_key", "Secret access key"), + CredentialField("region", "Region", secret=False, required=False), + CredentialField("endpoint_url", "Custom endpoint URL", secret=False, required=False), + ), + sync_ingestor="s3", + setup_fields=( + CredentialField("bucket", "Bucket", secret=False), + CredentialField("prefix", "Path prefix", secret=False, required=False), + ), + setup={"tools": "off", "sync": "ask"}, + docs_url=f"{_DOCS}#amazon-s3", + ), + ConnectorDefinition( + key="reddit", + name="Reddit", + description="Sync Reddit posts that match your searches into Sources.", + icon="reddit", + category="knowledge", + auth_kind="api_key", + capabilities=("sync",), + credential_fields=( + CredentialField("client_id", "Client ID", secret=False), + CredentialField("client_secret", "Client secret"), + CredentialField("user_agent", "User agent", secret=False), + ), + sync_ingestor="reddit", + setup_fields=( + CredentialField("search_queries", "Search queries", secret=False), + CredentialField("number_posts", "Number of posts", secret=False), + ), + setup={"tools": "off", "sync": "ask"}, + ), + ConnectorDefinition( + key="brave", + name="Brave Search", + description="Search the web and images with the Brave Search API.", + icon="tool_brave", + category="search", + auth_kind="api_key", + capabilities=("read",), + credential_fields=(CredentialField("token", "API key"),), + tool_templates=("brave",), + setup={"tools": "auto", "sync": "off"}, + ), + ConnectorDefinition( + key="telegram", + name="Telegram", + description="Send messages and images to a Telegram chat.", + icon="tool_telegram", + category="messaging", + auth_kind="api_key", + capabilities=("write",), + credential_fields=(CredentialField("token", "Bot token"),), + tool_templates=("telegram",), + setup={"tools": "auto", "sync": "off"}, + ), + ConnectorDefinition( + key="ntfy", + name="ntfy", + description="Send push notifications through an ntfy server.", + icon="tool_ntfy", + category="messaging", + auth_kind="api_key", + capabilities=("write",), + credential_fields=(CredentialField("token", "Access token"),), + tool_templates=("ntfy",), + setup={"tools": "auto", "sync": "off"}, + ), + ConnectorDefinition( + key="postgres", + name="PostgreSQL", + description="Read the schema and run SQL against a Postgres database.", + icon="tool_postgres", + category="database", + auth_kind="api_key", + capabilities=("read", "write"), + credential_fields=(CredentialField("token", "Connection string"),), + tool_templates=("postgres",), + setup={"tools": "auto", "sync": "off"}, + ), + ConnectorDefinition( + key="custom_mcp", + name="MCP server", + description="Connect any remote Model Context Protocol server.", + icon="tool_mcp_tool", + category="custom", + auth_kind="mcp", + capabilities=("read", "write"), + tool_templates=("mcp_tool",), + setup={"tools": "auto", "sync": "off"}, + publisher="custom", + ), + ConnectorDefinition( + key="custom_openapi", + name="OpenAPI / REST", + description="Import an OpenAPI spec and call its endpoints as tools.", + icon="tool_api_tool", + category="custom", + auth_kind="none", + capabilities=("read", "write"), + tool_templates=("api_tool",), + setup={"tools": "ask", "sync": "off"}, + publisher="custom", + ), +) + +_PRESETS_FILE = Path(__file__).parent / "presets" / "mcp.yaml" + + +def _load_presets(path: Optional[Path] = None) -> tuple[ConnectorDefinition, ...]: + """Read the curated MCP server presets shipped with the repository.""" + path = path or _PRESETS_FILE + if not path.exists(): + return () + with path.open(encoding="utf-8") as fh: + entries = yaml.safe_load(fh) or [] + presets = [] + for entry in entries: + presets.append( + ConnectorDefinition( + key=entry["key"], + name=entry["name"], + description=entry["description"], + icon=entry.get("icon") or "tool_mcp_tool", + category=entry.get("category", "knowledge"), + auth_kind=entry.get("auth_kind", "mcp_oauth"), + capabilities=tuple(entry.get("capabilities") or ("read", "write")), + tool_templates=("mcp_tool",), + setup={"tools": "auto", "sync": "off"}, + mcp_url=entry["mcp_url"], + publisher="preset", + docs_url=entry.get("docs_url"), + oauth_scopes=tuple(entry.get("oauth_scopes") or ()), + ) + ) + return tuple(presets) + + +_REGISTRY: dict[str, ConnectorDefinition] = {} + + +def _registry() -> dict[str, ConnectorDefinition]: + if not _REGISTRY: + for definition in (*_BUILT_IN, *_load_presets()): + _REGISTRY[definition.key] = definition + return _REGISTRY + + +def all_definitions() -> list[ConnectorDefinition]: + """Every catalog entry, built-ins first, then presets.""" + return list(_registry().values()) + + +def get_definition(key: Optional[str]) -> Optional[ConnectorDefinition]: + """The definition for ``key``, or None when it is not in the catalog.""" + if not key: + return None + return _registry().get(key) + + +def preset_for_url(url: Optional[str]) -> Optional[ConnectorDefinition]: + """The MCP preset whose server shares ``url``'s base URL, if any.""" + target = base_url(url) + if not target: + return None + for definition in _registry().values(): + if definition.publisher == "preset" and definition.mcp_base_url == target: + return definition + return None + + +def definition_for_tool(tool_name: str) -> Optional[ConnectorDefinition]: + """The built-in connector that provides the ``user_tools`` template ``tool_name``.""" + for definition in _BUILT_IN: + if definition.publisher == "built_in" and tool_name in definition.tool_templates: + return definition + return None + + +def connector_key_for_row(row: dict) -> Optional[str]: + """Catalog key for a ``connector_sessions`` row. + + Rows written before ``0038_connections`` carry no key; they are named from + ``provider``. A custom MCP row whose server matches a preset is reported + as that preset. + """ + key = row.get("connector_key") + provider = row.get("provider") or "" + if not key: + if provider.startswith("mcp:"): + key = "custom_mcp" + elif get_definition(provider): + key = provider + if key == "custom_mcp": + preset = preset_for_url(row.get("server_url") or provider[4:]) + if preset: + return preset.key + return key + + +def tool_connector_keys() -> set[str]: + """``user_tools`` names that belong to a built-in service connector.""" + return { + name + for definition in _BUILT_IN + if definition.publisher == "built_in" + for name in definition.tool_templates + } + + +def iter_by_category(definitions: Iterable[ConnectorDefinition]) -> dict[str, list[ConnectorDefinition]]: + """Group definitions by category, keeping :data:`CATEGORIES` order.""" + grouped: dict[str, list[ConnectorDefinition]] = {c: [] for c in CATEGORIES} + for definition in definitions: + grouped.setdefault(definition.category, []).append(definition) + return grouped + + +def reset_registry_for_tests() -> None: + """Drop the cached registry so a test can load a different presets file.""" + _REGISTRY.clear() + + +def to_public(definition: ConnectorDefinition, **extra: Any) -> dict: + """Definition dict plus request-specific fields (availability, counts).""" + return {**definition.to_dict(), **extra} diff --git a/docsgpt/connectors/permissions.py b/docsgpt/connectors/permissions.py new file mode 100644 index 00000000..b5afce78 --- /dev/null +++ b/docsgpt/connectors/permissions.py @@ -0,0 +1,83 @@ +"""Read / write classification and permissions for connection-backed tools. + +Every action a connection's tool offers is either a *read* (it only looks +something up) or a *write* (it changes or sends something). Reads default to +"Always allow"; writes default to "Needs approval". The permission is stored +on the action itself: ``active`` off means "Off", ``require_approval`` means +"Needs approval", neither means "Always allow". +""" + +from __future__ import annotations + +from typing import Optional + +ACCESS_READ = "read" +ACCESS_WRITE = "write" + +PERMISSION_ALWAYS = "always" +PERMISSION_ASK = "ask" +PERMISSION_OFF = "off" +PERMISSIONS = (PERMISSION_ALWAYS, PERMISSION_ASK, PERMISSION_OFF) + +_READ_WORDS = ("search", "query", "find", "list", "get", "read", "fetch", "lookup", "describe", "retrieve") + + +def action_access(tool_name: Optional[str], action: dict) -> str: + """Whether ``action`` reads or writes. + + Order of evidence: an explicit ``access`` on the action metadata, MCP tool + annotations (``readOnlyHint`` / ``destructiveHint``), the HTTP method of an + API tool action, then the action's name. + + Args: + tool_name: The ``user_tools`` name the action belongs to. + action: One entry of the tool's ``actions``. + + Returns: + ``read`` or ``write``. + """ + access = action.get("access") + if access in (ACCESS_READ, ACCESS_WRITE): + return access + annotations = action.get("annotations") or {} + if isinstance(annotations, dict): + if annotations.get("readOnlyHint") is True: + return ACCESS_READ + if annotations.get("destructiveHint") is True or annotations.get("readOnlyHint") is False: + return ACCESS_WRITE + method = (action.get("method") or "").upper() + if tool_name == "api_tool" and method: + return ACCESS_READ if method in ("GET", "HEAD", "OPTIONS") else ACCESS_WRITE + name = (action.get("name") or "").lower() + return ACCESS_READ if any(word in name for word in _READ_WORDS) else ACCESS_WRITE + + +def action_permission(action: dict) -> str: + """The action's permission: ``always``, ``ask`` or ``off``.""" + if action.get("active") is False: + return PERMISSION_OFF + if action.get("require_approval"): + return PERMISSION_ASK + return PERMISSION_ALWAYS + + +def apply_permission(action: dict, permission: str) -> dict: + """Return ``action`` with ``permission`` written onto its flags.""" + if permission not in PERMISSIONS: + raise ValueError(f"Unknown permission: {permission}") + updated = dict(action) + updated["active"] = permission != PERMISSION_OFF + updated["require_approval"] = permission == PERMISSION_ASK + return updated + + +def apply_default_permissions(tool_name: Optional[str], actions: list[dict]) -> list[dict]: + """Stamp ``access`` on each action and default writes to needing approval.""" + stamped = [] + for action in actions: + access = action_access(tool_name, action) + updated = {**action, "access": access} + if access == ACCESS_WRITE: + updated["require_approval"] = True + stamped.append(updated) + return stamped diff --git a/docsgpt/connectors/service.py b/docsgpt/connectors/service.py new file mode 100644 index 00000000..3bb16f72 --- /dev/null +++ b/docsgpt/connectors/service.py @@ -0,0 +1,264 @@ +"""Connection lifecycle: listing, status, and what each connection provides. + +A connection is one ``connector_sessions`` row: a signed-in OAuth account, an +MCP server or a set of API credentials. Sources and tools point at it through +``connection_id``. Everything the API returns about a connection goes through +:func:`serialize_connection`, which never includes credentials. +""" + +from __future__ import annotations + +import json +from typing import Any, Iterable, Optional + +from docsgpt.connectors import catalog +from docsgpt.connectors.catalog import ConnectorDefinition +from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository + +STATUS_CONNECTED = "connected" +STATUS_RECONNECT = "reconnect_needed" +STATUS_DISCONNECTED = "disconnected" +STATUS_ERROR = "error" +STATUS_PENDING = "pending" + +# Worst first: a card shows the worst status among the user's accounts. +_STATUS_SEVERITY = { + STATUS_RECONNECT: 4, + STATUS_ERROR: 3, + STATUS_DISCONNECTED: 2, + STATUS_CONNECTED: 1, + STATUS_PENDING: 0, +} + + +def _json(value: Any) -> Any: + if isinstance(value, str): + try: + return json.loads(value) + except ValueError: + return None + return value + + +def has_credentials(row: dict) -> bool: + """Whether a connection holds usable credentials (without decrypting them).""" + if row.get("encrypted_credentials"): + return True + token_info = _json(row.get("token_info")) or {} + if isinstance(token_info, dict) and token_info.get("access_token"): + return True + session_data = _json(row.get("session_data")) or {} + tokens = session_data.get("tokens") if isinstance(session_data, dict) else None + return bool(isinstance(tokens, dict) and tokens.get("access_token")) + + +def normalize_status(row: dict) -> str: + """Map a row's stored status onto the connection status set. + + Rows written before ``0038_connections`` use ``authorized`` for a + finished OAuth sign-in, and MCP rows carry no status at all; both are + read from whether the row holds credentials. + """ + raw = (row.get("status") or "").lower() + if raw in (STATUS_RECONNECT, STATUS_DISCONNECTED, STATUS_ERROR): + return raw + if raw == STATUS_PENDING and not has_credentials(row): + return STATUS_PENDING + if raw in ("authorized", STATUS_CONNECTED, "active") or has_credentials(row): + return STATUS_CONNECTED + return STATUS_PENDING + + +def worst_status(statuses: Iterable[str]) -> Optional[str]: + """The status that needs the most attention, or None for an empty list.""" + ranked = sorted(statuses, key=lambda s: _STATUS_SEVERITY.get(s, 0), reverse=True) + return ranked[0] if ranked else None + + +def account_label(row: dict) -> str: + """What identifies the account to its owner: an email, a workspace or a host.""" + return ( + row.get("account_label") + or row.get("user_email") + or row.get("display_name") + or catalog.base_url(row.get("server_url")).split("://")[-1] + or "" + ) + + +def _iso(value: Any) -> Optional[str]: + return value.isoformat() if hasattr(value, "isoformat") else value + + +def serialize_connection(row: dict, counts: Optional[dict] = None) -> dict: + """Public shape of a connection. Never includes tokens or secrets.""" + key = catalog.connector_key_for_row(row) + definition = catalog.get_definition(key) + counts = counts or {} + return { + "id": str(row["id"]), + "connector_key": key, + "name": (definition.name if definition and definition.publisher != "custom" else None) + or row.get("display_name") + or (definition.name if definition else key), + "display_name": row.get("display_name"), + "icon": definition.icon if definition else "tool_mcp_tool", + "account_label": account_label(row), + "auth_kind": row.get("auth_kind") or (definition.auth_kind if definition else None), + "status": normalize_status(row), + "server_url": row.get("server_url"), + "last_error": row.get("last_error"), + "created_at": _iso(row.get("created_at")), + "updated_at": _iso(row.get("updated_at")), + "last_used_at": _iso(row.get("last_used_at")), + "source_count": counts.get("sources", 0), + "tool_count": counts.get("tools", 0), + "credential_mode": row.get("credential_mode"), + } + + +def list_connections(conn, user_id: str) -> list[dict]: + """The user's connections, finished sign-ins only, with resource counts.""" + repo = ConnectorSessionsRepository(conn) + rows = [r for r in repo.list_for_user(user_id) if normalize_status(r) != STATUS_PENDING] + counts = repo.resource_counts([str(r["id"]) for r in rows]) + return [serialize_connection(r, counts.get(str(r["id"]))) for r in rows] + + +def serialize_source(row: dict, connection_status: Optional[str] = None) -> dict: + """A source linked to a connection, as the connection drawer lists it.""" + metadata = _json(row.get("metadata")) or {} + sync_state = metadata.get("sync_state") if isinstance(metadata, dict) else None + if connection_status in (STATUS_RECONNECT, STATUS_DISCONNECTED) and not sync_state: + sync_state = "paused_reconnect" + return { + "id": str(row["id"]), + "name": row.get("name"), + "type": row.get("type"), + "last_sync": _iso(row.get("date")), + "sync_frequency": row.get("sync_frequency") or "never", + "sync_state": sync_state or "active", + } + + +def serialize_tool(row: dict) -> dict: + """A tool linked to a connection, with its actions and their permissions.""" + from docsgpt.connectors.permissions import action_access, action_permission + + actions = [] + for action in _json(row.get("actions")) or []: + if not isinstance(action, dict): + continue + access = action_access(row.get("name"), action) + actions.append( + { + "name": action.get("name"), + "description": action.get("description", ""), + "access": access, + "permission": action_permission(action), + } + ) + return { + "id": str(row["id"]), + "name": row.get("name"), + "display_name": row.get("custom_name") or row.get("display_name") or row.get("name"), + "status": bool(row.get("status")), + "credential_mode": row.get("credential_mode") or "owner", + "actions": actions, + } + + +def connection_detail(conn, row: dict) -> dict: + """A connection with the sources and tools it feeds.""" + repo = ConnectorSessionsRepository(conn) + status = normalize_status(row) + sources = [serialize_source(s, status) for s in repo.list_sources(str(row["id"]))] + tools = [serialize_tool(t) for t in repo.list_tools(str(row["id"]))] + detail = serialize_connection(row, {"sources": len(sources), "tools": len(tools)}) + detail["sources"] = sources + detail["tools"] = tools + return detail + + +def _card_state(definition: ConnectorDefinition, available: bool, disabled: bool, status: Optional[str]) -> str: + if disabled: + return "disabled" + if status in (STATUS_RECONNECT, STATUS_ERROR): + return "reconnect" + if status == STATUS_CONNECTED: + return "connected" + if not available: + return "needs_setup" + if definition.publisher == "custom": + return "custom" + return "available" + + +def catalog_for_user(conn, user_id: str, *, is_admin: bool, policies: Optional[dict] = None) -> list[dict]: + """Catalog entries with availability and the caller's connection summary. + + Args: + conn: Open database connection. + user_id: The caller. + is_admin: Admins see the names of missing server settings; everyone + else only learns that setup is needed. + policies: ``connector_key`` to policy row, when admin policies exist. + """ + policies = policies or {} + by_key: dict[str, list[str]] = {} + for connection in list_connections(conn, user_id): + by_key.setdefault(connection["connector_key"], []).append(connection["status"]) + + entries = [] + for definition in catalog.all_definitions(): + policy = policies.get(definition.key) or {} + disabled = policy.get("enabled") is False + missing = definition.missing_settings + available = not missing and not disabled + statuses = by_key.get(definition.key, []) + status = worst_status(statuses) + entries.append( + { + **definition.to_dict(), + "available": available, + "disabled": disabled, + "needs_setup": bool(missing), + "missing_settings": missing if is_admin else [], + "connected_count": sum(1 for s in statuses if s == STATUS_CONNECTED), + "connection_count": len(statuses), + "status": status, + "state": _card_state(definition, available, disabled, status), + "credential_policy": policy.get("credential_mode") or "choose", + } + ) + return entries + + +def disconnect(conn, row: dict) -> dict: + """Forget a connection's credentials but keep the row and what it feeds. + + Sources keep their indexed content and stop syncing; tools stop working + until the account is reconnected. An MCP server's client registration is + kept so reconnecting skips dynamic client registration. + + Args: + conn: Open database connection inside a transaction. + row: The connection row, already authorised for the caller. + + Returns: + The connection's public shape after the change. + """ + repo = ConnectorSessionsRepository(conn) + session_data = _json(row.get("session_data")) or {} + if isinstance(session_data, dict): + session_data = {k: v for k, v in session_data.items() if k != "tokens"} + repo.update( + str(row["id"]), + { + "status": STATUS_DISCONNECTED, + "token_info": None, + "session_token": None, + "session_data": session_data or {}, + }, + ) + return serialize_connection(repo.get(str(row["id"]))) diff --git a/docsgpt/storage/db/models.py b/docsgpt/storage/db/models.py index ebd25567..147f4316 100644 --- a/docsgpt/storage/db/models.py +++ b/docsgpt/storage/db/models.py @@ -216,6 +216,9 @@ user_tools_table = Table( Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), Column("updated_at", DateTime(timezone=True), nullable=False, server_default=func.now()), Column("legacy_mongo_id", Text), + Column( + "connection_id", UUID(as_uuid=True), ForeignKey("connector_sessions.id", ondelete="SET NULL"), + ), ) token_usage_table = Table( @@ -338,6 +341,9 @@ sources_table = Table( Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), Column("updated_at", DateTime(timezone=True), nullable=False, server_default=func.now()), Column("legacy_mongo_id", Text), + Column( + "connection_id", UUID(as_uuid=True), ForeignKey("connector_sessions.id", ondelete="SET NULL"), + ), ) agents_table = Table( @@ -585,6 +591,13 @@ connector_sessions_table = Table( Column("expires_at", DateTime(timezone=True)), Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), Column("legacy_mongo_id", Text), + # Added in ``0038_connections``: each row is a connection (one signed-in + # account, one MCP server or one set of API credentials). + Column("connector_key", Text), + Column("display_name", Text), + Column("account_label", Text), + Column("auth_kind", Text), + Column("updated_at", DateTime(timezone=True), nullable=False, server_default=func.now()), ) diff --git a/docsgpt/storage/db/repositories/connector_sessions.py b/docsgpt/storage/db/repositories/connector_sessions.py index 7a95b9d1..a521406c 100644 --- a/docsgpt/storage/db/repositories/connector_sessions.py +++ b/docsgpt/storage/db/repositories/connector_sessions.py @@ -24,12 +24,13 @@ from typing import Any, Optional from sqlalchemy import Connection, text -from docsgpt.storage.db.base_repository import row_to_dict +from docsgpt.storage.db.base_repository import looks_like_uuid, row_to_dict from docsgpt.storage.db.serialization import PGNativeJSONEncoder _UPDATABLE_SCALARS = { "server_url", "session_token", "user_email", "status", "expires_at", + "connector_key", "display_name", "account_label", "auth_kind", } _UPDATABLE_JSONB = {"session_data", "token_info"} @@ -176,11 +177,83 @@ class ConnectorSessionsRepository: def list_for_user(self, user_id: str) -> list[dict]: result = self._conn.execute( - text("SELECT * FROM connector_sessions WHERE user_id = :user_id"), + text("SELECT * FROM connector_sessions WHERE user_id = :user_id ORDER BY created_at"), {"user_id": user_id}, ) return [row_to_dict(r) for r in result.fetchall()] + def get(self, connection_id: str) -> Optional[dict]: + """Fetch a connection by id, whoever owns it. Callers authorise.""" + if not looks_like_uuid(connection_id): + return None + result = self._conn.execute( + text("SELECT * FROM connector_sessions WHERE id = CAST(:id AS uuid)"), + {"id": str(connection_id)}, + ) + row = result.fetchone() + return row_to_dict(row) if row is not None else None + + def get_for_user(self, connection_id: str, user_id: str) -> Optional[dict]: + """Fetch a connection only when ``user_id`` owns it.""" + row = self.get(connection_id) + if row is None or row.get("user_id") != user_id: + return None + return row + + def get_for_update(self, connection_id: str) -> Optional[dict]: + """Fetch and row-lock a connection until the transaction ends. + + Token refresh holds this lock so two workers refreshing a rotating + refresh token (Microsoft, Atlassian) cannot both spend it. + """ + if not looks_like_uuid(connection_id): + return None + result = self._conn.execute( + text("SELECT * FROM connector_sessions WHERE id = CAST(:id AS uuid) FOR UPDATE"), + {"id": str(connection_id)}, + ) + row = result.fetchone() + return row_to_dict(row) if row is not None else None + + def resource_counts(self, connection_ids: list[str]) -> dict[str, dict[str, int]]: + """Number of sources and tools linked to each connection id.""" + ids = [str(i) for i in connection_ids if looks_like_uuid(str(i))] + counts: dict[str, dict[str, int]] = {i: {"sources": 0, "tools": 0} for i in ids} + if not ids: + return counts + for table, key in (("sources", "sources"), ("user_tools", "tools")): + result = self._conn.execute( + text( + f"SELECT connection_id, count(*) FROM {table} " + "WHERE connection_id = ANY(CAST(:ids AS uuid[])) GROUP BY connection_id" + ), + {"ids": ids}, + ) + for connection_id, count in result.fetchall(): + counts[str(connection_id)][key] = int(count) + return counts + + def list_sources(self, connection_id: str) -> list[dict]: + """Sources synced from a connection, newest first.""" + result = self._conn.execute( + text( + "SELECT id, name, type, date, sync_frequency, metadata, remote_data, user_id " + "FROM sources WHERE connection_id = CAST(:id AS uuid) ORDER BY date DESC" + ), + {"id": str(connection_id)}, + ) + return [row_to_dict(r) for r in result.fetchall()] + + def list_tools(self, connection_id: str) -> list[dict]: + """Tools a connection provides, oldest first.""" + result = self._conn.execute( + text( + "SELECT * FROM user_tools WHERE connection_id = CAST(:id AS uuid) ORDER BY created_at" + ), + {"id": str(connection_id)}, + ) + return [row_to_dict(r) for r in result.fetchall()] + def update(self, session_id: str, fields: dict) -> bool: """Partial update by PG UUID.""" filtered = { @@ -198,6 +271,7 @@ class ConnectorSessionsRepository: else: set_clauses.append(f"{col} = :{col}") params[col] = val + set_clauses.append("updated_at = now()") result = self._conn.execute( text( f"UPDATE connector_sessions SET {', '.join(set_clauses)} " diff --git a/tests/connectors/__init__.py b/tests/connectors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/connectors/test_catalog.py b/tests/connectors/test_catalog.py new file mode 100644 index 00000000..37e3b86e --- /dev/null +++ b/tests/connectors/test_catalog.py @@ -0,0 +1,116 @@ +"""Tests for the connector catalog.""" + +from __future__ import annotations + +import pytest + +from docsgpt.connectors import catalog + + +@pytest.fixture(autouse=True) +def _fresh_registry(): + catalog.reset_registry_for_tests() + yield + catalog.reset_registry_for_tests() + + +class TestDefinitions: + def test_built_ins_are_registered(self): + keys = {d.key for d in catalog.all_definitions()} + assert { + "google_drive", "share_point", "confluence", "s3", "reddit", + "brave", "telegram", "ntfy", "postgres", "custom_mcp", "custom_openapi", + } <= keys + + def test_every_definition_uses_known_values(self): + for definition in catalog.all_definitions(): + assert definition.category in catalog.CATEGORIES, definition.key + assert definition.auth_kind in catalog.AUTH_KINDS, definition.key + assert definition.publisher in catalog.PUBLISHERS, definition.key + assert set(definition.capabilities) <= {"sync", "read", "write"}, definition.key + + def test_tool_connectors_match_tool_config_requirements(self): + """Credential fields must be the keys the tool reads its secrets from.""" + import docsgpt.api.user # noqa: F401 (mcp_tool imports it; load it first) + from docsgpt.agents.tools.tool_manager import ToolManager + + tools = ToolManager(config={}).tools + for definition in catalog.all_definitions(): + if definition.publisher != "built_in": + continue + for tool_name in definition.tool_templates: + requirements = tools[tool_name].get_config_requirements() + secret_keys = {k for k, spec in requirements.items() if spec.get("secret")} + field_keys = {f.key for f in definition.credential_fields} + assert secret_keys <= field_keys, definition.key + + def test_to_dict_has_no_server_secrets(self): + payload = catalog.get_definition("google_drive").to_dict() + assert "required_settings" not in payload + assert payload["capabilities"] == ["sync"] + + +class TestAvailability: + def test_missing_settings_listed(self, monkeypatch): + from docsgpt.core.settings import settings + + monkeypatch.setattr(settings, "GOOGLE_CLIENT_ID", None) + monkeypatch.setattr(settings, "GOOGLE_CLIENT_SECRET", "secret") + definition = catalog.get_definition("google_drive") + assert definition.missing_settings == ["GOOGLE_CLIENT_ID"] + assert not definition.configured + + def test_configured_when_all_set(self, monkeypatch): + from docsgpt.core.settings import settings + + monkeypatch.setattr(settings, "GOOGLE_CLIENT_ID", "id") + monkeypatch.setattr(settings, "GOOGLE_CLIENT_SECRET", "secret") + assert catalog.get_definition("google_drive").configured + + def test_api_key_connectors_need_no_settings(self): + assert catalog.get_definition("telegram").configured + + +class TestRowMapping: + def test_oauth_provider_maps_to_key(self): + assert catalog.connector_key_for_row({"provider": "share_point"}) == "share_point" + + def test_legacy_mcp_row_is_custom(self): + row = {"provider": "mcp:https://mcp.unknown.dev", "server_url": "https://mcp.unknown.dev"} + assert catalog.connector_key_for_row(row) == "custom_mcp" + + def test_stored_key_wins(self): + assert catalog.connector_key_for_row({"provider": "telegram", "connector_key": "telegram"}) == "telegram" + + def test_unknown_provider(self): + assert catalog.connector_key_for_row({"provider": "nope"}) is None + + def test_definition_for_tool(self): + assert catalog.definition_for_tool("telegram").key == "telegram" + assert catalog.definition_for_tool("memory") is None + assert catalog.definition_for_tool("mcp_tool") is None + + +class TestPresets: + def test_presets_load_from_yaml(self, tmp_path, monkeypatch): + presets = tmp_path / "mcp.yaml" + presets.write_text( + "- key: mcp:example\n" + " name: Example\n" + " description: Example records.\n" + " icon: example\n" + " category: knowledge\n" + " mcp_url: https://mcp.example.com/mcp\n" + " auth_kind: mcp_oauth\n" + ) + monkeypatch.setattr(catalog, "_PRESETS_FILE", presets) + definition = catalog.get_definition("mcp:example") + assert definition.publisher == "preset" + assert definition.mcp_base_url == "https://mcp.example.com" + assert catalog.preset_for_url("https://mcp.example.com/other").key == "mcp:example" + row = {"provider": "mcp:https://mcp.example.com", "server_url": "https://mcp.example.com"} + assert catalog.connector_key_for_row(row) == "mcp:example" + + def test_base_url(self): + assert catalog.base_url("https://a.example.com:8443/x/y") == "https://a.example.com:8443" + assert catalog.base_url("not a url") == "" diff --git a/tests/connectors/test_permissions.py b/tests/connectors/test_permissions.py new file mode 100644 index 00000000..bb4318e1 --- /dev/null +++ b/tests/connectors/test_permissions.py @@ -0,0 +1,47 @@ +"""Tests for read / write classification and permissions.""" + +from __future__ import annotations + +import pytest + +from docsgpt.connectors import permissions as p + + +class TestActionAccess: + def test_explicit_access_wins(self): + assert p.action_access("telegram", {"name": "search", "access": "write"}) == "write" + + def test_mcp_annotations(self): + assert p.action_access("mcp_tool", {"name": "create_page", "annotations": {"readOnlyHint": True}}) == "read" + assert p.action_access("mcp_tool", {"name": "search", "annotations": {"destructiveHint": True}}) == "write" + + def test_api_tool_method(self): + assert p.action_access("api_tool", {"name": "x", "method": "get"}) == "read" + assert p.action_access("api_tool", {"name": "x", "method": "POST"}) == "write" + + @pytest.mark.parametrize( + "name,expected", + [("search_pages", "read"), ("list_issues", "read"), ("create_issue", "write"), ("send_message", "write")], + ) + def test_name_heuristic(self, name, expected): + assert p.action_access("mcp_tool", {"name": name}) == expected + + +class TestPermissions: + def test_permission_from_flags(self): + assert p.action_permission({"active": False, "require_approval": True}) == "off" + assert p.action_permission({"active": True, "require_approval": True}) == "ask" + assert p.action_permission({"active": True}) == "always" + + def test_apply_permission(self): + assert p.apply_permission({"name": "a"}, "ask") == {"name": "a", "active": True, "require_approval": True} + assert p.apply_permission({"name": "a"}, "off")["active"] is False + with pytest.raises(ValueError): + p.apply_permission({}, "sometimes") + + def test_defaults_writes_to_approval(self): + stamped = p.apply_default_permissions( + "mcp_tool", [{"name": "search"}, {"name": "delete_page"}] + ) + assert stamped[0] == {"name": "search", "access": "read"} + assert stamped[1]["access"] == "write" and stamped[1]["require_approval"] is True diff --git a/tests/connectors/test_service_read.py b/tests/connectors/test_service_read.py new file mode 100644 index 00000000..01c8a6d6 --- /dev/null +++ b/tests/connectors/test_service_read.py @@ -0,0 +1,263 @@ +"""Tests for the read side of the connection service and its API.""" + +from __future__ import annotations + +import json +from contextlib import contextmanager +from unittest.mock import patch + +import pytest +from flask import Flask +from sqlalchemy import text + +from docsgpt.connectors import service + + +def _session(conn, user="alice", provider="google_drive", **cols) -> str: + cols = {"status": "authorized", "user_email": f"{user}@example.com", **cols} + casts = {"token_info": "jsonb", "session_data": "jsonb"} + names = ", ".join(["user_id", "provider", *cols]) + values = ", ".join( + [":user_id", ":provider", *[f"CAST(:{k} AS {casts[k]})" if k in casts else f":{k}" for k in cols]] + ) + return str( + conn.execute( + text(f"INSERT INTO connector_sessions ({names}) VALUES ({values}) RETURNING id"), + {"user_id": user, "provider": provider, **cols}, + ).scalar() + ) + + +def _source(conn, connection_id, user="alice", name="Handbook") -> str: + return str( + conn.execute( + text( + "INSERT INTO sources (user_id, name, type, sync_frequency, connection_id) " + "VALUES (:u, :n, 'connector:file', 'weekly', CAST(:c AS uuid)) RETURNING id" + ), + {"u": user, "n": name, "c": connection_id}, + ).scalar() + ) + + +def _tool(conn, connection_id, user="alice") -> str: + actions = [ + {"name": "search_pages", "description": "Search", "active": True}, + {"name": "create_page", "description": "Create", "active": True, "require_approval": True}, + ] + return str( + conn.execute( + text( + "INSERT INTO user_tools (user_id, name, display_name, actions, connection_id) " + "VALUES (:u, 'mcp_tool', 'Notion', CAST(:a AS jsonb), CAST(:c AS uuid)) RETURNING id" + ), + {"u": user, "a": json.dumps(actions), "c": connection_id}, + ).scalar() + ) + + +class TestNormalizeStatus: + @pytest.mark.parametrize( + "row,expected", + [ + ({"status": "authorized"}, "connected"), + ({"status": "reconnect_needed"}, "reconnect_needed"), + ({"status": "pending"}, "pending"), + ({"status": "pending", "token_info": {"access_token": "x"}}, "connected"), + ({"status": None, "session_data": {"tokens": {"access_token": "x"}}}, "connected"), + ({"status": None, "session_data": {"client_info": {}}}, "pending"), + ({"status": None, "encrypted_credentials": "v2:..."}, "connected"), + ], + ) + def test_statuses(self, row, expected): + assert service.normalize_status(row) == expected + + def test_worst_status(self): + assert service.worst_status(["connected", "reconnect_needed"]) == "reconnect_needed" + assert service.worst_status([]) is None + + +class TestListing: + def test_lists_finished_connections_with_counts(self, pg_conn): + drive = _session(pg_conn) + _session(pg_conn, provider="confluence", status="pending", user_email=None) + _source(pg_conn, drive) + _source(pg_conn, drive, name="Wiki") + connections = service.list_connections(pg_conn, "alice") + assert [c["connector_key"] for c in connections] == ["google_drive"] + only = connections[0] + assert only["name"] == "Google Drive" + assert only["account_label"] == "alice@example.com" + assert only["status"] == "connected" + assert only["source_count"] == 2 + assert "token_info" not in only and "session_token" not in only + + def test_other_users_rows_hidden(self, pg_conn): + _session(pg_conn, user="bob") + assert service.list_connections(pg_conn, "alice") == [] + + def test_detail_lists_sources_and_tools(self, pg_conn): + from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository + + drive = _session(pg_conn, status="reconnect_needed") + _source(pg_conn, drive) + _tool(pg_conn, drive) + row = ConnectorSessionsRepository(pg_conn).get(drive) + detail = service.connection_detail(pg_conn, row) + assert detail["sources"][0]["sync_state"] == "paused_reconnect" + actions = {a["name"]: a for a in detail["tools"][0]["actions"]} + assert actions["search_pages"] == { + "name": "search_pages", "description": "Search", "access": "read", "permission": "always", + } + assert actions["create_page"]["access"] == "write" + assert actions["create_page"]["permission"] == "ask" + + +class TestCatalogForUser: + def test_states(self, pg_conn, monkeypatch): + from docsgpt.core.settings import settings + + monkeypatch.setattr(settings, "GOOGLE_CLIENT_ID", "id") + monkeypatch.setattr(settings, "GOOGLE_CLIENT_SECRET", "secret") + monkeypatch.setattr(settings, "MICROSOFT_CLIENT_ID", None) + _session(pg_conn) + _session(pg_conn, provider="confluence", status="reconnect_needed") + entries = {e["key"]: e for e in service.catalog_for_user(pg_conn, "alice", is_admin=False)} + assert entries["google_drive"]["state"] == "connected" + assert entries["google_drive"]["connected_count"] == 1 + assert entries["confluence"]["state"] == "reconnect" + assert entries["share_point"]["state"] == "needs_setup" + assert entries["share_point"]["missing_settings"] == [] + assert entries["telegram"]["state"] == "available" + assert entries["custom_mcp"]["state"] == "custom" + + def test_admin_sees_missing_setting_names(self, pg_conn, monkeypatch): + from docsgpt.core.settings import settings + + monkeypatch.setattr(settings, "MICROSOFT_CLIENT_ID", None) + entries = {e["key"]: e for e in service.catalog_for_user(pg_conn, "alice", is_admin=True)} + assert "MICROSOFT_CLIENT_ID" in entries["share_point"]["missing_settings"] + + def test_policy_disables(self, pg_conn): + entries = { + e["key"]: e + for e in service.catalog_for_user( + pg_conn, "alice", is_admin=False, policies={"telegram": {"enabled": False}} + ) + } + assert entries["telegram"]["state"] == "disabled" + assert entries["telegram"]["available"] is False + + +@contextmanager +def _patched_db(conn): + @contextmanager + def _yield(): + yield conn + + with patch("docsgpt.api.connector.connections.db_readonly", _yield): + yield + + +@pytest.fixture +def app(): + return Flask(__name__) + + +def _call(app, resource, path, token, *args): + with app.test_request_context(path): + from flask import request + + request.decoded_token = token + return resource().get(*args) + + +class TestRoutes: + def test_catalog_requires_auth(self, app): + from docsgpt.api.connector.connections import ConnectorCatalog + + assert _call(app, ConnectorCatalog, "/api/connectors/catalog", None).status_code == 401 + + def test_catalog(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectorCatalog + + with _patched_db(pg_conn): + resp = _call(app, ConnectorCatalog, "/api/connectors/catalog", {"sub": "alice"}) + assert resp.status_code == 200 + keys = {c["key"] for c in resp.get_json()["connectors"]} + assert "telegram" in keys + + def test_list_and_detail(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionDetail, ConnectionsList + + drive = _session(pg_conn) + with _patched_db(pg_conn): + listed = _call(app, ConnectionsList, "/api/connections", {"sub": "alice"}) + detail = _call(app, ConnectionDetail, f"/api/connections/{drive}", {"sub": "alice"}, drive) + assert listed.get_json()["connections"][0]["id"] == drive + assert detail.get_json()["connection"]["id"] == drive + + def test_detail_of_another_users_connection_is_404(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionDetail + + drive = _session(pg_conn, user="bob") + with _patched_db(pg_conn): + resp = _call(app, ConnectionDetail, f"/api/connections/{drive}", {"sub": "alice"}, drive) + assert resp.status_code == 404 + + def test_detail_of_bad_id_is_404(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionDetail + + with _patched_db(pg_conn): + resp = _call(app, ConnectionDetail, "/api/connections/nope", {"sub": "alice"}, "nope") + assert resp.status_code == 404 + + +class TestDisconnect: + def test_clears_credentials_keeps_resources(self, pg_conn): + from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository + + drive = _session( + pg_conn, session_token="tok", token_info=json.dumps({"access_token": "at", "refresh_token": "rt"}), + ) + source = _source(pg_conn, drive) + repo = ConnectorSessionsRepository(pg_conn) + result = service.disconnect(pg_conn, repo.get(drive)) + assert result["status"] == "disconnected" + row = repo.get(drive) + assert row["token_info"] is None and row["session_token"] is None + linked = pg_conn.execute( + text("SELECT connection_id FROM sources WHERE id = CAST(:id AS uuid)"), {"id": source} + ).scalar() + assert str(linked) == drive + + def test_mcp_keeps_client_registration(self, pg_conn): + from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository + + mcp = _session( + pg_conn, + provider="mcp:https://mcp.example.com", + status=None, + session_data=json.dumps({"tokens": {"access_token": "x"}, "client_info": {"client_id": "c"}}), + ) + repo = ConnectorSessionsRepository(pg_conn) + service.disconnect(pg_conn, repo.get(mcp)) + assert repo.get(mcp)["session_data"] == {"client_info": {"client_id": "c"}} + + def test_route_rejects_other_users(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionDisconnect + + drive = _session(pg_conn, user="bob") + + @contextmanager + def _yield(): + yield pg_conn + + with patch("docsgpt.api.connector.connections.db_session", _yield), app.test_request_context( + f"/api/connections/{drive}/disconnect", method="POST" + ): + from flask import request + + request.decoded_token = {"sub": "alice"} + resp = ConnectionDisconnect().post(drive) + assert resp.status_code == 404 diff --git a/tests/storage/db/test_migration_0038.py b/tests/storage/db/test_migration_0038.py new file mode 100644 index 00000000..5bdee6ff --- /dev/null +++ b/tests/storage/db/test_migration_0038.py @@ -0,0 +1,154 @@ +"""Migration tests for 0038_connections: columns, backfill and round trip.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from sqlalchemy import text + + +pytestmark = pytest.mark.integration + +_0037 = "0037_request_traces" + + +def _alembic_ini() -> Path: + return Path(__file__).resolve().parents[3] / "docsgpt" / "alembic.ini" + + +def _run_alembic(url: str, *args: str) -> None: + subprocess.check_call( + [sys.executable, "-m", "alembic", "-c", str(_alembic_ini()), *args], + timeout=120, + env={**os.environ, "POSTGRES_URI": url}, + ) + + +def _columns(conn, table: str) -> set[str]: + rows = conn.execute( + text("SELECT column_name FROM information_schema.columns WHERE table_name = :t"), + {"t": table}, + ).fetchall() + return {r[0] for r in rows} + + +def _seed_legacy(conn) -> dict: + """Rows as a pre-0038 install left them.""" + ids = {} + ids["drive"] = conn.execute( + text( + "INSERT INTO connector_sessions (user_id, provider, session_token, user_email, status, token_info) " + "VALUES ('alice', 'google_drive', 'tok-a', 'alice@example.com', 'authorized', " + "CAST(:ti AS jsonb)) RETURNING id" + ), + {"ti": json.dumps({"access_token": "at", "refresh_token": "rt"})}, + ).scalar() + ids["mcp"] = conn.execute( + text( + "INSERT INTO connector_sessions (user_id, provider, server_url, session_data) " + "VALUES ('alice', 'mcp:https://mcp.example.com', 'https://mcp.example.com', CAST(:sd AS jsonb)) " + "RETURNING id" + ), + {"sd": json.dumps({"tokens": {"access_token": "m"}})}, + ).scalar() + ids["source"] = conn.execute( + text( + "INSERT INTO sources (user_id, name, type, remote_data) " + "VALUES ('alice', 'Handbook', 'connector:file', CAST(:rd AS jsonb)) RETURNING id" + ), + {"rd": json.dumps({"provider": "google_drive", "file_ids": ["f1"]})}, + ).scalar() + ids["other_source"] = conn.execute( + text( + "INSERT INTO sources (user_id, name, type, remote_data) " + "VALUES ('bob', 'Bob files', 'connector:file', CAST(:rd AS jsonb)) RETURNING id" + ), + {"rd": json.dumps({"provider": "google_drive"})}, + ).scalar() + ids["mcp_tool"] = conn.execute( + text( + "INSERT INTO user_tools (user_id, name, config) " + "VALUES ('alice', 'mcp_tool', CAST(:c AS jsonb)) RETURNING id" + ), + {"c": json.dumps({"server_url": "https://mcp.example.com/mcp", "auth_type": "oauth"})}, + ).scalar() + ids["bearer_tool"] = conn.execute( + text( + "INSERT INTO user_tools (user_id, name, config) " + "VALUES ('alice', 'mcp_tool', CAST(:c AS jsonb)) RETURNING id" + ), + {"c": json.dumps({"server_url": "https://mcp.example.com/mcp", "auth_type": "bearer"})}, + ).scalar() + return ids + + +class TestMigration0038: + def test_head_has_connection_columns(self, pg_engine): + with pg_engine.connect() as conn: + assert {"connector_key", "display_name", "account_label", "auth_kind", "updated_at"} <= _columns( + conn, "connector_sessions" + ) + assert "connection_id" in _columns(conn, "sources") + assert "connection_id" in _columns(conn, "user_tools") + + def test_backfill_links_legacy_rows(self, pg_engine): + url = pg_engine.url.render_as_string(hide_password=False) + _run_alembic(url, "downgrade", _0037) + with pg_engine.begin() as conn: + ids = _seed_legacy(conn) + _run_alembic(url, "upgrade", "head") + + with pg_engine.connect() as conn: + drive = conn.execute( + text( + "SELECT connector_key, auth_kind, display_name, account_label " + "FROM connector_sessions WHERE id = :id" + ), + {"id": ids["drive"]}, + ).one() + assert tuple(drive) == ("google_drive", "oauth", "Google Drive", "alice@example.com") + + mcp = conn.execute( + text("SELECT connector_key, auth_kind, display_name FROM connector_sessions WHERE id = :id"), + {"id": ids["mcp"]}, + ).one() + assert tuple(mcp) == ("custom_mcp", "mcp_oauth", "mcp.example.com") + + linked = conn.execute( + text("SELECT connection_id FROM sources WHERE id = :id"), {"id": ids["source"]} + ).scalar() + assert linked == ids["drive"] + # Bob has no session for the provider: nothing to link. + assert ( + conn.execute( + text("SELECT connection_id FROM sources WHERE id = :id"), {"id": ids["other_source"]} + ).scalar() + is None + ) + tool = conn.execute( + text("SELECT connection_id FROM user_tools WHERE id = :id"), {"id": ids["mcp_tool"]} + ).scalar() + assert tool == ids["mcp"] + # A bearer MCP tool has no OAuth session to point at. + bearer = conn.execute( + text("SELECT connection_id FROM user_tools WHERE id = :id"), {"id": ids["bearer_tool"]} + ).scalar() + assert bearer is None + + def test_downgrade_then_upgrade_is_idempotent(self, pg_engine): + url = pg_engine.url.render_as_string(hide_password=False) + _run_alembic(url, "downgrade", _0037) + with pg_engine.connect() as conn: + assert "connector_key" not in _columns(conn, "connector_sessions") + assert "connection_id" not in _columns(conn, "sources") + _run_alembic(url, "upgrade", "head") + _run_alembic(url, "downgrade", _0037) + _run_alembic(url, "upgrade", "head") + with pg_engine.connect() as conn: + assert "connector_key" in _columns(conn, "connector_sessions") + assert "connection_id" in _columns(conn, "user_tools") From aad43c2fd0a8c7165137a28513b2d8b2fa0cdcd3 Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:30:56 +0100 Subject: [PATCH 002/162] Add the Connectors page Settings > Workspace gets a Connectors page between Sources and Tools: a searchable catalog filtered by category, one card per service with its capabilities (Sync, Read, Write) and state (Connect, Connected, Reconnect, Needs admin setup, Disabled by admin, Custom). A card opens a drawer listing the accounts, the sources each syncs and the tools each provides, with Reconnect and Disconnect. Tools and Sources show which connection an item comes from and when its sync is paused. Brand icons now draw in currentColor so they follow the theme, and all new copy is translated in every locale. --- frontend/src/api/endpoints.ts | 5 + .../src/api/services/connectorsService.ts | 29 ++ frontend/src/assets/confluence.svg | 2 +- frontend/src/assets/connectors/asana.svg | 1 + frontend/src/assets/connectors/atlassian.svg | 1 + frontend/src/assets/connectors/linear.svg | 1 + frontend/src/assets/connectors/notion.svg | 1 + frontend/src/assets/connectors/sentry.svg | 1 + frontend/src/assets/connectors/stripe.svg | 1 + frontend/src/assets/drive.svg | 2 +- frontend/src/assets/reddit.svg | 4 +- frontend/src/assets/s3.svg | 10 +- frontend/src/assets/sharepoint.svg | 2 +- frontend/src/connectors/ConnectionDrawer.tsx | 438 ++++++++++++++++++ frontend/src/connectors/ConnectorCard.tsx | 138 ++++++ frontend/src/connectors/ConnectorIcon.tsx | 70 +++ frontend/src/connectors/connectorsSlice.ts | 80 ++++ frontend/src/connectors/i18n.ts | 29 ++ frontend/src/connectors/types.ts | 114 +++++ .../src/connectors/useConnectorLauncher.tsx | 140 ++++++ frontend/src/locale/connectors.test.ts | 72 +++ frontend/src/locale/de.json | 128 ++++- frontend/src/locale/en.json | 128 ++++- frontend/src/locale/es.json | 128 ++++- frontend/src/locale/jp.json | 127 ++++- frontend/src/locale/ru.json | 130 +++++- frontend/src/locale/zh-TW.json | 127 ++++- frontend/src/locale/zh.json | 127 ++++- frontend/src/modals/ConfirmationModal.tsx | 4 + frontend/src/modals/MCPServerModal.tsx | 10 +- frontend/src/models/misc.ts | 2 + frontend/src/navigation/sections.ts | 7 + frontend/src/settings/Connectors.test.tsx | 217 +++++++++ frontend/src/settings/Connectors.tsx | 220 +++++++++ frontend/src/settings/Sources.tsx | 49 +- frontend/src/settings/Tools.tsx | 55 ++- frontend/src/settings/index.tsx | 2 + frontend/src/settings/types/index.ts | 2 + frontend/src/store.ts | 2 + 39 files changed, 2502 insertions(+), 104 deletions(-) create mode 100644 frontend/src/api/services/connectorsService.ts create mode 100644 frontend/src/assets/connectors/asana.svg create mode 100644 frontend/src/assets/connectors/atlassian.svg create mode 100644 frontend/src/assets/connectors/linear.svg create mode 100644 frontend/src/assets/connectors/notion.svg create mode 100644 frontend/src/assets/connectors/sentry.svg create mode 100644 frontend/src/assets/connectors/stripe.svg create mode 100644 frontend/src/connectors/ConnectionDrawer.tsx create mode 100644 frontend/src/connectors/ConnectorCard.tsx create mode 100644 frontend/src/connectors/ConnectorIcon.tsx create mode 100644 frontend/src/connectors/connectorsSlice.ts create mode 100644 frontend/src/connectors/i18n.ts create mode 100644 frontend/src/connectors/types.ts create mode 100644 frontend/src/connectors/useConnectorLauncher.tsx create mode 100644 frontend/src/locale/connectors.test.ts create mode 100644 frontend/src/settings/Connectors.test.tsx create mode 100644 frontend/src/settings/Connectors.tsx diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index 7989be60..d621dfea 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -87,6 +87,11 @@ const endpoints = { CONNECTOR_FILES: '/api/connectors/files', CONNECTOR_VALIDATE_SESSION: '/api/connectors/validate-session', CONNECTOR_DISCONNECT: '/api/connectors/disconnect', + CONNECTORS_CATALOG: '/api/connectors/catalog', + CONNECTIONS: '/api/connections', + CONNECTION: (id: string) => `/api/connections/${encodeURIComponent(id)}`, + CONNECTION_DISCONNECT: (id: string) => + `/api/connections/${encodeURIComponent(id)}/disconnect`, GET_CHUNKS: ( docId: string, page: number, diff --git a/frontend/src/api/services/connectorsService.ts b/frontend/src/api/services/connectorsService.ts new file mode 100644 index 00000000..19640996 --- /dev/null +++ b/frontend/src/api/services/connectorsService.ts @@ -0,0 +1,29 @@ +import apiClient, { throttledApiClient } from '../client'; +import endpoints from '../endpoints'; + +// apiClient resolves to the raw fetch Response; parse it here so the slice and +// components get plain data. A non-2xx body still parses (``success: false``). +const json = async (response: Response) => { + try { + return await response.json(); + } catch { + return { success: false }; + } +}; + +const connectorsService = { + getCatalog: async (token: string | null) => + json( + await throttledApiClient.get(endpoints.USER.CONNECTORS_CATALOG, token), + ), + listConnections: async (token: string | null) => + json(await throttledApiClient.get(endpoints.USER.CONNECTIONS, token)), + getConnection: async (id: string, token: string | null) => + json(await apiClient.get(endpoints.USER.CONNECTION(id), token)), + disconnect: async (id: string, token: string | null) => + json( + await apiClient.post(endpoints.USER.CONNECTION_DISCONNECT(id), {}, token), + ), +}; + +export default connectorsService; diff --git a/frontend/src/assets/confluence.svg b/frontend/src/assets/confluence.svg index 183807ad..34631723 100644 --- a/frontend/src/assets/confluence.svg +++ b/frontend/src/assets/confluence.svg @@ -1,4 +1,4 @@ - + diff --git a/frontend/src/assets/connectors/asana.svg b/frontend/src/assets/connectors/asana.svg new file mode 100644 index 00000000..3b2ba44c --- /dev/null +++ b/frontend/src/assets/connectors/asana.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/assets/connectors/atlassian.svg b/frontend/src/assets/connectors/atlassian.svg new file mode 100644 index 00000000..76d35504 --- /dev/null +++ b/frontend/src/assets/connectors/atlassian.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/assets/connectors/linear.svg b/frontend/src/assets/connectors/linear.svg new file mode 100644 index 00000000..583413fe --- /dev/null +++ b/frontend/src/assets/connectors/linear.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/assets/connectors/notion.svg b/frontend/src/assets/connectors/notion.svg new file mode 100644 index 00000000..5a8188cb --- /dev/null +++ b/frontend/src/assets/connectors/notion.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/assets/connectors/sentry.svg b/frontend/src/assets/connectors/sentry.svg new file mode 100644 index 00000000..4f08902e --- /dev/null +++ b/frontend/src/assets/connectors/sentry.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/assets/connectors/stripe.svg b/frontend/src/assets/connectors/stripe.svg new file mode 100644 index 00000000..3ffcab80 --- /dev/null +++ b/frontend/src/assets/connectors/stripe.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/assets/drive.svg b/frontend/src/assets/drive.svg index 8ec4dc9b..ea0c1607 100644 --- a/frontend/src/assets/drive.svg +++ b/frontend/src/assets/drive.svg @@ -1,3 +1,3 @@ - + diff --git a/frontend/src/assets/reddit.svg b/frontend/src/assets/reddit.svg index 705ef638..61d78727 100644 --- a/frontend/src/assets/reddit.svg +++ b/frontend/src/assets/reddit.svg @@ -1,4 +1,4 @@ - - + + diff --git a/frontend/src/assets/s3.svg b/frontend/src/assets/s3.svg index 3fdc41ff..c40cb06f 100644 --- a/frontend/src/assets/s3.svg +++ b/frontend/src/assets/s3.svg @@ -1,7 +1,7 @@ - - - - - + + + + + diff --git a/frontend/src/assets/sharepoint.svg b/frontend/src/assets/sharepoint.svg index 9a332f8e..8aa88c99 100644 --- a/frontend/src/assets/sharepoint.svg +++ b/frontend/src/assets/sharepoint.svg @@ -1,6 +1,6 @@ - + diff --git a/frontend/src/connectors/ConnectionDrawer.tsx b/frontend/src/connectors/ConnectionDrawer.tsx new file mode 100644 index 00000000..e37453a6 --- /dev/null +++ b/frontend/src/connectors/ConnectionDrawer.tsx @@ -0,0 +1,438 @@ +import { + CircleAlert, + ExternalLink, + Plus, + RefreshCw, + Unplug, +} from 'lucide-react'; +import { useCallback, useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useDispatch, useSelector } from 'react-redux'; + +import connectorsService from '../api/services/connectorsService'; +import { Alert, AlertDescription, AlertTitle } from '../components/ui/alert'; +import { Badge } from '../components/ui/badge'; +import { Button } from '../components/ui/button'; +import { Card } from '../components/ui/card'; +import { ActionMenu } from '../components/ui/dropdown-menu'; +import { EmptyState } from '../components/ui/empty-state'; +import { ListRow, ListRows } from '../components/ui/list-row'; +import { LoadingState } from '../components/ui/loading-state'; +import { SectionHeader } from '../components/ui/section-header'; +import { Sheet, SheetContent, SheetTitle } from '../components/ui/sheet'; +import ConfirmationModal from '../modals/ConfirmationModal'; +import { selectToken } from '../preferences/preferenceSlice'; +import type { AppDispatch } from '../store'; +import { formatDateTime } from '../utils/dateTimeUtils'; +import { CapabilityBadges } from './ConnectorCard'; +import ConnectorIcon from './ConnectorIcon'; +import { selectConnections, loadConnectors } from './connectorsSlice'; +import { connectorDescription, connectorName } from './i18n'; +import type { + ConnectionDetail, + ConnectionStatus, + ConnectionTool, + ConnectorDefinition, +} from './types'; + +const STATUS_VARIANT: Record< + ConnectionStatus, + 'success' | 'warning' | 'neutral' | 'destructive' +> = { + connected: 'success', + reconnect_needed: 'warning', + disconnected: 'neutral', + error: 'destructive', + pending: 'neutral', +}; + +const PERMISSION_VARIANT = { + always: 'success', + ask: 'warning', + off: 'neutral', +} as const; + +function ToolActions({ tool }: { tool: ConnectionTool }) { + const { t } = useTranslation(); + const groups = (['read', 'write'] as const) + .map((access) => ({ + access, + actions: tool.actions.filter((action) => action.access === access), + })) + .filter((group) => group.actions.length > 0); + return ( + +
+ + {tool.display_name} + + + {tool.status + ? t('settings.connectors.detail.toolOn') + : t('settings.connectors.detail.toolOff')} + +
+ {groups.map((group) => ( +
+ +
    + {group.actions.map((action) => ( +
  • + + {action.name} + + + {t(`settings.connectors.permission.${action.permission}`)} + +
    • + ))} +
+
+ ))} + + ); +} + +function AccountSection({ + detail, + onReconnect, + onDisconnect, +}: { + detail: ConnectionDetail; + onReconnect: () => void; + onDisconnect: (detail: ConnectionDetail) => void; +}) { + const { t } = useTranslation(); + return ( +
+ + + {detail.last_error} + ) : undefined + } + trailing={ +
+ + {t(`settings.connectors.connectionStatus.${detail.status}`)} + + {detail.status !== 'connected' && ( + + )} + {detail.status !== 'disconnected' && ( + onDisconnect(detail), + variant: 'destructive', + }, + ]} + /> + )} +
+ } + /> + + + {detail.sources.length > 0 && ( +
+ + + + {detail.sources.map((source) => ( + + {t('settings.connectors.detail.paused')} + + ) : undefined + } + /> + ))} + + +
+ )} + {detail.tools.length > 0 && ( +
+ + {detail.tools.map((tool) => ( + + ))} +
+ )} +
+ ); +} + +/** + * Everything about one connector: its accounts, the sources each syncs and + * the tools each provides. Opens from a Connectors page card. + */ +export default function ConnectionDrawer({ + connector, + onClose, + onConnect, +}: { + connector: ConnectorDefinition | null; + onClose: () => void; + onConnect: (connector: ConnectorDefinition) => void; +}) { + const { t } = useTranslation(); + const dispatch = useDispatch(); + const token = useSelector(selectToken); + const connections = useSelector(selectConnections); + const [details, setDetails] = useState([]); + const [loading, setLoading] = useState(false); + const [failed, setFailed] = useState(false); + const [reloadKey, setReloadKey] = useState(0); + const [toDisconnect, setToDisconnect] = useState( + null, + ); + + const accountIds = connections + .filter((connection) => connection.connector_key === connector?.key) + .map((connection) => connection.id) + .join(','); + + useEffect(() => { + if (!connector) return; + const ids = accountIds ? accountIds.split(',') : []; + let cancelled = false; + setLoading(true); + setFailed(false); + Promise.all(ids.map((id) => connectorsService.getConnection(id, token))) + .then((responses) => { + if (cancelled) return; + if (responses.some((response) => !response?.success)) { + setFailed(true); + return; + } + setDetails(responses.map((response) => response.connection)); + }) + .catch(() => !cancelled && setFailed(true)) + .finally(() => !cancelled && setLoading(false)); + return () => { + cancelled = true; + }; + }, [connector, accountIds, token, reloadKey]); + + const refresh = useCallback(() => { + dispatch(loadConnectors({ token })); + setReloadKey((key) => key + 1); + }, [dispatch, token]); + + const confirmDisconnect = () => { + if (!toDisconnect) return; + connectorsService.disconnect(toDisconnect.id, token).finally(() => { + setToDisconnect(null); + refresh(); + }); + }; + + if (!connector) return null; + const name = connectorName(t, connector); + + return ( + <> + !open && onClose()}> + +
+
+ + + +
+ {name} +

+ {t(`settings.connectors.publisher.${connector.publisher}`)} +

+
+
+

+ {connectorDescription(t, connector)} +

+ + + {connector.needs_setup && ( + + + + {t('settings.connectors.status.needsAdminSetup')} + + +
+ {connector.missing_settings.length > 0 ? ( + <> + {t('settings.connectors.setupSettings')} + + {connector.missing_settings.join(', ')} + + + ) : ( + {t('settings.connectors.askAdmin')} + )} + {connector.docs_url && ( + + )} +
+ + + )} + +
+ 0 ? ( + + ) : undefined + } + /> + {loading ? ( + + ) : failed ? ( + setReloadKey((key) => key + 1)} + > + {t('retry')} + + } + /> + ) : details.length === 0 ? ( + onConnect(connector)} + > + {t('settings.connectors.status.connect')} + + ) : undefined + } + /> + ) : ( +
+ {details.map((detail) => ( + onConnect(connector)} + onDisconnect={setToDisconnect} + /> + ))} +
+ )} +
+
+ + + state === 'INACTIVE' && setToDisconnect(null)} + handleSubmit={confirmDisconnect} + submitLabel={t('settings.connectors.detail.disconnect')} + variant="destructive" + /> + + ); +} diff --git a/frontend/src/connectors/ConnectorCard.tsx b/frontend/src/connectors/ConnectorCard.tsx new file mode 100644 index 00000000..40bf8703 --- /dev/null +++ b/frontend/src/connectors/ConnectorCard.tsx @@ -0,0 +1,138 @@ +import { Plus } from 'lucide-react'; +import { useTranslation } from 'react-i18next'; + +import { Badge } from '../components/ui/badge'; +import { + Card, + CardDescription, + CardFooter, + CardTitle, +} from '../components/ui/card'; +import ConnectorIcon from './ConnectorIcon'; +import { connectorDescription, connectorName } from './i18n'; +import type { Capability, ConnectorDefinition } from './types'; + +// Sync is blue, Read green, Write red: what the connection does to your data. +const CAPABILITY_VARIANT: Record< + Capability, + 'info' | 'success' | 'destructive' +> = { + sync: 'info', + read: 'success', + write: 'destructive', +}; + +export function CapabilityBadges({ + capabilities, +}: { + capabilities: Capability[]; +}) { + const { t } = useTranslation(); + return ( +
+ {capabilities.map((capability) => ( + + {t(`settings.connectors.capability.${capability}`)} + + ))} +
+ ); +} + +/** The state a catalog card ends on: a badge, or the Connect call to action. */ +export function ConnectorStateBadge({ + connector, +}: { + connector: ConnectorDefinition; +}) { + const { t } = useTranslation(); + switch (connector.state) { + case 'connected': + return ( + + {connector.connected_count > 1 + ? t('settings.connectors.status.connectedCount', { + count: connector.connected_count, + }) + : t('settings.connectors.status.connected')} + + ); + case 'reconnect': + return ( + + {t('settings.connectors.status.reconnect')} + + ); + case 'needs_setup': + return ( + + {t('settings.connectors.status.needsAdminSetup')} + + ); + case 'disabled': + return ( + + {t('settings.connectors.status.disabledByAdmin')} + + ); + default: + return ( + + + ); + } +} + +/** + * A catalog tile. The whole card is the one action: it connects an + * available service and opens the connection details for everything else. + */ +export default function ConnectorCard({ + connector, + onOpen, +}: { + connector: ConnectorDefinition; + onOpen: (connector: ConnectorDefinition) => void; +}) { + const { t } = useTranslation(); + const name = connectorName(t, connector); + return ( + + + + ); +} diff --git a/frontend/src/connectors/ConnectorIcon.tsx b/frontend/src/connectors/ConnectorIcon.tsx new file mode 100644 index 00000000..da2020a9 --- /dev/null +++ b/frontend/src/connectors/ConnectorIcon.tsx @@ -0,0 +1,70 @@ +import { Plug } from 'lucide-react'; +import * as React from 'react'; + +import { cn } from '@/lib/utils'; +import ConfluenceIcon from '../assets/confluence.svg?react'; +import DriveIcon from '../assets/drive.svg?react'; +import RedditIcon from '../assets/reddit.svg?react'; +import S3Icon from '../assets/s3.svg?react'; +import SharePointIcon from '../assets/sharepoint.svg?react'; + +type SvgComponent = React.FC>; + +// Preset logos live in assets/connectors; built-in tool services reuse the +// tool icons. Every asset draws in currentColor unless it is a multi-colour +// brand mark (Brave), so the icon follows the theme through `text-*`. +const presetModules = import.meta.glob('../assets/connectors/*.svg', { + query: '?react', + import: 'default', + eager: true, +}) as Record; +const toolModules = import.meta.glob('../assets/toolIcons/tool_*.svg', { + query: '?react', + import: 'default', + eager: true, +}) as Record; + +const ICONS: Record = { + drive: DriveIcon, + sharepoint: SharePointIcon, + confluence: ConfluenceIcon, + s3: S3Icon, + reddit: RedditIcon, +}; +for (const [path, Component] of Object.entries(presetModules)) { + const match = path.match(/connectors\/(.+)\.svg$/); + if (match) ICONS[match[1]] = Component; +} +for (const [path, Component] of Object.entries(toolModules)) { + const match = path.match(/(tool_.+)\.svg$/); + if (match) ICONS[match[1]] = Component; +} + +type ConnectorIconProps = { + /** Catalog `icon` key, e.g. `drive`, `tool_telegram` or `notion`. */ + icon: string; + className?: string; + /** Accessible name. Omit when the connector's name is next to the icon. */ + title?: string; +}; + +/** + * A connector's logo. Unknown keys (a custom MCP server) fall back to a plug, + * the same glyph the Connectors page uses in the navigation. + */ +export default function ConnectorIcon({ + icon, + className, + title, +}: ConnectorIconProps) { + const Icon = ICONS[icon]; + const a11y = title + ? { role: 'img' as const, 'aria-label': title } + : { 'aria-hidden': true as const }; + if (!Icon) { + return ( + + ); + } + return ; +} diff --git a/frontend/src/connectors/connectorsSlice.ts b/frontend/src/connectors/connectorsSlice.ts new file mode 100644 index 00000000..cc26674b --- /dev/null +++ b/frontend/src/connectors/connectorsSlice.ts @@ -0,0 +1,80 @@ +import { createAsyncThunk, createSlice } from '@reduxjs/toolkit'; + +import connectorsService from '../api/services/connectorsService'; +import type { Connection, ConnectorDefinition } from './types'; + +export type ConnectorsState = { + catalog: ConnectorDefinition[]; + connections: Connection[]; + loading: boolean; + loaded: boolean; + failed: boolean; +}; + +const initialState: ConnectorsState = { + catalog: [], + connections: [], + loading: false, + loaded: false, + failed: false, +}; + +/** + * Load the catalog and the caller's connections together. Every screen that + * shows a connector's state (the Connectors page, Add Source, Add Tool, the + * composer pickers) reads this one copy, so a connect anywhere updates all. + */ +export const loadConnectors = createAsyncThunk< + { catalog: ConnectorDefinition[]; connections: Connection[] }, + { token: string | null } +>('connectors/load', async ({ token }) => { + const [catalog, connections] = await Promise.all([ + connectorsService.getCatalog(token), + connectorsService.listConnections(token), + ]); + if (!catalog?.success || !connections?.success) { + throw new Error('Failed to load connectors'); + } + return { + catalog: catalog.connectors ?? [], + connections: connections.connections ?? [], + }; +}); + +const connectorsSlice = createSlice({ + name: 'connectors', + initialState, + reducers: {}, + extraReducers: (builder) => { + builder + .addCase(loadConnectors.pending, (state) => { + state.loading = true; + state.failed = false; + }) + .addCase(loadConnectors.fulfilled, (state, action) => { + state.loading = false; + state.loaded = true; + state.catalog = action.payload.catalog; + state.connections = action.payload.connections; + }) + .addCase(loadConnectors.rejected, (state) => { + state.loading = false; + state.failed = true; + }); + }, +}); + +type RootLike = { connectors: ConnectorsState }; + +export const selectConnectorCatalog = (state: RootLike) => + state.connectors.catalog; +export const selectConnections = (state: RootLike) => + state.connectors.connections; +export const selectConnectorsLoading = (state: RootLike) => + state.connectors.loading; +export const selectConnectorsLoaded = (state: RootLike) => + state.connectors.loaded; +export const selectConnectorsFailed = (state: RootLike) => + state.connectors.failed; + +export default connectorsSlice.reducer; diff --git a/frontend/src/connectors/i18n.ts b/frontend/src/connectors/i18n.ts new file mode 100644 index 00000000..565f5a67 --- /dev/null +++ b/frontend/src/connectors/i18n.ts @@ -0,0 +1,29 @@ +import type { TFunction } from 'i18next'; + +import type { ConnectorDefinition } from './types'; + +/** + * Locale key segment for a catalog key. Preset keys look like `mcp:notion`, + * and i18next reads `:` as a namespace separator, so it becomes `_`. + */ +export const connectorLocaleKey = (key: string) => key.replace(/[:.]/g, '_'); + +/** The card's one-line description, translated when the locale has it. */ +export const connectorDescription = ( + t: TFunction, + connector: Pick, +) => + t(`settings.connectors.descriptions.${connectorLocaleKey(connector.key)}`, { + defaultValue: connector.description, + }); + +/** The connector's name; only the two custom kinds have translated names. */ +export const connectorName = ( + t: TFunction, + connector: Pick, +) => + connector.publisher === 'custom' + ? t(`settings.connectors.custom.${connectorLocaleKey(connector.key)}`, { + defaultValue: connector.name, + }) + : connector.name; diff --git a/frontend/src/connectors/types.ts b/frontend/src/connectors/types.ts new file mode 100644 index 00000000..557e4491 --- /dev/null +++ b/frontend/src/connectors/types.ts @@ -0,0 +1,114 @@ +export type ConnectorCategory = + | 'files' + | 'knowledge' + | 'dev' + | 'messaging' + | 'database' + | 'search' + | 'custom'; + +export type Capability = 'sync' | 'read' | 'write'; + +export type ConnectorAuthKind = + 'oauth' | 'mcp_oauth' | 'api_key' | 'none' | 'mcp'; + +export type ConnectionStatus = + 'connected' | 'reconnect_needed' | 'disconnected' | 'error' | 'pending'; + +/** What a catalog card shows, worked out on the server. */ +export type ConnectorCardState = + | 'available' + | 'connected' + | 'reconnect' + | 'needs_setup' + | 'disabled' + | 'custom'; + +export type CredentialMode = 'owner' | 'member'; + +export type CredentialField = { + key: string; + label: string; + secret: boolean; + required: boolean; +}; + +export type ConnectorDefinition = { + key: string; + name: string; + description: string; + icon: string; + category: ConnectorCategory; + auth_kind: ConnectorAuthKind; + capabilities: Capability[]; + credential_fields: CredentialField[]; + setup_fields: CredentialField[]; + sync_ingestor: string | null; + default_sync_frequency: string; + tool_templates: string[]; + setup: { tools: 'auto' | 'ask' | 'off'; sync: 'auto' | 'ask' | 'off' }; + mcp_url: string | null; + publisher: 'built_in' | 'preset' | 'custom'; + docs_url: string | null; + oauth_scopes: string[]; + available: boolean; + disabled: boolean; + needs_setup: boolean; + missing_settings: string[]; + connected_count: number; + connection_count: number; + status: ConnectionStatus | null; + state: ConnectorCardState; + credential_policy: 'choose' | CredentialMode; +}; + +export type Connection = { + id: string; + connector_key: string; + name: string; + display_name: string | null; + icon: string; + account_label: string; + auth_kind: ConnectorAuthKind | null; + status: ConnectionStatus; + server_url: string | null; + last_error: string | null; + created_at: string | null; + updated_at: string | null; + last_used_at: string | null; + source_count: number; + tool_count: number; +}; + +export type ActionAccess = 'read' | 'write'; +export type ActionPermission = 'always' | 'ask' | 'off'; + +export type ConnectionToolAction = { + name: string; + description: string; + access: ActionAccess; + permission: ActionPermission; +}; + +export type ConnectionTool = { + id: string; + name: string; + display_name: string; + status: boolean; + credential_mode: CredentialMode; + actions: ConnectionToolAction[]; +}; + +export type ConnectionSource = { + id: string; + name: string; + type: string; + last_sync: string | null; + sync_frequency: string; + sync_state: 'active' | 'paused_reconnect'; +}; + +export type ConnectionDetail = Connection & { + sources: ConnectionSource[]; + tools: ConnectionTool[]; +}; diff --git a/frontend/src/connectors/useConnectorLauncher.tsx b/frontend/src/connectors/useConnectorLauncher.tsx new file mode 100644 index 00000000..df8d81d1 --- /dev/null +++ b/frontend/src/connectors/useConnectorLauncher.tsx @@ -0,0 +1,140 @@ +import { useCallback, useState, type ReactNode } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { useNavigate } from 'react-router-dom'; + +import userService from '../api/services/userService'; +import ConfigToolModal from '../modals/ConfigToolModal'; +import MCPServerModal from '../modals/MCPServerModal'; +import type { AvailableToolType } from '../modals/types'; +import type { ActiveState } from '../models/misc'; +import { selectToken } from '../preferences/preferenceSlice'; +import type { AppDispatch } from '../store'; +import Upload from '../upload/Upload'; +import type { IngestorType } from '../upload/types/ingestor'; +import { loadConnectors } from './connectorsSlice'; +import type { ConnectorDefinition } from './types'; + +type Launch = + | { kind: 'source'; ingestor: IngestorType } + | { kind: 'tool'; tool: AvailableToolType } + | { kind: 'mcp'; server?: Record } + | null; + +/** + * One way to start connecting any catalog entry, used by every entry point + * (the Connectors page, Add Source, Add Tool). Returns `launch` and the + * modals it drives; render `modals` once where the hook is used. + */ +export default function useConnectorLauncher({ + onConnected, +}: { onConnected?: () => void } = {}) { + const dispatch = useDispatch(); + const navigate = useNavigate(); + const token = useSelector(selectToken); + const [active, setActive] = useState(null); + + const finish = useCallback(() => { + setActive(null); + dispatch(loadConnectors({ token })); + onConnected?.(); + }, [dispatch, token, onConnected]); + + const launch = useCallback( + async (connector: ConnectorDefinition) => { + if (connector.sync_ingestor) { + setActive({ + kind: 'source', + ingestor: connector.sync_ingestor as IngestorType, + }); + return; + } + if ( + connector.auth_kind === 'mcp' || + connector.auth_kind === 'mcp_oauth' + ) { + setActive({ + kind: 'mcp', + server: connector.mcp_url + ? { + displayName: connector.name, + server_url: connector.mcp_url, + auth_type: + connector.auth_kind === 'mcp_oauth' ? 'oauth' : 'none', + preset: true, + } + : undefined, + }); + return; + } + const templateName = connector.tool_templates[0]; + if (!templateName) return; + const response = await userService.getAvailableTools(token); + const data = await response.json(); + const tool = (data.data as AvailableToolType[] | undefined)?.find( + (candidate) => candidate.name === templateName, + ); + if (!tool) return; + if (Object.keys(tool.configRequirements ?? {}).length === 0) { + // The OpenAPI connector: create the empty API tool and open it so + // the user can import a spec (today's API Tool flow). + const created = await userService.createTool( + { + name: tool.name, + displayName: tool.displayName, + description: tool.description, + config: {}, + actions: tool.actions, + status: true, + }, + token, + ); + const body = await created.json(); + if (body?.id) { + navigate('/settings/tools', { state: { openToolId: body.id } }); + } + return; + } + setActive({ kind: 'tool', tool }); + }, + [navigate, token], + ); + + const close = (state: ActiveState) => { + if (state === 'INACTIVE') setActive(null); + }; + + const modals: ReactNode = ( + <> + {active?.kind === 'source' && ( + setActive(null)} + initialIngestor={active.ingestor} + onSuccessfulUpload={finish} + selectUploadedDoc={false} + /> + )} + {active?.kind === 'tool' && ( + + )} + {active?.kind === 'mcp' && ( + + )} + + ); + + return { launch, modals }; +} diff --git a/frontend/src/locale/connectors.test.ts b/frontend/src/locale/connectors.test.ts new file mode 100644 index 00000000..7dbafc8c --- /dev/null +++ b/frontend/src/locale/connectors.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest'; + +import de from './de.json'; +import en from './en.json'; +import es from './es.json'; +import jp from './jp.json'; +import ru from './ru.json'; +import zhTW from './zh-TW.json'; +import zh from './zh.json'; + +type Tree = { [key: string]: string | Tree }; + +const PLURAL_SUFFIX = /_(zero|one|two|few|many|other)$/; + +const flatten = (tree: Tree, prefix = ''): string[] => + Object.entries(tree).flatMap(([key, value]) => + typeof value === 'string' + ? [prefix + key.replace(PLURAL_SUFFIX, '')] + : flatten(value, `${prefix}${key}.`), + ); + +const values = (tree: Tree): string[] => + Object.values(tree).flatMap((value) => + typeof value === 'string' ? [value] : values(value), + ); + +const block = (locale: object, path: string): Tree => + path + .split('.') + .reduce((node, key) => (node[key] as Tree) ?? {}, locale as Tree); + +const keysOf = (locale: object, path: string): string[] => + Array.from(new Set(flatten(block(locale, path)))).sort(); + +const LOCALES = { es, de, jp, ru, zh, zhTW }; + +// Every block the Connectors work adds strings to. +const BLOCKS = ['settings.connectors']; + +describe('connectors locale blocks', () => { + it.each(Object.entries(LOCALES))( + '%s has the same connector keys as en', + (_name, locale) => { + for (const path of BLOCKS) { + expect(keysOf(locale, path)).toEqual(keysOf(en, path)); + } + }, + ); + + it.each(Object.entries(LOCALES))( + '%s is translated, not an English copy', + (_name, locale) => { + const source = block(en, 'settings.connectors'); + const target = block(locale, 'settings.connectors'); + expect(target.subtitle).not.toBe(source.subtitle); + expect((target.status as Tree).connect).not.toBe( + (source.status as Tree).connect, + ); + }, + ); + + it.each(Object.entries({ en, ...LOCALES }))( + '%s has no em dash in connector strings', + (_name, locale) => { + for (const path of BLOCKS) { + for (const value of values(block(locale, path))) { + expect(value).not.toContain('—'); + } + } + }, + ); +}); diff --git a/frontend/src/locale/de.json b/frontend/src/locale/de.json index 7c35fe9b..ea2373ff 100644 --- a/frontend/src/locale/de.json +++ b/frontend/src/locale/de.json @@ -97,7 +97,7 @@ }, "sources": { "title": "Hier kannst du alle verfügbaren Quelldateien verwalten, die dir zur Verfügung stehen und die du hochgeladen hast.", - "subtitle": "Dokumente und Wissensquellen hochladen und verwalten, die deine Antworten antreiben", + "subtitle": "Inhalte, die DocsGPT durchsuchen und zitieren kann: Uploads, Websites, Wikis und aus deinen Verbindungen synchronisierte Inhalte.", "label": "Quellen", "name": "Quellenname", "date": "Vektor-Datum", @@ -354,7 +354,8 @@ "forbidden": "Du hast keine Berechtigung, die Konfiguration dieser Quelle zu bearbeiten.", "saveFailed": "Konfiguration konnte nicht gespeichert werden. Bitte versuche es erneut." } - } + }, + "paused": "Sync pausiert. Verbinde {{name}} neu, um fortzufahren." }, "analytics": { "label": "Analytik", @@ -651,11 +652,11 @@ }, "tools": { "label": "Werkzeuge", - "subtitle": "Werkzeuge und Integrationen durchsuchen, verbinden und konfigurieren, die deine Agenten antreiben", + "subtitle": "Aktionen, die deine Agenten ausführen können. Integrierte Werkzeuge findest du hier. Werkzeuge eines verbundenen Dienstes sind unter diesem Dienst gruppiert.", "searchPlaceholder": "Werkzeuge suchen...", "addTool": "Werkzeug hinzufügen", "noToolsFound": "Keine Werkzeuge gefunden", - "selectToolSetup": "Wähle ein Werkzeug zur Einrichtung", + "selectToolSetup": "Werkzeug hinzufügen", "settingsIconAlt": "Einstellungssymbol", "configureToolAria": "{{toolName}} konfigurieren", "toggleToolAria": "{{toolName}} umschalten", @@ -667,8 +668,8 @@ "builtIn": "Integriert", "authStatus": { "connected": "Verbunden", - "needsAuth": "Authentifizierung nötig", - "configured": "Konfiguriert" + "needsAuth": "Neu verbinden", + "configured": "Verbunden" }, "deleteWarning": "Bist du sicher, dass du das Werkzeug \"{{toolName}}\" löschen möchtest?", "unsavedChanges": "Du hast ungespeicherte Änderungen, die verloren gehen, wenn du ohne Speichern verlässt.", @@ -773,7 +774,11 @@ "maxTimeout": "Das maximale Timeout beträgt 300 Sekunden" }, "headerValuePlaceholder": "z. B. application/json", - "toolIconTitle": "{{name}}-Symbol" + "toolIconTitle": "{{name}}-Symbol", + "groupBuiltIn": "Integriert", + "groupService": "Aus einem Dienst", + "browseConnectors": "Suchst du Notion, Linear oder einen anderen Dienst? Konnektoren durchsuchen", + "viaConnection": "über {{name}}" }, "devices": { "label": "Geräte", @@ -1002,13 +1007,107 @@ "subtitle": "Das neue Geheimnis für \"{{name}}\" ist bereit. Das alte funktioniert nicht mehr." } } + }, + "connectors": { + "label": "Konnektoren", + "subtitle": "Verbinde DocsGPT mit den Diensten, die dein Team nutzt. Eine Verbindung kann Inhalte in Quellen synchronisieren und Agenten Werkzeuge geben.", + "search": "Konnektoren suchen...", + "addCustom": "Eigenen Konnektor hinzufügen", + "custom": { + "custom_mcp": "MCP-Server", + "custom_openapi": "OpenAPI / REST" + }, + "categoriesLabel": "Konnektoren nach Kategorie filtern", + "categories": { + "all": "Alle", + "connected": "Verbunden", + "files": "Dateien und Dokumente", + "knowledge": "Wissensdatenbanken", + "dev": "Entwicklerwerkzeuge", + "messaging": "Nachrichten", + "database": "Datenbanken", + "search": "Suche", + "custom": "Eigene" + }, + "capability": { + "sync": "Sync", + "read": "Lesen", + "write": "Schreiben" + }, + "status": { + "connect": "Verbinden", + "connected": "Verbunden", + "connectedCount": "{{count}} verbunden", + "reconnect": "Neu verbinden", + "needsAdminSetup": "Admin-Einrichtung nötig", + "disabledByAdmin": "Vom Admin deaktiviert", + "custom": "Eigener" + }, + "connectionStatus": { + "connected": "Verbunden", + "reconnect_needed": "Neu verbinden nötig", + "disconnected": "Getrennt", + "error": "Fehler", + "pending": "Wartet auf Anmeldung" + }, + "publisher": { + "built_in": "Integriert", + "preset": "Vorlage", + "custom": "Eigener" + }, + "descriptions": { + "google_drive": "Docs, Tabellen und PDFs in Quellen synchronisieren.", + "share_point": "Dateien aus SharePoint-Sites und OneDrive in Quellen synchronisieren.", + "confluence": "Confluence-Bereiche und -Seiten in Quellen synchronisieren.", + "s3": "Dokumente aus einem S3-Bucket in Quellen synchronisieren.", + "reddit": "Reddit-Beiträge zu deinen Suchen in Quellen synchronisieren.", + "brave": "Web und Bilder mit der Brave Search API durchsuchen.", + "telegram": "Nachrichten und Bilder an einen Telegram-Chat senden.", + "ntfy": "Push-Benachrichtigungen über einen ntfy-Server senden.", + "postgres": "Schema lesen und SQL auf einer Postgres-Datenbank ausführen.", + "custom_mcp": "Einen beliebigen Remote-Server für das Model Context Protocol verbinden.", + "custom_openapi": "Eine OpenAPI-Spezifikation importieren und ihre Endpunkte als Werkzeuge aufrufen." + }, + "empty": "Noch nichts verbunden. Verbinde einen Dienst, um seine Inhalte zu synchronisieren oder Agenten neue Werkzeuge zu geben.", + "noMatches": "Keine Konnektoren passen zu deiner Suche.", + "loadFailed": "Konnektoren konnten nicht geladen werden.", + "askAdmin": "Bitte einen Admin, dies einzurichten.", + "setupSettings": "Setze diese Servereinstellungen und starte DocsGPT neu:", + "setupGuide": "Einrichtungsanleitung", + "unverified": "Dieser Server wurde nicht von DocsGPT geprüft. Verbinde nur Server, denen du vertraust.", + "detail": { + "accounts": "Konten", + "connectedAs": "Verbunden als {{account}}", + "connectAnother": "Weiteres Konto verbinden", + "sources": "Quellen aus dieser Verbindung", + "tools": "Werkzeuge aus dieser Verbindung", + "noAccounts": "Noch kein Konto verbunden.", + "failed": "Diese Verbindung konnte nicht geladen werden.", + "lastSync": "Synchronisiert {{date}} · {{frequency}}", + "paused": "Sync pausiert", + "toolOn": "An", + "toolOff": "Aus", + "accountMenu": "Kontoaktionen", + "disconnect": "Trennen" + }, + "permission": { + "always": "Immer erlauben", + "ask": "Freigabe nötig", + "off": "Aus" + }, + "disconnect": { + "title": "{{name}} trennen?", + "body_zero": "Werkzeuge aus dieser Verbindung funktionieren erst wieder, wenn du neu verbindest.", + "body_one": "Deine Quelle behält ihren Inhalt, wird aber nicht mehr synchronisiert. Werkzeuge aus dieser Verbindung funktionieren erst wieder, wenn du neu verbindest.", + "body_other": "Deine {{count}} Quellen behalten ihren Inhalt, werden aber nicht mehr synchronisiert. Werkzeuge aus dieser Verbindung funktionieren erst wieder, wenn du neu verbindest." + } } }, "modals": { "uploadDoc": { - "label": "Neues Dokument hochladen", + "label": "Quelle hinzufügen", "select": "Wähle, wie du dein Dokument zu DocsGPT hochladen möchtest", - "selectSource": "Wähle die Art, wie du deine Quelle hinzufügen möchtest", + "selectSource": "Quelle hinzufügen", "selectedFiles": "Ausgewählte Dateien", "noFilesSelected": "Keine Dateien ausgewählt", "file": "Vom Gerät hochladen", @@ -1021,7 +1120,7 @@ "info": "Bitte lade .pdf, .txt, .rst, .csv, .xlsx, .xlsm, .xlsb, .xls, .ods, .docx, .docm, .doc, .odt, .rtf, .md, .html, .xhtml, .png, .jpg, .jpeg, .epub, .json, .pptx, .pptm, .ppt, .pps, .ppsx, .ppsm, .pot, .odp, .zip hoch (max. 25 MB)", "uploadedFiles": "Hochgeladene Dateien", "cancel": "Abbrechen", - "train": "Trainieren", + "train": "Quelle hinzufügen", "create": "Erstellen", "link": "Link", "urlLink": "URL-Link", @@ -1154,7 +1253,10 @@ "region": "AWS-Region", "endpointUrl": "Benutzerdefinierte Endpunkt-URL (optional)", "initialContent": "Anfangsinhalt (optional)" - } + }, + "groupUploadWeb": "Upload und Web", + "groupConnection": "Aus einer Verbindung", + "tileConnectedAs": "Verbunden als {{account}}" }, "createAPIKey": { "label": "Neuen API-Schlüssel erstellen", @@ -1190,8 +1292,8 @@ "configTool": { "title": "Werkzeug-Konfiguration", "type": "Typ", - "apiKeyLabel": "API-Schlüssel / OAuth", - "apiKeyPlaceholder": "API-Schlüssel / OAuth eingeben", + "apiKeyLabel": "API-Schlüssel", + "apiKeyPlaceholder": "API-Schlüssel eingeben", "addButton": "Werkzeug hinzufügen", "closeButton": "Schließen", "customNamePlaceholder": "Benutzerdefinierten Namen eingeben (optional)", diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index 986577a1..af1443a8 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -101,7 +101,7 @@ }, "sources": { "title": "Here you can manage all of the source file that are available to you and those you have uploaded.", - "subtitle": "Upload and manage the documents and knowledge sources used to power your responses", + "subtitle": "Content DocsGPT can search and cite: uploads, websites, wikis and content synced from your connections.", "label": "Sources", "name": "Source Name", "date": "Vector Date", @@ -359,7 +359,8 @@ "forbidden": "You don't have permission to edit this source's config.", "saveFailed": "Failed to save config. Please try again." } - } + }, + "paused": "Sync paused. Reconnect {{name}} to resume." }, "analytics": { "label": "Analytics", @@ -657,11 +658,11 @@ }, "tools": { "label": "Tools", - "subtitle": "Browse, connect, and configure the tools and integrations that power your agents", + "subtitle": "Actions your agents can take. Built-in tools live here. Tools from a connected service are grouped under that service.", "searchPlaceholder": "Search tools...", "addTool": "Add Tool", "noToolsFound": "No tools found", - "selectToolSetup": "Select a tool to set up", + "selectToolSetup": "Add a tool", "settingsIconAlt": "Settings icon", "configureToolAria": "Configure {{toolName}}", "toggleToolAria": "Toggle {{toolName}}", @@ -673,8 +674,8 @@ "builtIn": "Built-in", "authStatus": { "connected": "Connected", - "needsAuth": "Needs Auth", - "configured": "Configured" + "needsAuth": "Reconnect", + "configured": "Connected" }, "deleteWarning": "Are you sure you want to delete the tool \"{{toolName}}\"?", "unsavedChanges": "You have unsaved changes that will be lost if you leave without saving.", @@ -779,7 +780,11 @@ "maxTimeout": "Maximum timeout is 300 seconds" }, "headerValuePlaceholder": "e.g., application/json", - "toolIconTitle": "{{name}} icon" + "toolIconTitle": "{{name}} icon", + "groupBuiltIn": "Built in", + "groupService": "From a service", + "browseConnectors": "Looking for Notion, Linear or another service? Browse connectors", + "viaConnection": "via {{name}}" }, "devices": { "label": "Devices", @@ -1008,13 +1013,107 @@ "subtitle": "The new secret for \"{{name}}\" is ready. The old one no longer works." } } + }, + "connectors": { + "label": "Connectors", + "subtitle": "Connect DocsGPT to the services your team uses. A connection can sync content into Sources and give agents tools.", + "search": "Search connectors...", + "addCustom": "Add custom connector", + "custom": { + "custom_mcp": "MCP server", + "custom_openapi": "OpenAPI / REST" + }, + "categoriesLabel": "Filter connectors by category", + "categories": { + "all": "All", + "connected": "Connected", + "files": "Files and docs", + "knowledge": "Knowledge bases", + "dev": "Developer tools", + "messaging": "Messaging", + "database": "Databases", + "search": "Search", + "custom": "Custom" + }, + "capability": { + "sync": "Sync", + "read": "Read", + "write": "Write" + }, + "status": { + "connect": "Connect", + "connected": "Connected", + "connectedCount": "{{count}} connected", + "reconnect": "Reconnect", + "needsAdminSetup": "Needs admin setup", + "disabledByAdmin": "Disabled by admin", + "custom": "Custom" + }, + "connectionStatus": { + "connected": "Connected", + "reconnect_needed": "Reconnect needed", + "disconnected": "Disconnected", + "error": "Error", + "pending": "Waiting for sign-in" + }, + "publisher": { + "built_in": "Built in", + "preset": "Preset", + "custom": "Custom" + }, + "descriptions": { + "google_drive": "Sync Docs, Sheets and PDFs into Sources.", + "share_point": "Sync files from SharePoint sites and OneDrive into Sources.", + "confluence": "Sync Confluence spaces and pages into Sources.", + "s3": "Sync documents from an S3 bucket into Sources.", + "reddit": "Sync Reddit posts that match your searches into Sources.", + "brave": "Search the web and images with the Brave Search API.", + "telegram": "Send messages and images to a Telegram chat.", + "ntfy": "Send push notifications through an ntfy server.", + "postgres": "Read the schema and run SQL against a Postgres database.", + "custom_mcp": "Connect any remote Model Context Protocol server.", + "custom_openapi": "Import an OpenAPI spec and call its endpoints as tools." + }, + "empty": "Nothing connected yet. Connect a service to sync its content or give agents new tools.", + "noMatches": "No connectors match your search.", + "loadFailed": "Could not load connectors.", + "askAdmin": "Ask an admin to set this up.", + "setupSettings": "Set these server settings, then restart DocsGPT:", + "setupGuide": "Setup guide", + "unverified": "This server is not reviewed by DocsGPT. Only connect servers you trust.", + "detail": { + "accounts": "Accounts", + "connectedAs": "Connected as {{account}}", + "connectAnother": "Connect another account", + "sources": "Sources from this connection", + "tools": "Tools from this connection", + "noAccounts": "No account connected yet.", + "failed": "Could not load this connection.", + "lastSync": "Synced {{date}} · {{frequency}}", + "paused": "Sync paused", + "toolOn": "On", + "toolOff": "Off", + "accountMenu": "Account actions", + "disconnect": "Disconnect" + }, + "permission": { + "always": "Always allow", + "ask": "Needs approval", + "off": "Off" + }, + "disconnect": { + "title": "Disconnect {{name}}?", + "body_zero": "Tools from this connection stop working until you reconnect.", + "body_one": "Your source keeps its content but stops syncing. Tools from this connection stop working until you reconnect.", + "body_other": "Your {{count}} sources keep their content but stop syncing. Tools from this connection stop working until you reconnect." + } } }, "modals": { "uploadDoc": { - "label": "Upload new document", + "label": "Add a source", "select": "Choose how to upload your document to DocsGPT", - "selectSource": "Select the way to add your source", + "selectSource": "Add a source", "selectedFiles": "Selected Files", "noFilesSelected": "No files selected", "file": "Upload from device", @@ -1027,7 +1126,7 @@ "info": "Please upload .pdf, .txt, .rst, .csv, .xlsx, .xlsm, .xlsb, .xls, .ods, .docx, .docm, .doc, .odt, .rtf, .md, .html, .xhtml, .png, .jpg, .jpeg, .epub, .json, .pptx, .pptm, .ppt, .pps, .ppsx, .ppsm, .pot, .odp, .zip limited to 25mb", "uploadedFiles": "Uploaded Files", "cancel": "Cancel", - "train": "Train", + "train": "Add source", "create": "Create", "link": "Link", "urlLink": "URL Link", @@ -1160,7 +1259,10 @@ "region": "AWS Region", "endpointUrl": "Custom Endpoint URL (optional)", "initialContent": "Initial content (optional)" - } + }, + "groupUploadWeb": "Upload and web", + "groupConnection": "From a connection", + "tileConnectedAs": "Connected as {{account}}" }, "createAPIKey": { "label": "Create New API Key", @@ -1196,8 +1298,8 @@ "configTool": { "title": "Tool Config", "type": "Type", - "apiKeyLabel": "API Key / OAuth", - "apiKeyPlaceholder": "Enter API Key / OAuth", + "apiKeyLabel": "API key", + "apiKeyPlaceholder": "Enter API key", "addButton": "Add Tool", "closeButton": "Close", "customNamePlaceholder": "Enter custom name (optional)", diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index 681e2e0d..4f009258 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -97,7 +97,7 @@ }, "sources": { "title": "Aquí puedes gestionar todos los archivos fuente que están disponibles para ti y los que has subido.", - "subtitle": "Sube y gestiona los documentos y fuentes de conocimiento que impulsan tus respuestas", + "subtitle": "Contenido que DocsGPT puede buscar y citar: archivos subidos, sitios web, wikis y contenido sincronizado desde tus conexiones.", "label": "Fuentes", "name": "Nombre de la Fuente", "date": "Fecha de Vector", @@ -354,7 +354,8 @@ "forbidden": "No tienes permiso para editar la configuración de esta fuente.", "saveFailed": "No se pudo guardar la configuración. Inténtalo de nuevo." } - } + }, + "paused": "Sincronización en pausa. Reconecta {{name}} para reanudarla." }, "analytics": { "label": "Analítica", @@ -651,11 +652,11 @@ }, "tools": { "label": "Herramientas", - "subtitle": "Explora, conecta y configura las herramientas e integraciones que impulsan a tus agentes", + "subtitle": "Acciones que pueden realizar tus agentes. Aquí están las herramientas integradas. Las herramientas de un servicio conectado se agrupan bajo ese servicio.", "searchPlaceholder": "Buscar...", "addTool": "Agregar Herramienta", "noToolsFound": "No se encontraron herramientas", - "selectToolSetup": "Seleccione una herramienta para configurar", + "selectToolSetup": "Añadir una herramienta", "settingsIconAlt": "Icono de configuración", "configureToolAria": "Configurar {{toolName}}", "toggleToolAria": "Alternar {{toolName}}", @@ -667,8 +668,8 @@ "builtIn": "Integrada", "authStatus": { "connected": "Conectado", - "needsAuth": "Requiere autenticación", - "configured": "Configurado" + "needsAuth": "Reconectar", + "configured": "Conectado" }, "deleteWarning": "¿Estás seguro de que deseas eliminar la herramienta \"{{toolName}}\"?", "unsavedChanges": "Tienes cambios sin guardar que se perderán si sales sin guardar.", @@ -773,7 +774,11 @@ "maxTimeout": "El tiempo de espera máximo es de 300 segundos" }, "headerValuePlaceholder": "p. ej., application/json", - "toolIconTitle": "Icono de {{name}}" + "toolIconTitle": "Icono de {{name}}", + "groupBuiltIn": "Integradas", + "groupService": "De un servicio", + "browseConnectors": "¿Buscas Notion, Linear u otro servicio? Explora los conectores", + "viaConnection": "mediante {{name}}" }, "devices": { "label": "Dispositivos", @@ -1002,13 +1007,107 @@ "subtitle": "El nuevo secreto de \"{{name}}\" está listo. El anterior ya no funciona." } } + }, + "connectors": { + "label": "Conectores", + "subtitle": "Conecta DocsGPT a los servicios que usa tu equipo. Una conexión puede sincronizar contenido en Fuentes y dar herramientas a los agentes.", + "search": "Buscar conectores...", + "addCustom": "Añadir conector personalizado", + "custom": { + "custom_mcp": "Servidor MCP", + "custom_openapi": "OpenAPI / REST" + }, + "categoriesLabel": "Filtrar conectores por categoría", + "categories": { + "all": "Todos", + "connected": "Conectados", + "files": "Archivos y documentos", + "knowledge": "Bases de conocimiento", + "dev": "Herramientas de desarrollo", + "messaging": "Mensajería", + "database": "Bases de datos", + "search": "Búsqueda", + "custom": "Personalizados" + }, + "capability": { + "sync": "Sincronizar", + "read": "Leer", + "write": "Escribir" + }, + "status": { + "connect": "Conectar", + "connected": "Conectado", + "connectedCount": "{{count}} conectadas", + "reconnect": "Reconectar", + "needsAdminSetup": "Requiere configuración del administrador", + "disabledByAdmin": "Desactivado por el administrador", + "custom": "Personalizado" + }, + "connectionStatus": { + "connected": "Conectado", + "reconnect_needed": "Hay que reconectar", + "disconnected": "Desconectado", + "error": "Error", + "pending": "Esperando el inicio de sesión" + }, + "publisher": { + "built_in": "Integrado", + "preset": "Predefinido", + "custom": "Personalizado" + }, + "descriptions": { + "google_drive": "Sincroniza Docs, Hojas de cálculo y PDF en Fuentes.", + "share_point": "Sincroniza archivos de sitios de SharePoint y OneDrive en Fuentes.", + "confluence": "Sincroniza espacios y páginas de Confluence en Fuentes.", + "s3": "Sincroniza documentos de un bucket de S3 en Fuentes.", + "reddit": "Sincroniza en Fuentes las publicaciones de Reddit que coinciden con tus búsquedas.", + "brave": "Busca en la web e imágenes con la API de Brave Search.", + "telegram": "Envía mensajes e imágenes a un chat de Telegram.", + "ntfy": "Envía notificaciones push mediante un servidor ntfy.", + "postgres": "Lee el esquema y ejecuta SQL en una base de datos Postgres.", + "custom_mcp": "Conecta cualquier servidor remoto de Model Context Protocol.", + "custom_openapi": "Importa una especificación OpenAPI y usa sus endpoints como herramientas." + }, + "empty": "Aún no hay nada conectado. Conecta un servicio para sincronizar su contenido o dar nuevas herramientas a los agentes.", + "noMatches": "Ningún conector coincide con tu búsqueda.", + "loadFailed": "No se pudieron cargar los conectores.", + "askAdmin": "Pide a un administrador que lo configure.", + "setupSettings": "Define estos ajustes del servidor y reinicia DocsGPT:", + "setupGuide": "Guía de configuración", + "unverified": "DocsGPT no ha revisado este servidor. Conecta solo servidores de confianza.", + "detail": { + "accounts": "Cuentas", + "connectedAs": "Conectado como {{account}}", + "connectAnother": "Conectar otra cuenta", + "sources": "Fuentes de esta conexión", + "tools": "Herramientas de esta conexión", + "noAccounts": "Aún no hay ninguna cuenta conectada.", + "failed": "No se pudo cargar esta conexión.", + "lastSync": "Sincronizado {{date}} · {{frequency}}", + "paused": "Sincronización en pausa", + "toolOn": "Activada", + "toolOff": "Desactivada", + "accountMenu": "Acciones de la cuenta", + "disconnect": "Desconectar" + }, + "permission": { + "always": "Permitir siempre", + "ask": "Requiere aprobación", + "off": "Desactivada" + }, + "disconnect": { + "title": "¿Desconectar {{name}}?", + "body_zero": "Las herramientas de esta conexión dejarán de funcionar hasta que vuelvas a conectarla.", + "body_one": "Tu fuente conserva su contenido pero deja de sincronizarse. Las herramientas de esta conexión dejarán de funcionar hasta que vuelvas a conectarla.", + "body_other": "Tus {{count}} fuentes conservan su contenido pero dejan de sincronizarse. Las herramientas de esta conexión dejarán de funcionar hasta que vuelvas a conectarla." + } } }, "modals": { "uploadDoc": { - "label": "Subir nuevo documento", + "label": "Añadir una fuente", "select": "Elige cómo cargar tu documento en DocsGPT", - "selectSource": "Selecciona la forma de agregar tu fuente", + "selectSource": "Añadir una fuente", "selectedFiles": "Archivos Seleccionados", "noFilesSelected": "No hay archivos seleccionados", "file": "Subir desde el dispositivo", @@ -1021,7 +1120,7 @@ "info": "Por favor, sube archivos .pdf, .txt, .rst, .csv, .xlsx, .xlsm, .xlsb, .xls, .ods, .docx, .docm, .doc, .odt, .rtf, .md, .html, .xhtml, .png, .jpg, .jpeg, .epub, .json, .pptx, .pptm, .ppt, .pps, .ppsx, .ppsm, .pot, .odp, .zip limitados a 25MB", "uploadedFiles": "Archivos Subidos", "cancel": "Cancelar", - "train": "Entrenar", + "train": "Añadir fuente", "create": "Crear", "link": "Enlace", "urlLink": "Enlace URL", @@ -1154,7 +1253,10 @@ "region": "Región de AWS", "endpointUrl": "URL de endpoint personalizada (opcional)", "initialContent": "Contenido inicial (opcional)" - } + }, + "groupUploadWeb": "Subida y web", + "groupConnection": "Desde una conexión", + "tileConnectedAs": "Conectado como {{account}}" }, "createAPIKey": { "label": "Crear Nueva Clave de API", @@ -1190,8 +1292,8 @@ "configTool": { "title": "Configuración de la Herramienta", "type": "Tipo", - "apiKeyLabel": "Clave API / OAuth", - "apiKeyPlaceholder": "Ingrese la Clave API / OAuth", + "apiKeyLabel": "Clave de API", + "apiKeyPlaceholder": "Introduce la clave de API", "addButton": "Agregar Herramienta", "closeButton": "Cerrar", "customNamePlaceholder": "Enter custom name (optional)", diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index 9e0fa1dd..2595f2f3 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -97,7 +97,7 @@ }, "sources": { "title": "ここでは、利用可能なすべてのソースファイルとアップロードしたファイルを管理できます。", - "subtitle": "レスポンスを機能させるドキュメントとナレッジソースをアップロードして管理", + "subtitle": "DocsGPT が検索・引用できるコンテンツ：アップロード、ウェブサイト、Wiki、接続から同期したコンテンツ。", "label": "ソース", "name": "ソース名", "date": "ベクトル日付", @@ -353,7 +353,8 @@ "forbidden": "このソースの設定を編集する権限がありません。", "saveFailed": "設定の保存に失敗しました。もう一度お試しください。" } - } + }, + "paused": "同期を一時停止中です。再開するには {{name}} を再接続してください。" }, "analytics": { "label": "分析", @@ -650,11 +651,11 @@ }, "tools": { "label": "ツール", - "subtitle": "エージェントを強化するツールやインテグレーションを検索、接続、設定", + "subtitle": "エージェントが実行できる操作です。組み込みツールはここにあります。接続したサービスのツールは、そのサービスの下にまとめて表示されます。", "searchPlaceholder": "検索...", "addTool": "ツールを追加", "noToolsFound": "ツールが見つかりません", - "selectToolSetup": "設定するツールを選択してください", + "selectToolSetup": "ツールを追加", "settingsIconAlt": "設定アイコン", "configureToolAria": "{{toolName}}を設定", "toggleToolAria": "{{toolName}}を切り替え", @@ -666,8 +667,8 @@ "builtIn": "ビルトイン", "authStatus": { "connected": "接続済み", - "needsAuth": "要認証", - "configured": "設定済み" + "needsAuth": "再接続", + "configured": "接続済み" }, "deleteWarning": "ツール \"{{toolName}}\" を削除してもよろしいですか？", "unsavedChanges": "保存されていない変更があります。保存せずに離れると失われます。", @@ -772,7 +773,11 @@ "maxTimeout": "タイムアウトの上限は300秒です" }, "headerValuePlaceholder": "例: application/json", - "toolIconTitle": "{{name}}のアイコン" + "toolIconTitle": "{{name}}のアイコン", + "groupBuiltIn": "組み込み", + "groupService": "サービスから", + "browseConnectors": "Notion、Linear などのサービスをお探しですか？コネクタを見る", + "viaConnection": "{{name}} 経由" }, "devices": { "label": "デバイス", @@ -1001,13 +1006,106 @@ "subtitle": "「{{name}}」の新しいシークレットの準備ができました。以前のものは使用できません。" } } + }, + "connectors": { + "label": "コネクタ", + "subtitle": "DocsGPT をチームで使っているサービスに接続します。接続ごとに、コンテンツをソースへ同期したり、エージェントにツールを提供したりできます。", + "search": "コネクタを検索...", + "addCustom": "カスタムコネクタを追加", + "custom": { + "custom_mcp": "MCP サーバー", + "custom_openapi": "OpenAPI / REST" + }, + "categoriesLabel": "カテゴリでコネクタを絞り込む", + "categories": { + "all": "すべて", + "connected": "接続済み", + "files": "ファイルとドキュメント", + "knowledge": "ナレッジベース", + "dev": "開発者ツール", + "messaging": "メッセージ", + "database": "データベース", + "search": "検索", + "custom": "カスタム" + }, + "capability": { + "sync": "同期", + "read": "読み取り", + "write": "書き込み" + }, + "status": { + "connect": "接続", + "connected": "接続済み", + "connectedCount": "{{count}} 件接続済み", + "reconnect": "再接続", + "needsAdminSetup": "管理者の設定が必要", + "disabledByAdmin": "管理者により無効", + "custom": "カスタム" + }, + "connectionStatus": { + "connected": "接続済み", + "reconnect_needed": "再接続が必要", + "disconnected": "切断済み", + "error": "エラー", + "pending": "サインイン待ち" + }, + "publisher": { + "built_in": "組み込み", + "preset": "プリセット", + "custom": "カスタム" + }, + "descriptions": { + "google_drive": "ドキュメント、スプレッドシート、PDF をソースに同期します。", + "share_point": "SharePoint サイトと OneDrive のファイルをソースに同期します。", + "confluence": "Confluence のスペースとページをソースに同期します。", + "s3": "S3 バケットのドキュメントをソースに同期します。", + "reddit": "検索に一致する Reddit の投稿をソースに同期します。", + "brave": "Brave Search API でウェブと画像を検索します。", + "telegram": "Telegram のチャットにメッセージや画像を送信します。", + "ntfy": "ntfy サーバー経由でプッシュ通知を送信します。", + "postgres": "Postgres データベースのスキーマを読み取り、SQL を実行します。", + "custom_mcp": "任意のリモート Model Context Protocol サーバーに接続します。", + "custom_openapi": "OpenAPI 仕様をインポートし、そのエンドポイントをツールとして呼び出します。" + }, + "empty": "まだ何も接続されていません。サービスを接続すると、コンテンツを同期したり、エージェントに新しいツールを提供したりできます。", + "noMatches": "検索に一致するコネクタはありません。", + "loadFailed": "コネクタを読み込めませんでした。", + "askAdmin": "管理者に設定を依頼してください。", + "setupSettings": "次のサーバー設定を行い、DocsGPT を再起動してください：", + "setupGuide": "設定ガイド", + "unverified": "このサーバーは DocsGPT による確認を受けていません。信頼できるサーバーにのみ接続してください。", + "detail": { + "accounts": "アカウント", + "connectedAs": "{{account}} として接続中", + "connectAnother": "別のアカウントを接続", + "sources": "この接続のソース", + "tools": "この接続のツール", + "noAccounts": "まだアカウントが接続されていません。", + "failed": "この接続を読み込めませんでした。", + "lastSync": "{{date}} に同期 · {{frequency}}", + "paused": "同期を一時停止中", + "toolOn": "オン", + "toolOff": "オフ", + "accountMenu": "アカウントの操作", + "disconnect": "切断" + }, + "permission": { + "always": "常に許可", + "ask": "承認が必要", + "off": "オフ" + }, + "disconnect": { + "title": "{{name}} を切断しますか？", + "body_zero": "この接続のツールは、再接続するまで使えなくなります。", + "body_other": "{{count}} 件のソースはコンテンツを保持しますが、同期は停止します。この接続のツールは、再接続するまで使えなくなります。" + } } }, "modals": { "uploadDoc": { - "label": "新しい文書をアップロードする", + "label": "ソースを追加", "select": "ドキュメントを DocsGPT にアップロードする方法を選択します", - "selectSource": "ソースを追加する方法を選択してください", + "selectSource": "ソースを追加", "selectedFiles": "選択されたファイル", "noFilesSelected": "ファイルが選択されていません", "file": "デバイスからアップロード", @@ -1020,7 +1118,7 @@ "info": "25MBまでの.pdf、.txt、.rst、.csv、.xlsx、.xlsm、.xlsb、.xls、.ods、.docx、.docm、.doc、.odt、.rtf、.md、.html、.xhtml、.png、.jpg、.jpeg、.epub、.json、.pptx、.pptm、.ppt、.pps、.ppsx、.ppsm、.pot、.odp、.zipファイルをアップロードしてください", "uploadedFiles": "アップロードされたファイル", "cancel": "キャンセル", - "train": "トレーニング", + "train": "ソースを追加", "create": "作成", "link": "リンク", "urlLink": "URLリンク", @@ -1153,7 +1251,10 @@ "region": "AWS リージョン", "endpointUrl": "カスタムエンドポイント URL（任意）", "initialContent": "初期コンテンツ（任意）" - } + }, + "groupUploadWeb": "アップロードとウェブ", + "groupConnection": "接続から", + "tileConnectedAs": "{{account}} として接続中" }, "createAPIKey": { "label": "新しいAPIキーを作成", @@ -1189,8 +1290,8 @@ "configTool": { "title": "ツール設定", "type": "タイプ", - "apiKeyLabel": "APIキー / OAuth", - "apiKeyPlaceholder": "APIキー / OAuthを入力してください", + "apiKeyLabel": "API キー", + "apiKeyPlaceholder": "API キーを入力", "addButton": "ツールを追加", "closeButton": "閉じる", "customNamePlaceholder": "Enter custom name (optional)", diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index c00face0..b3cb6d3d 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -97,7 +97,7 @@ }, "sources": { "title": "Здесь вы можете управлять всеми исходными файлами, которые доступны вам и которые вы загрузили.", - "subtitle": "Загружайте и управляйте документами и источниками знаний, которые обеспечивают ваши ответы", + "subtitle": "Контент, по которому DocsGPT может искать и который может цитировать: загрузки, сайты, вики и контент, синхронизированный из ваших подключений.", "label": "Источники", "name": "Название источника", "date": "Дата вектора", @@ -366,7 +366,8 @@ "forbidden": "У вас нет прав на изменение настроек этого источника.", "saveFailed": "Не удалось сохранить настройки. Пожалуйста, попробуйте ещё раз." } - } + }, + "paused": "Синхронизация приостановлена. Переподключите {{name}}, чтобы продолжить." }, "analytics": { "label": "Аналитика", @@ -677,11 +678,11 @@ }, "tools": { "label": "Инструменты", - "subtitle": "Просматривайте, подключайте и настраивайте инструменты и интеграции для ваших агентов", + "subtitle": "Действия, которые могут выполнять ваши агенты. Здесь находятся встроенные инструменты. Инструменты подключённого сервиса сгруппированы под этим сервисом.", "searchPlaceholder": "Поиск...", "addTool": "Добавить инструмент", "noToolsFound": "Инструменты не найдены", - "selectToolSetup": "Выберите инструмент для настройки", + "selectToolSetup": "Добавить инструмент", "settingsIconAlt": "Значок настроек", "configureToolAria": "Настроить {{toolName}}", "toggleToolAria": "Переключить {{toolName}}", @@ -693,8 +694,8 @@ "builtIn": "Встроенный", "authStatus": { "connected": "Подключено", - "needsAuth": "Требуется авторизация", - "configured": "Настроено" + "needsAuth": "Переподключить", + "configured": "Подключено" }, "deleteWarning": "Вы уверены, что хотите удалить инструмент \"{{toolName}}\"?", "unsavedChanges": "У вас есть несохраненные изменения, которые будут потеряны, если вы уйдете без сохранения.", @@ -799,7 +800,11 @@ "maxTimeout": "Максимальный тайм-аут — 300 секунд" }, "headerValuePlaceholder": "например, application/json", - "toolIconTitle": "Значок {{name}}" + "toolIconTitle": "Значок {{name}}", + "groupBuiltIn": "Встроенные", + "groupService": "Из сервиса", + "browseConnectors": "Ищете Notion, Linear или другой сервис? Смотрите коннекторы", + "viaConnection": "через {{name}}" }, "devices": { "label": "Устройства", @@ -1042,13 +1047,109 @@ "subtitle": "Новый секрет для «{{name}}» готов. Старый больше не работает." } } + }, + "connectors": { + "label": "Коннекторы", + "subtitle": "Подключите DocsGPT к сервисам, которыми пользуется ваша команда. Подключение может синхронизировать контент в Источники и давать агентам инструменты.", + "search": "Поиск коннекторов...", + "addCustom": "Добавить свой коннектор", + "custom": { + "custom_mcp": "MCP-сервер", + "custom_openapi": "OpenAPI / REST" + }, + "categoriesLabel": "Фильтр коннекторов по категории", + "categories": { + "all": "Все", + "connected": "Подключённые", + "files": "Файлы и документы", + "knowledge": "Базы знаний", + "dev": "Инструменты разработчика", + "messaging": "Сообщения", + "database": "Базы данных", + "search": "Поиск", + "custom": "Свои" + }, + "capability": { + "sync": "Синхронизация", + "read": "Чтение", + "write": "Запись" + }, + "status": { + "connect": "Подключить", + "connected": "Подключено", + "connectedCount": "Подключено: {{count}}", + "reconnect": "Переподключить", + "needsAdminSetup": "Нужна настройка администратором", + "disabledByAdmin": "Отключено администратором", + "custom": "Свой" + }, + "connectionStatus": { + "connected": "Подключено", + "reconnect_needed": "Нужно переподключить", + "disconnected": "Отключено", + "error": "Ошибка", + "pending": "Ожидание входа" + }, + "publisher": { + "built_in": "Встроенный", + "preset": "Готовый", + "custom": "Свой" + }, + "descriptions": { + "google_drive": "Синхронизирует Документы, Таблицы и PDF в Источники.", + "share_point": "Синхронизирует файлы с сайтов SharePoint и из OneDrive в Источники.", + "confluence": "Синхронизирует пространства и страницы Confluence в Источники.", + "s3": "Синхронизирует документы из бакета S3 в Источники.", + "reddit": "Синхронизирует в Источники посты Reddit по вашим поисковым запросам.", + "brave": "Поиск в интернете и по картинкам через Brave Search API.", + "telegram": "Отправляет сообщения и изображения в чат Telegram.", + "ntfy": "Отправляет push-уведомления через сервер ntfy.", + "postgres": "Читает схему и выполняет SQL в базе данных Postgres.", + "custom_mcp": "Подключает любой удалённый сервер Model Context Protocol.", + "custom_openapi": "Импортирует спецификацию OpenAPI и вызывает её эндпоинты как инструменты." + }, + "empty": "Пока ничего не подключено. Подключите сервис, чтобы синхронизировать его контент или дать агентам новые инструменты.", + "noMatches": "Нет коннекторов, подходящих под запрос.", + "loadFailed": "Не удалось загрузить коннекторы.", + "askAdmin": "Попросите администратора настроить это.", + "setupSettings": "Задайте эти настройки сервера и перезапустите DocsGPT:", + "setupGuide": "Инструкция по настройке", + "unverified": "DocsGPT не проверял этот сервер. Подключайте только серверы, которым доверяете.", + "detail": { + "accounts": "Аккаунты", + "connectedAs": "Подключено как {{account}}", + "connectAnother": "Подключить другой аккаунт", + "sources": "Источники из этого подключения", + "tools": "Инструменты из этого подключения", + "noAccounts": "Аккаунт ещё не подключён.", + "failed": "Не удалось загрузить это подключение.", + "lastSync": "Синхронизировано {{date}} · {{frequency}}", + "paused": "Синхронизация приостановлена", + "toolOn": "Вкл.", + "toolOff": "Выкл.", + "accountMenu": "Действия с аккаунтом", + "disconnect": "Отключить" + }, + "permission": { + "always": "Всегда разрешать", + "ask": "Нужно одобрение", + "off": "Выкл." + }, + "disconnect": { + "title": "Отключить {{name}}?", + "body_zero": "Инструменты из этого подключения перестанут работать, пока вы не переподключитесь.", + "body_one": "Ваш {{count}} источник сохранит контент, но перестанет синхронизироваться. Инструменты из этого подключения перестанут работать, пока вы не переподключитесь.", + "body_few": "Ваши {{count}} источника сохранят контент, но перестанут синхронизироваться. Инструменты из этого подключения перестанут работать, пока вы не переподключитесь.", + "body_many": "Ваши {{count}} источников сохранят контент, но перестанут синхронизироваться. Инструменты из этого подключения перестанут работать, пока вы не переподключитесь.", + "body_other": "Ваши {{count}} источника сохранят контент, но перестанут синхронизироваться. Инструменты из этого подключения перестанут работать, пока вы не переподключитесь." + } } }, "modals": { "uploadDoc": { - "label": "Загрузить новый документ", + "label": "Добавить источник", "select": "Выберите способ загрузки документа в DocsGPT", - "selectSource": "Выберите способ добавления источника", + "selectSource": "Добавить источник", "selectedFiles": "Выбранные файлы", "noFilesSelected": "Файлы не выбраны", "file": "Загрузить с устройства", @@ -1061,7 +1162,7 @@ "info": "Пожалуйста, загрузите файлы .pdf, .txt, .rst, .csv, .xlsx, .xlsm, .xlsb, .xls, .ods, .docx, .docm, .doc, .odt, .rtf, .md, .html, .xhtml, .png, .jpg, .jpeg, .epub, .json, .pptx, .pptm, .ppt, .pps, .ppsx, .ppsm, .pot, .odp, .zip размером до 25 МБ", "uploadedFiles": "Загруженные файлы", "cancel": "Отмена", - "train": "Тренировка", + "train": "Добавить источник", "create": "Создать", "link": "Ссылка", "urlLink": "URL ссылка", @@ -1194,7 +1295,10 @@ "region": "Регион AWS", "endpointUrl": "Пользовательский URL эндпоинта (необязательно)", "initialContent": "Исходное содержимое (необязательно)" - } + }, + "groupUploadWeb": "Загрузка и веб", + "groupConnection": "Из подключения", + "tileConnectedAs": "Подключено как {{account}}" }, "createAPIKey": { "label": "Создать новый API ключ", @@ -1230,8 +1334,8 @@ "configTool": { "title": "Настройка инструмента", "type": "Тип", - "apiKeyLabel": "API ключ / OAuth", - "apiKeyPlaceholder": "Введите API ключ / OAuth", + "apiKeyLabel": "API-ключ", + "apiKeyPlaceholder": "Введите API-ключ", "addButton": "Добавить инструмент", "closeButton": "Закрыть", "customNamePlaceholder": "Enter custom name (optional)", diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index 3ca0570b..183deb66 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -97,7 +97,7 @@ }, "sources": { "title": "在這裡您可以管理所有可用的來源檔案以及您上傳的檔案。", - "subtitle": "上傳並管理您的回答所依賴的文件和知識來源", + "subtitle": "DocsGPT 可以搜尋和引用的內容：上傳的檔案、網站、Wiki，以及從連線同步的內容。", "label": "來源", "name": "來源名稱", "date": "向量日期", @@ -353,7 +353,8 @@ "forbidden": "您沒有編輯此來源設定的權限。", "saveFailed": "儲存設定失敗。請再試一次。" } - } + }, + "paused": "同步已暫停。重新連線 {{name}} 以繼續。" }, "analytics": { "label": "分析", @@ -650,11 +651,11 @@ }, "tools": { "label": "工具", - "subtitle": "瀏覽、連接並設定驅動您的智能體的工具和整合", + "subtitle": "代理可以執行的操作。內建工具在這裡。已連線服務提供的工具歸在該服務之下。", "searchPlaceholder": "搜尋工具...", "addTool": "新增工具", "noToolsFound": "找不到工具", - "selectToolSetup": "選擇要設定的工具", + "selectToolSetup": "新增工具", "settingsIconAlt": "設定圖示", "configureToolAria": "設定 {{toolName}}", "toggleToolAria": "切換 {{toolName}}", @@ -666,8 +667,8 @@ "builtIn": "內建", "authStatus": { "connected": "已連線", - "needsAuth": "需要驗證", - "configured": "已設定" + "needsAuth": "重新連線", + "configured": "已連線" }, "deleteWarning": "您確定要刪除工具 \"{{toolName}}\" 嗎？", "unsavedChanges": "您有未儲存的變更，如果不儲存就離開將會遺失。", @@ -772,7 +773,11 @@ "maxTimeout": "逾時時間最長為 300 秒" }, "headerValuePlaceholder": "例如：application/json", - "toolIconTitle": "{{name}} 圖示" + "toolIconTitle": "{{name}} 圖示", + "groupBuiltIn": "內建", + "groupService": "來自服務", + "browseConnectors": "在找 Notion、Linear 或其他服務嗎？瀏覽連接器", + "viaConnection": "透過 {{name}}" }, "devices": { "label": "裝置", @@ -1001,13 +1006,106 @@ "subtitle": "「{{name}}」的新密鑰已就緒，舊密鑰已失效。" } } + }, + "connectors": { + "label": "連接器", + "subtitle": "將 DocsGPT 連接到團隊使用的服務。一個連線可以把內容同步到來源，也可以為代理提供工具。", + "search": "搜尋連接器...", + "addCustom": "新增自訂連接器", + "custom": { + "custom_mcp": "MCP 伺服器", + "custom_openapi": "OpenAPI / REST" + }, + "categoriesLabel": "依類別篩選連接器", + "categories": { + "all": "全部", + "connected": "已連線", + "files": "檔案和文件", + "knowledge": "知識庫", + "dev": "開發者工具", + "messaging": "訊息", + "database": "資料庫", + "search": "搜尋", + "custom": "自訂" + }, + "capability": { + "sync": "同步", + "read": "讀取", + "write": "寫入" + }, + "status": { + "connect": "連線", + "connected": "已連線", + "connectedCount": "已連線 {{count}} 個", + "reconnect": "重新連線", + "needsAdminSetup": "需要管理員設定", + "disabledByAdmin": "已被管理員停用", + "custom": "自訂" + }, + "connectionStatus": { + "connected": "已連線", + "reconnect_needed": "需要重新連線", + "disconnected": "已中斷連線", + "error": "錯誤", + "pending": "等待登入" + }, + "publisher": { + "built_in": "內建", + "preset": "預設", + "custom": "自訂" + }, + "descriptions": { + "google_drive": "將文件、試算表和 PDF 同步到來源。", + "share_point": "將 SharePoint 網站和 OneDrive 中的檔案同步到來源。", + "confluence": "將 Confluence 空間和頁面同步到來源。", + "s3": "將 S3 儲存貯體中的文件同步到來源。", + "reddit": "將符合搜尋條件的 Reddit 貼文同步到來源。", + "brave": "使用 Brave Search API 搜尋網頁和圖片。", + "telegram": "傳送訊息和圖片到 Telegram 聊天。", + "ntfy": "透過 ntfy 伺服器傳送推播通知。", + "postgres": "讀取 Postgres 資料庫的結構並執行 SQL。", + "custom_mcp": "連接任何遠端 Model Context Protocol 伺服器。", + "custom_openapi": "匯入 OpenAPI 規格，並將其端點作為工具呼叫。" + }, + "empty": "尚未連線任何服務。連線一個服務以同步其內容，或為代理提供新工具。", + "noMatches": "沒有符合搜尋的連接器。", + "loadFailed": "無法載入連接器。", + "askAdmin": "請管理員完成設定。", + "setupSettings": "設定以下伺服器設定，然後重新啟動 DocsGPT：", + "setupGuide": "設定指南", + "unverified": "此伺服器未經 DocsGPT 審查。請只連接你信任的伺服器。", + "detail": { + "accounts": "帳號", + "connectedAs": "已以 {{account}} 連線", + "connectAnother": "連線其他帳號", + "sources": "來自此連線的來源", + "tools": "來自此連線的工具", + "noAccounts": "尚未連線帳號。", + "failed": "無法載入此連線。", + "lastSync": "已於 {{date}} 同步 · {{frequency}}", + "paused": "同步已暫停", + "toolOn": "開啟", + "toolOff": "關閉", + "accountMenu": "帳號操作", + "disconnect": "中斷連線" + }, + "permission": { + "always": "一律允許", + "ask": "需要核准", + "off": "關閉" + }, + "disconnect": { + "title": "要中斷 {{name}} 的連線嗎？", + "body_zero": "在重新連線之前，此連線的工具將無法使用。", + "body_other": "你的 {{count}} 個來源會保留內容，但會停止同步。在重新連線之前，此連線的工具將無法使用。" + } } }, "modals": { "uploadDoc": { - "label": "上傳新文件", + "label": "新增來源", "select": "選擇如何將文件上傳到 DocsGPT", - "selectSource": "選擇新增來源的方式", + "selectSource": "新增來源", "selectedFiles": "已選擇的檔案", "noFilesSelected": "未選擇檔案", "file": "從檔案", @@ -1020,7 +1118,7 @@ "info": "請上傳限制為25MB的.pdf、.txt、.rst、.csv、.xlsx、.xlsm、.xlsb、.xls、.ods、.docx、.docm、.doc、.odt、.rtf、.md、.html、.xhtml、.png、.jpg、.jpeg、.epub、.json、.pptx、.pptm、.ppt、.pps、.ppsx、.ppsm、.pot、.odp、.zip檔案", "uploadedFiles": "已上傳檔案", "cancel": "取消", - "train": "訓練", + "train": "新增來源", "create": "建立", "link": "連結", "urlLink": "URL 連結", @@ -1153,7 +1251,10 @@ "region": "AWS 區域", "endpointUrl": "自訂端點 URL（選填）", "initialContent": "初始內容（選填）" - } + }, + "groupUploadWeb": "上傳和網頁", + "groupConnection": "來自連線", + "tileConnectedAs": "已以 {{account}} 連線" }, "createAPIKey": { "label": "建立新的 API 金鑰", @@ -1189,8 +1290,8 @@ "configTool": { "title": "工具設定", "type": "類型", - "apiKeyLabel": "API 金鑰 / OAuth", - "apiKeyPlaceholder": "輸入 API 金鑰 / OAuth", + "apiKeyLabel": "API 金鑰", + "apiKeyPlaceholder": "輸入 API 金鑰", "addButton": "新增工具", "closeButton": "關閉", "customNamePlaceholder": "Enter custom name (optional)", diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index d41815c1..29b4880b 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -97,7 +97,7 @@ }, "sources": { "title": "在这里您可以管理所有可用的源文件以及您上传的文件。", - "subtitle": "上传并管理您的回答所依赖的文档和知识来源", + "subtitle": "DocsGPT 可以搜索和引用的内容：上传的文件、网站、Wiki，以及从连接同步的内容。", "label": "来源", "name": "来源名称", "date": "向量日期", @@ -353,7 +353,8 @@ "forbidden": "您没有权限编辑此来源的配置。", "saveFailed": "保存配置失败，请重试。" } - } + }, + "paused": "同步已暂停。重新连接 {{name}} 以继续。" }, "analytics": { "label": "分析", @@ -650,11 +651,11 @@ }, "tools": { "label": "工具", - "subtitle": "浏览、连接并配置驱动您的智能体的工具和集成", + "subtitle": "智能体可以执行的操作。内置工具在这里。已连接服务提供的工具归在该服务下。", "searchPlaceholder": "搜索工具...", "addTool": "添加工具", "noToolsFound": "未找到工具", - "selectToolSetup": "选择要设置的工具", + "selectToolSetup": "添加工具", "settingsIconAlt": "设置图标", "configureToolAria": "配置 {{toolName}}", "toggleToolAria": "切换 {{toolName}}", @@ -666,8 +667,8 @@ "builtIn": "内置", "authStatus": { "connected": "已连接", - "needsAuth": "需要认证", - "configured": "已配置" + "needsAuth": "重新连接", + "configured": "已连接" }, "deleteWarning": "您确定要删除工具 \"{{toolName}}\" 吗？", "unsavedChanges": "您有未保存的更改，如果不保存就离开将会丢失。", @@ -772,7 +773,11 @@ "maxTimeout": "超时时间最长为 300 秒" }, "headerValuePlaceholder": "例如：application/json", - "toolIconTitle": "{{name}} 图标" + "toolIconTitle": "{{name}} 图标", + "groupBuiltIn": "内置", + "groupService": "来自服务", + "browseConnectors": "在找 Notion、Linear 或其他服务？浏览连接器", + "viaConnection": "通过 {{name}}" }, "devices": { "label": "设备", @@ -1001,13 +1006,106 @@ "subtitle": "“{{name}}”的新密钥已就绪，旧密钥已失效。" } } + }, + "connectors": { + "label": "连接器", + "subtitle": "将 DocsGPT 连接到团队使用的服务。一个连接可以把内容同步到来源，也可以为智能体提供工具。", + "search": "搜索连接器...", + "addCustom": "添加自定义连接器", + "custom": { + "custom_mcp": "MCP 服务器", + "custom_openapi": "OpenAPI / REST" + }, + "categoriesLabel": "按类别筛选连接器", + "categories": { + "all": "全部", + "connected": "已连接", + "files": "文件和文档", + "knowledge": "知识库", + "dev": "开发者工具", + "messaging": "消息", + "database": "数据库", + "search": "搜索", + "custom": "自定义" + }, + "capability": { + "sync": "同步", + "read": "读取", + "write": "写入" + }, + "status": { + "connect": "连接", + "connected": "已连接", + "connectedCount": "已连接 {{count}} 个", + "reconnect": "重新连接", + "needsAdminSetup": "需要管理员设置", + "disabledByAdmin": "已被管理员停用", + "custom": "自定义" + }, + "connectionStatus": { + "connected": "已连接", + "reconnect_needed": "需要重新连接", + "disconnected": "已断开", + "error": "错误", + "pending": "等待登录" + }, + "publisher": { + "built_in": "内置", + "preset": "预设", + "custom": "自定义" + }, + "descriptions": { + "google_drive": "将文档、表格和 PDF 同步到来源。", + "share_point": "将 SharePoint 站点和 OneDrive 中的文件同步到来源。", + "confluence": "将 Confluence 空间和页面同步到来源。", + "s3": "将 S3 存储桶中的文档同步到来源。", + "reddit": "将符合搜索条件的 Reddit 帖子同步到来源。", + "brave": "使用 Brave Search API 搜索网页和图片。", + "telegram": "向 Telegram 聊天发送消息和图片。", + "ntfy": "通过 ntfy 服务器发送推送通知。", + "postgres": "读取 Postgres 数据库的结构并执行 SQL。", + "custom_mcp": "连接任意远程 Model Context Protocol 服务器。", + "custom_openapi": "导入 OpenAPI 规范，并将其端点作为工具调用。" + }, + "empty": "尚未连接任何服务。连接一个服务以同步其内容，或为智能体提供新工具。", + "noMatches": "没有与搜索匹配的连接器。", + "loadFailed": "无法加载连接器。", + "askAdmin": "请让管理员完成设置。", + "setupSettings": "设置以下服务器配置，然后重启 DocsGPT：", + "setupGuide": "设置指南", + "unverified": "此服务器未经 DocsGPT 审核。请只连接你信任的服务器。", + "detail": { + "accounts": "账号", + "connectedAs": "已作为 {{account}} 连接", + "connectAnother": "连接其他账号", + "sources": "来自此连接的来源", + "tools": "来自此连接的工具", + "noAccounts": "尚未连接账号。", + "failed": "无法加载此连接。", + "lastSync": "已于 {{date}} 同步 · {{frequency}}", + "paused": "同步已暂停", + "toolOn": "开启", + "toolOff": "关闭", + "accountMenu": "账号操作", + "disconnect": "断开连接" + }, + "permission": { + "always": "始终允许", + "ask": "需要批准", + "off": "关闭" + }, + "disconnect": { + "title": "断开 {{name}}？", + "body_zero": "在重新连接之前，此连接的工具将无法使用。", + "body_other": "你的 {{count}} 个来源会保留内容，但将停止同步。在重新连接之前，此连接的工具将无法使用。" + } } }, "modals": { "uploadDoc": { - "label": "上传新文档", + "label": "添加来源", "select": "选择如何将文档上传到 DocsGPT", - "selectSource": "选择添加源的方式", + "selectSource": "添加来源", "selectedFiles": "已选择的文件", "noFilesSelected": "未选择文件", "file": "从设备上传", @@ -1020,7 +1118,7 @@ "info": "请上传限制为25MB的.pdf、.txt、.rst、.csv、.xlsx、.xlsm、.xlsb、.xls、.ods、.docx、.docm、.doc、.odt、.rtf、.md、.html、.xhtml、.png、.jpg、.jpeg、.epub、.json、.pptx、.pptm、.ppt、.pps、.ppsx、.ppsm、.pot、.odp、.zip文件", "uploadedFiles": "已上传文件", "cancel": "取消", - "train": "训练", + "train": "添加来源", "create": "创建", "link": "链接", "urlLink": "URL 链接", @@ -1153,7 +1251,10 @@ "region": "AWS 区域", "endpointUrl": "自定义端点 URL（可选）", "initialContent": "初始内容（可选）" - } + }, + "groupUploadWeb": "上传和网页", + "groupConnection": "来自连接", + "tileConnectedAs": "已作为 {{account}} 连接" }, "createAPIKey": { "label": "创建新的 API 密钥", @@ -1189,8 +1290,8 @@ "configTool": { "title": "工具配置", "type": "类型", - "apiKeyLabel": "API 密钥 / OAuth", - "apiKeyPlaceholder": "输入 API 密钥 / OAuth", + "apiKeyLabel": "API 密钥", + "apiKeyPlaceholder": "输入 API 密钥", "addButton": "添加工具", "closeButton": "关闭", "customNamePlaceholder": "Enter custom name (optional)", diff --git a/frontend/src/modals/ConfirmationModal.tsx b/frontend/src/modals/ConfirmationModal.tsx index 4e986a03..1cfbdfe6 100644 --- a/frontend/src/modals/ConfirmationModal.tsx +++ b/frontend/src/modals/ConfirmationModal.tsx @@ -5,6 +5,7 @@ import { ActiveState } from '../models/misc'; export default function ConfirmationModal({ message, + description, modalState, setModalState, submitLabel, @@ -14,6 +15,8 @@ export default function ConfirmationModal({ variant = 'default', }: { message: string; + /** A muted line under the title that says what the action does. */ + description?: string; modalState: ActiveState; setModalState: (state: ActiveState) => void; submitLabel: string; @@ -41,6 +44,7 @@ export default function ConfirmationModal({ if (!open) setModalState('INACTIVE'); }} title={message} + description={description} footer={
+ {!server?.preset && ( + + + )} {server?.has_encrypted_credentials && formData.auth_type !== 'oauth' && ( diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts index 5c118244..2c4c5647 100644 --- a/frontend/src/models/misc.ts +++ b/frontend/src/models/misc.ts @@ -85,6 +85,8 @@ export type Doc = { // Access level when shared via a team: 'viewer' (read-only) or 'editor' // (full write). Null/absent for sources the caller owns. team_access?: 'viewer' | 'editor' | null; + // The connection a synced source comes from (Google Drive account, S3 keys). + connectionId?: string | null; }; export type GetDocsResponse = { diff --git a/frontend/src/navigation/sections.ts b/frontend/src/navigation/sections.ts index 5eb73b39..0334f18c 100644 --- a/frontend/src/navigation/sections.ts +++ b/frontend/src/navigation/sections.ts @@ -11,6 +11,7 @@ import { LayoutDashboard, LayoutGrid, LayoutTemplate, + Plug, ScrollText, Settings2, ShieldCheck, @@ -115,6 +116,12 @@ export const SETTINGS_SECTION: Section = { labelKey: 'settings.sources.label', icon: Database, }, + { + key: 'connectors', + path: '/settings/connectors', + labelKey: 'settings.connectors.label', + icon: Plug, + }, { key: 'tools', path: '/settings/tools', diff --git a/frontend/src/settings/Connectors.test.tsx b/frontend/src/settings/Connectors.test.tsx new file mode 100644 index 00000000..112c86c2 --- /dev/null +++ b/frontend/src/settings/Connectors.test.tsx @@ -0,0 +1,217 @@ +import { configureStore } from '@reduxjs/toolkit'; +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { Provider } from 'react-redux'; +import { MemoryRouter } from 'react-router-dom'; + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ + t: (key: string, opts?: { defaultValue?: string; count?: number }) => + opts?.count !== undefined ? `${key}:${opts.count}` : key, + }), +})); + +const service = vi.hoisted(() => ({ + getCatalog: vi.fn(), + listConnections: vi.fn(), + getConnection: vi.fn(), + disconnect: vi.fn(), +})); +vi.mock('../api/services/connectorsService', () => ({ default: service })); + +import connectorsReducer from '../connectors/connectorsSlice'; +import type { ConnectorDefinition } from '../connectors/types'; +import Connectors from './Connectors'; + +Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + +const definition = ( + overrides: Partial, +): ConnectorDefinition => ({ + key: 'telegram', + name: 'Telegram', + description: 'Send messages.', + icon: 'tool_telegram', + category: 'messaging', + auth_kind: 'api_key', + capabilities: ['write'], + credential_fields: [], + setup_fields: [], + sync_ingestor: null, + default_sync_frequency: 'weekly', + tool_templates: ['telegram'], + setup: { tools: 'auto', sync: 'off' }, + mcp_url: null, + publisher: 'built_in', + docs_url: null, + oauth_scopes: [], + available: true, + disabled: false, + needs_setup: false, + missing_settings: [], + connected_count: 0, + connection_count: 0, + status: null, + state: 'available', + credential_policy: 'choose', + ...overrides, +}); + +const CATALOG = [ + definition({}), + definition({ + key: 'google_drive', + name: 'Google Drive', + category: 'files', + capabilities: ['sync'], + state: 'connected', + connected_count: 2, + connection_count: 2, + status: 'connected', + }), + definition({ + key: 'confluence', + name: 'Confluence', + category: 'knowledge', + capabilities: ['sync'], + state: 'reconnect', + connection_count: 1, + status: 'reconnect_needed', + }), + definition({ + key: 'share_point', + name: 'SharePoint', + category: 'files', + state: 'needs_setup', + available: false, + needs_setup: true, + }), + definition({ + key: 'brave', + name: 'Brave Search', + category: 'search', + state: 'disabled', + available: false, + disabled: true, + }), + definition({ + key: 'custom_mcp', + name: 'MCP server', + category: 'custom', + publisher: 'custom', + auth_kind: 'mcp', + state: 'custom', + }), +]; + +describe('Connectors page', () => { + let container: HTMLDivElement; + let root: Root; + + beforeEach(() => { + service.getCatalog.mockResolvedValue({ + success: true, + connectors: CATALOG, + }); + service.listConnections.mockResolvedValue({ + success: true, + connections: [], + }); + container = document.createElement('div'); + document.body.appendChild(container); + root = createRoot(container); + }); + + afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + }); + + const render = async (path = '/settings/connectors') => { + const store = configureStore({ + reducer: { + connectors: connectorsReducer, + preference: (state = { token: null }) => state, + }, + }); + await act(async () => { + root.render( + + + + + , + ); + }); + }; + + const card = (key: string) => + container.querySelector( + `[data-testid="connector-card-${key}"]`, + ); + + it('renders every card state', async () => { + await render(); + expect(card('telegram')!.textContent).toContain( + 'settings.connectors.status.connect', + ); + expect(card('google_drive')!.textContent).toContain( + 'settings.connectors.status.connectedCount:2', + ); + expect(card('confluence')!.textContent).toContain( + 'settings.connectors.status.reconnect', + ); + expect(card('share_point')!.textContent).toContain( + 'settings.connectors.status.needsAdminSetup', + ); + expect(card('brave')!.textContent).toContain( + 'settings.connectors.status.disabledByAdmin', + ); + expect(card('brave')!.disabled).toBe(true); + expect(card('custom_mcp')!.textContent).toContain( + 'settings.connectors.publisher.custom', + ); + }); + + it('sorts connections that need attention first', async () => { + await render(); + const keys = Array.from( + container.querySelectorAll('[data-testid^="connector-card-"]'), + ).map((el) => el.getAttribute('data-testid')); + expect(keys[0]).toBe('connector-card-confluence'); + expect(keys[1]).toBe('connector-card-google_drive'); + expect(keys[keys.length - 1]).toBe('connector-card-brave'); + }); + + it('shows capability chips', async () => { + await render(); + expect(card('google_drive')!.textContent).toContain( + 'settings.connectors.capability.sync', + ); + expect(card('telegram')!.textContent).toContain( + 'settings.connectors.capability.write', + ); + }); + + it('filters to connected services', async () => { + await render('/settings/connectors?filter=connected'); + expect(card('google_drive')).not.toBeNull(); + expect(card('confluence')).not.toBeNull(); + expect(card('telegram')).toBeNull(); + }); + + it('shows the empty state when nothing is connected', async () => { + service.getCatalog.mockResolvedValue({ + success: true, + connectors: [definition({})], + }); + await render('/settings/connectors?filter=connected'); + expect(container.textContent).toContain('settings.connectors.empty'); + }); + + it('shows a retry when the catalog fails to load', async () => { + service.getCatalog.mockResolvedValue({ success: false }); + await render(); + expect(container.textContent).toContain('settings.connectors.loadFailed'); + }); +}); diff --git a/frontend/src/settings/Connectors.tsx b/frontend/src/settings/Connectors.tsx new file mode 100644 index 00000000..cf2bdfd2 --- /dev/null +++ b/frontend/src/settings/Connectors.tsx @@ -0,0 +1,220 @@ +import { ChevronDown, Plus } from 'lucide-react'; +import { useEffect, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useDispatch, useSelector } from 'react-redux'; +import { useSearchParams } from 'react-router-dom'; + +import PageToolbar from '../components/PageToolbar'; +import SearchInput from '../components/SearchInput'; +import SkeletonLoader from '../components/SkeletonLoader'; +import { Button } from '../components/ui/button'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from '../components/ui/dropdown-menu'; +import { EmptyState } from '../components/ui/empty-state'; +import { ToggleGroup, ToggleGroupItem } from '../components/ui/toggle-group'; +import ConnectionDrawer from '../connectors/ConnectionDrawer'; +import ConnectorCard from '../connectors/ConnectorCard'; +import ConnectorIcon from '../connectors/ConnectorIcon'; +import { + loadConnectors, + selectConnectorCatalog, + selectConnectorsFailed, + selectConnectorsLoaded, +} from '../connectors/connectorsSlice'; +import { connectorDescription, connectorName } from '../connectors/i18n'; +import type { ConnectorDefinition } from '../connectors/types'; +import useConnectorLauncher from '../connectors/useConnectorLauncher'; +import { selectToken } from '../preferences/preferenceSlice'; +import type { AppDispatch } from '../store'; + +const FILTERS = [ + 'all', + 'connected', + 'files', + 'knowledge', + 'dev', + 'messaging', + 'database', + 'search', + 'custom', +] as const; +type Filter = (typeof FILTERS)[number]; + +// Connected and needing attention first, then what can be connected, then +// what an admin still has to set up. +const STATE_ORDER: Record = { + reconnect: 0, + connected: 1, + available: 2, + custom: 3, + needs_setup: 4, + disabled: 5, +}; + +const isConnected = (connector: ConnectorDefinition) => + connector.connection_count > 0; + +export default function Connectors() { + const { t } = useTranslation(); + const dispatch = useDispatch(); + const token = useSelector(selectToken); + const catalog = useSelector(selectConnectorCatalog); + const loaded = useSelector(selectConnectorsLoaded); + const failed = useSelector(selectConnectorsFailed); + const [searchParams, setSearchParams] = useSearchParams(); + const [search, setSearch] = useState(''); + const initialFilter = searchParams.get('filter') as Filter | null; + const [filter, setFilter] = useState( + initialFilter && FILTERS.includes(initialFilter) ? initialFilter : 'all', + ); + const [openKey, setOpenKey] = useState( + searchParams.get('connector'), + ); + const { launch, modals } = useConnectorLauncher(); + + useEffect(() => { + dispatch(loadConnectors({ token })); + }, [dispatch, token]); + + const custom = catalog.filter((c) => c.publisher === 'custom'); + const openConnector = catalog.find((c) => c.key === openKey) ?? null; + + const visible = useMemo(() => { + const query = search.trim().toLowerCase(); + return catalog + .filter((connector) => { + if (filter === 'connected') return isConnected(connector); + if (filter !== 'all') return connector.category === filter; + return true; + }) + .filter( + (connector) => + !query || + connectorName(t, connector).toLowerCase().includes(query) || + connectorDescription(t, connector).toLowerCase().includes(query), + ) + .sort((a, b) => STATE_ORDER[a.state] - STATE_ORDER[b.state]); + }, [catalog, filter, search, t]); + + const open = (connector: ConnectorDefinition) => { + if (connector.state === 'available' || connector.state === 'custom') { + launch(connector); + return; + } + setOpenKey(connector.key); + }; + + const closeDrawer = () => { + setOpenKey(null); + if (searchParams.has('connector')) { + searchParams.delete('connector'); + setSearchParams(searchParams, { replace: true }); + } + }; + + return ( +
+ setSearch(e.target.value)} + /> + } + action={ + + + + + + {custom.map((connector) => ( + launch(connector)} + > + + {connectorName(t, connector)} + + ))} + + + } + > + value && setFilter(value as Filter)} + aria-label={t('settings.connectors.categoriesLabel')} + className="mb-6" + > + {FILTERS.map((key) => ( + + {t(`settings.connectors.categories.${key}`)} + + ))} + + + + {!loaded && !failed ? ( +
+ +
+ ) : failed && catalog.length === 0 ? ( + dispatch(loadConnectors({ token }))} + > + {t('retry')} + + } + /> + ) : visible.length === 0 ? ( + + ) : ( +
+ {visible.map((connector) => ( + + ))} +
+ )} + + { + closeDrawer(); + launch(connector); + }} + /> + {modals} +
+ ); +} diff --git a/frontend/src/settings/Sources.tsx b/frontend/src/settings/Sources.tsx index 38aa5297..a95c53ed 100644 --- a/frontend/src/settings/Sources.tsx +++ b/frontend/src/settings/Sources.tsx @@ -48,6 +48,13 @@ import { import { formatDate } from '../utils/dateTimeUtils'; import FileTree from '../components/FileTree'; import ConnectorTree from '../components/ConnectorTree'; +import ConnectorIcon from '../connectors/ConnectorIcon'; +import { + loadConnectors, + selectConnections, + selectConnectorsLoaded, +} from '../connectors/connectorsSlice'; +import type { AppDispatch } from '../store'; import Chunks from '../components/Chunks'; import WikiViewer from '../components/WikiViewer'; import GraphView from '../components/GraphView'; @@ -78,9 +85,15 @@ export default function Sources({ handleDeleteDocument, }: DocumentsProps) { const { t } = useTranslation(); - const dispatch = useDispatch(); + const dispatch = useDispatch(); const token = useSelector(selectToken); const uploadTasks = useSelector(selectUploadTasks); + const connections = useSelector(selectConnections); + const connectorsLoaded = useSelector(selectConnectorsLoaded); + + useEffect(() => { + if (!connectorsLoaded) dispatch(loadConnectors({ token })); + }, [connectorsLoaded, dispatch, token]); const [searchTerm, setSearchTerm] = useState(''); const debouncedSearchTerm = useDebouncedValue(searchTerm, 500); @@ -620,6 +633,12 @@ export default function Sources({
{currentDocuments.map((document, index) => { const docId = document.id ? document.id.toString() : ''; + const connection = document.connectionId + ? connections.find((c) => c.id === document.connectionId) + : undefined; + const paused = + connection?.status === 'reconnect_needed' || + connection?.status === 'disconnected'; return (
@@ -670,6 +689,17 @@ export default function Sources({ : t('teamAccess.viewer')} )} + {connection && paused && ( + + {t('settings.connectors.detail.paused')} + + )} {document.ingestStatus === 'failed' && ( {t('settings.sources.ingestFailed')} @@ -713,6 +743,23 @@ export default function Sources({ ); })()} + {connection && ( + + + + {t('settings.tools.viaConnection', { + name: connection.account_label, + interpolation: { escapeValue: false }, + })} + + + )} {document.date ? formatDate(document.date) : ''} diff --git a/frontend/src/settings/Tools.tsx b/frontend/src/settings/Tools.tsx index eb408ad4..1e7c6731 100644 --- a/frontend/src/settings/Tools.tsx +++ b/frontend/src/settings/Tools.tsx @@ -1,7 +1,8 @@ import { Pencil, RefreshCw, Trash2, Users } from 'lucide-react'; import React from 'react'; import { useTranslation } from 'react-i18next'; -import { useSelector } from 'react-redux'; +import { useDispatch, useSelector } from 'react-redux'; +import { Link, useLocation, useNavigate } from 'react-router-dom'; import devicesService from '../api/services/devicesService'; import userService from '../api/services/userService'; @@ -15,12 +16,18 @@ import { Card, CardDescription, CardTitle } from '../components/ui/card'; import { Switch } from '../components/ui/switch'; import { ActionMenu, type MenuOption } from '../components/ui/dropdown-menu'; import { EmptyState } from '../components/ui/empty-state'; +import ConnectorIcon from '../connectors/ConnectorIcon'; +import { + loadConnectors, + selectConnections, +} from '../connectors/connectorsSlice'; import { useLoaderState } from '../hooks'; import AddToolModal from '../modals/AddToolModal'; import ConfirmationModal from '../modals/ConfirmationModal'; import MCPServerModal from '../modals/MCPServerModal'; import { ActiveState } from '../models/misc'; import { selectToken } from '../preferences/preferenceSlice'; +import type { AppDispatch } from '../store'; import ShareToTeamModal from '../teams/ShareToTeamModal'; import RemoteDeviceConfig from './RemoteDeviceConfig'; import ToolConfig from './ToolConfig'; @@ -29,6 +36,10 @@ import { APIToolType, UserToolType } from './types'; export default function Tools() { const { t } = useTranslation(); const token = useSelector(selectToken); + const dispatch = useDispatch(); + const connections = useSelector(selectConnections); + const location = useLocation(); + const navigate = useNavigate(); const [searchTerm, setSearchTerm] = React.useState(''); const [addToolModalState, setAddToolModalState] = @@ -236,7 +247,18 @@ export default function Tools() { React.useEffect(() => { getUserTools(); fetchMcpStatuses(); + dispatch(loadConnectors({ token })); }, []); + + // The Connectors page creates an OpenAPI tool and sends its id here so + // the spec import opens straight away. + const openToolId = (location.state as { openToolId?: string } | null) + ?.openToolId; + React.useEffect(() => { + if (!openToolId) return; + handleToolAdded(openToolId); + navigate(location.pathname, { replace: true, state: null }); + }, [openToolId]); return (
{selectedTool ? ( @@ -363,6 +385,37 @@ export default function Tools() { > {tool.customName || tool.displayName} + {(() => { + const connection = tool.connection_id + ? connections.find( + (c) => c.id === tool.connection_id, + ) + : undefined; + if (!connection) return null; + return ( + + ); + })()} } /> } /> + } /> } /> getDefaultMiddleware().concat( From 45c4d928c44c979f6b58624f5d552e92843ff4fe Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:30:56 +0100 Subject: [PATCH 003/162] Group Add Source and Add Tool by where things come from Add Source splits into Upload and web, and From a connection. Connector tiles are always shown and say whether an account is connected or the server still needs setup, instead of disappearing when a VITE_ client id is unset; without the Google Picker key, Drive falls back to the server-side file browser. The name follows the first picked item and the button reads Add source. Add Tool splits into Built in and From a service. The MCP and API tool cards move to Connectors > Add custom connector. --- frontend/src/components/FilePicker.tsx | 11 + frontend/src/components/GoogleDrivePicker.tsx | 10 + frontend/src/modals/AddToolModal.tsx | 162 +++++++------- frontend/src/modals/types/index.ts | 3 + frontend/src/upload/Upload.test.tsx | 87 +++++++- frontend/src/upload/Upload.tsx | 202 +++++++++++++++--- frontend/src/upload/types/ingestor.ts | 30 +-- 7 files changed, 389 insertions(+), 116 deletions(-) diff --git a/frontend/src/components/FilePicker.tsx b/frontend/src/components/FilePicker.tsx index f9570bc0..1e0d1ac1 100644 --- a/frontend/src/components/FilePicker.tsx +++ b/frontend/src/components/FilePicker.tsx @@ -75,6 +75,8 @@ interface CloudFilePickerProps { selectedFolderIds?: string[], ) => void; onDisconnect?: () => void; + /** Called with the first item's name when the selection goes from empty to one. */ + onFirstPickName?: (name: string) => void; provider: string; token: string | null; initialSelectedFiles?: string[]; @@ -84,6 +86,7 @@ interface CloudFilePickerProps { export const FilePicker: React.FC = ({ onSelectionChange, onDisconnect, + onFirstPickName, provider, token, initialSelectedFiles = [], @@ -404,6 +407,14 @@ export const FilePicker: React.FC = ({ }; const handleFileSelect = (fileId: string, isFolder: boolean) => { + if ( + selectedFiles.length === 0 && + selectedFolders.length === 0 && + onFirstPickName + ) { + const picked = files.find((file) => file.id === fileId); + if (picked) onFirstPickName(picked.name); + } if (isFolder) { const newSelectedFolders = selectedFolders.includes(fileId) ? selectedFolders.filter((id) => id !== fileId) diff --git a/frontend/src/components/GoogleDrivePicker.tsx b/frontend/src/components/GoogleDrivePicker.tsx index e172040f..99f57abc 100644 --- a/frontend/src/components/GoogleDrivePicker.tsx +++ b/frontend/src/components/GoogleDrivePicker.tsx @@ -33,11 +33,14 @@ interface PickerFile { interface GoogleDrivePickerProps { token: string | null; onSelectionChange: (fileIds: string[], folderIds?: string[]) => void; + /** Called with the first item's name when the selection goes from empty to one. */ + onFirstPickName?: (name: string) => void; } const GoogleDrivePicker: React.FC = ({ token, onSelectionChange, + onFirstPickName, }) => { const { t } = useTranslation(); const [selectedFiles, setSelectedFiles] = useState([]); @@ -191,6 +194,13 @@ const GoogleDrivePicker: React.FC = ({ ); return [...prevFolders, ...uniqueNewFolders]; }); + if ( + selectedFiles.length === 0 && + selectedFolders.length === 0 && + docs.length > 0 + ) { + onFirstPickName?.(docs[0].name); + } onSelectionChange( [...selectedFiles, ...newFiles].map((file) => file.id), [...selectedFolders, ...newFolders].map((folder) => folder.id), diff --git a/frontend/src/modals/AddToolModal.tsx b/frontend/src/modals/AddToolModal.tsx index 81ae3771..421831fe 100644 --- a/frontend/src/modals/AddToolModal.tsx +++ b/frontend/src/modals/AddToolModal.tsx @@ -1,18 +1,21 @@ +import { ArrowRight, Plus } from 'lucide-react'; import React from 'react'; import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; +import { Link } from 'react-router-dom'; import userService from '../api/services/userService'; import SkeletonLoader from '../components/SkeletonLoader'; import ToolIcon from '../components/ToolIcon'; +import { Button } from '../components/ui/button'; import { Card, CardDescription, CardTitle } from '../components/ui/card'; import { Modal } from '../components/ui/modal'; +import { SectionHeader } from '../components/ui/section-header'; import { useLoaderState } from '../hooks'; import PairDeviceModal from '../settings/PairDeviceModal'; import { ActiveState } from '../models/misc'; import { selectToken } from '../preferences/preferenceSlice'; import ConfigToolModal from './ConfigToolModal'; -import MCPServerModal from './MCPServerModal'; import { AvailableToolType } from './types'; export default function AddToolModal({ @@ -39,8 +42,6 @@ export default function AddToolModal({ React.useState(null); const [configModalState, setConfigModalState] = React.useState('INACTIVE'); - const [mcpModalState, setMcpModalState] = - React.useState('INACTIVE'); const [pairModalState, setPairModalState] = React.useState('INACTIVE'); const [loading, setLoading] = useLoaderState(false); @@ -53,7 +54,11 @@ export default function AddToolModal({ return res.json(); }) .then((data) => { - setAvailableTools(data.data); + setAvailableTools( + (data.data as AvailableToolType[]).filter( + (tool) => tool.group !== 'custom', + ), + ); setLoading(false); }); }; @@ -96,9 +101,6 @@ export default function AddToolModal({ .catch((error) => { console.error('Failed to create tool:', error); }); - } else if (tool.name === 'mcp_tool') { - setModalState('INACTIVE'); - setMcpModalState('ACTIVE'); } else { setModalState('INACTIVE'); setConfigModalState('ACTIVE'); @@ -109,11 +111,6 @@ export default function AddToolModal({ if (modalState === 'ACTIVE') getAvailableTools(); }, [modalState]); - const handleMcpServerAdded = () => { - getUserTools(); - setMcpModalState('INACTIVE'); - }; - return ( <> -
-
-
- {loading ? ( -
- -
- ) : ( -
- {availableTools.map((tool, index) => ( - - - - ))} -
- )} +
+ {loading ? ( +
+
+ ) : ( + (['built_in', 'service'] as const).map((group) => { + const tools = availableTools.filter( + (tool) => (tool.group ?? 'built_in') === group, + ); + if (tools.length === 0) return null; + return ( +
+ +
+ {tools.map((tool) => ( + + + + ))} +
+
+ ); + }) + )} +
+ +
@@ -187,11 +210,6 @@ export default function AddToolModal({ tool={selectedTool} getUserTools={getUserTools} /> - ({ useTranslation: () => ({ t: (key: string) => key }), })); +const connectorsState = vi.hoisted(() => ({ + catalog: [] as Record[], + connections: [] as Record[], +})); + vi.mock('react-redux', () => ({ - useSelector: () => null, + useSelector: (selector: (state: unknown) => unknown) => { + const state = { + preference: { token: null, selectedDocs: [] }, + connectors: { + catalog: connectorsState.catalog, + connections: connectorsState.connections, + loaded: true, + loading: false, + failed: false, + }, + }; + try { + return selector(state); + } catch { + return null; + } + }, useDispatch: () => vi.fn(), useStore: () => ({ getState: () => ({}) }), })); @@ -84,6 +105,70 @@ describe('Upload source-type tiles', () => { ); }); + it('groups upload and web apart from connections', async () => { + await render(); + expect(document.body.textContent).toContain( + 'modals.uploadDoc.groupUploadWeb', + ); + expect(document.body.textContent).toContain( + 'modals.uploadDoc.groupConnection', + ); + }); + + it('shows connector tiles even when the server is not set up', async () => { + connectorsState.catalog = [ + { + key: 'google_drive', + icon: 'drive', + sync_ingestor: 'google_drive', + available: false, + missing_settings: ['GOOGLE_CLIENT_ID'], + }, + ]; + await render(); + const drive = tiles().find((tile) => + tile.textContent?.includes( + 'modals.uploadDoc.ingestors.google_drive.label', + ), + ); + expect(drive).toBeDefined(); + expect(drive!.textContent).toContain( + 'settings.connectors.status.needsAdminSetup', + ); + await act(async () => drive!.click()); + expect(document.body.textContent).toContain('GOOGLE_CLIENT_ID'); + connectorsState.catalog = []; + }); + + it('names the connected account on a connection tile', async () => { + connectorsState.catalog = [ + { + key: 'confluence', + icon: 'confluence', + sync_ingestor: 'confluence', + available: true, + missing_settings: [], + }, + ]; + connectorsState.connections = [ + { + id: 'c1', + connector_key: 'confluence', + status: 'connected', + account_label: 'alex@example.com', + }, + ]; + await render(); + const confluence = tiles().find((tile) => + tile.textContent?.includes('modals.uploadDoc.ingestors.confluence.label'), + ); + expect(confluence!.textContent).toContain( + 'modals.uploadDoc.tileConnectedAs', + ); + connectorsState.catalog = []; + connectorsState.connections = []; + }); + it('leaves the disabled Train button on the default variant', async () => { await render(); const crawler = tiles().find((tile) => diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 257032c5..9c02bacb 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -1,4 +1,4 @@ -import { ChevronLeft } from 'lucide-react'; +import { ChevronLeft, CircleAlert } from 'lucide-react'; import { envVar } from '@/env'; import { cn } from '@/lib/utils'; import { useCallback, useEffect, useState } from 'react'; @@ -28,6 +28,16 @@ import { Textarea } from '../components/ui/textarea'; import { Modal } from '../components/ui/modal'; import { Separator } from '../components/ui/separator'; import { OptionCard } from '../components/ui/option-card'; +import { SectionHeader } from '../components/ui/section-header'; +import { Alert, AlertDescription, AlertTitle } from '../components/ui/alert'; +import ConnectorIcon from '../connectors/ConnectorIcon'; +import { + loadConnectors, + selectConnections, + selectConnectorCatalog, + selectConnectorsLoaded, +} from '../connectors/connectorsSlice'; +import type { AppDispatch } from '../store'; import { ActiveState, Doc } from '../models/misc'; import { getDocs } from '../preferences/preferenceApi'; @@ -38,10 +48,12 @@ import { setSourceDocs, } from '../preferences/preferenceSlice'; import { + CONNECTION_INGESTORS, IngestorDefaultConfigs, IngestorFormSchemas, getIngestorSchema, IngestorOption, + UPLOAD_AND_WEB_INGESTORS, } from '../upload/types/ingestor'; import { addUploadTask, updateUploadTask } from './uploadSlice'; @@ -65,6 +77,7 @@ function Upload({ close, onSuccessfulUpload = () => undefined, selectUploadedDoc = true, + initialIngestor, }: { receivedFile: File[]; setModalState: (state: ActiveState) => void; @@ -79,9 +92,14 @@ function Upload({ * uploading never repoints the conversation the user left open. */ selectUploadedDoc?: boolean; + /** Open straight on this source type's form (Connect from a connector card). */ + initialIngestor?: IngestorType; }) { const token = useSelector(selectToken); const selectedDocs = useSelector(selectSelectedDocs); + const connectorCatalog = useSelector(selectConnectorCatalog); + const connections = useSelector(selectConnections); + const connectorsLoaded = useSelector(selectConnectorsLoaded); const [files, setfiles] = useState(receivedFile); const [activeTab, setActiveTab] = useState(true); @@ -334,6 +352,7 @@ function Upload({ setSelectedFiles(selectedFileIds); setSelectedFolders(selectedFolderIds); }} + onFirstPickName={prefillName} provider={ingestor.type as unknown as string} token={token} initialSelectedFiles={selectedFiles} @@ -341,7 +360,9 @@ function Upload({ /> ); case 'google_drive_picker': - return ( + // Google's own picker needs the client id in the browser; without it + // the server-side file browser lists the same Drive. + return envVar('VITE_GOOGLE_CLIENT_ID') ? ( + ) : ( + { + setSelectedFiles(selectedFileIds); + setSelectedFolders(selectedFolderIds); + }} + onFirstPickName={prefillName} + provider="google_drive" + token={token} + initialSelectedFiles={selectedFiles} + initialSelectedFolders={selectedFolders} + /> ); case 'share_point_picker': return ( @@ -365,6 +403,7 @@ function Upload({ setSelectedFiles(selectedFileIds); setSelectedFolders(selectedFolderIds); }} + onFirstPickName={prefillName} provider="share_point" token={token} initialSelectedFiles={selectedFiles} @@ -382,6 +421,7 @@ function Upload({ setSelectedFiles(selectedFileIds); setSelectedFolders(selectedFolderIds); }} + onFirstPickName={prefillName} provider="confluence" token={token} initialSelectedFiles={selectedFiles} @@ -402,17 +442,39 @@ function Upload({ const [nameTouched, setNameTouched] = useState(false); const { t } = useTranslation(); - const dispatch = useDispatch(); + const dispatch = useDispatch(); const store = useStore(); - const ingestorOptions: IngestorOption[] = IngestorFormSchemas.filter( - (schema) => (schema.validate ? schema.validate() : true), - ).map((schema) => ({ - label: schema.label, - value: schema.key, - icon: schema.icon, - heading: schema.heading, - })); + useEffect(() => { + if (!connectorsLoaded) dispatch(loadConnectors({ token })); + }, [connectorsLoaded, dispatch, token]); + + useEffect(() => { + if (initialIngestor) handleIngestorTypeChange(initialIngestor); + // Only the type the modal opened with; later picks are the user's. + }, []); + + /** The name field follows the first picked item until the user edits it. */ + const prefillName = (name: string) => { + if (nameTouched || !name) return; + setIngestor((prev) => (prev.name ? prev : { ...prev, name })); + }; + + const connectorFor = (type: IngestorType | null) => + type && CONNECTION_INGESTORS.includes(type) + ? connectorCatalog.find((c) => c.sync_ingestor === type) + : undefined; + const selectedConnector = connectorFor(ingestor.type); + const needsSetup = !!selectedConnector && !selectedConnector.available; + + const ingestorOptions: IngestorOption[] = IngestorFormSchemas.map( + (schema) => ({ + label: schema.label, + value: schema.key, + icon: schema.icon, + heading: schema.heading, + }), + ); const resetUploaderState = useCallback(() => { setIngestor({ type: null, name: '', config: {} }); @@ -923,6 +985,7 @@ function Upload({ if (!isPrescreenConfigValid(retrievalOptions)) return true; if (!ingestor.type) return true; + if (needsSetup) return true; const ingestorSchemaForValidation = getIngestorSchema( ingestor.type as IngestorType, ); @@ -1026,28 +1089,109 @@ function Upload({ } }; + const connectionTileState = (type: IngestorType) => { + const connector = connectorFor(type); + if (!connector) return undefined; + if (!connector.available) + return t('settings.connectors.status.needsAdminSetup'); + const accounts = connections.filter( + (c) => c.connector_key === connector.key && c.status === 'connected', + ); + if (accounts.length === 1) + return t('modals.uploadDoc.tileConnectedAs', { + account: accounts[0].account_label, + interpolation: { escapeValue: false }, + }); + if (accounts.length > 1) + return t('settings.connectors.status.connectedCount', { + count: accounts.length, + }); + return t('settings.connectors.status.connect'); + }; + const renderIngestorSelection = () => { + const optionsFor = (types: IngestorType[]) => + types + .map((type) => ingestorOptions.find((o) => o.value === type)) + .filter((option): option is IngestorOption => !!option); return ( -
- {ingestorOptions.map((option) => ( - - } - title={t(`modals.uploadDoc.ingestors.${option.value}.label`)} - onClick={() => - handleIngestorTypeChange(option.value as IngestorType) - } +
+
+ - ))} +
+ {optionsFor(UPLOAD_AND_WEB_INGESTORS).map((option) => ( + + } + title={t(`modals.uploadDoc.ingestors.${option.value}.label`)} + onClick={() => handleIngestorTypeChange(option.value)} + /> + ))} +
+
+
+ +
+ {optionsFor(CONNECTION_INGESTORS).map((option) => { + const connector = connectorFor(option.value); + return ( + + } + title={t(`modals.uploadDoc.ingestors.${option.value}.label`)} + description={connectionTileState(option.value)} + onClick={() => handleIngestorTypeChange(option.value)} + /> + ); + })} +
+
); }; + + const renderSetupNotice = () => + selectedConnector && needsSetup ? ( + + + + {t('settings.connectors.status.needsAdminSetup')} + + +
+ {selectedConnector.missing_settings.length > 0 ? ( + <> + {t('settings.connectors.setupSettings')} + + {selectedConnector.missing_settings.join(', ')} + + + ) : ( + {t('settings.connectors.askAdmin')} + )} +
+ + + ) : null; return ( - {renderFormFields()} - {ingestor.type !== 'wiki' && ( + {needsSetup ? renderSetupNotice() : renderFormFields()} + {ingestor.type !== 'wiki' && !needsSetup && ( boolean; fields: FormField[]; } +/** Add Source groups: what needs no account, and what comes from a connection. */ +export const UPLOAD_AND_WEB_INGESTORS: IngestorType[] = [ + 'local_file', + 'url', + 'crawler', + 'github', + 'wiki', +]; +export const CONNECTION_INGESTORS: IngestorType[] = [ + 'google_drive', + 'share_point', + 'confluence', + 's3', + 'reddit', +]; + export const IngestorFormSchemas: IngestorSchema[] = [ { key: 'local_file', @@ -175,10 +189,6 @@ export const IngestorFormSchemas: IngestorSchema[] = [ label: 'Google Drive', icon: DriveIcon, heading: 'Upload from Google Drive', - validate: () => { - const googleClientId = envVar('VITE_GOOGLE_CLIENT_ID'); - return !!googleClientId; - }, fields: [ { name: 'files', @@ -243,10 +253,6 @@ export const IngestorFormSchemas: IngestorSchema[] = [ label: 'Share Point', icon: SharePoint, heading: 'Upload from Share Point', - validate: () => { - const sharePointClientId = envVar('VITE_SHARE_POINT_CLIENT_ID'); - return !!sharePointClientId; - }, fields: [ { name: 'files', @@ -261,10 +267,6 @@ export const IngestorFormSchemas: IngestorSchema[] = [ label: 'Confluence', icon: ConfluenceIcon, heading: 'Upload from Confluence', - validate: () => { - const confluenceClientId = envVar('VITE_CONFLUENCE_CLIENT_ID'); - return !!confluenceClientId; - }, fields: [ { name: 'files', From 667d4bbab0ef228a99301ec219895cefba318ddd Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:03:02 +0100 Subject: [PATCH 004/162] Encrypt connection credentials with an owner-bound AES-GCM envelope Adds a v2 credential envelope next to the v1 tool-secret helpers: AES-256-GCM, a master key derived once per process from ENCRYPTION_SECRET_KEY, and a per-record key from HKDF over the owner's id, which is also the associated data, so a blob moved onto another user's row does not decrypt. The envelope names its key, so ENCRYPTION_SECRET_KEY_PREVIOUS keeps old rows readable during a rotation. Log redaction now also covers token_info, tokens and client_info, and the API warns at startup when the public default key is in use. --- docs/content/Deploying/Settings-Reference.mdx | 8 +- docsgpt/app.py | 21 +++ docsgpt/core/settings/auth.py | 12 +- docsgpt/security/encryption.py | 137 ++++++++++++++++++ docsgpt/storage/db/redaction.py | 6 +- tests/security/test_encryption_v2.py | 94 ++++++++++++ 6 files changed, 275 insertions(+), 3 deletions(-) create mode 100644 tests/security/test_encryption_v2.py diff --git a/docs/content/Deploying/Settings-Reference.mdx b/docs/content/Deploying/Settings-Reference.mdx index b08af68a..4f44bb47 100644 --- a/docs/content/Deploying/Settings-Reference.mdx +++ b/docs/content/Deploying/Settings-Reference.mdx @@ -34,7 +34,13 @@ Signing key for session tokens and other signed capabilities. Required on every Type `str`, default `default-docsgpt-encryption-key`. -Key used to encrypt stored credentials such as tool and connector secrets. +Key used to encrypt stored credentials such as tool and connector secrets. Set your own value before connecting services on a multi-user install; the default is public. + +### `ENCRYPTION_SECRET_KEY_PREVIOUS` + +Type `str`, default unset. + +Previous ENCRYPTION_SECRET_KEY, tried when a stored credential was encrypted with it. Set it while rotating the key, run `docsgpt connectors reencrypt`, then remove it. ### `INTERNAL_KEY` diff --git a/docsgpt/app.py b/docsgpt/app.py index 5a45ab31..1e018bf8 100644 --- a/docsgpt/app.py +++ b/docsgpt/app.py @@ -184,6 +184,27 @@ if settings.AUTH_TYPE == "simple_jwt": print(f"Generated Simple JWT Token: {SIMPLE_JWT_TOKEN}") +def _warn_default_encryption_key() -> None: + """Say when stored credentials are sealed with the public default key.""" + from docsgpt.security.encryption import is_default_encryption_key + + if not is_default_encryption_key(): + return + if settings.AUTH_TYPE: + logging.getLogger(__name__).warning( + "ENCRYPTION_SECRET_KEY is the public default: connecting services is refused until you set your " + "own value (then run `docsgpt connectors reencrypt`)." + ) + else: + logging.getLogger(__name__).warning( + "ENCRYPTION_SECRET_KEY is the public default. Stored connector credentials are only as safe as " + "that key; set your own value before exposing this install." + ) + + +_warn_default_encryption_key() + + @app.route("/") def home(): if request.remote_addr in ("0.0.0.0", "127.0.0.1", "localhost", "172.18.0.1"): diff --git a/docsgpt/core/settings/auth.py b/docsgpt/core/settings/auth.py index 706a3339..9c6deacb 100644 --- a/docsgpt/core/settings/auth.py +++ b/docsgpt/core/settings/auth.py @@ -29,7 +29,17 @@ class AuthSettings(SettingsGroup): ) ENCRYPTION_SECRET_KEY: str = Field( default="default-docsgpt-encryption-key", - description="Key used to encrypt stored credentials such as tool and connector secrets.", + description=( + "Key used to encrypt stored credentials such as tool and connector secrets. Set your own value before " + "connecting services on a multi-user install; the default is public." + ), + ) + ENCRYPTION_SECRET_KEY_PREVIOUS: Optional[str] = Field( + default=None, + description=( + "Previous ENCRYPTION_SECRET_KEY, tried when a stored credential was encrypted with it. Set it while " + "rotating the key, run `docsgpt connectors reencrypt`, then remove it." + ), ) INTERNAL_KEY: Optional[str] = Field( default=None, description="Internal API key for worker-to-backend authentication." diff --git a/docsgpt/security/encryption.py b/docsgpt/security/encryption.py index aa77069d..5793fc7c 100644 --- a/docsgpt/security/encryption.py +++ b/docsgpt/security/encryption.py @@ -1,11 +1,17 @@ import base64 +import functools +import hashlib +import hmac import json import logging import os +from typing import Optional from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.ciphers import algorithms, Cipher, modes +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC from docsgpt.core.settings import settings @@ -86,3 +92,134 @@ def _pad_data(data: bytes) -> bytes: def _unpad_data(data: bytes) -> bytes: padding_len = data[-1] return data[:-padding_len] + + +# --------------------------------------------------------------------------- +# Envelope v2: connection credentials +# --------------------------------------------------------------------------- +# +# ``v2::`` +# +# AES-256-GCM, so a tampered blob fails to decrypt instead of returning +# garbage. The master key is derived once per process from +# ENCRYPTION_SECRET_KEY (PBKDF2, cached); each record gets its own key from +# HKDF(master, salt, owner id), which keeps the v1 owner binding without +# paying 100k PBKDF2 iterations on every token read in the worker. The owner +# id is also the GCM associated data, so a blob copied onto another user's +# row does not decrypt. ``key_id`` names the master key, so a blob written +# under ENCRYPTION_SECRET_KEY_PREVIOUS is still readable during a rotation. + +_V2_PREFIX = "v2" +_V2_MASTER_SALT = b"docsgpt-credentials-v2" +_V2_ITERATIONS = 200_000 +_V2_SALT_BYTES = 16 +_V2_NONCE_BYTES = 12 +DEFAULT_ENCRYPTION_KEY = "default-docsgpt-encryption-key" + + +class CredentialDecryptionError(Exception): + """A stored credential could not be decrypted (wrong key, tampering, bad format).""" + + +@functools.lru_cache(maxsize=8) +def _master_key(secret: str) -> bytes: + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=32, + salt=_V2_MASTER_SALT, + iterations=_V2_ITERATIONS, + backend=default_backend(), + ) + return kdf.derive(secret.encode()) + + +def _key_id(master: bytes) -> str: + return hmac.new(master, b"docsgpt-key-id", hashlib.sha256).hexdigest()[:8] + + +def _record_key(master: bytes, owner_id: str, salt: bytes) -> bytes: + return HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + info=b"docsgpt-v2|" + owner_id.encode(), + backend=default_backend(), + ).derive(master) + + +def _candidate_keys() -> dict[str, bytes]: + """Master keys this process can decrypt with, by key id (current first).""" + keys: dict[str, bytes] = {} + for secret in (settings.ENCRYPTION_SECRET_KEY, settings.ENCRYPTION_SECRET_KEY_PREVIOUS): + if secret: + master = _master_key(secret) + keys.setdefault(_key_id(master), master) + return keys + + +def current_key_id() -> str: + """Key id of ENCRYPTION_SECRET_KEY, as written into new v2 blobs.""" + return _key_id(_master_key(settings.ENCRYPTION_SECRET_KEY)) + + +def is_default_encryption_key() -> bool: + """Whether ENCRYPTION_SECRET_KEY is still the public default.""" + return settings.ENCRYPTION_SECRET_KEY == DEFAULT_ENCRYPTION_KEY + + +def encrypt_json(data: dict, owner_id: str) -> str: + """Encrypt ``data`` for ``owner_id`` into a v2 envelope. + + Args: + data: JSON-serialisable credentials. + owner_id: The user the credentials belong to; decryption needs it. + + Returns: + The ``v2::`` string. + """ + master = _master_key(settings.ENCRYPTION_SECRET_KEY) + key_id = _key_id(master) + salt = os.urandom(_V2_SALT_BYTES) + nonce = os.urandom(_V2_NONCE_BYTES) + key = _record_key(master, owner_id, salt) + plaintext = json.dumps(data, separators=(",", ":")).encode() + ciphertext = AESGCM(key).encrypt(nonce, plaintext, owner_id.encode()) + payload = base64.b64encode(salt + nonce + ciphertext).decode() + return f"{_V2_PREFIX}:{key_id}:{payload}" + + +def envelope_key_id(blob: str) -> Optional[str]: + """The key id a v2 blob was written with, or None for anything else.""" + parts = (blob or "").split(":", 2) + if len(parts) != 3 or parts[0] != _V2_PREFIX: + return None + return parts[1] + + +def decrypt_json(blob: str, owner_id: str) -> dict: + """Decrypt a v2 envelope written for ``owner_id``. + + Raises: + CredentialDecryptionError: The blob is malformed, was written with a + key this process does not have, belongs to another owner, or was + tampered with. + """ + key_id = envelope_key_id(blob) + if key_id is None: + raise CredentialDecryptionError("Not a v2 credential envelope") + master = _candidate_keys().get(key_id) + if master is None: + raise CredentialDecryptionError("Credential was encrypted with an unknown key") + try: + raw = base64.b64decode(blob.split(":", 2)[2].encode(), validate=True) + salt = raw[:_V2_SALT_BYTES] + nonce = raw[_V2_SALT_BYTES:_V2_SALT_BYTES + _V2_NONCE_BYTES] + ciphertext = raw[_V2_SALT_BYTES + _V2_NONCE_BYTES:] + key = _record_key(master, owner_id, salt) + plaintext = AESGCM(key).decrypt(nonce, ciphertext, owner_id.encode()) + data = json.loads(plaintext.decode()) + except Exception as exc: + raise CredentialDecryptionError("Credential could not be decrypted") from exc + if not isinstance(data, dict): + raise CredentialDecryptionError("Credential payload is not an object") + return data diff --git a/docsgpt/storage/db/redaction.py b/docsgpt/storage/db/redaction.py index cb1e4754..483c7564 100644 --- a/docsgpt/storage/db/redaction.py +++ b/docsgpt/storage/db/redaction.py @@ -33,13 +33,17 @@ _SECRET_SUBSTRINGS = ( "credential", "authorization", "bearer", + # Connection secrets: an OAuth token_info blob, MCP token and client + # registration dicts (``client_secret`` is covered by ``secret``). + "token_info", + "client_info", ) def is_secret_key(key: str) -> bool: """True when ``key`` names a credential that must not be persisted/returned.""" k = key.lower() - if k == "token": + if k in ("token", "tokens"): return True return any(s in k for s in _SECRET_SUBSTRINGS) diff --git a/tests/security/test_encryption_v2.py b/tests/security/test_encryption_v2.py new file mode 100644 index 00000000..1551c45b --- /dev/null +++ b/tests/security/test_encryption_v2.py @@ -0,0 +1,94 @@ +"""Tests for the v2 credential envelope.""" + +from __future__ import annotations + +import base64 + +import pytest + +from docsgpt.security import encryption as enc + + +@pytest.fixture(autouse=True) +def _keys(monkeypatch): + from docsgpt.core.settings import settings + + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", "current-key-for-tests") + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY_PREVIOUS", None) + yield + + +class TestEnvelope: + def test_round_trip(self): + blob = enc.encrypt_json({"access_token": "at", "n": 1}, "alice") + assert blob.startswith("v2:") + assert enc.decrypt_json(blob, "alice") == {"access_token": "at", "n": 1} + + def test_plaintext_never_in_blob(self): + blob = enc.encrypt_json({"refresh_token": "very-secret-refresh"}, "alice") + assert "very-secret-refresh" not in blob + assert "very-secret-refresh".encode() not in base64.b64decode(blob.split(":", 2)[2]) + + def test_each_blob_is_unique(self): + assert enc.encrypt_json({"a": 1}, "alice") != enc.encrypt_json({"a": 1}, "alice") + + def test_bound_to_owner(self): + blob = enc.encrypt_json({"a": 1}, "alice") + with pytest.raises(enc.CredentialDecryptionError): + enc.decrypt_json(blob, "bob") + + def test_tampering_is_detected(self): + blob = enc.encrypt_json({"a": 1}, "alice") + prefix, key_id, payload = blob.split(":", 2) + raw = bytearray(base64.b64decode(payload)) + raw[-1] ^= 0x01 + tampered = f"{prefix}:{key_id}:{base64.b64encode(bytes(raw)).decode()}" + with pytest.raises(enc.CredentialDecryptionError): + enc.decrypt_json(tampered, "alice") + + @pytest.mark.parametrize("blob", ["", "v1:abc", "v2:only-two", "v2:deadbeef:%%%not-base64"]) + def test_malformed(self, blob): + with pytest.raises(enc.CredentialDecryptionError): + enc.decrypt_json(blob, "alice") + + def test_key_id_names_the_key(self): + blob = enc.encrypt_json({"a": 1}, "alice") + assert enc.envelope_key_id(blob) == enc.current_key_id() + assert enc.envelope_key_id("not an envelope") is None + + +class TestRotation: + def test_unknown_key_fails(self, monkeypatch): + from docsgpt.core.settings import settings + + blob = enc.encrypt_json({"a": 1}, "alice") + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", "a-new-key") + with pytest.raises(enc.CredentialDecryptionError): + enc.decrypt_json(blob, "alice") + + def test_previous_key_still_decrypts(self, monkeypatch): + from docsgpt.core.settings import settings + + blob = enc.encrypt_json({"a": 1}, "alice") + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", "a-new-key") + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY_PREVIOUS", "current-key-for-tests") + assert enc.decrypt_json(blob, "alice") == {"a": 1} + assert enc.envelope_key_id(enc.encrypt_json({"a": 1}, "alice")) == enc.current_key_id() + assert enc.envelope_key_id(blob) != enc.current_key_id() + + +class TestDefaultKey: + def test_detects_default(self, monkeypatch): + from docsgpt.core.settings import settings + + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", enc.DEFAULT_ENCRYPTION_KEY) + assert enc.is_default_encryption_key() + + def test_custom_key(self): + assert not enc.is_default_encryption_key() + + +class TestLegacyV1Unchanged: + def test_v1_round_trip_still_works(self): + blob = enc.encrypt_credentials({"token": "t"}, "alice") + assert enc.decrypt_credentials(blob, "alice") == {"token": "t"} From 3511004f50827b19593fbae8a64b6a6eddeb7f9d Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:03:09 +0100 Subject: [PATCH 005/162] Connections own their credentials Migration 0038 moves every stored secret (OAuth tokens, MCP OAuth tokens and client registrations, API keys) into the connection's encrypted envelope, links API-key tools to one connection per distinct credential, allows several accounts per provider, and adds credential_mode to sources and tools. OAuth MCP tools keep resolving each member's own token, as they did before. docsgpt.connectors.service is now the only reader of OAuth tokens: get_valid_token_info refreshes under a row lock and persists rotated refresh tokens, and a revoked grant flags the connection, pauses its sources and notifies the owner. Loaders build from a connection (BaseConnectorLoader.from_connection), so scheduled sync covers Drive, SharePoint and Confluence sources with no browser. S3 and Reddit keys stay on the connection instead of in remote_data. New endpoints: POST /api/connections, /setup, /reconnect, /picker-token, /claim, DELETE /api/connections/, per-action permissions and MCP refresh-tools. Upload, file listing, sync and validate-session take a connection_id; session tokens keep working for this release. The tool executor reads credentials from the resolved connection (owner or member mode) and pauses on a Connect card when a connection needs signing in. docsgpt connectors reencrypt rewrites stored credentials after a key rotation. --- docsgpt/agents/tool_executor.py | 124 ++- docsgpt/agents/tools/brave.py | 2 + docsgpt/agents/tools/mcp_tool.py | 97 +- docsgpt/agents/tools/ntfy.py | 1 + docsgpt/agents/tools/postgres.py | 2 + docsgpt/agents/tools/telegram.py | 2 + docsgpt/alembic/versions/0038_connections.py | 297 +++++- docsgpt/api/connector/connections.py | 346 ++++++- docsgpt/api/connector/routes.py | 217 ++-- docsgpt/api/user/sources/routes.py | 103 +- docsgpt/api/user/sources/upload.py | 64 +- docsgpt/api/user/tasks.py | 26 +- docsgpt/api/user/tools/mcp.py | 106 +- docsgpt/api/user/tools/routes.py | 88 ++ docsgpt/cli.py | 30 + docsgpt/connectors/mcp.py | 63 ++ docsgpt/connectors/resolve.py | 128 +++ docsgpt/connectors/service.py | 944 +++++++++++++++++- docsgpt/llm/handlers/base.py | 4 + docsgpt/parser/connectors/base.py | 47 +- docsgpt/parser/connectors/confluence/auth.py | 27 - .../parser/connectors/confluence/loader.py | 30 +- .../parser/connectors/google_drive/auth.py | 34 - .../parser/connectors/google_drive/loader.py | 6 +- docsgpt/parser/connectors/share_point/auth.py | 36 - .../parser/connectors/share_point/loader.py | 24 +- docsgpt/storage/db/models.py | 10 + .../db/repositories/connector_sessions.py | 94 +- docsgpt/storage/db/repositories/user_tools.py | 20 +- docsgpt/worker.py | 154 ++- tests/agents/tools/test_mcp_tool.py | 12 +- tests/api/test_connector_oauth_security.py | 6 +- tests/api/test_connector_routes_happy.py | 8 +- .../user/sources/test_upload_idempotency.py | 4 + tests/api/user/test_tasks.py | 6 +- tests/connectors/test_connections_api.py | 323 ++++++ tests/connectors/test_runtime.py | 254 +++++ tests/connectors/test_service_credentials.py | 420 ++++++++ tests/connectors/test_service_read.py | 4 +- .../confluence/test_auth_session.py | 80 -- .../connectors/confluence/test_loader.py | 59 +- .../connectors/test_auth_token_leakage.py | 50 +- .../connectors/test_connector_creator.py | 11 +- .../connectors/test_google_drive_auth.py | 65 -- .../connectors/test_google_drive_loader.py | 11 +- .../connectors/test_share_point_auth.py | 60 -- .../connectors/test_share_point_loader.py | 34 +- tests/parser/connectors/token_patch.py | 23 + .../parser/remote/test_share_point_loader.py | 12 +- tests/storage/db/test_migration_0038.py | 174 ++++ tests/test_extra_coverage.py | 10 +- tests/worker/test_ingest_connector.py | 3 +- tests/worker/test_remote_worker.py | 3 +- tests/worker/test_worker_sse_publishes.py | 3 +- 54 files changed, 4007 insertions(+), 754 deletions(-) create mode 100644 docsgpt/connectors/mcp.py create mode 100644 docsgpt/connectors/resolve.py create mode 100644 tests/connectors/test_connections_api.py create mode 100644 tests/connectors/test_runtime.py create mode 100644 tests/connectors/test_service_credentials.py create mode 100644 tests/parser/connectors/token_patch.py diff --git a/docsgpt/agents/tool_executor.py b/docsgpt/agents/tool_executor.py index bc7f809a..2e9faf9f 100644 --- a/docsgpt/agents/tool_executor.py +++ b/docsgpt/agents/tool_executor.py @@ -485,6 +485,9 @@ class ToolExecutor: self._tool_to_name: Dict[Tuple[str, str], str] = {} # Filled by the LLMHandler.handle_tool_calls headless loop. self.headless_denials: List[Dict] = [] + # Per-turn connection resolution for connection-backed tools, keyed + # by tool row id, so check_pause and execute share one lookup. + self._connections: Dict[str, Any] = {} def get_tools(self) -> Dict[str, Dict]: """Load tool configs from DB based on user context. @@ -785,6 +788,33 @@ class ToolExecutor: return result return resolve_tool_result(result, decision) + def _resolve_connection(self, tool_data: Dict): + """The connection a connection-backed tool runs with this turn, or None.""" + if not tool_data.get("connection_id") or tool_data.get("client_side"): + return None + key = str(tool_data.get("id") or tool_data.get("connection_id")) + if key not in self._connections: + from docsgpt.connectors.resolve import resolve_connection + + try: + self._connections[key] = resolve_connection(tool_data, self.user) + except Exception: + logger.exception("connection resolution failed for tool %s", key) + self._connections[key] = None + return self._connections[key] + + @staticmethod + def _connection_payload(resolved) -> Dict: + """What the chat's Connect card needs; never an account or a secret.""" + return { + "connector_key": resolved.connector_key, + "connector_name": resolved.connector_name, + "status": ( + "missing" if resolved.row is None + else (resolved.row.get("status") or "reconnect_needed") + ), + } + def check_pause(self, tools_dict: Dict, call, llm_class_name: str) -> Optional[Dict]: """Return a pending-action dict (approval / client / headless_denied) or None. @@ -831,6 +861,41 @@ class ToolExecutor: "thought_signature": getattr(call, "thought_signature", None), } + # A tool whose connection needs signing in pauses on a Connect card + # (the approval card's connection variant) instead of failing; the + # user connects, then continues, and the pending call resumes. + resolved = self._resolve_connection(tool_data) + if resolved is not None and not resolved.available: + if self.headless: + return { + "call_id": call_id, + "name": llm_name, + "tool_name": tool_data.get("name", "unknown"), + "tool_id": tool_id, + "action_name": action_name, + "llm_name": llm_name, + "arguments": arguments, + "pause_type": "headless_denied", + "deny_reason": ( + f"{resolved.connector_name or 'This service'} needs to be connected " + "before this tool can run." + ), + "error_type": "connection_required", + "thought_signature": getattr(call, "thought_signature", None), + } + return { + "call_id": call_id, + "name": llm_name, + "tool_name": tool_data.get("name", "unknown"), + "tool_id": tool_id, + "action_name": action_name, + "llm_name": llm_name, + "arguments": arguments, + "pause_type": "awaiting_approval", + "connection_required": self._connection_payload(resolved), + "thought_signature": getattr(call, "thought_signature", None), + } + # Approval required if tool_data["name"] == "api_tool": action_data = tool_data.get("config", {}).get("actions", {}).get(action_name, {}) @@ -1285,16 +1350,24 @@ class ToolExecutor: target_dict[param] = value # Load tool (with caching) - tool = self._get_or_load_tool( - tool_data, - tool_id, - action_name, - headers=headers, - query_params=query_params, - ) + from docsgpt.connectors.service import ConnectionUnavailable + + connection_error = None + try: + tool = self._get_or_load_tool( + tool_data, + tool_id, + action_name, + headers=headers, + query_params=query_params, + ) + except ConnectionUnavailable as exc: + tool, connection_error = None, str(exc) if tool is None: - error_message = ( + error_message = connection_error and ( + f"{connection_error}. Ask the user to connect it in Settings > Connectors, then try again." + ) or ( f"Failed to load tool '{tool_data.get('name')}' (tool_id key={tool_id}): missing 'id' on tool row." ) logger.error( @@ -1476,7 +1549,10 @@ class ToolExecutor: # silently decrypt-failing. Falls back to self.user for the # agentless path where the tool row carries no user_id. tool_owner = tool_data.get("user_id") or self.user - if tool_config.get("encrypted_credentials") and tool_owner: + resolved = self._resolve_connection(tool_data) + if resolved is not None: + self._apply_connection(tool_data, tool_id, tool_config, resolved) + elif tool_config.get("encrypted_credentials") and tool_owner: if tool_owner != self.user: # Credential delegation: the invoker is running a shared # tool with the owner's secrets. Audit it (the agent-run @@ -1540,6 +1616,36 @@ class ToolExecutor: return tool + def _apply_connection(self, tool_data: Dict, tool_id: str, tool_config: Dict, resolved) -> None: + """Merge a connection's credentials into ``tool_config``. + + Raises: + ConnectionUnavailable: The connection needs signing in again. + """ + from docsgpt.connectors import service + from docsgpt.connectors.resolve import audit_delegation + + if not resolved.available or resolved.row is None: + raise service.ConnectionUnavailable( + f"{resolved.connector_name or 'This service'} needs to be connected", + connection_id=resolved.connection_id, + ) + audit_delegation( + resolved, + invoker=self.user, + resource_type="tool", + resource_id=str(tool_data.get("id") or tool_id), + agent_id=self.agent_id, + ) + tool_config.pop("encrypted_credentials", None) + if (resolved.row.get("auth_kind") or "") in ("api_key", "none"): + credentials = service.get_credentials(resolved.row) + tool_config.update(credentials) + tool_config["auth_credentials"] = credentials + if tool_data.get("name") == "mcp_tool": + # MCP OAuth tokens are read by connection id inside the tool. + tool_config["connection_id"] = resolved.connection_id + # Keys the client needs that are not part of the fixed shape below. They are # small and optional, and are copied only when present so an ordinary tool # call does not grow null columns in every persisted row. diff --git a/docsgpt/agents/tools/brave.py b/docsgpt/agents/tools/brave.py index 532e310f..c58d784a 100644 --- a/docsgpt/agents/tools/brave.py +++ b/docsgpt/agents/tools/brave.py @@ -136,6 +136,7 @@ class BraveSearchTool(Tool): return [ { "name": "brave_web_search", + "access": "read", "description": ( "Search the web with Brave Search. Returns result titles, " "URLs, and snippets. Use it for current events or " @@ -163,6 +164,7 @@ class BraveSearchTool(Tool): }, { "name": "brave_image_search", + "access": "read", "description": ( "Search for images with Brave Search. Returns image " "titles, page URLs, and thumbnail URLs." diff --git a/docsgpt/agents/tools/mcp_tool.py b/docsgpt/agents/tools/mcp_tool.py index 061f4c6d..c3cfdc91 100644 --- a/docsgpt/agents/tools/mcp_tool.py +++ b/docsgpt/agents/tools/mcp_tool.py @@ -31,6 +31,19 @@ logger = logging.getLogger(__name__) _mcp_clients_cache = {} +_ANNOTATION_HINTS = ("readOnlyHint", "destructiveHint", "idempotentHint", "openWorldHint") + + +def _annotation_hints(annotations: Any) -> Dict[str, bool]: + """The boolean MCP tool annotation hints, from a model or a dict.""" + if annotations is None: + return {} + if hasattr(annotations, "model_dump"): + annotations = annotations.model_dump() + if not isinstance(annotations, dict): + return {} + return {k: annotations[k] for k in _ANNOTATION_HINTS if isinstance(annotations.get(k), bool)} + class MCPTool(Tool): """ @@ -38,6 +51,8 @@ class MCPTool(Tool): Connect to remote Model Context Protocol (MCP) servers to access dynamic tools and resources. """ + connection_id: Optional[str] = None + def __init__(self, config: Dict[str, Any], user_id: Optional[str] = None): """ Initialize the MCP Tool with configuration. @@ -76,6 +91,10 @@ class MCPTool(Tool): self.oauth_scopes = config.get("oauth_scopes", []) self.oauth_task_id = config.get("oauth_task_id", None) self.oauth_client_name = config.get("oauth_client_name", "DocsGPT-MCP") + # The connection whose OAuth tokens this tool uses. Set by the tool + # executor for connection-backed tools, so a shared tool in ``owner`` + # mode signs in with its owner's account rather than the invoker's. + self.connection_id = config.get("connection_id") self.redirect_uri = self._resolve_redirect_uri(config.get("redirect_uri")) # Pulled out of ``config`` (rather than left in ``self.config``) # because it is a callable supplied by the OAuth worker — not @@ -125,7 +144,9 @@ class MCPTool(Tool): auth_key = "" if self.auth_type == "oauth": scopes_str = ",".join(self.oauth_scopes) if self.oauth_scopes else "none" - oauth_identity = self.user_id or self.oauth_task_id or "anonymous" + # A connection-backed tool shares a client only with calls that + # use the same connection's tokens. + oauth_identity = self.connection_id or self.user_id or self.oauth_task_id or "anonymous" auth_key = ( f"oauth:{oauth_identity}:{self.oauth_client_name}:{scopes_str}:{self.redirect_uri}" ) @@ -165,6 +186,7 @@ class MCPTool(Tool): redis_client=redis_client, redirect_uri=self.redirect_uri, user_id=self.user_id, + connection_id=self.connection_id, ) else: auth = DocsGPTOAuth( @@ -175,6 +197,7 @@ class MCPTool(Tool): task_id=self.oauth_task_id, user_id=self.user_id, redirect_publish=self.oauth_redirect_publish, + connection_id=self.connection_id, ) elif self.auth_type == "bearer": token = self.auth_credentials.get( @@ -245,6 +268,9 @@ class MCPTool(Tool): } if hasattr(tool, "inputSchema"): tool_dict["inputSchema"] = tool.inputSchema + annotations = _annotation_hints(getattr(tool, "annotations", None)) + if annotations: + tool_dict["annotations"] = annotations tools_dict.append(tool_dict) elif isinstance(tool, dict): tools_dict.append(tool) @@ -493,7 +519,7 @@ class MCPTool(Tool): def _test_oauth_connection(self) -> Dict: storage = DBTokenStorage( - server_url=self.server_url, user_id=self.user_id, + server_url=self.server_url, user_id=self.user_id, connection_id=self.connection_id, ) loop = asyncio.new_event_loop() try: @@ -580,6 +606,11 @@ class MCPTool(Tool): "description": tool.get("description", ""), "parameters": parameters_schema, } + # ``readOnlyHint`` / ``destructiveHint`` decide whether the action + # is a read (always allowed) or a write (needs approval). + annotations = _annotation_hints(tool.get("annotations")) + if annotations: + action["annotations"] = annotations actions.append(action) return actions @@ -688,6 +719,7 @@ class DocsGPTOAuth(OAuthClientProvider): additional_client_metadata: dict[str, Any] | None = None, skip_redirect_validation: bool = False, redirect_publish=None, + connection_id: Optional[str] = None, ): self.redirect_uri = redirect_uri self.redis_client = redis_client @@ -717,6 +749,7 @@ class DocsGPTOAuth(OAuthClientProvider): server_url=self.server_base_url, user_id=self.user_id, expected_redirect_uri=None if skip_redirect_validation else redirect_uri, + connection_id=connection_id, ) super().__init__( @@ -836,15 +869,26 @@ class NonInteractiveOAuth(DocsGPTOAuth): class DBTokenStorage(TokenStorage): + """MCP OAuth tokens and client registration, kept encrypted on the connection. + + Reads and writes go through ``docsgpt.connectors.service``, which stores + them in the connection's owner-bound ``encrypted_credentials``. A tool + that runs with a specific connection (``owner`` mode on a shared tool) + passes its ``connection_id``; otherwise the invoking user's connection + for the server's base URL is used. + """ + def __init__( self, server_url: str, user_id: str, expected_redirect_uri: Optional[str] = None, + connection_id: Optional[str] = None, ): self.server_url = server_url self.user_id = user_id self.expected_redirect_uri = expected_redirect_uri + self.connection_id = connection_id @staticmethod def get_base_url(url: str) -> str: @@ -855,26 +899,12 @@ class DBTokenStorage(TokenStorage): return f"mcp:{self.get_base_url(self.server_url)}" def _fetch_session_data(self) -> dict: - """Read the JSONB ``session_data`` blob for this MCP server row.""" - from docsgpt.storage.db.repositories.connector_sessions import ( - ConnectorSessionsRepository, - ) - from docsgpt.storage.db.session import db_readonly + """The decrypted ``tokens`` / ``client_info`` for this MCP server.""" + from docsgpt.connectors import service - base_url = self.get_base_url(self.server_url) - with db_readonly() as conn: - row = ConnectorSessionsRepository(conn).get_by_user_and_server_url( - self.user_id, base_url, - ) - if not row: - return {} - data = row.get("session_data") or {} - if isinstance(data, str): - try: - data = json.loads(data) - except ValueError: - return {} - return data if isinstance(data, dict) else {} + return service.read_mcp_secrets( + self.user_id, self.get_base_url(self.server_url), self.connection_id, + ) async def get_tokens(self) -> OAuthToken | None: data = await asyncio.to_thread(self._fetch_session_data) @@ -887,22 +917,19 @@ class DBTokenStorage(TokenStorage): return None def _merge(self, patch: dict) -> None: - """Shallow-merge ``patch`` into this row's ``session_data``. + """Merge ``patch`` into the connection's secrets; ``None`` drops a key.""" + from docsgpt.connectors import service - Threads ``server_url`` through to the repository so it lands in - the scalar column — ``get_by_user_and_server_url`` needs that to - resolve the row (``NULL = 'https://...'`` is UNKNOWN in SQL). - """ - from docsgpt.storage.db.repositories.connector_sessions import ( - ConnectorSessionsRepository, + status = None + if patch.get("tokens"): + status = service.STATUS_CONNECTED + service.update_mcp_secrets( + self.user_id, + self.get_base_url(self.server_url), + patch, + connection_id=self.connection_id, + status=status, ) - from docsgpt.storage.db.session import db_session - - base_url = self.get_base_url(self.server_url) - with db_session() as conn: - ConnectorSessionsRepository(conn).merge_session_data( - self.user_id, self._pg_provider(), base_url, patch, - ) def _delete(self) -> None: from docsgpt.storage.db.repositories.connector_sessions import ( diff --git a/docsgpt/agents/tools/ntfy.py b/docsgpt/agents/tools/ntfy.py index 8d52d3f5..d719211f 100644 --- a/docsgpt/agents/tools/ntfy.py +++ b/docsgpt/agents/tools/ntfy.py @@ -89,6 +89,7 @@ class NtfyTool(Tool): return [ { "name": "ntfy_send_message", + "access": "write", "description": ( "Send a push notification to an ntfy topic on the " "configured server. Provide the message text; title and " diff --git a/docsgpt/agents/tools/postgres.py b/docsgpt/agents/tools/postgres.py index 35aaab81..d465faee 100644 --- a/docsgpt/agents/tools/postgres.py +++ b/docsgpt/agents/tools/postgres.py @@ -136,6 +136,7 @@ class PostgresTool(Tool): return [ { "name": "postgres_execute_sql", + "access": "write", "description": "Execute an SQL query against the PostgreSQL database and return the results. Use this tool to interact with the database, e.g., retrieve specific data or perform updates. Only SELECT queries will return data, other queries will return execution status.", "parameters": { "type": "object", @@ -151,6 +152,7 @@ class PostgresTool(Tool): }, { "name": "postgres_get_schema", + "access": "read", "description": "Retrieve the schema of the PostgreSQL database, including tables and their columns. Use this to understand the database structure before executing queries. db_name is 'default' if not provided.", "parameters": { "type": "object", diff --git a/docsgpt/agents/tools/telegram.py b/docsgpt/agents/tools/telegram.py index cf8ca67a..138a0240 100644 --- a/docsgpt/agents/tools/telegram.py +++ b/docsgpt/agents/tools/telegram.py @@ -45,6 +45,7 @@ class TelegramTool(Tool): return [ { "name": "telegram_send_message", + "access": "write", "description": ( "Send a text message to the configured Telegram chat via " "the bot. Compose the final message text before sending." @@ -67,6 +68,7 @@ class TelegramTool(Tool): }, { "name": "telegram_send_image", + "access": "write", "description": ( "Send an image to the configured Telegram chat. Requires " "a publicly accessible image URL." diff --git a/docsgpt/alembic/versions/0038_connections.py b/docsgpt/alembic/versions/0038_connections.py index 3ce6aa43..42586c75 100644 --- a/docsgpt/alembic/versions/0038_connections.py +++ b/docsgpt/alembic/versions/0038_connections.py @@ -11,7 +11,16 @@ and lets sources and tools point at the row they use: resources a connection feeds. ``ON DELETE SET NULL`` keeps a source's indexed content when its connection is removed. -Backfill (idempotent, only fills NULLs): +Connections own their credentials. Every secret of a connection (OAuth +tokens, MCP OAuth tokens and the dynamic client registration, API keys) +moves into ``encrypted_credentials``, a v2 envelope from +``docsgpt.security.encryption`` bound to the owner. Plain columns +(``status``, ``has_refresh_token``, ``scopes``, ``expires_at``) keep status +checks from ever decrypting. The unique index gains ``account_label`` so one +user can connect several accounts of the same service, and +``credential_mode`` says whose account a shared tool or source uses. + +Backfill (idempotent, only fills NULLs or unconverted rows): 1. ``connector_key``, ``auth_kind``, ``display_name`` from ``provider``. 2. ``account_label`` from ``user_email`` for OAuth rows. @@ -19,6 +28,17 @@ Backfill (idempotent, only fills NULLs): owner's only row for ``remote_data->>'provider'``. 4. ``user_tools.connection_id`` for OAuth MCP tools, matched to the owner's row for the tool's server base URL. +5. API-key tools (Brave, Telegram, ntfy, PostgreSQL, custom MCP with a key, + bearer token or basic auth) get one connection per distinct credential, + re-encrypted into v2. The tool keeps its v1 copy for one release so a + rollback still works; the executor prefers the connection. +6. ``token_info`` and the secret parts of ``session_data`` (``tokens``, + ``client_info``) are encrypted into ``encrypted_credentials`` and removed + from the plaintext columns. This needs ``ENCRYPTION_SECRET_KEY`` set to + the value the app will run with. Downgrade decrypts them back. +7. ``credential_mode`` is ``owner`` everywhere except OAuth MCP tools, which + resolved each invoking member's own token before this migration and keep + doing so (``member``); owners can switch them in the share dialog. Revision ID: 0038_connections Revises: 0037_request_traces @@ -42,7 +62,22 @@ _OAUTH_PROVIDERS = { } +_TOOL_CONNECTORS = { + "brave": "Brave Search", + "telegram": "Telegram", + "ntfy": "ntfy", + "postgres": "PostgreSQL", +} +_MCP_SECRET_AUTH = ("api_key", "bearer", "basic") +_BATCH = 500 + + def upgrade() -> None: + _upgrade_links() + _upgrade_credentials() + + +def _upgrade_links() -> None: op.execute( """ ALTER TABLE connector_sessions @@ -119,7 +154,267 @@ def upgrade() -> None: ) +def _credential_hint(credentials: dict) -> str: + """``…abcd``: the last four characters of the first secret, never more.""" + for value in credentials.values(): + if isinstance(value, str) and len(value) >= 8: + return "\u2026" + value[-4:] + return "\u2026" + + +def _upgrade_credentials() -> None: + op.execute( + """ + ALTER TABLE connector_sessions + ADD COLUMN IF NOT EXISTS encrypted_credentials TEXT, + ADD COLUMN IF NOT EXISTS has_refresh_token BOOLEAN NOT NULL DEFAULT false, + ADD COLUMN IF NOT EXISTS scopes JSONB NOT NULL DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS last_error TEXT, + ADD COLUMN IF NOT EXISTS last_used_at TIMESTAMPTZ; + """ + ) + for table in ("sources", "user_tools"): + op.execute( + f"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS credential_mode TEXT NOT NULL DEFAULT 'owner' " + f"CONSTRAINT {table}_credential_mode_chk CHECK (credential_mode IN ('owner', 'member'));" + ) + op.execute("DROP INDEX IF EXISTS connector_sessions_user_endpoint_uidx;") + op.execute( + "CREATE UNIQUE INDEX IF NOT EXISTS connector_sessions_account_uidx ON connector_sessions " + "(user_id, provider, COALESCE(server_url, ''), COALESCE(account_label, ''));" + ) + + # Statuses become connected / pending / reconnect_needed / disconnected / + # error. MCP rows never had one; name them before their tokens are sealed. + op.execute("UPDATE connector_sessions SET status = 'connected' WHERE status = 'authorized';") + op.execute( + """ + UPDATE connector_sessions SET status = + CASE WHEN session_data ? 'tokens' THEN 'connected' ELSE 'pending' END + WHERE provider LIKE 'mcp:%' AND status IS NULL; + """ + ) + + bind = op.get_bind() + _encrypt_session_secrets(bind) + _link_api_key_tools(bind) + + # 7: keep OAuth MCP tools on each member's own token, as before. + op.execute( + """ + UPDATE user_tools SET credential_mode = 'member' + WHERE name = 'mcp_tool' AND config->>'auth_type' = 'oauth'; + """ + ) + + +def _encrypt_session_secrets(bind) -> None: + """6: move plaintext tokens into the owner-bound v2 envelope.""" + import json + + from sqlalchemy import text + + from docsgpt.security.encryption import encrypt_json + + while True: + rows = bind.execute( + text( + """ + SELECT id, user_id, token_info, session_data FROM connector_sessions + WHERE encrypted_credentials IS NULL + AND (token_info IS NOT NULL OR session_data ? 'tokens' OR session_data ? 'client_info') + LIMIT :batch + """ + ), + {"batch": _BATCH}, + ).fetchall() + if not rows: + return + for row in rows: + token_info = row.token_info if isinstance(row.token_info, dict) else None + session_data = dict(row.session_data or {}) + secrets = {} + if token_info: + secrets["token_info"] = token_info + for key in ("tokens", "client_info"): + if key in session_data: + secrets[key] = session_data.pop(key) + tokens = secrets.get("tokens") if isinstance(secrets.get("tokens"), dict) else {} + has_refresh = bool((token_info or {}).get("refresh_token") or tokens.get("refresh_token")) + scopes = (token_info or {}).get("scopes") or [] + if isinstance(scopes, str): + scopes = scopes.split() + bind.execute( + text( + """ + UPDATE connector_sessions SET + encrypted_credentials = :blob, + has_refresh_token = :has_refresh, + scopes = CAST(:scopes AS jsonb), + token_info = NULL, + session_data = CAST(:session_data AS jsonb) + WHERE id = :id + """ + ), + { + "blob": encrypt_json(secrets, row.user_id), + "has_refresh": has_refresh, + "scopes": json.dumps(list(scopes)), + "session_data": json.dumps(session_data), + "id": row.id, + }, + ) + + +def _link_api_key_tools(bind) -> None: + """5: one connection per distinct API credential, linked from its tools.""" + from urllib.parse import urlparse + + from sqlalchemy import text + + from docsgpt.security.encryption import decrypt_credentials, encrypt_json + + rows = bind.execute( + text( + """ + SELECT id, user_id, name, custom_name, display_name, config FROM user_tools + WHERE connection_id IS NULL + AND config ? 'encrypted_credentials' + AND ( + name = ANY(:names) + OR (name = 'mcp_tool' AND config->>'auth_type' = ANY(:mcp_auth)) + ) + """ + ), + {"names": list(_TOOL_CONNECTORS), "mcp_auth": list(_MCP_SECRET_AUTH)}, + ).fetchall() + for row in rows: + config = row.config or {} + credentials = decrypt_credentials(config.get("encrypted_credentials") or "", row.user_id) + if not credentials: + # Written with a different key; the tool keeps its v1 copy. + continue + if row.name == "mcp_tool": + parsed = urlparse(config.get("server_url") or "") + server_url = f"{parsed.scheme}://{parsed.netloc}" if parsed.netloc else None + connector_key = "custom_mcp" + display_name = row.custom_name or row.display_name or parsed.netloc or "MCP server" + else: + server_url = None + connector_key = row.name + display_name = _TOOL_CONNECTORS[row.name] + label = _credential_hint(credentials) + existing = bind.execute( + text( + """ + SELECT id, encrypted_credentials FROM connector_sessions + WHERE user_id = :user_id AND provider = :provider + AND COALESCE(server_url, '') = COALESCE(:server_url, '') + AND COALESCE(account_label, '') = :label + """ + ), + {"user_id": row.user_id, "provider": connector_key, "server_url": server_url, "label": label}, + ).fetchone() + if existing is None: + connection_id = bind.execute( + text( + """ + INSERT INTO connector_sessions ( + user_id, provider, server_url, connector_key, display_name, account_label, + auth_kind, status, encrypted_credentials, session_data + ) VALUES ( + :user_id, :provider, :server_url, :provider, :display_name, :label, + 'api_key', 'connected', :blob, '{}'::jsonb + ) RETURNING id + """ + ), + { + "user_id": row.user_id, + "provider": connector_key, + "server_url": server_url, + "display_name": display_name, + "label": label, + "blob": encrypt_json({"credentials": credentials}, row.user_id), + }, + ).scalar() + else: + connection_id = existing.id + bind.execute( + text("UPDATE user_tools SET connection_id = :cid WHERE id = :id"), + {"cid": connection_id, "id": row.id}, + ) + + +def _downgrade_credentials() -> None: + """Decrypt the envelopes back into the pre-0038 plaintext columns.""" + import json + + from sqlalchemy import text + + from docsgpt.security.encryption import CredentialDecryptionError, decrypt_json + + bind = op.get_bind() + # Tools keep their v1 secrets; connections made only for them go away. + bind.execute(text("UPDATE user_tools SET connection_id = NULL WHERE connection_id IN " + "(SELECT id FROM connector_sessions WHERE auth_kind = 'api_key')")) + bind.execute(text("DELETE FROM connector_sessions WHERE auth_kind = 'api_key'")) + rows = bind.execute( + text( + "SELECT id, user_id, session_data, encrypted_credentials FROM connector_sessions " + "WHERE encrypted_credentials IS NOT NULL" + ) + ).fetchall() + for row in rows: + try: + secrets = decrypt_json(row.encrypted_credentials, row.user_id) + except CredentialDecryptionError: + continue + session_data = dict(row.session_data or {}) + for key in ("tokens", "client_info"): + if key in secrets: + session_data[key] = secrets[key] + bind.execute( + text( + "UPDATE connector_sessions SET token_info = CAST(:token_info AS jsonb), " + "session_data = CAST(:session_data AS jsonb) WHERE id = :id" + ), + { + "token_info": json.dumps(secrets["token_info"]) if "token_info" in secrets else None, + "session_data": json.dumps(session_data), + "id": row.id, + }, + ) + # Several accounts per provider cannot survive the old unique index: keep + # the most recently updated one. + op.execute( + """ + DELETE FROM connector_sessions c USING connector_sessions newer + WHERE c.user_id = newer.user_id AND c.provider = newer.provider + AND COALESCE(c.server_url, '') = COALESCE(newer.server_url, '') + AND (c.updated_at, c.id) < (newer.updated_at, newer.id); + """ + ) + op.execute("DROP INDEX IF EXISTS connector_sessions_account_uidx;") + op.execute( + "CREATE UNIQUE INDEX IF NOT EXISTS connector_sessions_user_endpoint_uidx " + "ON connector_sessions (user_id, COALESCE(server_url, ''), provider);" + ) + for table in ("sources", "user_tools"): + op.execute(f"ALTER TABLE {table} DROP COLUMN IF EXISTS credential_mode;") + op.execute( + """ + ALTER TABLE connector_sessions + DROP COLUMN IF EXISTS last_used_at, + DROP COLUMN IF EXISTS last_error, + DROP COLUMN IF EXISTS scopes, + DROP COLUMN IF EXISTS has_refresh_token, + DROP COLUMN IF EXISTS encrypted_credentials; + """ + ) + + def downgrade() -> None: + _downgrade_credentials() op.execute("DROP INDEX IF EXISTS user_tools_connection_idx;") op.execute("DROP INDEX IF EXISTS sources_connection_idx;") op.execute("ALTER TABLE user_tools DROP COLUMN IF EXISTS connection_id;") diff --git a/docsgpt/api/connector/connections.py b/docsgpt/api/connector/connections.py index 46cb9edc..32bbd0af 100644 --- a/docsgpt/api/connector/connections.py +++ b/docsgpt/api/connector/connections.py @@ -11,12 +11,16 @@ from __future__ import annotations from flask import current_app, jsonify, make_response, request from flask_restx import Namespace, Resource +import uuid + from docsgpt.api import api from docsgpt.api.user.authz import ROLE_ADMIN, has_role -from docsgpt.connectors import service +from docsgpt.connectors import catalog, service from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository from docsgpt.storage.db.session import db_readonly, db_session +_FREQUENCIES = ("never", "daily", "weekly", "monthly") + connections_ns = Namespace("connections", description="Connectors and connections", path="/api") api.add_namespace(connections_ns) @@ -52,8 +56,64 @@ class ConnectorCatalog(Resource): return make_response(jsonify({"success": True, "connectors": entries}), 200) +def _json_body() -> dict: + body = request.get_json(silent=True) + return body if isinstance(body, dict) else {} + + +def _owned(conn, connection_id: str, user_id: str): + return ConnectorSessionsRepository(conn).get_for_user(connection_id, user_id) + + +def _error(message: str, status: int, **extra): + return make_response(jsonify({"success": False, "error": message, **extra}), status) + + @connections_ns.route("/connections") class ConnectionsList(Resource): + @api.doc( + description=( + "Create a connection from pasted credentials: " + "{connector_key, credentials, label?}. Same credentials reuse the same connection." + ) + ) + def post(self): + user_id = _user_id() + if not user_id: + return _unauthorized() + body = _json_body() + definition = catalog.get_definition(body.get("connector_key")) + if definition is None or definition.auth_kind != "api_key": + return _error("This connector does not take pasted credentials", 400) + if definition.missing_settings: + return _error("This connector needs admin setup", 400, code="needs_setup") + credentials = body.get("credentials") + if not isinstance(credentials, dict): + return _error("credentials must be an object", 400) + try: + with db_session() as conn: + row, created = service.create_api_key_connection( + conn, user_id, definition, credentials, label=(body.get("label") or None), + ) + except service.EncryptionKeyNotConfigured as err: + return _error(str(err), 400, code="encryption_key_default") + except ValueError as err: + return _error(str(err), 400) + except Exception as err: + current_app.logger.error(f"Error creating connection: {err}", exc_info=True) + return _error("Failed to create connection", 500) + return make_response( + jsonify( + { + "success": True, + "created": created, + "connection": service.serialize_connection(row), + "setup": dict(definition.setup), + } + ), + 201 if created else 200, + ) + @api.doc(description="The caller's connections with status and linked resource counts") def get(self): user_id = _user_id() @@ -86,6 +146,35 @@ class ConnectionDetail(Resource): return make_response(jsonify({"success": False, "error": "Failed to load connection"}), 500) return make_response(jsonify({"success": True, "connection": detail}), 200) + @api.doc( + description=( + "Remove a connection: {sources: keep | delete, tools: delete | keep}. " + "Kept sources keep their content and stop syncing." + ) + ) + def delete(self, connection_id: str): + from docsgpt.api.user.sources.routes import delete_source + + user_id = _user_id() + if not user_id: + return _unauthorized() + body = _json_body() + sources_mode = body.get("sources", "keep") + tools_mode = body.get("tools", "delete") + if sources_mode not in ("keep", "delete") or tools_mode not in ("keep", "delete"): + return _error("sources and tools must be keep or delete", 400) + try: + with db_session() as conn: + row = _owned(conn, connection_id, user_id) + if row is None: + return _not_found() + to_delete = service.remove_connection(conn, row, sources=sources_mode, tools=tools_mode) + failed = [str(doc["id"]) for doc in to_delete if not delete_source(user_id, doc)] + except Exception as err: + current_app.logger.error(f"Error removing connection: {err}", exc_info=True) + return _error("Failed to remove connection", 500) + return make_response(jsonify({"success": True, "failed_sources": failed}), 200) + @connections_ns.route("/connections//disconnect") class ConnectionDisconnect(Resource): @@ -104,3 +193,258 @@ class ConnectionDisconnect(Resource): current_app.logger.error(f"Error disconnecting connection: {err}", exc_info=True) return make_response(jsonify({"success": False, "error": "Failed to disconnect"}), 500) return make_response(jsonify({"success": True, "connection": connection}), 200) + + +@connections_ns.route("/connections//setup") +class ConnectionSetup(Resource): + @api.doc( + description=( + "Apply the connect wizard's choices: {create_tools, tool_permissions?, " + "sync?: {items, frequency, name?}}. Honours an Idempotency-Key header for the sync." + ) + ) + def post(self, connection_id: str): + user_id = _user_id() + if not user_id: + return _unauthorized() + body = _json_body() + try: + with db_session() as conn: + row = _owned(conn, connection_id, user_id) + if row is None: + return _not_found() + if service.normalize_status(row) != service.STATUS_CONNECTED: + return _error("Reconnect before setting up", 409, code="reconnect") + tools = [] + if body.get("create_tools", True): + tools = service.ensure_connection_tools( + conn, user_id, row, permissions=body.get("tool_permissions") or None, + ) + tool_payload = [service.serialize_tool(tool) for tool in tools] + sources = [] + if body.get("sync"): + started = _start_sync(user_id, row, body["sync"]) + if isinstance(started, tuple): + return _error(*started) + sources.append(started) + except Exception as err: + current_app.logger.error(f"Error setting up connection: {err}", exc_info=True) + return _error("Failed to set up connection", 500) + return make_response(jsonify({"success": True, "tools": tool_payload, "sources": sources}), 200) + + +def _start_sync(user_id: str, row: dict, sync: dict): + """Queue the first ingest of a source synced from ``row``. + + Returns the source summary, or ``(message, status)`` on a bad request. + """ + from docsgpt.api.user.sources.upload import ( + _claim_task_or_get_cached, + _derive_source_id, + _read_idempotency_key, + _scoped_idempotency_key, + ) + from docsgpt.api.user.tasks import ingest_connector_task, ingest_remote + + definition = catalog.get_definition(catalog.connector_key_for_row(row)) + if definition is None or not definition.sync_ingestor: + return ("This connector does not sync content", 400) + items = sync.get("items") or {} + if not isinstance(items, dict): + return ("items must be an object", 400) + frequency = sync.get("frequency") or definition.default_sync_frequency + if frequency not in _FREQUENCIES: + return ("Unknown sync frequency", 400) + name = (sync.get("name") or "").strip() or definition.name + idempotency_key, _ = _read_idempotency_key() + scoped_key = _scoped_idempotency_key(idempotency_key, user_id) + task_id = None + if scoped_key: + task_id, cached = _claim_task_or_get_cached(scoped_key, "connection_setup_sync") + if cached is not None: + return {"id": cached.get("source_id"), "task_id": cached.get("task_id"), "name": name} + source_id = str(_derive_source_id(scoped_key)) if scoped_key else str(uuid.uuid4()) + options = {"task_id": task_id} if task_id else {} + if definition.auth_kind == "oauth": + file_ids = [str(i) for i in items.get("file_ids") or [] if i] + folder_ids = [str(i) for i in items.get("folder_ids") or [] if i] + if not file_ids and not folder_ids: + return ("Pick at least one file or folder", 400) + task = ingest_connector_task.apply_async( + kwargs={ + "job_name": name, + "user": user_id, + "source_type": definition.sync_ingestor, + "connection_id": str(row["id"]), + "file_ids": file_ids, + "folder_ids": folder_ids, + "recursive": bool(items.get("recursive", True)), + "sync_frequency": frequency, + "idempotency_key": scoped_key, + "source_id": source_id, + }, + **options, + ) + else: + fields = {f.key for f in definition.setup_fields} + source_data = {k: v for k, v in items.items() if k in fields and v not in (None, "")} + missing = [f.label for f in definition.setup_fields if f.required and f.key not in source_data] + if missing: + return (f"Missing: {', '.join(missing)}", 400) + task = ingest_remote.apply_async( + kwargs={ + "source_data": source_data, + "job_name": name, + "user": user_id, + "loader": definition.sync_ingestor, + "connection_id": str(row["id"]), + "sync_frequency": frequency, + "idempotency_key": scoped_key, + "source_id": source_id, + }, + **options, + ) + return {"id": source_id, "task_id": task_id or task.id, "name": name, "sync_frequency": frequency} + + +@connections_ns.route("/connections//reconnect") +class ConnectionReconnect(Resource): + @api.doc( + description=( + "OAuth: returns an authorization URL for the same account. " + "API key: accepts {credentials} and replaces the stored ones." + ) + ) + def post(self, connection_id: str): + from docsgpt.api.connector.routes import build_authorization + + user_id = _user_id() + if not user_id: + return _unauthorized() + body = _json_body() + try: + with db_readonly() as conn: + row = _owned(conn, connection_id, user_id) + if row is None: + return _not_found() + key = catalog.connector_key_for_row(row) + definition = catalog.get_definition(key) + auth_kind = row.get("auth_kind") or (definition.auth_kind if definition else None) + if auth_kind == "oauth": + started = build_authorization(row["provider"], user_id, connection_id) + return make_response(jsonify({"success": True, "kind": "oauth", **started}), 200) + if auth_kind == "mcp_oauth": + # The MCP client runs the OAuth dance (dynamic registration, + # PKCE); the frontend starts it through /api/mcp_server/test. + return make_response( + jsonify({"success": True, "kind": "mcp_oauth", "server_url": row.get("server_url")}), 200, + ) + credentials = body.get("credentials") + if not isinstance(credentials, dict) or not credentials: + return _error("credentials are required", 400) + service.ensure_can_store_credentials() + with db_session() as conn: + locked = ConnectorSessionsRepository(conn).get_for_update(connection_id) + stored = service.load_secrets(locked) or {} + merged = {**(stored.get("credentials") or {}), **{k: v for k, v in credentials.items() if v}} + service.write_secrets( + conn, locked, {**stored, "credentials": merged}, + status=service.STATUS_CONNECTED, last_error=None, + ) + service.resume_sources(conn, connection_id) + connection = service.serialize_connection(ConnectorSessionsRepository(conn).get(connection_id)) + except service.EncryptionKeyNotConfigured as err: + return _error(str(err), 400, code="encryption_key_default") + except Exception as err: + current_app.logger.error(f"Error reconnecting: {err}", exc_info=True) + return _error("Failed to reconnect", 500) + return make_response(jsonify({"success": True, "kind": "api_key", "connection": connection}), 200) + + +@connections_ns.route("/connections//picker-token") +class ConnectionPickerToken(Resource): + @api.doc(description="A short-lived access token for a browser-side file picker. Owner only; never a refresh token.") + def post(self, connection_id: str): + user_id = _user_id() + if not user_id: + return _unauthorized() + with db_readonly() as conn: + row = _owned(conn, connection_id, user_id) + if row is None or (row.get("auth_kind") or "oauth") != "oauth": + return _not_found() + try: + token = service.picker_token(connection_id) + except service.ConnectionUnavailable: + return _error("Reconnect to continue", 409, code="reconnect") + except service.TransientConnectionError: + return _error("The provider is not responding. Try again.", 503) + return make_response(jsonify({"success": True, **token}), 200) + + +@connections_ns.route("/connections/claim") +class ConnectionClaim(Resource): + @api.doc( + description=( + "One-time link of a legacy browser session token ({provider, session_token}) " + "to the caller's connection. Removed next release." + ) + ) + def post(self): + user_id = _user_id() + if not user_id: + return _unauthorized() + body = _json_body() + provider, token = body.get("provider"), body.get("session_token") + if not provider or not token: + return _error("provider and session_token are required", 400) + with db_readonly() as conn: + row = service.claim_session_token(conn, user_id, str(provider), str(token)) + if row is None: + return _not_found() + return make_response(jsonify({"success": True, "connection_id": str(row["id"])}), 200) + + +@connections_ns.route("/connections//tools//permissions") +class ConnectionToolPermissions(Resource): + @api.doc(description="Set per-action permissions: {permissions: {action: always | ask | off}}") + def put(self, connection_id: str, tool_id: str): + from docsgpt.connectors.permissions import PERMISSIONS + + user_id = _user_id() + if not user_id: + return _unauthorized() + permissions = _json_body().get("permissions") + if not isinstance(permissions, dict) or any(p not in PERMISSIONS for p in permissions.values()): + return _error("permissions must map action names to always, ask or off", 400) + with db_session() as conn: + row = _owned(conn, connection_id, user_id) + if row is None: + return _not_found() + tool = service.set_tool_permissions(conn, user_id, tool_id, permissions) + if tool is None or str(tool.get("connection_id")) != connection_id: + return _not_found() + payload = service.serialize_tool(tool) + return make_response(jsonify({"success": True, "tool": payload}), 200) + + +@connections_ns.route("/connections//refresh-tools") +class ConnectionRefreshTools(Resource): + @api.doc(description="MCP: re-scan the server's actions and return what was added and removed") + def post(self, connection_id: str): + from docsgpt.connectors.mcp import refresh_mcp_tools + + user_id = _user_id() + if not user_id: + return _unauthorized() + with db_readonly() as conn: + row = _owned(conn, connection_id, user_id) + if row is None: + return _not_found() + try: + diff = refresh_mcp_tools(user_id, row) + except service.ConnectionUnavailable: + return _error("Reconnect to continue", 409, code="reconnect") + except Exception as err: + current_app.logger.error(f"Error refreshing MCP tools: {err}", exc_info=True) + return _error("Failed to refresh tools", 502) + return make_response(jsonify({"success": True, **diff}), 200) diff --git a/docsgpt/api/connector/routes.py b/docsgpt/api/connector/routes.py index 5240ee58..03822102 100644 --- a/docsgpt/api/connector/routes.py +++ b/docsgpt/api/connector/routes.py @@ -1,7 +1,6 @@ import base64 import html import json -import uuid from typing import Optional from urllib.parse import urlencode, urlsplit @@ -20,11 +19,11 @@ from docsgpt.api import api from docsgpt.api.user.tasks import ( ingest_connector_task, ) +from docsgpt.connectors import service from docsgpt.core.settings import settings from docsgpt.parser.connectors.connector_creator import ConnectorCreator from docsgpt.storage.db.repositories.connector_sessions import ( ConnectorSessionsRepository, - owns_connector_session, ) from docsgpt.storage.db.repositories.sources import SourcesRepository from docsgpt.storage.db.session import db_readonly, db_session @@ -100,6 +99,7 @@ def _js_literal(value) -> str: def _render_callback_page( status: str, message: str, provider_raw: str, session_token: str = "", user_email: str = "", + connection_id: str = "", ): """Popup page that reports an OAuth result to the opener on allowed origins only.""" status = status if status in ("success", "error", "cancelled") else "error" @@ -110,7 +110,14 @@ def _render_callback_page( ) payload = None if provider_key and status == "success" and session_token: - payload = {"type": f"{provider_key}_auth_success", "session_token": session_token, "user_email": user_email} + payload = { + "type": f"{provider_key}_auth_success", + # The connection id is what current frontends use; the session + # token is kept for one release for frontends from before it. + "connection_id": connection_id, + "session_token": session_token, + "user_email": user_email, + } elif provider_key and status == "error": # The frontend shows its own localized failure message; cancellations are # reported when the popup closes. @@ -170,6 +177,27 @@ def _render_callback_page( +def build_authorization(provider: str, user_id: str, connection_id: Optional[str] = None) -> dict: + """Start an OAuth sign-in for ``provider`` and return its authorization URL. + + Raises: + service.EncryptionKeyNotConfigured: See ``ensure_can_store_credentials``. + service.ConnectionUnavailable: ``connection_id`` is not the caller's. + """ + service.ensure_can_store_credentials() + with db_session() as conn: + session_row = service.begin_oauth(conn, user_id, provider, connection_id) + state = base64.urlsafe_b64encode( + json.dumps({"provider": provider, "object_id": str(session_row["id"])}).encode() + ).decode() + auth = ConnectorCreator.create_auth(provider) + return { + "authorization_url": auth.get_authorization_url(state=state), + "state": state, + "callback_origin": _origin_of(settings.CONNECTOR_REDIRECT_BASE_URI), + } + + @connectors_ns.route("/api/connectors/auth") class ConnectorAuth(Resource): @api.doc(description="Get connector OAuth authorization URL", params={"provider": "Connector provider (e.g., google_drive)"}) @@ -186,20 +214,14 @@ class ConnectorAuth(Resource): if not decoded_token: return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401) user_id = decoded_token.get('sub') - - with db_session() as conn: - session_row = ConnectorSessionsRepository(conn).upsert( - user_id, provider, status="pending", + try: + started = build_authorization(provider, user_id, request.args.get("connection_id") or None) + except service.EncryptionKeyNotConfigured as err: + return make_response( + jsonify({"success": False, "error": str(err), "code": "encryption_key_default"}), 400, ) - session_pg_id = str(session_row["id"]) - state_dict = { - "provider": provider, - "object_id": session_pg_id, - } - state = base64.urlsafe_b64encode(json.dumps(state_dict).encode()).decode() - - auth = ConnectorCreator.create_auth(provider) - authorization_url = auth.get_authorization_url(state=state) + except service.ConnectionUnavailable: + return make_response(jsonify({"success": False, "error": "Connection not found"}), 404) # The popup drops results for origins outside the allowlist, which the # user only sees as a cancelled sign-in; name the missing origin here. request_origin = _origin_of(request.headers.get("Origin")) @@ -208,12 +230,7 @@ class ConnectorAuth(Resource): f"Connector sign-in requested from {request_origin}, which cannot receive the result; " "add it to CONNECTOR_ALLOWED_ORIGINS" ) - return make_response(jsonify({ - "success": True, - "authorization_url": authorization_url, - "state": state, - "callback_origin": _origin_of(settings.CONNECTOR_REDIRECT_BASE_URI), - }), 200) + return make_response(jsonify({"success": True, **started}), 200) except Exception as e: current_app.logger.error(f"Error generating connector auth URL: {e}", exc_info=True) return make_response(jsonify({"success": False, "error": "Failed to generate authorization URL"}), 500) @@ -269,13 +286,11 @@ class ConnectorsCallback(Resource): auth = ConnectorCreator.create_auth(provider) token_info = auth.exchange_code_for_tokens(authorization_code) - session_token = str(uuid.uuid4()) - try: if provider == "google_drive": credentials = auth.create_credentials_from_token_info(token_info) - service = auth.build_drive_service(credentials) - user_info = service.about().get(fields="user").execute() + drive_service = auth.build_drive_service(credentials) + user_info = drive_service.about().get(fields="user").execute() user_email = user_info.get('user', {}).get('emailAddress', 'Connected User') else: user_email = token_info.get('user_info', {}).get('email', 'Connected User') @@ -288,29 +303,23 @@ class ConnectorsCallback(Resource): # ``object_id`` in the OAuth state is the PG session row # UUID (new flow) or a legacy Mongo ObjectId (pre-cutover - # issued state). Try UUID update first; fall back to - # legacy id path. - patch = { - "session_token": session_token, - "token_info": sanitized_token_info, - "user_email": user_email, - "status": "authorized", - } + # issued state). with db_session() as conn: repo = ConnectorSessionsRepository(conn) - if state_object_id: - value = str(state_object_id) - updated = False - if len(value) == 36 and "-" in value: - updated = repo.update(value, patch) - if not updated: - repo.update_by_legacy_id(value, patch) + value = str(state_object_id or "") + state_row = repo.get(value) or (repo.get_by_legacy_id(value) if value else None) + if state_row is None or state_row.get("provider") != provider: + raise ValueError("OAuth state names no pending connection") + connection = service.complete_oauth( + conn, state_row, provider, sanitized_token_info, user_email, + ) # Render instead of redirecting so the session token never # lands in a URL (browser history, access logs, Referer). return _render_callback_page( "success", "Authentication successful", provider, - session_token=session_token, user_email=user_email, + session_token=connection.get("session_token") or "", user_email=user_email, + connection_id=str(connection["id"]), ) except Exception as e: @@ -333,7 +342,8 @@ class ConnectorsCallback(Resource): class ConnectorFiles(Resource): @api.expect(api.model("ConnectorFilesModel", { "provider": fields.String(required=True), - "session_token": fields.String(required=True), + "connection_id": fields.String(required=False), + "session_token": fields.String(required=False, description="Legacy; use connection_id"), "folder_id": fields.String(required=False), "limit": fields.Integer(required=False), "page_token": fields.String(required=False), @@ -344,26 +354,29 @@ class ConnectorFiles(Resource): try: data = request.get_json() provider = data.get('provider') - session_token = data.get('session_token') limit = data.get('limit', 10) - if not provider or not session_token: - return make_response(jsonify({"success": False, "error": "provider and session_token are required"}), 400) + if not provider or not (data.get('connection_id') or data.get('session_token')): + return make_response( + jsonify({"success": False, "error": "provider and connection_id are required"}), 400, + ) decoded_token = request.decoded_token if not decoded_token: return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401) user = decoded_token.get('sub') - with db_readonly() as conn: - session = ConnectorSessionsRepository(conn).get_by_session_token( - session_token, - ) - if not owns_connector_session(session, user, provider): + session = service.resolve_request_connection(user, provider, data) + if session is None: return make_response(jsonify({"success": False, "error": "Invalid or unauthorized session"}), 401) - loader = ConnectorCreator.create_connector(provider, session_token) + try: + loader = ConnectorCreator.create_connector(provider, connection_id=str(session["id"])) + except service.ConnectionUnavailable: + return make_response( + jsonify({"success": False, "error": "Reconnect to continue", "reconnect": True}), 401, + ) - generic_keys = {'provider', 'session_token'} + generic_keys = {'provider', 'session_token', 'connection_id'} input_config = { k: v for k, v in data.items() if k not in generic_keys } @@ -408,65 +421,49 @@ class ConnectorFiles(Resource): @connectors_ns.route("/api/connectors/validate-session") class ConnectorValidateSession(Resource): - @api.expect(api.model("ConnectorValidateSessionModel", {"provider": fields.String(required=True), "session_token": fields.String(required=True)})) - @api.doc(description="Validate connector session token and return user info and access token") + @api.expect(api.model("ConnectorValidateSessionModel", { + "provider": fields.String(required=True), + "connection_id": fields.String(required=False), + "session_token": fields.String(required=False, description="Legacy; use connection_id"), + })) + @api.doc(description="Validate a connection and return the account and a short-lived access token") def post(self): try: - data = request.get_json() + data = request.get_json() or {} provider = data.get('provider') - session_token = data.get('session_token') - if not provider or not session_token: - return make_response(jsonify({"success": False, "error": "provider and session_token are required"}), 400) + if not provider or not (data.get('connection_id') or data.get('session_token')): + return make_response( + jsonify({"success": False, "error": "provider and connection_id are required"}), 400, + ) decoded_token = request.decoded_token if not decoded_token: return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401) user = decoded_token.get('sub') - with db_readonly() as conn: - session = ConnectorSessionsRepository(conn).get_by_session_token( - session_token, - ) - if not owns_connector_session(session, user, provider) or not session.get("token_info"): + session = service.resolve_request_connection(user, provider, data) + if session is None: return make_response(jsonify({"success": False, "error": "Invalid or expired session"}), 401) - - token_info = session["token_info"] - auth = ConnectorCreator.create_auth(provider) - is_expired = auth.is_token_expired(token_info) - - if is_expired and token_info.get('refresh_token'): - try: - refreshed_token_info = auth.refresh_access_token(token_info.get('refresh_token')) - sanitized_token_info = auth.sanitize_token_info(refreshed_token_info) - with db_session() as conn: - repo = ConnectorSessionsRepository(conn) - row = repo.get_by_session_token(session_token) - if row: - repo.update(str(row["id"]), {"token_info": sanitized_token_info}) - token_info = sanitized_token_info - is_expired = False - except Exception as refresh_error: - current_app.logger.error(f"Failed to refresh token: {refresh_error}") - - if is_expired: + try: + token = service.picker_token(str(session["id"])) + except service.ConnectionUnavailable: return make_response(jsonify({ "success": False, "expired": True, "error": "Session token has expired. Please reconnect." }), 401) + except service.TransientConnectionError: + return make_response( + jsonify({"success": False, "error": "The provider is not responding. Try again."}), 503, + ) - _base_fields = {"access_token", "refresh_token", "token_uri", "expiry"} - provider_extras = {k: v for k, v in token_info.items() if k not in _base_fields} - - response_data = { + return make_response(jsonify({ "success": True, "expired": False, - "user_email": session.get('user_email', 'Connected User'), - "access_token": token_info.get('access_token'), - **provider_extras, - } - - return make_response(jsonify(response_data), 200) + "connection_id": str(session["id"]), + "user_email": session.get('account_label') or session.get('user_email') or 'Connected User', + **token, + }), 200) except Exception as e: current_app.logger.error(f"Error validating connector session: {e}", exc_info=True) return make_response(jsonify({"success": False, "error": "Failed to validate session"}), 500) @@ -483,15 +480,13 @@ class ConnectorDisconnect(Resource): try: data = request.get_json() provider = data.get('provider') - session_token = data.get('session_token') if not provider: return make_response(jsonify({"success": False, "error": "provider is required"}), 400) - if session_token: + session = service.resolve_request_connection(decoded_token.get('sub'), provider, data) + if session is not None: with db_session() as conn: - ConnectorSessionsRepository(conn).delete_by_session_token( - session_token, decoded_token.get('sub'), - ) + service.disconnect(conn, session) return make_response(jsonify({"success": True}), 200) except Exception as e: @@ -506,7 +501,10 @@ class ConnectorSync(Resource): "ConnectorSyncModel", { "source_id": fields.String(required=True, description="Source ID to sync"), - "session_token": fields.String(required=True, description="Authentication token") + "connection_id": fields.String( + required=False, description="Connection to sync with; defaults to the source's own", + ), + "session_token": fields.String(required=False, description="Legacy; use connection_id") }, ) ) @@ -517,16 +515,15 @@ class ConnectorSync(Resource): return make_response(jsonify({"success": False}), 401) try: - data = request.get_json() + data = request.get_json() or {} source_id = data.get('source_id') - session_token = data.get('session_token') - if not all([source_id, session_token]): + if not source_id: return make_response( jsonify({ "success": False, - "error": "source_id and session_token are required" - }), + "error": "source_id is required" + }), 400 ) user_id = decoded_token.get('sub') @@ -562,9 +559,11 @@ class ConnectorSync(Resource): 400 ) - with db_readonly() as conn: - session = ConnectorSessionsRepository(conn).get_by_session_token(session_token) - if not owns_connector_session(session, user_id, source_type): + lookup = dict(data) + if not (lookup.get('connection_id') or lookup.get('session_token')) and source.get('connection_id'): + lookup['connection_id'] = str(source['connection_id']) + session = service.resolve_request_connection(user_id, source_type, lookup) + if session is None: return make_response( jsonify({"success": False, "error": "Invalid or unauthorized session"}), 401, @@ -580,7 +579,7 @@ class ConnectorSync(Resource): job_name=source.get('name'), user=decoded_token.get('sub'), source_type=source_type, - session_token=session_token, + connection_id=str(session["id"]), file_ids=file_ids, folder_ids=folder_ids, recursive=recursive, diff --git a/docsgpt/api/user/sources/routes.py b/docsgpt/api/user/sources/routes.py index 7ca3b339..8b13abd1 100644 --- a/docsgpt/api/user/sources/routes.py +++ b/docsgpt/api/user/sources/routes.py @@ -231,6 +231,63 @@ class PaginatedSources(Resource): return make_response(jsonify({"success": False}), 400) +def delete_source(user: str, doc: dict) -> bool: + """Delete a source's index, stored files and row. Returns whether it worked. + + Args: + user: The owner deleting it. + doc: The source row, already authorised for ``user``. + """ + storage = StorageCreator.get_storage() + resolved_id = str(doc["id"]) + source_id = resolved_id + + try: + if settings.VECTOR_STORE == "faiss": + index_path = f"indexes/{resolved_id}" + # index.pkl is the legacy sidecar; index.json the current one. + # Older sources have only the former, so clear whichever exist. + for index_file in ("index.faiss", "index.json", "index.pkl"): + if storage.file_exists(f"{index_path}/{index_file}"): + storage.delete_file(f"{index_path}/{index_file}") + else: + vectorstore = VectorCreator.create_vectorstore( + settings.VECTOR_STORE, source_id=source_id + ) + vectorstore.delete_index() + if "file_path" in doc and doc["file_path"]: + file_path = doc["file_path"] + if storage.is_directory(file_path): + files = storage.list_files(file_path) + for f in files: + storage.delete_file(f) + else: + storage.delete_file(file_path) + except FileNotFoundError: + pass + except Exception as err: + current_app.logger.error( + f"Error deleting files and indexes: {err}", exc_info=True + ) + return False + try: + with db_session() as conn: + SourcesRepository(conn).delete(resolved_id, user) + record_event( + conn, + "source.deleted", + actor=user, + source_id=resolved_id, + name=doc.get("name"), + ) + except Exception as err: + current_app.logger.error( + f"Error deleting source row: {err}", exc_info=True + ) + return False + return True + + @sources_ns.route("/delete_old") class DeleteOldIndexes(Resource): @api.doc( @@ -255,51 +312,7 @@ class DeleteOldIndexes(Resource): return make_response(jsonify({"success": False}), 400) if not doc: return make_response(jsonify({"status": "not found"}), 404) - storage = StorageCreator.get_storage() - resolved_id = str(doc["id"]) - - try: - if settings.VECTOR_STORE == "faiss": - index_path = f"indexes/{resolved_id}" - # index.pkl is the legacy sidecar; index.json the current one. - # Older sources have only the former, so clear whichever exist. - for index_file in ("index.faiss", "index.json", "index.pkl"): - if storage.file_exists(f"{index_path}/{index_file}"): - storage.delete_file(f"{index_path}/{index_file}") - else: - vectorstore = VectorCreator.create_vectorstore( - settings.VECTOR_STORE, source_id=resolved_id - ) - vectorstore.delete_index() - if "file_path" in doc and doc["file_path"]: - file_path = doc["file_path"] - if storage.is_directory(file_path): - files = storage.list_files(file_path) - for f in files: - storage.delete_file(f) - else: - storage.delete_file(file_path) - except FileNotFoundError: - pass - except Exception as err: - current_app.logger.error( - f"Error deleting files and indexes: {err}", exc_info=True - ) - return make_response(jsonify({"success": False}), 400) - try: - with db_session() as conn: - SourcesRepository(conn).delete(resolved_id, user) - record_event( - conn, - "source.deleted", - actor=user, - source_id=resolved_id, - name=doc.get("name"), - ) - except Exception as err: - current_app.logger.error( - f"Error deleting source row: {err}", exc_info=True - ) + if not delete_source(user, doc): return make_response(jsonify({"success": False}), 400) return make_response(jsonify({"success": True}), 200) diff --git a/docsgpt/api/user/sources/upload.py b/docsgpt/api/user/sources/upload.py index f15bee65..52b62fd1 100644 --- a/docsgpt/api/user/sources/upload.py +++ b/docsgpt/api/user/sources/upload.py @@ -28,7 +28,6 @@ from docsgpt.security.zip_archive import ( ) from docsgpt.storage.db.repositories.connector_sessions import ( ConnectorSessionsRepository, - owns_connector_session, ) from docsgpt.storage.db.repositories.idempotency import IdempotencyRepository from docsgpt.storage.db.repositories.sources import SourcesRepository @@ -451,6 +450,43 @@ class UploadFile(Resource): return make_response(jsonify(response_payload), 200) +def _remote_credentials(user, source, config): + """Split an S3 / Reddit request into loader config and the connection holding its keys. + + A request naming a ``connection_id`` uses that connection's stored keys. + A request carrying keys (the form before connections) stores them on a + connection, so they are entered once and never land in + ``sources.remote_data``. When a multi-user install still runs on the + public default encryption key, the keys stay with the source as before. + + Returns: + ``(source_data, connection_id, error_response)``. + """ + from docsgpt.connectors import catalog, service + + definition = catalog.get_definition(source) + credential_keys = {f.key for f in definition.credential_fields} + public = {k: v for k, v in config.items() if k not in credential_keys and k != "connection_id"} + connection_id = config.get("connection_id") + if connection_id: + with db_readonly() as conn: + row = ConnectorSessionsRepository(conn).get_for_user(str(connection_id), user) + if row is None or catalog.connector_key_for_row(row) != source: + return None, None, make_response( + jsonify({"success": False, "error": "Invalid or unauthorized connection"}), 401, + ) + return public, str(row["id"]), None + provided = {k: config[k] for k in credential_keys if config.get(k) not in (None, "")} + if not provided: + return config, None, None + try: + with db_session() as conn: + row, _ = service.create_api_key_connection(conn, user, definition, provided) + except (service.EncryptionKeyNotConfigured, ValueError): + return config, None, None + return public, str(row["id"]), None + + @sources_upload_ns.route("/remote") class UploadRemote(Resource): @api.expect( @@ -520,32 +556,35 @@ class UploadRemote(Resource): try: config = json.loads(data["data"]) source_data = None + connection_id = None if data["source"] == "github": source_data = config.get("repo_url") elif data["source"] in ["crawler", "url", "sitemap"]: source_data = config.get("url") - elif data["source"] == "reddit": - source_data = config - elif data["source"] == "s3": - source_data = config + elif data["source"] in ("reddit", "s3"): + source_data, connection_id, error = _remote_credentials(user, data["source"], config) + if error is not None: + if scoped_key: + _release_claim(scoped_key) + return error elif data["source"] in ConnectorCreator.get_supported_connectors(): - session_token = config.get("session_token") - if not session_token: + if not (config.get("connection_id") or config.get("session_token")): if scoped_key: _release_claim(scoped_key) return make_response( jsonify( { "success": False, - "error": f"Missing session_token in {data['source']} configuration", + "error": f"Missing connection_id in {data['source']} configuration", } ), 400, ) - with db_readonly() as conn: - connector_session = ConnectorSessionsRepository(conn).get_by_session_token(session_token) - if not owns_connector_session(connector_session, user, data["source"]): + from docsgpt.connectors import service as connection_service + + connector_session = connection_service.resolve_request_connection(user, data["source"], config) + if connector_session is None: if scoped_key: _release_claim(scoped_key) return make_response( @@ -576,7 +615,7 @@ class UploadRemote(Resource): "job_name": data["name"], "user": user, "source_type": data["source"], - "session_token": session_token, + "connection_id": str(connector_session["id"]), "file_ids": file_ids, "folder_ids": folder_ids, "recursive": config.get("recursive", False), @@ -611,6 +650,7 @@ class UploadRemote(Resource): remote_kwargs = { "kwargs": { "source_data": source_data, + "connection_id": connection_id, "job_name": data["name"], "user": user, "loader": data["source"], diff --git a/docsgpt/api/user/tasks.py b/docsgpt/api/user/tasks.py index b34945fe..94f5fcb6 100644 --- a/docsgpt/api/user/tasks.py +++ b/docsgpt/api/user/tasks.py @@ -79,6 +79,12 @@ DURABLE_TASK = dict( ) +def _connection_unavailable(): + from docsgpt.connectors.service import ConnectionUnavailable + + return ConnectionUnavailable + + def durable_task(**overrides) -> Dict: """Return ``DURABLE_TASK`` with per-task overrides applied. @@ -167,13 +173,16 @@ def ingest( @with_idempotency(task_name="ingest_remote", on_poison=_emit_ingest_poison_event) def ingest_remote( self, source_data, job_name, user, loader, - config=None, idempotency_key=None, source_id=None, + config=None, idempotency_key=None, source_id=None, connection_id=None, + sync_frequency="never", ): resp = remote_worker( self, source_data, job_name, user, loader, + sync_frequency=sync_frequency, config=config, idempotency_key=idempotency_key, source_id=source_id, + connection_id=connection_id, ) return resp @@ -249,6 +258,15 @@ def schedule_syncs(self, frequency): return resp +@celery.task(bind=True, acks_late=True, autoretry_for=(Exception,), max_retries=3, retry_backoff=60, + dont_autoretry_for=(_connection_unavailable(),)) +def sync_connector_source(self, source_id): + """Re-sync one connector source from its connection, with no browser involved.""" + from docsgpt.worker import sync_connector_source as run + + return run(self, source_id) + + @celery.task(bind=True) def sync_source( self, @@ -408,7 +426,9 @@ except Exception: pass -@celery.task(**DURABLE_TASK) +# A revoked or disconnected connection will not heal by retrying; the +# service has already paused the source and told its owner to reconnect. +@celery.task(**durable_task(dont_autoretry_for=(DocumentParseError, _connection_unavailable()))) @with_idempotency( task_name="ingest_connector_task", on_poison=_emit_ingest_poison_event, ) @@ -428,6 +448,7 @@ def ingest_connector_task( config=None, idempotency_key=None, source_id=None, + connection_id=None, ): from docsgpt.worker import ingest_connector @@ -437,6 +458,7 @@ def ingest_connector_task( user, source_type, session_token=session_token, + connection_id=connection_id, file_ids=file_ids, folder_ids=folder_ids, recursive=recursive, diff --git a/docsgpt/api/user/tools/mcp.py b/docsgpt/api/user/tools/mcp.py index bcc99084..cd69bf75 100644 --- a/docsgpt/api/user/tools/mcp.py +++ b/docsgpt/api/user/tools/mcp.py @@ -75,6 +75,43 @@ def _validate_mcp_server_url(config: dict) -> None: raise ValueError(f"Invalid server URL: {exc}") from exc +def _mcp_connection(user, config, auth_type, auth_credentials, display_name): + """The connection an MCP tool runs with; created on first save. + + OAuth servers already have one (the sign-in stored its tokens there). + Key, bearer and basic auth store their secret on a connection; servers + with no auth get a credential-less connection so they still appear on + the Connectors page. Returns None when a multi-user install runs on the + default encryption key, which keeps the legacy per-tool secret. + """ + from docsgpt.connectors import catalog, service + + base = catalog.base_url(config.get("server_url")) + if not base: + return None + if auth_type == "oauth": + with db_readonly() as conn: + row = service._mcp_row(conn, user, base, None) + return str(row["id"]) if row else None + definition = catalog.get_definition("custom_mcp") + host = base.split("://")[-1] + try: + with db_session() as conn: + if auth_credentials: + row, _ = service.create_api_key_connection( + conn, user, definition, auth_credentials, server_url=base, display_name=display_name, + ) + else: + repo = ConnectorSessionsRepository(conn) + row = repo.find_account(user, "custom_mcp", server_url=base, account_label=host) or repo.create( + user, "custom_mcp", connector_key="custom_mcp", auth_kind="none", + display_name=display_name, account_label=host, server_url=base, + ) + except service.EncryptionKeyNotConfigured: + return None + return str(row["id"]) if row else None + + @tools_mcp_ns.route("/mcp_server/test") class TestMCPServerConfig(Resource): @api.expect( @@ -283,9 +320,19 @@ class MCPServerSave(Resource): "redirect_uri", ]: storage_config.pop(field, None) - transformed_actions = transform_actions(actions_metadata) + from docsgpt.connectors.permissions import apply_default_permissions + + transformed_actions = apply_default_permissions( + "mcp_tool", transform_actions(actions_metadata), + ) display_name = data["displayName"] + connection_id = _mcp_connection( + user, storage_config, auth_type, auth_credentials, display_name, + ) or (str(existing_doc["connection_id"]) if existing_doc and existing_doc.get("connection_id") else None) + if connection_id and auth_type != "oauth": + # The secret lives on the connection only. + storage_config.pop("encrypted_credentials", None) description = f"MCP Server: {storage_config.get('server_url', 'Unknown')}" status_bool = bool(data.get("status", True)) @@ -301,6 +348,7 @@ class MCPServerSave(Resource): "config": storage_config, "actions": transformed_actions, "status": status_bool, + "connection_id": connection_id, }, ) saved_id = str(existing_doc["id"]) @@ -328,6 +376,7 @@ class MCPServerSave(Resource): "config": storage_config, "actions": transformed_actions, "status": status_bool, + "connection_id": connection_id, }, ) saved_id = str(existing_by_name["id"]) @@ -347,6 +396,7 @@ class MCPServerSave(Resource): config_requirements={}, actions=transformed_actions, status=status_bool, + connection_id=connection_id, ) saved_id = str(created["id"]) response_data = { @@ -461,50 +511,34 @@ class MCPAuthStatus(Resource): jsonify({"success": True, "statuses": {}}), 200 ) - oauth_server_urls: dict = {} + from docsgpt.connectors import service + + # Read from connection status alone: status checks never + # decrypt credentials. statuses: dict = {} for tool in mcp_tools: tool_id = str(tool["id"]) config = tool.get("config") or {} auth_type = config.get("auth_type", "none") - if auth_type == "oauth": - server_url = config.get("server_url", "") - if server_url: - parsed = urlparse(server_url) - base_url = f"{parsed.scheme}://{parsed.netloc}" - oauth_server_urls[tool_id] = base_url + row = None + if tool.get("connection_id"): + row = sessions_repo.get(str(tool["connection_id"])) + elif auth_type == "oauth" and config.get("server_url"): + parsed = urlparse(config["server_url"]) + row = sessions_repo.get_by_user_provider( + user, service.mcp_provider(f"{parsed.scheme}://{parsed.netloc}"), + ) + if row is not None: + connected = service.normalize_status(row) == service.STATUS_CONNECTED + if auth_type == "oauth" or not connected: + statuses[tool_id] = "connected" if connected else "needs_auth" else: - statuses[tool_id] = "needs_auth" + statuses[tool_id] = "configured" + elif auth_type == "oauth": + statuses[tool_id] = "needs_auth" else: statuses[tool_id] = "configured" - if oauth_server_urls: - # Look up a session per distinct base URL. MCP sessions - # are stored with ``provider = "mcp:"`` - # and the URL in ``server_url``; reuse the repo's - # per-URL accessor rather than an ad-hoc $in query. - url_has_tokens: dict = {} - for base_url in set(oauth_server_urls.values()): - session = sessions_repo.get_by_user_and_server_url( - user, base_url, - ) - tokens = ( - (session or {}).get("session_data", {}) or {} - ).get("tokens", {}) or {} - # MCP code also stashes tokens into token_info on - # the row; consider either present as "connected". - token_info = (session or {}).get("token_info") or {} - url_has_tokens[base_url] = bool( - tokens.get("access_token") - or token_info.get("access_token") - ) - - for tool_id, base_url in oauth_server_urls.items(): - if url_has_tokens.get(base_url): - statuses[tool_id] = "connected" - else: - statuses[tool_id] = "needs_auth" - return make_response(jsonify({"success": True, "statuses": statuses}), 200) except Exception as e: current_app.logger.error( diff --git a/docsgpt/api/user/tools/routes.py b/docsgpt/api/user/tools/routes.py index c4464b68..db72a847 100644 --- a/docsgpt/api/user/tools/routes.py +++ b/docsgpt/api/user/tools/routes.py @@ -271,6 +271,10 @@ class GetTools(Resource): ): tool_copy["config"]["has_encrypted_credentials"] = True tool_copy["config"].pop("encrypted_credentials", None) + if tool_copy.get("connection_id"): + # The secret lives on the connection; the form must not + # ask for it again. + tool_copy.setdefault("config", {})["has_encrypted_credentials"] = True tool_copy["ownership"] = ownership return tool_copy @@ -383,6 +387,11 @@ class CreateTool(Resource): f"Error getting tool actions: {err}", exc_info=True ) return make_response(jsonify({"success": False}), 400) + definition = definition_for_tool(data["name"]) + if definition is not None: + connected = _create_connected_tool(user, data, definition, tool_instance) + if connected is not None: + return connected try: config_requirements = tool_instance.get_config_requirements() if config_requirements: @@ -422,6 +431,74 @@ class CreateTool(Resource): return make_response(jsonify({"id": new_id}), 200) +def _create_connected_tool(user, data, definition, tool_instance): + """Create a service tool whose secret lives on a connection, not the tool. + + Uses ``connection_id`` when given, otherwise stores the pasted secret on + a connection (reusing an identical one). Returns None to fall back to the + legacy path when a multi-user install still runs on the default key. + """ + from docsgpt.connectors import catalog as connector_catalog + from docsgpt.connectors import service as connection_service + from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository + + config_requirements = tool_instance.get_config_requirements() + public, secrets = connection_service.split_secrets(data.get("config") or {}, config_requirements) + connection_id = data.get("connection_id") + if not connection_id: + validation_errors = _validate_config(data.get("config") or {}, config_requirements) + if validation_errors: + return make_response( + jsonify({"success": False, "message": "Validation failed", "errors": validation_errors}), 400, + ) + try: + with db_session() as conn: + if connection_id: + connection = ConnectorSessionsRepository(conn).get_for_user(str(connection_id), user) + if connection is None or connector_catalog.connector_key_for_row(connection) != definition.key: + return make_response(jsonify({"success": False, "message": "Connection not found"}), 404) + else: + connection, _ = connection_service.create_api_key_connection(conn, user, definition, secrets) + created = connection_service.create_tool_for_connection( + conn, + user, + connection, + template=data["name"], + display_name=data.get("customName") or data.get("displayName") or definition.name, + config=public, + status=bool(data.get("status", True)), + ) + except connection_service.EncryptionKeyNotConfigured: + return None + except ValueError as err: + return make_response(jsonify({"success": False, "message": str(err)}), 400) + return make_response(jsonify({"id": str(created["id"]), "connection_id": str(connection["id"])}), 200) + + +def _update_connection_secrets(conn, user, tool_doc, config, config_requirements): + """Write changed secrets onto the tool's connection; return the tool's public config. + + Returns None when the caller does not own the connection (an editor on a + team share may change actions, never the owner's credentials). + """ + from docsgpt.connectors import service as connection_service + from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository + + public, secrets = connection_service.split_secrets(config or {}, config_requirements) + if secrets: + connection = ConnectorSessionsRepository(conn).get_for_user(str(tool_doc["connection_id"]), user) + if connection is None: + return None + stored = connection_service.read_secrets(connection) + credentials = {**(stored.get("credentials") or {}), **secrets} + connection_service.write_secrets( + conn, connection, {**stored, "credentials": credentials}, + status=connection_service.STATUS_CONNECTED, last_error=None, + ) + connection_service.resume_sources(conn, str(connection["id"])) + return public + + @tools_ns.route("/update_tool") class UpdateTool(Resource): @api.expect( @@ -528,6 +605,17 @@ class UpdateTool(Resource): ) existing_config = tool_doc.get("config", {}) or {} has_existing_secrets = "encrypted_credentials" in existing_config + if tool_doc.get("connection_id"): + has_existing_secrets = True + new_config = _update_connection_secrets( + conn, user, tool_doc, data["config"], config_requirements, + ) + if new_config is None: + return make_response( + jsonify({"success": False, "message": "Only the owner can change the credentials"}), + 403, + ) + data = {**data, "config": new_config} if config_requirements: validation_errors = _validate_config( diff --git a/docsgpt/cli.py b/docsgpt/cli.py index d837642d..c1264dc7 100644 --- a/docsgpt/cli.py +++ b/docsgpt/cli.py @@ -285,6 +285,28 @@ def _add_deploy_commands(commands) -> None: dev.set_defaults(func=_deploy("dev"), deploy=True) +def _connectors(args: argparse.Namespace) -> int: + """``docsgpt connectors reencrypt``: move every credential onto the current key.""" + if getattr(args, "connectors_action", None) != "reencrypt": + print("usage: docsgpt connectors reencrypt", file=sys.stderr) + return 2 + from docsgpt.connectors.service import reencrypt_all + + counts = reencrypt_all() + print( + f"docsgpt: re-encrypted {counts['rewritten']} connection(s), " + f"{counts['current']} already current, {counts['failed']} unreadable", + file=sys.stderr, + ) + if counts["failed"]: + print( + "docsgpt: unreadable connections were marked 'Reconnect needed'; " + "their owners must reconnect them.", + file=sys.stderr, + ) + return 1 if counts["failed"] else 0 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="docsgpt", description="DocsGPT: private AI for agents, assistants and search.") parser.add_argument("--version", action="version", version=f"docsgpt {__version__}") @@ -315,6 +337,14 @@ def build_parser() -> argparse.ArgumentParser: migrate.add_argument("--no-create", dest="create_db", action="store_false", help="fail instead of creating a missing database") migrate.set_defaults(func=_migrate) + connectors = commands.add_parser("connectors", help="manage stored connector credentials") + connector_actions = connectors.add_subparsers(dest="connectors_action", metavar="") + connector_actions.add_parser( + "reencrypt", + help="rewrite every stored credential with ENCRYPTION_SECRET_KEY (after a key rotation)", + ) + connectors.set_defaults(func=_connectors) + for name, (module, help_text) in SCRIPTS.items(): commands.add_parser(name, help=f"{help_text} (docsgpt.scripts.{module})", add_help=False) diff --git a/docsgpt/connectors/mcp.py b/docsgpt/connectors/mcp.py new file mode 100644 index 00000000..8ae389d7 --- /dev/null +++ b/docsgpt/connectors/mcp.py @@ -0,0 +1,63 @@ +"""MCP helpers for connections: re-scanning a server's actions.""" + +from __future__ import annotations + +from docsgpt.connectors import service +from docsgpt.connectors.permissions import apply_default_permissions +from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository +from docsgpt.storage.db.repositories.user_tools import UserToolsRepository +from docsgpt.storage.db.session import db_readonly, db_session + + +def _discover(user_id: str, connection: dict, tool: dict) -> list[dict]: + from docsgpt.agents.tools.mcp_tool import MCPTool + + config = {k: v for k, v in (tool.get("config") or {}).items() if k != "encrypted_credentials"} + if (connection.get("auth_kind") or "") in ("api_key", "none"): + config["auth_credentials"] = service.get_credentials(connection) + elif service.normalize_status(connection) != service.STATUS_CONNECTED: + raise service.ConnectionUnavailable("Reconnect to continue", connection_id=str(connection["id"])) + config["connection_id"] = str(connection["id"]) + config["query_mode"] = True + mcp_tool = MCPTool(config, user_id) + mcp_tool.discover_tools() + return mcp_tool.get_actions_metadata() + + +def refresh_mcp_tools(user_id: str, connection: dict) -> dict: + """Re-read the actions of every MCP tool on a connection. + + Actions that still exist keep the permissions the user chose; new ones + get the defaults from their annotations (reads always allowed, writes + need approval); removed ones disappear. + + Returns: + ``{"added": [...], "removed": [...], "tools": [...]}``. + """ + with db_readonly() as conn: + tools = [t for t in ConnectorSessionsRepository(conn).list_tools(str(connection["id"])) + if t.get("name") == "mcp_tool"] + added: set[str] = set() + removed: set[str] = set() + refreshed = [] + for tool in tools: + fresh = apply_default_permissions("mcp_tool", service._transform_actions(_discover(user_id, connection, tool))) + previous = {a.get("name"): a for a in (tool.get("actions") or []) if isinstance(a, dict)} + merged = [] + for action in fresh: + old = previous.get(action.get("name")) + if old is not None: + action = { + **action, + "active": old.get("active", True), + "require_approval": bool(old.get("require_approval")), + } + merged.append(action) + names = {a.get("name") for a in merged} + added |= names - set(previous) + removed |= set(previous) - names + with db_session() as conn: + repo = UserToolsRepository(conn) + repo.update(str(tool["id"]), user_id, {"actions": merged}) + refreshed.append(service.serialize_tool(repo.get_any(str(tool["id"]), user_id))) + return {"added": sorted(added), "removed": sorted(removed), "tools": refreshed} diff --git a/docsgpt/connectors/resolve.py b/docsgpt/connectors/resolve.py new file mode 100644 index 00000000..8580b842 --- /dev/null +++ b/docsgpt/connectors/resolve.py @@ -0,0 +1,128 @@ +"""Which connection a shared tool or source uses at runtime. + +A resource that points at a connection runs either with its owner's account +(``owner`` mode, the default for new shares) or with the invoking member's +own account for the same service (``member`` mode). Resolution never returns +credentials; callers read them from the resolved row through +``docsgpt.connectors.service``. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Optional + +from docsgpt.connectors import catalog, service +from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository +from docsgpt.storage.db.session import db_readonly + +logger = logging.getLogger(__name__) + +MODE_OWNER = "owner" +MODE_MEMBER = "member" + + +@dataclass(frozen=True) +class ResolvedConnection: + """The connection a call runs with, or why there is none. + + Attributes: + row: The ``connector_sessions`` row, None when missing. + available: Whether it can be used right now. + connector_key: Catalog key of the service. + connector_name: Name shown to the user ("Notion", or a custom label). + delegated: The row belongs to someone other than the invoker. + """ + + row: Optional[dict] + available: bool + connector_key: Optional[str] + connector_name: Optional[str] + delegated: bool = False + + @property + def connection_id(self) -> Optional[str]: + return str(self.row["id"]) if self.row else None + + +def _name_for(row: Optional[dict], fallback_key: Optional[str]) -> Optional[str]: + if row is not None: + return service.serialize_connection(row)["name"] + definition = catalog.get_definition(fallback_key) + return definition.name if definition else None + + +def resolve_connection(resource: dict, invoker_user_id: Optional[str]) -> Optional[ResolvedConnection]: + """Pick the connection a tool or source uses for ``invoker_user_id``. + + ``owner`` mode uses ``resource.connection_id``. ``member`` mode uses the + invoker's own connection to the same service (and, for MCP, the same + server), falling back to the owner's when the invoker is the owner. + + Args: + resource: A ``user_tools`` or ``sources`` row. + invoker_user_id: Who is running it. + + Returns: + None when the resource has no connection at all; otherwise the + resolution, possibly with ``available=False``. + """ + connection_id = resource.get("connection_id") + if not connection_id: + return None + mode = resource.get("credential_mode") or MODE_OWNER + owner = resource.get("user_id") + with db_readonly() as conn: + repo = ConnectorSessionsRepository(conn) + owned = repo.get(str(connection_id)) + if owned is not None and owner and owned.get("user_id") != owner: + # A resource may only point at its own owner's connection. + logger.warning( + "resource %s points at a connection it does not own", resource.get("id"), + ) + owned = None + row = owned + if mode == MODE_MEMBER and invoker_user_id and invoker_user_id != owner: + row = _member_connection(repo, owned, invoker_user_id) + key = catalog.connector_key_for_row(row or owned or {}) + available = row is not None and service.normalize_status(row) == service.STATUS_CONNECTED + return ResolvedConnection( + row=row, + available=available, + connector_key=key, + connector_name=_name_for(row or owned, key), + delegated=bool(row and invoker_user_id and row.get("user_id") != invoker_user_id), + ) + + +def _member_connection(repo: ConnectorSessionsRepository, owned: Optional[dict], invoker: str) -> Optional[dict]: + """The invoker's own connection to the service the owner's connection is for.""" + if owned is None: + return None + for row in repo.list_for_user(invoker): + if row.get("provider") != owned.get("provider"): + continue + if (row.get("server_url") or "") != (owned.get("server_url") or ""): + continue + if service.normalize_status(row) == service.STATUS_CONNECTED: + return row + return None + + +def audit_delegation(resolved: ResolvedConnection, *, invoker: Optional[str], resource_type: str, + resource_id: Optional[str], agent_id: Optional[str] = None) -> None: + """Log a call that runs with someone else's account (``owner`` mode).""" + if not resolved.delegated or resolved.row is None: + return + logger.info( + "tool_credential_delegation", + extra={ + "invoker": invoker, + "tool_owner": resolved.row.get("user_id"), + "connection_id": resolved.connection_id, + "resource_type": resource_type, + "resource_id": resource_id, + "agent_id": agent_id, + }, + ) diff --git a/docsgpt/connectors/service.py b/docsgpt/connectors/service.py index 3bb16f72..52c9f31d 100644 --- a/docsgpt/connectors/service.py +++ b/docsgpt/connectors/service.py @@ -13,7 +13,11 @@ from typing import Any, Iterable, Optional from docsgpt.connectors import catalog from docsgpt.connectors.catalog import ConnectorDefinition -from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository +from docsgpt.storage.db.repositories.connector_sessions import ( + ConnectorSessionsRepository, + owns_connector_session, +) +from docsgpt.storage.db.session import db_readonly, db_session STATUS_CONNECTED = "connected" STATUS_RECONNECT = "reconnect_needed" @@ -44,6 +48,11 @@ def has_credentials(row: dict) -> bool: """Whether a connection holds usable credentials (without decrypting them).""" if row.get("encrypted_credentials"): return True + return _has_plaintext_tokens(row) + + +def _has_plaintext_tokens(row: dict) -> bool: + """Legacy plaintext tokens on a row not yet converted by migration 0038.""" token_info = _json(row.get("token_info")) or {} if isinstance(token_info, dict) and token_info.get("access_token"): return True @@ -62,11 +71,12 @@ def normalize_status(row: dict) -> str: raw = (row.get("status") or "").lower() if raw in (STATUS_RECONNECT, STATUS_DISCONNECTED, STATUS_ERROR): return raw - if raw == STATUS_PENDING and not has_credentials(row): - return STATUS_PENDING - if raw in ("authorized", STATUS_CONNECTED, "active") or has_credentials(row): + if raw in ("authorized", STATUS_CONNECTED, "active"): return STATUS_CONNECTED - return STATUS_PENDING + if raw == STATUS_PENDING: + # A legacy row whose plaintext tokens landed after the pending mark. + return STATUS_CONNECTED if _has_plaintext_tokens(row) else STATUS_PENDING + return STATUS_CONNECTED if has_credentials(row) else STATUS_PENDING def worst_status(statuses: Iterable[str]) -> Optional[str]: @@ -237,9 +247,11 @@ def catalog_for_user(conn, user_id: str, *, is_admin: bool, policies: Optional[d def disconnect(conn, row: dict) -> dict: """Forget a connection's credentials but keep the row and what it feeds. - Sources keep their indexed content and stop syncing; tools stop working - until the account is reconnected. An MCP server's client registration is - kept so reconnecting skips dynamic client registration. + Revokes the grant at the provider where that is supported (Google); + a failed revocation never blocks forgetting the tokens here. Sources keep + their indexed content and pause syncing; tools stop working until the + account is reconnected. An MCP server's client registration is kept so + reconnecting skips dynamic client registration. Args: conn: Open database connection inside a transaction. @@ -248,17 +260,911 @@ def disconnect(conn, row: dict) -> dict: Returns: The connection's public shape after the change. """ - repo = ConnectorSessionsRepository(conn) + from sqlalchemy import text + + from docsgpt.security.encryption import CredentialDecryptionError + + try: + secrets = read_secrets(row) + except CredentialDecryptionError: + secrets = {} + revoke_at_provider(row, secrets) + kept = {"client_info": secrets["client_info"]} if secrets.get("client_info") else {} + write_secrets( + conn, row, kept, status=STATUS_DISCONNECTED, session_token=None, last_error=None, + ) + conn.execute( + text( + "UPDATE sources SET metadata = metadata || '{\"sync_state\": \"paused_reconnect\"}'::jsonb " + "WHERE connection_id = CAST(:id AS uuid)" + ), + {"id": str(row["id"])}, + ) + return serialize_connection(ConnectorSessionsRepository(conn).get(str(row["id"]))) + + +def revoke_at_provider(row: dict, secrets: dict) -> bool: + """Best-effort revocation of an OAuth grant. Returns whether it succeeded. + + Only Google offers a public revocation endpoint for these apps; Microsoft + and Atlassian tokens are simply forgotten locally. + """ + import logging + + import requests + + token_info = secrets.get("token_info") or {} + token = token_info.get("refresh_token") or token_info.get("access_token") + if row.get("provider") != "google_drive" or not token: + return False + try: + response = requests.post( + "https://oauth2.googleapis.com/revoke", + data={"token": token}, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=10, + ) + return response.status_code == 200 + except requests.RequestException as exc: + logging.getLogger(__name__).warning("Google token revocation failed: %s", type(exc).__name__) + return False + + +# --------------------------------------------------------------------------- +# Credentials +# --------------------------------------------------------------------------- +# +# A connection's secrets are one dict, encrypted as a whole into +# ``encrypted_credentials``: +# +# token_info OAuth tokens of an ingest provider (Drive, SharePoint, Confluence) +# tokens MCP OAuth tokens +# client_info MCP dynamic client registration (client_id / client_secret) +# credentials API keys and other pasted secrets, by credential field key +# +# This module is the only caller of encrypt_json / decrypt_json for +# connections, and get_valid_token_info the only reader of OAuth tokens. + +DECRYPT_ERROR = "Stored credentials could not be decrypted. Reconnect to continue." +_SECRET_SESSION_KEYS = ("tokens", "client_info") + + +class ConnectionUnavailable(ValueError): + """A connection cannot be used until its owner reconnects it. + + Attributes: + connection_id: The connection, when known. + status: ``reconnect_needed``, ``disconnected`` or ``missing``. + """ + + def __init__(self, message: str, *, connection_id: Optional[str] = None, status: str = STATUS_RECONNECT): + super().__init__(message) + self.connection_id = connection_id + self.status = status + + +class TransientConnectionError(Exception): + """The provider failed in a way worth retrying (5xx, rate limit, network).""" + + +class EncryptionKeyNotConfigured(Exception): + """New credentials are refused while a multi-user install uses the public default key.""" + + +def ensure_can_store_credentials() -> None: + """Refuse to store new credentials under the public default encryption key. + + Only multi-user installs (any ``AUTH_TYPE``) are refused; a single-user + local install keeps working and logs a warning at startup instead. + + Raises: + EncryptionKeyNotConfigured: When the key is the default and auth is on. + """ + from docsgpt.core.settings import settings + from docsgpt.security.encryption import is_default_encryption_key + + if settings.AUTH_TYPE and is_default_encryption_key(): + raise EncryptionKeyNotConfigured("Set ENCRYPTION_SECRET_KEY before connecting services.") + + +def read_secrets(row: dict) -> dict: + """Decrypt a connection's secrets. + + Rows not yet converted by migration 0038 (or written by an older process + during a rolling deploy) still carry plaintext ``token_info`` and + ``session_data`` tokens; those are read as they are. + + Raises: + CredentialDecryptionError: The envelope cannot be decrypted. + """ + from docsgpt.security.encryption import decrypt_json + + blob = row.get("encrypted_credentials") + if blob: + return decrypt_json(blob, row["user_id"]) + secrets: dict = {} + token_info = _json(row.get("token_info")) + if isinstance(token_info, dict) and token_info: + secrets["token_info"] = token_info session_data = _json(row.get("session_data")) or {} if isinstance(session_data, dict): - session_data = {k: v for k, v in session_data.items() if k != "tokens"} - repo.update( - str(row["id"]), - { - "status": STATUS_DISCONNECTED, - "token_info": None, - "session_token": None, - "session_data": session_data or {}, - }, + for key in _SECRET_SESSION_KEYS: + if key in session_data: + secrets[key] = session_data[key] + return secrets + + +def _has_refresh(secrets: dict) -> bool: + token_info = secrets.get("token_info") or {} + tokens = secrets.get("tokens") or {} + return bool( + (isinstance(token_info, dict) and token_info.get("refresh_token")) + or (isinstance(tokens, dict) and tokens.get("refresh_token")) ) - return serialize_connection(repo.get(str(row["id"]))) + + +def write_secrets(conn, row: dict, secrets: dict, **fields: Any) -> None: + """Replace a connection's secrets and clear every plaintext copy. + + Args: + conn: Open connection inside a transaction. + row: The connection row (only ``id``, ``user_id`` and ``session_data`` + are read). + secrets: The complete secrets dict; empty clears the credentials. + **fields: Other columns to update in the same statement. + """ + from docsgpt.security.encryption import encrypt_json + + session_data = _json(row.get("session_data")) or {} + if isinstance(session_data, dict): + session_data = {k: v for k, v in session_data.items() if k not in _SECRET_SESSION_KEYS} + else: + session_data = {} + update = { + "encrypted_credentials": encrypt_json(secrets, row["user_id"]) if secrets else None, + "has_refresh_token": _has_refresh(secrets), + "token_info": None, + "session_data": session_data, + **fields, + } + token_info = secrets.get("token_info") + if isinstance(token_info, dict) and token_info.get("scopes") and "scopes" not in fields: + scopes = token_info["scopes"] + update["scopes"] = scopes.split() if isinstance(scopes, str) else list(scopes) + ConnectorSessionsRepository(conn).update(str(row["id"]), update) + + +def credential_hint(credentials: dict) -> str: + """``…abcd``: the last four characters of the first long secret, never more.""" + for value in credentials.values(): + if isinstance(value, str) and len(value) >= 8: + return "…" + value[-4:] + return "…" + + +def mark_reconnect_needed(connection_id: str, error: str) -> None: + """Flag a connection whose credentials stopped working and tell its owner. + + Pauses syncing on every source the connection feeds (no retry storm) and + publishes a ``connection.reconnect_needed`` user event, which the + frontend shows as a toast with a Reconnect action. Runs in its own + transaction so it sticks even when the caller's transaction rolls back. + """ + from sqlalchemy import text + + from docsgpt.events.publisher import publish_user_event + with db_session() as conn: + repo = ConnectorSessionsRepository(conn) + row = repo.get(connection_id) + if row is None: + return + already = normalize_status(row) == STATUS_RECONNECT + repo.update(connection_id, {"status": STATUS_RECONNECT, "last_error": error[:500]}) + conn.execute( + text( + "UPDATE sources SET metadata = metadata || '{\"sync_state\": \"paused_reconnect\"}'::jsonb " + "WHERE connection_id = CAST(:id AS uuid)" + ), + {"id": connection_id}, + ) + if not already: + publish_user_event( + row["user_id"], + "connection.reconnect_needed", + { + "connection_id": connection_id, + "connector_key": catalog.connector_key_for_row(row), + "name": serialize_connection(row)["name"], + }, + scope={"kind": "connection", "id": connection_id}, + ) + + +def resume_sources(conn, connection_id: str) -> None: + """Lift the reconnect pause from a connection's sources.""" + from sqlalchemy import text + + conn.execute( + text( + "UPDATE sources SET metadata = metadata - 'sync_state' " + "WHERE connection_id = CAST(:id AS uuid) AND metadata ? 'sync_state'" + ), + {"id": connection_id}, + ) + + +def load_secrets(row: dict) -> Optional[dict]: + """Decrypt a row's secrets; on failure flag it for reconnect and return None.""" + from docsgpt.security.encryption import CredentialDecryptionError + + try: + return read_secrets(row) + except CredentialDecryptionError: + mark_reconnect_needed(str(row["id"]), DECRYPT_ERROR) + return None + + +def _is_auth_failure(exc: BaseException) -> bool: + """Whether a refresh failure means the grant is gone (vs. a retryable blip).""" + import requests + + response = getattr(exc, "response", None) + status = getattr(response, "status_code", None) + if isinstance(exc, requests.exceptions.HTTPError) and status is not None: + return status in (400, 401, 403) + if isinstance(exc, (requests.exceptions.ConnectionError, requests.exceptions.Timeout)): + return False + message = str(exc).lower() + if any(word in message for word in ("timed out", "temporarily", "rate limit", "503", "502", "504")): + return False + try: + from google.auth.exceptions import RefreshError, TransportError + + if isinstance(exc, TransportError): + return False + if isinstance(exc, RefreshError): + return True + except ImportError: # pragma: no cover - google libs are core deps + pass + return True + + +def connection_id_for_session_token(session_token: Optional[str]) -> str: + """The connection behind a legacy browser session token. + + Raises: + ConnectionUnavailable: No connection holds that token. + """ + from docsgpt.parser.connectors._auth_utils import session_token_fingerprint + row = None + if session_token: + with db_readonly() as conn: + row = ConnectorSessionsRepository(conn).get_by_session_token(session_token) + if row is None: + raise ConnectionUnavailable( + f"Invalid session token ({session_token_fingerprint(session_token or '')})", status="missing", + ) + return str(row["id"]) + + +def get_valid_token_info(connection_id: str, *, rejected_access_token: Optional[str] = None) -> dict: + """An unexpired OAuth ``token_info`` for an ingest connection. + + Holds a row lock (``SELECT ... FOR UPDATE``) while it checks expiry and + refreshes, and writes the rotated refresh token back in the same + transaction, so two workers refreshing a rotating token (Microsoft, + Atlassian) never spend the same one twice: the second waits and then + reads the first one's fresh token. + + Args: + connection_id: The connection to use. + rejected_access_token: An access token the provider just answered + with 401. The token is refreshed when it is still the stored one; + when another worker already replaced it, the new one is returned. + + Returns: + The decrypted token info, refreshed if it was about to expire. + + Raises: + ConnectionUnavailable: The connection is missing, disconnected, cannot + be decrypted, or its grant was revoked (it is then flagged + ``reconnect_needed``). + TransientConnectionError: The provider failed in a retryable way. + """ + from docsgpt.parser.connectors.connector_creator import ConnectorCreator + from docsgpt.security.encryption import CredentialDecryptionError + failure: Optional[str] = None + transient: Optional[BaseException] = None + token_info: Optional[dict] = None + with db_session() as conn: + repo = ConnectorSessionsRepository(conn) + row = repo.get_for_update(connection_id) + if row is None: + raise ConnectionUnavailable("Connection not found", connection_id=connection_id, status="missing") + status = normalize_status(row) + if status == STATUS_DISCONNECTED: + raise ConnectionUnavailable( + "Connection is disconnected", connection_id=connection_id, status=STATUS_DISCONNECTED + ) + try: + secrets = read_secrets(row) + except CredentialDecryptionError: + secrets, failure = {}, DECRYPT_ERROR + token_info = secrets.get("token_info") if not failure else None + if not failure and not token_info: + failure = "No stored sign-in for this connection. Reconnect to continue." + if not failure: + auth = ConnectorCreator.create_auth(row["provider"]) + rejected = bool(rejected_access_token) and token_info.get("access_token") == rejected_access_token + if rejected or auth.is_token_expired(token_info): + refresh_token = token_info.get("refresh_token") + if not refresh_token: + failure = "The sign-in expired and cannot be renewed. Reconnect to continue." + else: + try: + refreshed = auth.refresh_access_token(refresh_token) + except Exception as exc: # classified below + if _is_auth_failure(exc): + failure = f"The provider rejected the stored sign-in: {type(exc).__name__}" + else: + transient = exc + else: + merged = {**token_info, **{k: v for k, v in refreshed.items() if v is not None}} + token_info = auth.sanitize_token_info(merged) + write_secrets( + conn, row, {**secrets, "token_info": token_info}, + status=STATUS_CONNECTED, last_error=None, + ) + if not failure and transient is None: + repo.update(connection_id, {"last_used_at": _now()}) + if failure: + mark_reconnect_needed(connection_id, failure) + raise ConnectionUnavailable(failure, connection_id=connection_id) + if transient is not None: + raise TransientConnectionError(str(transient)) from transient + return token_info + + +def _now(): + import datetime + + return datetime.datetime.now(datetime.timezone.utc) + + +def get_credentials(row: dict) -> dict: + """The pasted credentials (API keys) of an ``api_key`` connection. + + Raises: + ConnectionUnavailable: Disconnected, flagged, or undecryptable. + """ + status = normalize_status(row) + if status in (STATUS_DISCONNECTED, STATUS_RECONNECT): + raise ConnectionUnavailable(f"Connection is {status}", connection_id=str(row["id"]), status=status) + secrets = load_secrets(row) + if secrets is None: + raise ConnectionUnavailable(DECRYPT_ERROR, connection_id=str(row["id"])) + return dict(secrets.get("credentials") or {}) + + +def create_api_key_connection( + conn, + user_id: str, + definition, + credentials: dict, + *, + label: Optional[str] = None, + server_url: Optional[str] = None, + display_name: Optional[str] = None, +) -> tuple[dict, bool]: + """Store a connection for pasted credentials, or reuse the matching one. + + The same credentials for the same service map to one connection + ("enter secrets once"): adding a second tool or bucket reuses it. + + Args: + conn: Open connection inside a transaction. + user_id: The owner. + definition: The catalog entry (``api_key`` or ``custom_mcp``). + credentials: Field key to value, as the user entered them. + label: What the account is called; defaults to a hint of the key. + server_url: Base URL, for custom MCP servers. + display_name: Name shown for custom connectors. + + Returns: + ``(row, created)``. + + Raises: + ValueError: A required credential field is missing. + EncryptionKeyNotConfigured: See :func:`ensure_can_store_credentials`. + """ + fields = {f.key: f for f in definition.credential_fields} + if fields: + missing = [f.label for f in fields.values() if f.required and not str(credentials.get(f.key) or "").strip()] + if missing: + raise ValueError(f"Missing credentials: {', '.join(missing)}") + credentials = {k: v for k, v in credentials.items() if k in fields and v not in (None, "")} + else: + credentials = {k: v for k, v in credentials.items() if v not in (None, "")} + ensure_can_store_credentials() + secret_values = {k: v for k, v in credentials.items() if (fields.get(k).secret if fields.get(k) else True)} + account_label = label or credential_hint(secret_values or credentials) + repo = ConnectorSessionsRepository(conn) + existing = repo.find_account(user_id, definition.key, server_url=server_url, account_label=account_label) + if existing is not None: + write_secrets(conn, existing, {"credentials": credentials}, status=STATUS_CONNECTED, last_error=None) + resume_sources(conn, str(existing["id"])) + return repo.get(str(existing["id"])), False + from docsgpt.security.encryption import encrypt_json + + row = repo.create( + user_id, + definition.key, + connector_key=definition.key, + auth_kind="api_key", + display_name=display_name or definition.name, + account_label=account_label, + server_url=server_url, + encrypted_credentials=encrypt_json({"credentials": credentials}, user_id), + ) + if row is None: # lost a race with an identical insert + row = repo.find_account(user_id, definition.key, server_url=server_url, account_label=account_label) + return row, False + return row, True + + +# --------------------------------------------------------------------------- +# OAuth sign-in +# --------------------------------------------------------------------------- + + +def begin_oauth(conn, user_id: str, provider: str, connection_id: Optional[str] = None) -> dict: + """The row an OAuth sign-in writes into when it completes. + + Reconnecting passes the connection being reconnected. A new sign-in + gets a pending row with no account label; the callback moves the tokens + onto the existing row for that account if there is one. + + Raises: + ConnectionUnavailable: ``connection_id`` is not the caller's + connection for ``provider``. + """ + from sqlalchemy import text + + repo = ConnectorSessionsRepository(conn) + if connection_id: + row = repo.get_for_user(connection_id, user_id) + if row is None or row.get("provider") != provider: + raise ConnectionUnavailable("Connection not found", connection_id=connection_id, status="missing") + return row + definition = catalog.get_definition(provider) + result = conn.execute( + text( + """ + INSERT INTO connector_sessions (user_id, provider, status, connector_key, auth_kind, display_name) + VALUES (:user_id, :provider, 'pending', :provider, 'oauth', :display_name) + ON CONFLICT (user_id, provider, COALESCE(server_url, ''), COALESCE(account_label, '')) + DO UPDATE SET status = CASE + WHEN connector_sessions.status IN ('connected', 'authorized') THEN connector_sessions.status + ELSE 'pending' END + RETURNING * + """ + ), + {"user_id": user_id, "provider": provider, "display_name": definition.name if definition else provider}, + ) + from docsgpt.storage.db.base_repository import row_to_dict + + return row_to_dict(result.fetchone()) + + +def complete_oauth(conn, state_row: dict, provider: str, token_info: dict, account: str) -> dict: + """Store a finished OAuth sign-in and return the connection it belongs to. + + Signing in to an account that already has a connection updates that + connection (and drops the pending placeholder), so reconnecting from any + entry point heals every source and tool of the account. + + Args: + conn: Open connection inside a transaction. + state_row: The row named in the OAuth ``state``. + provider: ``google_drive``, ``share_point`` or ``confluence``. + token_info: Sanitised token info from the provider. + account: The account's email or name, shown as "Connected as". + """ + import uuid + + repo = ConnectorSessionsRepository(conn) + target = state_row + existing = repo.find_account(state_row["user_id"], provider, server_url=None, account_label=account) + if existing is not None and str(existing["id"]) != str(state_row["id"]): + target = existing + if not has_credentials(state_row) and normalize_status(state_row) == STATUS_PENDING: + repo.delete_by_id(str(state_row["id"])) + definition = catalog.get_definition(provider) + write_secrets( + conn, + target, + {"token_info": token_info}, + status=STATUS_CONNECTED, + # Kept for frontends from before connections; new ones use the id. + session_token=str(uuid.uuid4()), + user_email=account, + account_label=account, + connector_key=provider, + auth_kind="oauth", + display_name=definition.name if definition else provider, + last_error=None, + ) + resume_sources(conn, str(target["id"])) + return repo.get(str(target["id"])) + + +def picker_token(connection_id: str) -> dict: + """What a browser-side picker needs: a short-lived access token, never the refresh token.""" + token_info = get_valid_token_info(connection_id) + return { + "access_token": token_info.get("access_token"), + "expiry": token_info.get("expiry"), + "allows_shared_content": bool(token_info.get("allows_shared_content")), + } + + +def claim_session_token(conn, user_id: str, provider: str, session_token: str) -> Optional[dict]: + """Link a legacy browser session token to its connection, once. + + Returns the connection when the token belongs to ``user_id`` and + ``provider``. The token itself keeps working for the session-token + routes during this release, so an older tab does not break. + """ + repo = ConnectorSessionsRepository(conn) + row = repo.get_by_session_token(session_token) + if not row or row.get("user_id") != user_id or (row.get("provider") or "").lower() != provider.lower(): + return None + return row + + +def resolve_request_connection(user_id: str, provider: Optional[str], data: dict) -> Optional[dict]: + """The caller's connection a request names, by ``connection_id`` or legacy ``session_token``. + + Returns None unless the caller owns the connection and it belongs to + ``provider`` (when given). + """ + connection_id = data.get("connection_id") + session_token = data.get("session_token") + with db_readonly() as conn: + repo = ConnectorSessionsRepository(conn) + if connection_id: + row = repo.get_for_user(str(connection_id), user_id) + elif session_token: + row = repo.get_by_session_token(session_token) + if not owns_connector_session(row, user_id, provider): + return None + else: + return None + if row is None: + return None + if provider and (row.get("provider") or "").lower() != provider.lower(): + return None + return row + + +# --------------------------------------------------------------------------- +# MCP OAuth token storage +# --------------------------------------------------------------------------- + + +def mcp_provider(base_url: str) -> str: + """``provider`` value MCP OAuth connections are stored under.""" + return f"mcp:{base_url}" + + +def _mcp_row(conn, user_id: str, base_url: str, connection_id: Optional[str], *, lock: bool = False): + from sqlalchemy import text + + from docsgpt.storage.db.base_repository import row_to_dict + + repo = ConnectorSessionsRepository(conn) + if connection_id: + return repo.get_for_update(connection_id) if lock else repo.get(connection_id) + result = conn.execute( + text( + "SELECT * FROM connector_sessions WHERE user_id = :user_id AND provider = :provider " + "ORDER BY updated_at DESC LIMIT 1" + (" FOR UPDATE" if lock else "") + ), + {"user_id": user_id, "provider": mcp_provider(base_url)}, + ) + row = result.fetchone() + return row_to_dict(row) if row is not None else None + + +def read_mcp_secrets(user_id: str, base_url: str, connection_id: Optional[str] = None) -> dict: + """The MCP OAuth ``tokens`` and ``client_info`` of a server connection.""" + with db_readonly() as conn: + row = _mcp_row(conn, user_id, base_url, connection_id) + if row is None: + return {} + return load_secrets(row) or {} + + +def update_mcp_secrets( + user_id: str, + base_url: str, + patch: dict, + *, + connection_id: Optional[str] = None, + status: Optional[str] = None, +) -> dict: + """Merge ``patch`` into an MCP connection's secrets (``None`` drops a key). + + Creates the connection on first use, named after the matching preset or + the server's host. + + Returns: + The connection row after the update. + """ + from docsgpt.security.encryption import CredentialDecryptionError + + with db_session() as conn: + repo = ConnectorSessionsRepository(conn) + row = _mcp_row(conn, user_id, base_url, connection_id, lock=True) + if row is None: + row = repo.merge_session_data(user_id, mcp_provider(base_url), base_url, {}) + try: + secrets = read_secrets(row) + except CredentialDecryptionError: + secrets = {} + for key, value in patch.items(): + if value is None: + secrets.pop(key, None) + else: + secrets[key] = value + fields: dict = {} + if status: + fields["status"] = status + if status == STATUS_CONNECTED: + fields["last_error"] = None + elif not row.get("status"): + fields["status"] = STATUS_CONNECTED if secrets.get("tokens") else STATUS_PENDING + if not row.get("connector_key"): + preset = catalog.preset_for_url(base_url) + fields["connector_key"] = preset.key if preset else "custom_mcp" + fields["auth_kind"] = "mcp_oauth" + fields["display_name"] = preset.name if preset else base_url.split("://")[-1] + write_secrets(conn, row, secrets, **fields) + if fields.get("status") == STATUS_CONNECTED: + resume_sources(conn, str(row["id"])) + return repo.get(str(row["id"])) + + +# --------------------------------------------------------------------------- +# Tools created from a connection +# --------------------------------------------------------------------------- + + +def _transform_actions(actions: list) -> list: + """``transform_actions`` from the tools API: active, LLM-filled parameters.""" + transformed = [] + for action in actions: + action = dict(action) + action["active"] = True + parameters = action.get("parameters") + if isinstance(parameters, dict): + for details in (parameters.get("properties") or {}).values(): + if isinstance(details, dict): + details["filled_by_llm"] = True + details["value"] = "" + transformed.append(action) + return transformed + + +def create_tool_for_connection( + conn, + user_id: str, + connection: dict, + *, + template: Optional[str] = None, + display_name: Optional[str] = None, + config: Optional[dict] = None, + actions: Optional[list] = None, + permissions: Optional[dict] = None, + status: bool = True, +) -> dict: + """Create the tool a connection provides, with read / write defaults. + + Writes default to "Needs approval", reads to "Always allow"; the + ``permissions`` map (action name to ``always`` / ``ask`` / ``off``) + overrides them. Secrets are never copied onto the tool: the executor + reads them from the connection at run time. + + Args: + conn: Open connection inside a transaction. + user_id: The owner of the connection and the new tool. + connection: The connection row. + template: ``user_tools`` name; defaults to the connector's first template. + display_name: Name shown for the tool. + config: Non-secret tool configuration (an MCP server URL, say). + actions: Action metadata; defaults to the tool class's own. + permissions: Per-action permission overrides. + status: Whether the tool starts enabled. + + Returns: + The new ``user_tools`` row. + """ + from docsgpt.agents.tools.tool_manager import ToolManager + from docsgpt.connectors.permissions import apply_default_permissions, apply_permission + from docsgpt.storage.db.repositories.user_tools import UserToolsRepository + + key = catalog.connector_key_for_row(connection) + definition = catalog.get_definition(key) + template = template or (definition.tool_templates[0] if definition and definition.tool_templates else None) + if not template: + raise ValueError(f"Connector {key} provides no tool") + tool = ToolManager(config={}).tools.get(template) + if tool is None: + raise ValueError(f"Unknown tool template: {template}") + doc = (tool.__doc__ or template).strip().split("\n", 1) + if actions is None: + actions = tool.get_actions_metadata() + actions = apply_default_permissions(template, _transform_actions(actions)) + for index, action in enumerate(actions): + permission = (permissions or {}).get(action.get("name")) + if permission: + actions[index] = apply_permission(action, permission) + name = display_name or (serialize_connection(connection)["name"] if connection else doc[0].strip()) + return UserToolsRepository(conn).create( + user_id, + template, + config=dict(config or {}), + custom_name=name, + display_name=name, + description=doc[1].strip() if len(doc) > 1 else "", + config_requirements=tool.get_config_requirements(), + actions=actions, + status=status, + connection_id=str(connection["id"]), + ) + + +def split_secrets(config: dict, config_requirements: dict) -> tuple[dict, dict]: + """``(public, secrets)``: a tool config split by its secret requirements.""" + secret_keys = {k for k, spec in (config_requirements or {}).items() if spec.get("secret")} + public = {k: v for k, v in config.items() if k not in secret_keys} + secrets = {k: v for k, v in config.items() if k in secret_keys and v not in (None, "")} + return public, secrets + + +def ensure_connection_tools(conn, user_id: str, connection: dict, permissions: Optional[dict] = None) -> list[dict]: + """Create the connector's tools once; later calls return the existing ones. + + This is what makes the setup step idempotent for tools: a retried or + repeated setup never creates a second Telegram tool for the same bot. + """ + repo = ConnectorSessionsRepository(conn) + existing = repo.list_tools(str(connection["id"])) + key = catalog.connector_key_for_row(connection) + definition = catalog.get_definition(key) + if not definition or not definition.tool_templates: + return existing + have = {tool.get("name") for tool in existing} + created = [] + for template in definition.tool_templates: + if template in have or template in ("mcp_tool", "api_tool"): + # MCP tools are created by the MCP save flow, which has the + # discovered actions; OpenAPI tools come from an imported spec. + continue + created.append( + create_tool_for_connection(conn, user_id, connection, template=template, permissions=permissions) + ) + return existing + created + + +def remove_connection(conn, row: dict, *, sources: str = "keep", tools: str = "delete") -> list[dict]: + """Delete a connection, choosing what happens to what it feeds. + + Args: + conn: Open connection inside a transaction. + row: The connection, already authorised for its owner. + sources: ``keep`` (content stays, syncing stops) or ``delete``. + tools: ``delete`` (the default: they cannot work without it) or ``keep``. + + Returns: + The source rows the caller must delete with their indexes (empty + when ``sources`` is ``keep``). Index files live outside the database, + so the caller deletes them after this transaction commits. + """ + from sqlalchemy import text + + from docsgpt.security.encryption import CredentialDecryptionError + + repo = ConnectorSessionsRepository(conn) + connection_id = str(row["id"]) + linked_sources = repo.list_sources(connection_id) + try: + revoke_at_provider(row, read_secrets(row)) + except CredentialDecryptionError: + pass + if tools == "delete": + conn.execute( + text("DELETE FROM user_tools WHERE connection_id = CAST(:id AS uuid) AND user_id = :user_id"), + {"id": connection_id, "user_id": row["user_id"]}, + ) + else: + conn.execute( + text("UPDATE user_tools SET status = false WHERE connection_id = CAST(:id AS uuid)"), + {"id": connection_id}, + ) + conn.execute( + text( + "UPDATE sources SET sync_frequency = 'never', metadata = metadata - 'sync_state' " + "WHERE connection_id = CAST(:id AS uuid)" + ), + {"id": connection_id}, + ) + repo.delete_by_id(connection_id) + return linked_sources if sources == "delete" else [] + + +def set_tool_permissions(conn, user_id: str, tool_id: str, permissions: dict) -> Optional[dict]: + """Apply ``{action: always | ask | off}`` to a tool the user owns.""" + from docsgpt.connectors.permissions import apply_permission + from docsgpt.storage.db.repositories.user_tools import UserToolsRepository + + tools = UserToolsRepository(conn) + tool = tools.get_any(tool_id, user_id) + if tool is None or tool.get("user_id") != user_id: + return None + actions = [ + apply_permission(action, permissions[action.get("name")]) if action.get("name") in permissions else action + for action in (_json(tool.get("actions")) or []) + ] + tools.update(str(tool["id"]), user_id, {"actions": actions}) + return tools.get_any(str(tool["id"]), user_id) + + +def reencrypt_all(batch_size: int = 500) -> dict: + """Rewrite every connection's credentials with the current key. + + Run after rotating ENCRYPTION_SECRET_KEY with the old value in + ENCRYPTION_SECRET_KEY_PREVIOUS; afterwards the previous key can go. + + Returns: + ``{"rewritten": n, "current": n, "failed": n}``; failed rows (neither + key opens them) are flagged ``reconnect_needed``. + """ + from sqlalchemy import text + + from docsgpt.security.encryption import CredentialDecryptionError, current_key_id, envelope_key_id + + key_id = current_key_id() + counts = {"rewritten": 0, "current": 0, "failed": 0} + failed_ids: list[str] = [] + last_id = "00000000-0000-0000-0000-000000000000" + while True: + with db_session() as conn: + rows = conn.execute( + text( + "SELECT * FROM connector_sessions WHERE encrypted_credentials IS NOT NULL " + "AND id > CAST(:last AS uuid) ORDER BY id LIMIT :batch FOR UPDATE" + ), + {"last": last_id, "batch": batch_size}, + ).fetchall() + if not rows: + break + from docsgpt.storage.db.base_repository import row_to_dict + + for raw in rows: + row = row_to_dict(raw) + last_id = str(row["id"]) + if envelope_key_id(row["encrypted_credentials"]) == key_id: + counts["current"] += 1 + continue + try: + secrets = read_secrets(row) + except CredentialDecryptionError: + counts["failed"] += 1 + failed_ids.append(last_id) + continue + write_secrets(conn, row, secrets) + counts["rewritten"] += 1 + for connection_id in failed_ids: + mark_reconnect_needed(connection_id, DECRYPT_ERROR) + return counts diff --git a/docsgpt/llm/handlers/base.py b/docsgpt/llm/handlers/base.py index 0954dd53..a8090056 100644 --- a/docsgpt/llm/handlers/base.py +++ b/docsgpt/llm/handlers/base.py @@ -1240,6 +1240,10 @@ class LLMHandler(ABC): # can wire the sticky "don't ask again" button. if pause_info.get("device_id"): pause_data["device_id"] = pause_info["device_id"] + # A connection-backed tool whose account needs signing in: the + # approval card becomes a Connect card. + if pause_info.get("connection_required"): + pause_data["connection_required"] = pause_info["connection_required"] trace_unexecuted_tool_call(call, pause_data) yield {"type": "tool_call", "data": pause_data} pending_actions.append(pause_info) diff --git a/docsgpt/parser/connectors/base.py b/docsgpt/parser/connectors/base.py index 5d2dd706..1b0f3c4d 100644 --- a/docsgpt/parser/connectors/base.py +++ b/docsgpt/parser/connectors/base.py @@ -6,7 +6,7 @@ interface for external knowledge base connectors. """ from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from docsgpt.parser.schema.base import Document @@ -88,18 +88,55 @@ class BaseConnectorLoader(ABC): Abstract base class for connector loaders. Defines the minimal interface that all connector loader - implementations must follow. + implementations must follow. A loader reads its OAuth tokens through + ``docsgpt.connectors.service`` from the connection it was built for, + either directly (``connection_id``, what background sync uses) or through + a legacy browser ``session_token`` that names the connection. """ - + + connection_id: Optional[str] = None + @abstractmethod - def __init__(self, session_token: str): + def __init__(self, session_token: Optional[str] = None, *, connection_id: Optional[str] = None): """ Initialize the connector loader. Args: - session_token: Authentication session token + session_token: Legacy browser session token naming the connection. + connection_id: The connection to read tokens from. """ pass + + @classmethod + def from_connection(cls, connection_id: str) -> "BaseConnectorLoader": + """Build a loader that reads its tokens from ``connection_id``.""" + return cls(connection_id=connection_id) + + def _load_token_info( + self, session_token: Optional[str], connection_id: Optional[str], + ) -> Tuple[str, Dict[str, Any]]: + """Resolve the connection and return ``(connection_id, token_info)``. + + Raises: + ValueError: The connection is missing or needs reconnecting. + """ + from docsgpt.connectors import service + + resolved = connection_id or service.connection_id_for_session_token(session_token) + self.connection_id = resolved + return resolved, service.get_valid_token_info(resolved) + + def _refresh_rejected_token(self, access_token: Optional[str]) -> Dict[str, Any]: + """Token info after the provider answered 401 to ``access_token``. + + Refreshes under the connection's row lock and persists the rotated + refresh token, or returns the token another worker already renewed. + """ + from docsgpt.connectors import service + + if not self.connection_id: + raise ValueError("Loader has no connection to refresh") + return service.get_valid_token_info(self.connection_id, rejected_access_token=access_token) @abstractmethod def load_data(self, inputs: Dict[str, Any]) -> List[Document]: diff --git a/docsgpt/parser/connectors/confluence/auth.py b/docsgpt/parser/connectors/confluence/auth.py index 431e99c7..6a5cf0fe 100644 --- a/docsgpt/parser/connectors/confluence/auth.py +++ b/docsgpt/parser/connectors/confluence/auth.py @@ -6,7 +6,6 @@ from urllib.parse import urlencode import requests from docsgpt.core.settings import settings -from docsgpt.parser.connectors._auth_utils import session_token_fingerprint from docsgpt.parser.connectors.base import BaseConnectorAuth logger = logging.getLogger(__name__) @@ -152,32 +151,6 @@ class ConfluenceAuth(BaseConnectorAuth): except Exception: return True - def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]: - from docsgpt.storage.db.repositories.connector_sessions import ( - ConnectorSessionsRepository, - ) - from docsgpt.storage.db.session import db_readonly - - with db_readonly() as conn: - session = ConnectorSessionsRepository(conn).get_by_session_token( - session_token - ) - if not session: - raise ValueError( - f"Invalid session token ({session_token_fingerprint(session_token)})" - ) - - token_info = session.get("token_info") - if not token_info: - raise ValueError("Session missing token information") - - required = ["access_token", "refresh_token", "cloud_id"] - missing = [f for f in required if not token_info.get(f)] - if missing: - raise ValueError(f"Missing required token fields: {missing}") - - return token_info - def sanitize_token_info( self, token_info: Dict[str, Any], **extra_fields ) -> Dict[str, Any]: diff --git a/docsgpt/parser/connectors/confluence/loader.py b/docsgpt/parser/connectors/confluence/loader.py index 38c379dc..646913b0 100644 --- a/docsgpt/parser/connectors/confluence/loader.py +++ b/docsgpt/parser/connectors/confluence/loader.py @@ -44,12 +44,8 @@ def _retry_on_auth_failure(func): "Auth failure in %s, refreshing token and retrying", func.__name__ ) try: - new_token_info = self.auth.refresh_access_token(self.refresh_token) + new_token_info = self._refresh_rejected_token(self.access_token) self.access_token = new_token_info["access_token"] - self.refresh_token = new_token_info.get( - "refresh_token", self.refresh_token - ) - self._persist_refreshed_tokens(new_token_info) except Exception as refresh_err: raise ValueError( f"Authentication failed and could not be refreshed: {refresh_err}" @@ -62,13 +58,15 @@ def _retry_on_auth_failure(func): class ConfluenceLoader(BaseConnectorLoader): - def __init__(self, session_token: str): + def __init__(self, session_token: Optional[str] = None, *, connection_id: Optional[str] = None): self.auth = ConfluenceAuth() self.session_token = session_token - token_info = self.auth.get_token_info_from_session(session_token) + _, token_info = self._load_token_info(session_token, connection_id) + missing = [f for f in ("access_token", "cloud_id") if not token_info.get(f)] + if missing: + raise ValueError(f"Missing required token fields: {missing}") self.access_token = token_info["access_token"] - self.refresh_token = token_info["refresh_token"] self.cloud_id = token_info["cloud_id"] self.base_url = API_V2.format(cloud_id=self.cloud_id) @@ -81,22 +79,6 @@ class ConfluenceLoader(BaseConnectorLoader): "Accept": "application/json", } - def _persist_refreshed_tokens(self, token_info: Dict[str, Any]) -> None: - try: - from docsgpt.storage.db.repositories.connector_sessions import ( - ConnectorSessionsRepository, - ) - from docsgpt.storage.db.session import db_session - - sanitized = self.auth.sanitize_token_info(token_info) - with db_session() as conn: - repo = ConnectorSessionsRepository(conn) - session = repo.get_by_session_token(self.session_token) - if session: - repo.update(str(session["id"]), {"token_info": sanitized}) - except Exception as e: - logger.warning("Failed to persist refreshed tokens: %s", e) - @_retry_on_auth_failure def load_data(self, inputs: Dict[str, Any]) -> List[Document]: folder_id = inputs.get("folder_id") diff --git a/docsgpt/parser/connectors/google_drive/auth.py b/docsgpt/parser/connectors/google_drive/auth.py index ddff3e4e..0a10d203 100644 --- a/docsgpt/parser/connectors/google_drive/auth.py +++ b/docsgpt/parser/connectors/google_drive/auth.py @@ -8,7 +8,6 @@ from googleapiclient.discovery import build from googleapiclient.errors import HttpError from docsgpt.core.settings import settings -from docsgpt.parser.connectors._auth_utils import session_token_fingerprint from docsgpt.parser.connectors.base import BaseConnectorAuth @@ -213,39 +212,6 @@ class GoogleDriveAuth(BaseConnectorAuth): return True - def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]: - try: - from docsgpt.storage.db.repositories.connector_sessions import ( - ConnectorSessionsRepository, - ) - from docsgpt.storage.db.session import db_readonly - - with db_readonly() as conn: - session = ConnectorSessionsRepository(conn).get_by_session_token( - session_token - ) - if not session: - raise ValueError( - f"Invalid session token ({session_token_fingerprint(session_token)})" - ) - - token_info = session.get("token_info") - if not token_info: - raise ValueError("Session missing token information") - - required_fields = ["access_token", "refresh_token"] - missing_fields = [field for field in required_fields if field not in token_info or not token_info.get(field)] - if missing_fields: - raise ValueError(f"Missing required token fields: {missing_fields}") - - if 'token_uri' not in token_info: - token_info['token_uri'] = 'https://oauth2.googleapis.com/token' - - return token_info - - except Exception as e: - raise ValueError(f"Failed to retrieve Google Drive token information: {str(e)}") - def validate_credentials(self, credentials: Credentials) -> bool: """ Validate Google Drive credentials by making a test API call. diff --git a/docsgpt/parser/connectors/google_drive/loader.py b/docsgpt/parser/connectors/google_drive/loader.py index 9a8d7a97..f37a6560 100644 --- a/docsgpt/parser/connectors/google_drive/loader.py +++ b/docsgpt/parser/connectors/google_drive/loader.py @@ -48,11 +48,13 @@ class GoogleDriveLoader(BaseConnectorLoader): 'application/vnd.google-apps.spreadsheet': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' } - def __init__(self, session_token: str): + def __init__(self, session_token: Optional[str] = None, *, connection_id: Optional[str] = None): self.auth = GoogleDriveAuth() self.session_token = session_token - token_info = self.auth.get_token_info_from_session(session_token) + _, token_info = self._load_token_info(session_token, connection_id) + # Google refresh tokens do not rotate, so the credentials object may + # renew the access token in memory for the rest of this run. self.credentials = self.auth.create_credentials_from_token_info(token_info) try: diff --git a/docsgpt/parser/connectors/share_point/auth.py b/docsgpt/parser/connectors/share_point/auth.py index ec006740..4188f143 100644 --- a/docsgpt/parser/connectors/share_point/auth.py +++ b/docsgpt/parser/connectors/share_point/auth.py @@ -5,7 +5,6 @@ from typing import Optional, Dict, Any from msal import ConfidentialClientApplication from docsgpt.core.settings import settings -from docsgpt.parser.connectors._auth_utils import session_token_fingerprint from docsgpt.parser.connectors.base import BaseConnectorAuth logger = logging.getLogger(__name__) @@ -76,41 +75,6 @@ class SharePointAuth(BaseConnectorAuth): return self.map_token_response(result) - def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]: - try: - from docsgpt.storage.db.repositories.connector_sessions import ( - ConnectorSessionsRepository, - ) - from docsgpt.storage.db.session import db_readonly - - with db_readonly() as conn: - session = ConnectorSessionsRepository(conn).get_by_session_token( - session_token - ) - - if not session: - raise ValueError( - f"Invalid session token ({session_token_fingerprint(session_token)})" - ) - - token_info = session.get("token_info") - if not token_info: - raise ValueError("Session missing token information") - - required_fields = ["access_token", "refresh_token"] - missing_fields = [field for field in required_fields if field not in token_info or not token_info.get(field)] - if missing_fields: - raise ValueError(f"Missing required token fields: {missing_fields}") - - if 'token_uri' not in token_info: - token_info['token_uri'] = f"https://login.microsoftonline.com/{settings.MICROSOFT_TENANT_ID}/oauth2/v2.0/token" - - return token_info - - except Exception as e: - logger.error("Failed to retrieve token from session: %s", e) - raise ValueError(f"Failed to retrieve SharePoint token information: {str(e)}") - def is_token_expired(self, token_info: Dict[str, Any]) -> bool: if not token_info: return True diff --git a/docsgpt/parser/connectors/share_point/loader.py b/docsgpt/parser/connectors/share_point/loader.py index 6fa3e2c3..4690b303 100644 --- a/docsgpt/parser/connectors/share_point/loader.py +++ b/docsgpt/parser/connectors/share_point/loader.py @@ -26,8 +26,7 @@ def _retry_on_auth_failure(func): if e.response is not None and e.response.status_code in (401, 403): logging.info(f"Auth failure in {func.__name__}, refreshing token and retrying") try: - new_token_info = self.auth.refresh_access_token(self.refresh_token) - self.access_token = new_token_info.get('access_token') + self._apply_token_info(self._refresh_rejected_token(self.access_token)) except Exception as refresh_error: raise ValueError( f"Authentication failed and could not be refreshed: {refresh_error}" @@ -63,13 +62,12 @@ class SharePointLoader(BaseConnectorLoader): GRAPH_API_BASE = "https://graph.microsoft.com/v1.0" - def __init__(self, session_token: str): + def __init__(self, session_token: Optional[str] = None, *, connection_id: Optional[str] = None): self.auth = SharePointAuth() self.session_token = session_token - token_info = self.auth.get_token_info_from_session(session_token) - self.access_token = token_info.get('access_token') - self.refresh_token = token_info.get('refresh_token') + _, token_info = self._load_token_info(session_token, connection_id) + self._apply_token_info(token_info) self.allows_shared_content = token_info.get('allows_shared_content', False) if not self.access_token: @@ -77,6 +75,10 @@ class SharePointLoader(BaseConnectorLoader): self.next_page_token = None + def _apply_token_info(self, token_info: Dict[str, Any]) -> None: + self.access_token = token_info.get('access_token') + self.expiry = token_info.get('expiry') + def _get_headers(self) -> Dict[str, str]: return { 'Authorization': f'Bearer {self.access_token}', @@ -87,12 +89,14 @@ class SharePointLoader(BaseConnectorLoader): if not self.access_token: raise ValueError("No access token available") - token_info = {'access_token': self.access_token, 'expiry': None} - if self.auth.is_token_expired(token_info): + # The connection service refreshes under a row lock and stores the + # rotated refresh token; refreshing here would spend it and lose it. + if self.auth.is_token_expired({'access_token': self.access_token, 'expiry': self.expiry}): logging.info("Token expired, attempting refresh") try: - new_token_info = self.auth.refresh_access_token(self.refresh_token) - self.access_token = new_token_info.get('access_token') + from docsgpt.connectors import service + + self._apply_token_info(service.get_valid_token_info(self.connection_id)) except Exception: raise ValueError("Failed to refresh access token") diff --git a/docsgpt/storage/db/models.py b/docsgpt/storage/db/models.py index 147f4316..4e8d052a 100644 --- a/docsgpt/storage/db/models.py +++ b/docsgpt/storage/db/models.py @@ -219,6 +219,8 @@ user_tools_table = Table( Column( "connection_id", UUID(as_uuid=True), ForeignKey("connector_sessions.id", ondelete="SET NULL"), ), + # Whose account a shared resource runs with: the owner's, or each member's. + Column("credential_mode", Text, nullable=False, server_default="owner"), ) token_usage_table = Table( @@ -344,6 +346,8 @@ sources_table = Table( Column( "connection_id", UUID(as_uuid=True), ForeignKey("connector_sessions.id", ondelete="SET NULL"), ), + # Whose account a shared resource runs with: the owner's, or each member's. + Column("credential_mode", Text, nullable=False, server_default="owner"), ) agents_table = Table( @@ -598,6 +602,12 @@ connector_sessions_table = Table( Column("account_label", Text), Column("auth_kind", Text), Column("updated_at", DateTime(timezone=True), nullable=False, server_default=func.now()), + # Every secret of the connection, as one owner-bound v2 envelope. + Column("encrypted_credentials", Text), + Column("has_refresh_token", Boolean, nullable=False, server_default="false"), + Column("scopes", JSONB, nullable=False, server_default="[]"), + Column("last_error", Text), + Column("last_used_at", DateTime(timezone=True)), ) diff --git a/docsgpt/storage/db/repositories/connector_sessions.py b/docsgpt/storage/db/repositories/connector_sessions.py index a521406c..1ec86927 100644 --- a/docsgpt/storage/db/repositories/connector_sessions.py +++ b/docsgpt/storage/db/repositories/connector_sessions.py @@ -9,8 +9,13 @@ Shape notes: unique constraint on ``session_token``. * MCP sessions key off ``server_url`` instead — a single user may have multiple MCP servers, one row each. The composite unique index - ``(user_id, COALESCE(server_url, ''), provider)`` makes both patterns - coexist without collision. + ``(user_id, provider, COALESCE(server_url, ''), COALESCE(account_label, ''))`` + makes both patterns coexist, and lets one user connect several accounts + of the same provider (each row carries its own ``account_label``). +* Every secret lives in ``encrypted_credentials``, written and read only by + ``docsgpt.connectors.service``; ``token_info`` and the ``tokens`` / + ``client_info`` keys of ``session_data`` are legacy plaintext that + migration 0038 moved into it. * ``session_data`` remains a catch-all JSONB for driver-specific state (tokens that don't fit anywhere else, per-provider scratch data). Promoted columns (``session_token``, ``user_email``, ``status``, @@ -31,8 +36,9 @@ from docsgpt.storage.db.serialization import PGNativeJSONEncoder _UPDATABLE_SCALARS = { "server_url", "session_token", "user_email", "status", "expires_at", "connector_key", "display_name", "account_label", "auth_kind", + "encrypted_credentials", "has_refresh_token", "last_error", "last_used_at", } -_UPDATABLE_JSONB = {"session_data", "token_info"} +_UPDATABLE_JSONB = {"session_data", "token_info", "scopes"} def _jsonb(value: Any) -> Any: @@ -71,7 +77,8 @@ class ConnectorSessionsRepository: ) -> dict: """Insert or update a connector session row. - Conflict key is ``(user_id, COALESCE(server_url, ''), provider)`` + Conflict key is the account index + ``(user_id, provider, COALESCE(server_url, ''), COALESCE(account_label, ''))`` so MCP rows (per-server) and OAuth rows (per-provider) both get idempotent upsert semantics. """ @@ -87,7 +94,7 @@ class ConnectorSessionsRepository: :status, CAST(:token_info AS jsonb), CAST(:session_data AS jsonb), :expires_at, :legacy_mongo_id ) - ON CONFLICT (user_id, COALESCE(server_url, ''), provider) + ON CONFLICT (user_id, provider, COALESCE(server_url, ''), COALESCE(account_label, '')) DO UPDATE SET session_token = COALESCE(EXCLUDED.session_token, connector_sessions.session_token), user_email = COALESCE(EXCLUDED.user_email, connector_sessions.user_email), @@ -182,6 +189,77 @@ class ConnectorSessionsRepository: ) return [row_to_dict(r) for r in result.fetchall()] + def create( + self, + user_id: str, + provider: str, + *, + connector_key: str, + auth_kind: str, + display_name: Optional[str] = None, + account_label: Optional[str] = None, + server_url: Optional[str] = None, + status: str = "connected", + encrypted_credentials: Optional[str] = None, + has_refresh_token: bool = False, + ) -> Optional[dict]: + """Insert a connection; return None when that account already exists.""" + result = self._conn.execute( + text( + """ + INSERT INTO connector_sessions ( + user_id, provider, server_url, connector_key, auth_kind, display_name, + account_label, status, encrypted_credentials, has_refresh_token, session_data + ) + VALUES ( + :user_id, :provider, :server_url, :connector_key, :auth_kind, :display_name, + :account_label, :status, :encrypted_credentials, :has_refresh_token, '{}'::jsonb + ) + ON CONFLICT (user_id, provider, COALESCE(server_url, ''), COALESCE(account_label, '')) + DO NOTHING + RETURNING * + """ + ), + { + "user_id": user_id, + "provider": provider, + "server_url": server_url, + "connector_key": connector_key, + "auth_kind": auth_kind, + "display_name": display_name, + "account_label": account_label, + "status": status, + "encrypted_credentials": encrypted_credentials, + "has_refresh_token": has_refresh_token, + }, + ) + row = result.fetchone() + return row_to_dict(row) if row is not None else None + + def find_account( + self, user_id: str, provider: str, *, server_url: Optional[str], account_label: Optional[str], + ) -> Optional[dict]: + """The connection for one account, matching the account unique index.""" + result = self._conn.execute( + text( + "SELECT * FROM connector_sessions WHERE user_id = :user_id AND provider = :provider " + "AND COALESCE(server_url, '') = COALESCE(:server_url, '') " + "AND COALESCE(account_label, '') = COALESCE(:account_label, '')" + ), + {"user_id": user_id, "provider": provider, "server_url": server_url, "account_label": account_label}, + ) + row = result.fetchone() + return row_to_dict(row) if row is not None else None + + def delete_by_id(self, connection_id: str) -> bool: + """Delete a connection row. Linked sources and tools keep existing (SET NULL).""" + if not looks_like_uuid(connection_id): + return False + result = self._conn.execute( + text("DELETE FROM connector_sessions WHERE id = CAST(:id AS uuid)"), {"id": str(connection_id)}, + ) + return result.rowcount > 0 + def get(self, connection_id: str) -> Optional[dict]: """Fetch a connection by id, whoever owns it. Callers authorise.""" if not looks_like_uuid(connection_id): @@ -237,7 +315,7 @@ class ConnectorSessionsRepository: """Sources synced from a connection, newest first.""" result = self._conn.execute( text( - "SELECT id, name, type, date, sync_frequency, metadata, remote_data, user_id " + "SELECT id, name, type, date, sync_frequency, metadata, remote_data, user_id, file_path " "FROM sources WHERE connection_id = CAST(:id AS uuid) ORDER BY date DESC" ), {"id": str(connection_id)}, @@ -337,7 +415,7 @@ class ConnectorSessionsRepository: Notes: The conflict target matches the table's composite unique - constraint ``(user_id, COALESCE(server_url, ''), provider)`` + index ``(user_id, provider, COALESCE(server_url, ''), COALESCE(account_label, ''))`` so MCP's per-URL rows and OAuth's single-row-per-user rows both upsert idempotently. """ @@ -358,7 +436,7 @@ class ConnectorSessionsRepository: :user_id, :provider, :server_url, CAST(:patch AS jsonb) ) - ON CONFLICT (user_id, COALESCE(server_url, ''), provider) + ON CONFLICT (user_id, provider, COALESCE(server_url, ''), COALESCE(account_label, '')) DO UPDATE SET server_url = COALESCE(EXCLUDED.server_url, connector_sessions.server_url), session_data = diff --git a/docsgpt/storage/db/repositories/user_tools.py b/docsgpt/storage/db/repositories/user_tools.py index ff235858..0c8e5ca7 100644 --- a/docsgpt/storage/db/repositories/user_tools.py +++ b/docsgpt/storage/db/repositories/user_tools.py @@ -23,8 +23,11 @@ from docsgpt.storage.db.base_repository import looks_like_uuid, row_to_dict _JSONB_COLUMNS = {"config", "config_requirements", "actions"} -_SCALAR_COLUMNS = {"name", "custom_name", "display_name", "description", "status"} -_ALLOWED_COLUMNS = _SCALAR_COLUMNS | _JSONB_COLUMNS +_SCALAR_COLUMNS = {"name", "custom_name", "display_name", "description", "status", "credential_mode"} +# Set by server code only (tool creation, connection setup); route handlers +# must not pass client input here. +_UUID_COLUMNS = {"connection_id"} +_ALLOWED_COLUMNS = _SCALAR_COLUMNS | _JSONB_COLUMNS | _UUID_COLUMNS def _encode_jsonb(value: Any) -> Any: @@ -60,6 +63,8 @@ class UserToolsRepository: status: bool = True, extra: Optional[dict] = None, legacy_mongo_id: Optional[str] = None, + connection_id: Optional[str] = None, + credential_mode: str = "owner", ) -> dict: """Insert a new tool row. ``extra`` is merged into the config JSONB.""" cfg = config or {} @@ -70,14 +75,16 @@ class UserToolsRepository: """ INSERT INTO user_tools ( user_id, name, custom_name, display_name, description, - config, config_requirements, actions, status, legacy_mongo_id + config, config_requirements, actions, status, legacy_mongo_id, + connection_id, credential_mode ) VALUES ( :user_id, :name, :custom_name, :display_name, :description, CAST(:config AS jsonb), CAST(:config_requirements AS jsonb), CAST(:actions AS jsonb), - :status, :legacy_mongo_id + :status, :legacy_mongo_id, + CAST(:connection_id AS uuid), :credential_mode ) RETURNING * """ @@ -93,6 +100,8 @@ class UserToolsRepository: "actions": _encode_jsonb(actions or []), "status": status, "legacy_mongo_id": legacy_mongo_id, + "connection_id": str(connection_id) if connection_id else None, + "credential_mode": credential_mode, }, ) return row_to_dict(result.fetchone()) @@ -213,6 +222,9 @@ class UserToolsRepository: if col in _JSONB_COLUMNS: set_clauses.append(f"{col} = CAST(:{col} AS jsonb)") params[col] = _encode_jsonb(val) + elif col in _UUID_COLUMNS: + set_clauses.append(f"{col} = CAST(:{col} AS uuid)") + params[col] = str(val) if val else None else: set_clauses.append(f"{col} = :{col}") params[col] = val diff --git a/docsgpt/worker.py b/docsgpt/worker.py index f3311859..83f7abc0 100755 --- a/docsgpt/worker.py +++ b/docsgpt/worker.py @@ -1237,6 +1237,7 @@ def remote_worker( config=None, idempotency_key=None, source_id=None, + connection_id=None, ): safe_user = safe_filename(user) full_path = os.path.join(directory, safe_user, uuid.uuid4().hex) @@ -1283,7 +1284,14 @@ def remote_worker( self.update_state(state="PROGRESS", meta={"current": 1}) logging.info("Initializing remote loader with type: %s", loader) remote_loader = RemoteCreator.create_loader(loader) - raw_docs = remote_loader.load_data(source_data) + loader_input = source_data + if connection_id: + loader_input = _with_connection_credentials(source_data, connection_id) + if loader_input is None: + from docsgpt.connectors.service import ConnectionUnavailable + + raise ConnectionUnavailable("Reconnect to continue", connection_id=str(connection_id)) + raw_docs = remote_loader.load_data(loader_input) cfg = SourceConfig.parse(config) chunker = ChunkerCreator.create_chunker( @@ -1414,6 +1422,8 @@ def remote_worker( f"Failed to update last_sync for source {source_id_for_events}: {upd_err}" ) upload_index(full_path, file_data) + if connection_id: + _link_source_to_connection(source_id_for_events, str(connection_id)) publish_user_event( user, "source.ingest.completed", @@ -1467,6 +1477,7 @@ def sync( retriever, doc_id=None, directory="temp", + connection_id=None, ): try: remote_worker( @@ -1480,6 +1491,7 @@ def sync( sync_frequency, "sync", doc_id, + connection_id=connection_id, ) except Exception as e: logging.error(f"Error during sync: {e}", exc_info=True) @@ -1494,7 +1506,7 @@ def sync_worker(self, frequency): with db_readonly() as conn: result = conn.execute( sql_text( - "SELECT id, name, user_id, type, remote_data, retriever " + "SELECT id, name, user_id, type, remote_data, retriever, connection_id " "FROM sources WHERE sync_frequency = :freq" ), {"freq": frequency}, @@ -1511,10 +1523,17 @@ def sync_worker(self, frequency): sync_counts["total_sync_count"] += 1 - # Connector sources have no RemoteCreator loader and need an OAuth - # token to sync, which a scheduled task lacks — skip them. + # Connector sources sync from their connection, whose token the + # worker can refresh. Legacy ones with no connection still need the + # browser, so they are skipped as before. if source_type and source_type.startswith("connector"): - sync_counts["sync_skipped"] += 1 + if doc.get("connection_id"): + from docsgpt.api.user.tasks import sync_connector_source as sync_task + + sync_task.delay(doc_id) + sync_counts["sync_dispatched"] += 1 + else: + sync_counts["sync_skipped"] += 1 continue source_data = normalize_remote_data(source_type, doc.get("remote_data")) @@ -1525,7 +1544,8 @@ def sync_worker(self, frequency): continue resp = sync( - self, source_data, name, user, source_type, frequency, retriever, doc_id + self, source_data, name, user, source_type, frequency, retriever, doc_id, + connection_id=str(doc["connection_id"]) if doc.get("connection_id") else None, ) sync_counts[ "sync_success" if resp["status"] == "success" else "sync_failure" @@ -1533,7 +1553,7 @@ def sync_worker(self, frequency): return { key: sync_counts[key] for key in [ - "total_sync_count", "sync_success", "sync_failure", "sync_skipped", + "total_sync_count", "sync_success", "sync_failure", "sync_skipped", "sync_dispatched", ] } @@ -2161,6 +2181,113 @@ def _webhook_tool_allowlist(agent_config): return [] +def _with_connection_credentials(source_data, connection_id: str): + """Loader input with the connection's stored keys merged in, or None. + + S3 and Reddit sources made from a connection keep their keys on the + connection only, never in ``sources.remote_data``. Returns None when the + connection is gone or needs reconnecting. + """ + from docsgpt.connectors import service + from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository + + with db_readonly() as conn: + row = ConnectorSessionsRepository(conn).get(str(connection_id)) + if row is None: + return None + try: + credentials = service.get_credentials(row) + except service.ConnectionUnavailable: + return None + as_text = isinstance(source_data, str) + data = json.loads(source_data) if as_text else dict(source_data or {}) + data.update(credentials) + return json.dumps(data) if as_text else data + + +def _link_source_to_connection(source_id: str, connection_id: str) -> None: + """Point a source at the connection it syncs from, and lift any reconnect pause.""" + from sqlalchemy import text as sql_text + + try: + with db_session() as conn: + conn.execute( + sql_text( + "UPDATE sources SET connection_id = CAST(:cid AS uuid), " + "metadata = metadata - 'sync_state' WHERE id = CAST(:sid AS uuid)" + ), + {"cid": str(connection_id), "sid": str(source_id)}, + ) + except Exception: + logging.warning("Could not link source %s to connection %s", source_id, connection_id, exc_info=True) + + +def sync_connector_source(self, source_id: str) -> Dict[str, Any]: + """Re-download and re-index a connector source from its connection. + + Runs as the connection owner with no browser: the connection service + refreshes the token under a row lock. When the grant was revoked the + service flags the connection and pauses its sources, and this returns + ``paused`` rather than failing again on every schedule. + + Args: + self: The bound Celery task. + source_id: The source to sync. + + Returns: + ``{"status": "success" | "paused" | "skipped"}`` plus the ingest result. + """ + from docsgpt.connectors.service import ConnectionUnavailable, normalize_status + from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository + + with db_readonly() as conn: + from sqlalchemy import text as sql_text + + row = conn.execute( + sql_text( + "SELECT id, name, user_id, remote_data, retriever, sync_frequency, config, connection_id " + "FROM sources WHERE id = CAST(:id AS uuid)" + ), + {"id": str(source_id)}, + ).fetchone() + source = dict(row._mapping) if row else None + connection = ( + ConnectorSessionsRepository(conn).get(str(source["connection_id"])) + if source and source.get("connection_id") + else None + ) + if not source or not connection: + return {"status": "skipped"} + if normalize_status(connection) != "connected": + return {"status": "paused"} + remote_data = source.get("remote_data") or {} + if isinstance(remote_data, str): + try: + remote_data = json.loads(remote_data) + except json.JSONDecodeError: + remote_data = {} + provider = remote_data.get("provider") or connection.get("provider") + try: + result = ingest_connector( + self, + source.get("name"), + source.get("user_id"), + provider, + connection_id=str(connection["id"]), + file_ids=remote_data.get("file_ids") or [], + folder_ids=remote_data.get("folder_ids") or [], + recursive=remote_data.get("recursive", True), + retriever=source.get("retriever") or "classic", + operation_mode="sync", + doc_id=str(source["id"]), + sync_frequency=source.get("sync_frequency") or "never", + config=source.get("config") or None, + ) + except ConnectionUnavailable: + return {"status": "paused"} + return {"status": "success", "result": result} + + def ingest_connector( self, job_name: str, @@ -2177,6 +2304,7 @@ def ingest_connector( config=None, idempotency_key=None, source_id=None, + connection_id=None, ) -> Dict[str, Any]: """ Ingestion for internal knowledge bases (GoogleDrive, etc.). @@ -2185,7 +2313,8 @@ def ingest_connector( job_name: Name of the ingestion job user: User identifier source_type: Type of remote source ("google_drive", "dropbox", etc.) - session_token: Authentication token for the service + session_token: Legacy browser session token naming the connection + connection_id: The connection whose account the files are read with file_ids: List of file IDs to download folder_ids: List of folder IDs to download recursive: Whether to recursively download folders @@ -2247,8 +2376,8 @@ def ingest_connector( meta={"current": 10, "status": "Initializing connector"}, ) - if not session_token: - raise ValueError(f"{source_type} connector requires session_token") + if not session_token and not connection_id: + raise ValueError(f"{source_type} connector requires a connection") if not ConnectorCreator.is_supported(source_type): raise ValueError( @@ -2256,8 +2385,9 @@ def ingest_connector( ) remote_loader = ConnectorCreator.create_connector( - source_type, session_token + source_type, session_token, connection_id=connection_id ) + connection_id = remote_loader.connection_id # Create a clean config for storage api_source_config = { @@ -2411,6 +2541,8 @@ def ingest_connector( ) upload_index(vector_store_path, file_data) + if connection_id: + _link_source_to_connection(source_id_for_events, connection_id) # Ensure we mark the task as complete self.update_state( diff --git a/tests/agents/tools/test_mcp_tool.py b/tests/agents/tools/test_mcp_tool.py index d418ef0a..d24da614 100644 --- a/tests/agents/tools/test_mcp_tool.py +++ b/tests/agents/tools/test_mcp_tool.py @@ -1010,6 +1010,10 @@ class TestDBTokenStorage: monkeypatch.setattr(session_mod, "db_session", _yield) monkeypatch.setattr(session_mod, "db_readonly", _yield) + import docsgpt.connectors.service as service_mod + + monkeypatch.setattr(service_mod, "db_session", _yield) + monkeypatch.setattr(service_mod, "db_readonly", _yield) def test_get_base_url(self): from docsgpt.agents.tools.mcp_tool import DBTokenStorage @@ -1132,7 +1136,13 @@ class TestDBTokenStorage: # ``server_url`` must NOT be duplicated inside the JSONB blob. session_data = row["session_data"] or {} assert "server_url" not in session_data - assert session_data.get("tokens", {}).get("access_token") == "at" + # Tokens live only in the encrypted envelope, never in plaintext. + assert "tokens" not in session_data + assert row["encrypted_credentials"].startswith("v2:") + from docsgpt.connectors.service import read_secrets + + assert read_secrets(row)["tokens"]["access_token"] == "at" + assert row["status"] == "connected" def test_clear_removes_row(self, monkeypatch, pg_conn): from mcp.shared.auth import OAuthToken diff --git a/tests/api/test_connector_oauth_security.py b/tests/api/test_connector_oauth_security.py index f8b2e521..d91641c7 100644 --- a/tests/api/test_connector_oauth_security.py +++ b/tests/api/test_connector_oauth_security.py @@ -21,7 +21,11 @@ def _patch_db(conn, module="docsgpt.api.connector.routes"): def _yield(): yield conn - with patch(f"{module}.db_session", _yield), patch(f"{module}.db_readonly", _yield): + with patch(f"{module}.db_session", _yield), patch(f"{module}.db_readonly", _yield), patch( + "docsgpt.api.connector.routes.db_readonly", _yield + ), patch("docsgpt.connectors.service.db_session", _yield), patch( + "docsgpt.connectors.service.db_readonly", _yield + ): yield diff --git a/tests/api/test_connector_routes_happy.py b/tests/api/test_connector_routes_happy.py index 141f87b0..52176f0a 100644 --- a/tests/api/test_connector_routes_happy.py +++ b/tests/api/test_connector_routes_happy.py @@ -30,6 +30,10 @@ def _patch_db(conn): "docsgpt.api.connector.routes.db_session", _yield ), patch( "docsgpt.api.connector.routes.db_readonly", _yield + ), patch( + "docsgpt.connectors.service.db_session", _yield + ), patch( + "docsgpt.connectors.service.db_readonly", _yield ): yield @@ -487,8 +491,10 @@ class TestConnectorSync: def test_returns_400_missing_fields(self, app): from docsgpt.api.connector.routes import ConnectorSync + # The source's own connection is used when none is named, so only + # the source id is required. with app.test_request_context( - "/api/connectors/sync", method="POST", json={"source_id": "x"} + "/api/connectors/sync", method="POST", json={} ): from flask import request request.decoded_token = {"sub": "u"} diff --git a/tests/api/user/sources/test_upload_idempotency.py b/tests/api/user/sources/test_upload_idempotency.py index f5b069bc..fe251ff4 100644 --- a/tests/api/user/sources/test_upload_idempotency.py +++ b/tests/api/user/sources/test_upload_idempotency.py @@ -24,6 +24,10 @@ def _patch_db(conn): "docsgpt.api.user.sources.upload.db_session", _yield ), patch( "docsgpt.api.user.sources.upload.db_readonly", _yield + ), patch( + "docsgpt.connectors.service.db_session", _yield + ), patch( + "docsgpt.connectors.service.db_readonly", _yield ): yield diff --git a/tests/api/user/test_tasks.py b/tests/api/user/test_tasks.py index a3ff2a19..dcf5be29 100644 --- a/tests/api/user/test_tasks.py +++ b/tests/api/user/test_tasks.py @@ -67,7 +67,8 @@ class TestIngestRemoteTask: mock_worker.assert_called_once_with( ANY, {"url": "http://x"}, "job1", "user1", "web", - config=None, idempotency_key=None, source_id=None, + sync_frequency="never", + config=None, idempotency_key=None, source_id=None, connection_id=None, ) assert result == {"status": "ok"} @@ -223,6 +224,7 @@ class TestIngestConnectorTask: "user1", "gdrive", session_token=None, + connection_id=None, file_ids=None, folder_ids=None, recursive=True, @@ -248,6 +250,7 @@ class TestIngestConnectorTask: "user1", "sharepoint", session_token="tok", + connection_id=None, file_ids=["f1"], folder_ids=["d1"], recursive=False, @@ -263,6 +266,7 @@ class TestIngestConnectorTask: "user1", "sharepoint", session_token="tok", + connection_id=None, file_ids=["f1"], folder_ids=["d1"], recursive=False, diff --git a/tests/connectors/test_connections_api.py b/tests/connectors/test_connections_api.py new file mode 100644 index 00000000..e02144c8 --- /dev/null +++ b/tests/connectors/test_connections_api.py @@ -0,0 +1,323 @@ +"""Tests for the connection write endpoints.""" + +from __future__ import annotations + +import json +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +import pytest +from flask import Flask +from sqlalchemy import text + +from docsgpt.connectors import service +from docsgpt.security.encryption import encrypt_json + + +@pytest.fixture +def app(): + return Flask(__name__) + + +@contextmanager +def _db(conn): + @contextmanager + def _yield(): + yield conn + + with patch.multiple("docsgpt.api.connector.connections", db_session=_yield, db_readonly=_yield), \ + patch.multiple("docsgpt.connectors.service", db_session=_yield, db_readonly=_yield), \ + patch.multiple("docsgpt.api.connector.routes", db_session=_yield, db_readonly=_yield): + yield + + +def _call(app, resource, method, path, user="alice", body=None, headers=None, args=()): + with app.test_request_context(path, method=method.upper(), json=body, headers=headers or {}): + from flask import request + + request.decoded_token = {"sub": user} if user else None + return getattr(resource(), method)(*args) + + +def _connection(conn, user="alice", provider="telegram", auth_kind="api_key", secrets=None, status="connected", + **cols) -> str: + values = { + "user_id": user, "provider": provider, "connector_key": provider, "auth_kind": auth_kind, + "status": status, "account_label": cols.pop("account_label", "…abcd"), + "encrypted_credentials": encrypt_json(secrets, user) if secrets is not None else None, **cols, + } + names = ", ".join(values) + params = ", ".join(f":{k}" for k in values) + return str(conn.execute( + text(f"INSERT INTO connector_sessions ({names}) VALUES ({params}) RETURNING id"), values, + ).scalar()) + + +class TestCreate: + def test_telegram_zero_question_setup(self, app, pg_conn): + """Appendix C of the connectors spec: create, then set up, tools appear.""" + from docsgpt.api.connector.connections import ConnectionSetup, ConnectionsList + + with _db(pg_conn): + created = _call(app, ConnectionsList, "post", "/api/connections", + body={"connector_key": "telegram", "credentials": {"token": "123:abcdefgh"}}) + assert created.status_code == 201 + payload = created.get_json() + assert payload["setup"] == {"tools": "auto", "sync": "off"} + assert "token" not in json.dumps(payload) + cid = payload["connection"]["id"] + setup = _call(app, ConnectionSetup, "post", f"/api/connections/{cid}/setup", + body={"create_tools": True}, args=[cid]) + assert setup.status_code == 200 + tools = setup.get_json()["tools"] + assert [t["name"] for t in tools] == ["telegram"] + actions = {a["name"]: a for a in tools[0]["actions"]} + assert actions["telegram_send_message"]["access"] == "write" + assert actions["telegram_send_message"]["permission"] == "ask" + + def test_setup_is_idempotent_for_tools(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionSetup + + cid = _connection(pg_conn, secrets={"credentials": {"token": "t"}}) + with _db(pg_conn): + for _ in range(2): + _call(app, ConnectionSetup, "post", f"/api/connections/{cid}/setup", + body={"create_tools": True}, args=[cid]) + count = pg_conn.execute( + text("SELECT count(*) FROM user_tools WHERE connection_id = CAST(:c AS uuid)"), {"c": cid}, + ).scalar() + assert count == 1 + + def test_rejects_oauth_connector(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionsList + + with _db(pg_conn): + resp = _call(app, ConnectionsList, "post", "/api/connections", + body={"connector_key": "google_drive", "credentials": {}}) + assert resp.status_code == 400 + + def test_default_key_refused(self, app, pg_conn, monkeypatch): + from docsgpt.api.connector.connections import ConnectionsList + from docsgpt.core.settings import settings + from docsgpt.security.encryption import DEFAULT_ENCRYPTION_KEY + + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", DEFAULT_ENCRYPTION_KEY) + monkeypatch.setattr(settings, "AUTH_TYPE", "oidc") + with _db(pg_conn): + resp = _call(app, ConnectionsList, "post", "/api/connections", + body={"connector_key": "brave", "credentials": {"token": "some-long-key"}}) + assert resp.status_code == 400 + assert resp.get_json()["code"] == "encryption_key_default" + + +class TestSetupSync: + def test_oauth_source_is_queued_with_connection(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionSetup + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth", secrets={"token_info": {}}) + task = MagicMock(id="task-1") + with _db(pg_conn), patch("docsgpt.api.user.tasks.ingest_connector_task.apply_async", + return_value=task) as apply: + resp = _call(app, ConnectionSetup, "post", f"/api/connections/{cid}/setup", body={ + "create_tools": True, + "sync": {"items": {"folder_ids": ["f1"]}, "frequency": "weekly", "name": "Handbook"}, + }, args=[cid]) + assert resp.status_code == 200 + source = resp.get_json()["sources"][0] + assert source["name"] == "Handbook" and source["sync_frequency"] == "weekly" + kwargs = apply.call_args.kwargs["kwargs"] + assert kwargs["connection_id"] == cid + assert kwargs["folder_ids"] == ["f1"] + assert "session_token" not in kwargs + # Drive has no tools; nothing was created. + assert resp.get_json()["tools"] == [] + + def test_nothing_picked_is_a_bad_request(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionSetup + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth", secrets={"token_info": {}}) + with _db(pg_conn): + resp = _call(app, ConnectionSetup, "post", f"/api/connections/{cid}/setup", + body={"sync": {"items": {}}}, args=[cid]) + assert resp.status_code == 400 + + def test_s3_keys_stay_on_the_connection(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionSetup + + cid = _connection(pg_conn, provider="s3", secrets={"credentials": { + "aws_access_key_id": "AKIA", "aws_secret_access_key": "shh-secret"}}) + with _db(pg_conn), patch("docsgpt.api.user.tasks.ingest_remote.apply_async", + return_value=MagicMock(id="t")) as apply: + resp = _call(app, ConnectionSetup, "post", f"/api/connections/{cid}/setup", body={ + "sync": {"items": {"bucket": "docs", "aws_secret_access_key": "smuggled"}}, + }, args=[cid]) + assert resp.status_code == 200 + kwargs = apply.call_args.kwargs["kwargs"] + assert kwargs["source_data"] == {"bucket": "docs"} + assert kwargs["connection_id"] == cid + + def test_flagged_connection_must_reconnect_first(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionSetup + + cid = _connection(pg_conn, status="reconnect_needed", secrets={}) + with _db(pg_conn): + resp = _call(app, ConnectionSetup, "post", f"/api/connections/{cid}/setup", + body={}, args=[cid]) + assert resp.status_code == 409 + + +class TestReconnect: + def test_api_key_reconnect_replaces_and_resumes(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionReconnect + + cid = _connection(pg_conn, status="reconnect_needed", secrets={"credentials": {"token": "old"}}) + pg_conn.execute(text( + "INSERT INTO sources (user_id, name, connection_id, metadata) " + "VALUES ('alice', 's', CAST(:c AS uuid), '{\"sync_state\": \"paused_reconnect\"}')" + ), {"c": cid}) + with _db(pg_conn): + resp = _call(app, ConnectionReconnect, "post", f"/api/connections/{cid}/reconnect", + body={"credentials": {"token": "new"}}, args=[cid]) + assert resp.status_code == 200 + assert resp.get_json()["connection"]["status"] == "connected" + row = pg_conn.execute(text("SELECT * FROM connector_sessions WHERE id = CAST(:c AS uuid)"), + {"c": cid}).one()._mapping + assert service.read_secrets(dict(row))["credentials"]["token"] == "new" + meta = pg_conn.execute(text("SELECT metadata FROM sources WHERE user_id = 'alice'")).scalar() + assert "sync_state" not in meta + + def test_oauth_reconnect_returns_authorization_url(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionReconnect + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth", secrets={}) + fake_auth = MagicMock() + fake_auth.get_authorization_url.return_value = "https://accounts.example/auth" + with _db(pg_conn), patch("docsgpt.api.connector.routes.ConnectorCreator.create_auth", + return_value=fake_auth): + resp = _call(app, ConnectionReconnect, "post", f"/api/connections/{cid}/reconnect", + body={}, args=[cid]) + assert resp.get_json()["authorization_url"] == "https://accounts.example/auth" + state = fake_auth.get_authorization_url.call_args.kwargs["state"] + import base64 + + assert json.loads(base64.urlsafe_b64decode(state))["object_id"] == cid + + def test_cannot_reconnect_someone_elses(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionReconnect + + cid = _connection(pg_conn, user="bob", secrets={}) + with _db(pg_conn): + resp = _call(app, ConnectionReconnect, "post", f"/api/connections/{cid}/reconnect", + body={"credentials": {"token": "x"}}, args=[cid]) + assert resp.status_code == 404 + + +class TestPickerToken: + def test_returns_access_token_only(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionPickerToken + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth", + secrets={"token_info": {"access_token": "at", "refresh_token": "rt"}}) + fake_auth = MagicMock() + fake_auth.is_token_expired.return_value = False + with _db(pg_conn), patch("docsgpt.parser.connectors.connector_creator.ConnectorCreator.create_auth", + return_value=fake_auth): + resp = _call(app, ConnectionPickerToken, "post", f"/api/connections/{cid}/picker-token", + args=[cid]) + payload = resp.get_json() + assert payload["access_token"] == "at" + assert "rt" not in json.dumps(payload) + + def test_owner_only(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionPickerToken + + cid = _connection(pg_conn, user="bob", provider="google_drive", auth_kind="oauth", secrets={}) + with _db(pg_conn): + resp = _call(app, ConnectionPickerToken, "post", f"/api/connections/{cid}/picker-token", + args=[cid]) + assert resp.status_code == 404 + + +class TestClaim: + def test_claims_own_token(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionClaim + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth", secrets={}, session_token="legacy") + with _db(pg_conn): + resp = _call(app, ConnectionClaim, "post", "/api/connections/claim", + body={"provider": "google_drive", "session_token": "legacy"}) + assert resp.get_json()["connection_id"] == cid + + def test_other_users_token_is_not_found(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionClaim + + _connection(pg_conn, user="bob", provider="google_drive", auth_kind="oauth", secrets={}, session_token="b") + with _db(pg_conn): + resp = _call(app, ConnectionClaim, "post", "/api/connections/claim", + body={"provider": "google_drive", "session_token": "b"}) + assert resp.status_code == 404 + + +class TestPermissions: + def test_set_permissions(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionToolPermissions + + cid = _connection(pg_conn, secrets={"credentials": {"token": "t"}}) + with _db(pg_conn): + tool = service.ensure_connection_tools(pg_conn, "alice", service.ConnectorSessionsRepository( + pg_conn).get(cid))[0] + resp = _call(app, ConnectionToolPermissions, "put", + f"/api/connections/{cid}/tools/{tool['id']}/permissions", + body={"permissions": {"telegram_send_image": "off", "telegram_send_message": "always"}}, + args=[cid, str(tool["id"])]) + actions = {a["name"]: a["permission"] for a in resp.get_json()["tool"]["actions"]} + assert actions == {"telegram_send_message": "always", "telegram_send_image": "off"} + + def test_rejects_unknown_permission(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionToolPermissions + + cid = _connection(pg_conn, secrets={}) + with _db(pg_conn): + resp = _call(app, ConnectionToolPermissions, "put", "/x", body={"permissions": {"a": "maybe"}}, + args=[cid, "t"]) + assert resp.status_code == 400 + + +class TestDelete: + def test_delete_with_source_removal(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionDetail + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth", secrets={}) + pg_conn.execute(text("INSERT INTO sources (user_id, name, connection_id) VALUES ('alice', 's', CAST(:c AS uuid))"), + {"c": cid}) + with _db(pg_conn), patch("docsgpt.connectors.service.revoke_at_provider"), patch( + "docsgpt.api.user.sources.routes.delete_source", return_value=True + ) as delete_source: + resp = _call(app, ConnectionDetail, "delete", f"/api/connections/{cid}", + body={"sources": "delete"}, args=[cid]) + assert resp.status_code == 200 + assert delete_source.call_count == 1 + + def test_bad_mode(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionDetail + + cid = _connection(pg_conn, secrets={}) + with _db(pg_conn): + resp = _call(app, ConnectionDetail, "delete", "/x", body={"sources": "burn"}, args=[cid]) + assert resp.status_code == 400 + + +class TestNoSecretsInResponses: + def test_list_and_detail(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionDetail, ConnectionsList + + cid = _connection(pg_conn, secrets={"credentials": {"token": "super-secret-value"}}, + session_token="sess-tok") + with _db(pg_conn): + listed = _call(app, ConnectionsList, "get", "/api/connections") + detail = _call(app, ConnectionDetail, "get", f"/api/connections/{cid}", args=[cid]) + for resp in (listed, detail): + body = json.dumps(resp.get_json()) + assert "super-secret-value" not in body + assert "sess-tok" not in body + assert "v2:" not in body diff --git a/tests/connectors/test_runtime.py b/tests/connectors/test_runtime.py new file mode 100644 index 00000000..b02f2442 --- /dev/null +++ b/tests/connectors/test_runtime.py @@ -0,0 +1,254 @@ +"""Runtime use of connections: tool execution, sharing modes and scheduled sync.""" + +from __future__ import annotations + +import logging +from contextlib import contextmanager +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from sqlalchemy import text + +from docsgpt.security.encryption import encrypt_json + + +@contextmanager +def _service_db(conn): + @contextmanager + def _yield(): + yield conn + + with patch.multiple("docsgpt.connectors.service", db_session=_yield, db_readonly=_yield), \ + patch.multiple("docsgpt.connectors.resolve", db_readonly=_yield): + yield + + +def _connection(conn, user="alice", provider="telegram", status="connected", secrets=None, + auth_kind="api_key", server_url=None) -> str: + return str(conn.execute( + text( + "INSERT INTO connector_sessions (user_id, provider, connector_key, auth_kind, status, " + "account_label, server_url, encrypted_credentials) VALUES (:u, :p, :p, :a, :s, :l, :url, :e) RETURNING id" + ), + {"u": user, "p": provider, "a": auth_kind, "s": status, "l": f"{user}-label", "url": server_url, + "e": encrypt_json(secrets or {"credentials": {"token": f"{user}-token"}}, user)}, + ).scalar()) + + +def _tool(connection_id, *, user="alice", name="telegram", mode="owner", tool_id="tool-1"): + return { + "id": tool_id, + "user_id": user, + "name": name, + "config": {}, + "actions": [{"name": "telegram_send_message", "active": True, "require_approval": False}], + "connection_id": connection_id, + "credential_mode": mode, + } + + +def _call(action="telegram_send_message"): + return SimpleNamespace(id="call-1", name=action, arguments="{}", thought_signature=None) + + +def _executor(user="alice", headless=False): + from docsgpt.agents.tool_executor import ToolExecutor + + executor = ToolExecutor(user=user, headless=headless) + return executor + + +def _pause(executor, tool): + with patch("docsgpt.agents.tool_executor.ToolActionParser") as parser: + parser.return_value.parse_args.return_value = ("t1", "telegram_send_message", {}) + return executor.check_pause({"t1": tool}, _call(), "OpenAILLM") + + +class TestResolution: + def test_owner_mode_uses_owner_connection(self, pg_conn): + from docsgpt.connectors.resolve import resolve_connection + + cid = _connection(pg_conn) + with _service_db(pg_conn): + resolved = resolve_connection(_tool(cid), "bob") + assert resolved.available and resolved.connection_id == cid + assert resolved.delegated is True + assert resolved.connector_name == "Telegram" + + def test_member_mode_uses_invokers_connection(self, pg_conn): + from docsgpt.connectors.resolve import resolve_connection + + owner = _connection(pg_conn) + bobs = _connection(pg_conn, user="bob") + with _service_db(pg_conn): + resolved = resolve_connection(_tool(owner, mode="member"), "bob") + assert resolved.connection_id == bobs + assert resolved.delegated is False + + def test_member_mode_without_own_connection_is_unavailable(self, pg_conn): + from docsgpt.connectors.resolve import resolve_connection + + owner = _connection(pg_conn) + with _service_db(pg_conn): + resolved = resolve_connection(_tool(owner, mode="member"), "bob") + assert resolved.available is False + assert resolved.connector_name == "Telegram" + + def test_member_mode_matches_mcp_server(self, pg_conn): + from docsgpt.connectors.resolve import resolve_connection + + owner = _connection(pg_conn, provider="mcp:https://a.example.com", auth_kind="mcp_oauth", + server_url="https://a.example.com", secrets={"tokens": {"access_token": "x"}}) + _connection(pg_conn, user="bob", provider="mcp:https://b.example.com", auth_kind="mcp_oauth", + server_url="https://b.example.com", secrets={"tokens": {"access_token": "y"}}) + with _service_db(pg_conn): + resolved = resolve_connection(_tool(owner, name="mcp_tool", mode="member"), "bob") + assert resolved.available is False + + def test_resource_cannot_borrow_another_users_connection(self, pg_conn): + from docsgpt.connectors.resolve import resolve_connection + + mallorys_target = _connection(pg_conn, user="victim") + with _service_db(pg_conn): + resolved = resolve_connection(_tool(mallorys_target, user="mallory"), "mallory") + assert resolved.available is False and resolved.row is None + + +class TestExecutor: + def test_credentials_come_from_the_connection(self, pg_conn): + cid = _connection(pg_conn) + executor = _executor() + tool = _tool(cid) + with _service_db(pg_conn), patch("docsgpt.agents.tool_executor.ToolManager") as manager: + executor._get_or_load_tool(tool, "t1", "telegram_send_message") + config = manager.return_value.load_tool.call_args.kwargs["tool_config"] + assert config["token"] == "alice-token" + assert "encrypted_credentials" not in config + + def test_owner_mode_delegation_is_audited(self, pg_conn, caplog): + cid = _connection(pg_conn) + executor = _executor(user="bob") + with _service_db(pg_conn), patch("docsgpt.agents.tool_executor.ToolManager"), \ + caplog.at_level(logging.INFO, logger="docsgpt.connectors.resolve"): + executor._get_or_load_tool(_tool(cid), "t1", "telegram_send_message") + record = next(r for r in caplog.records if r.message == "tool_credential_delegation") + assert record.connection_id == cid and record.invoker == "bob" + + def test_needs_reconnect_pauses_on_connect_card(self, pg_conn): + cid = _connection(pg_conn, status="reconnect_needed") + with _service_db(pg_conn): + pause = _pause(_executor(), _tool(cid)) + assert pause["pause_type"] == "awaiting_approval" + assert pause["connection_required"] == { + "connector_key": "telegram", "connector_name": "Telegram", "status": "reconnect_needed", + } + + def test_member_without_connection_pauses(self, pg_conn): + cid = _connection(pg_conn) + with _service_db(pg_conn): + pause = _pause(_executor(user="bob"), _tool(cid, mode="member")) + assert pause["connection_required"]["status"] == "missing" + + def test_headless_run_is_denied_not_paused(self, pg_conn): + cid = _connection(pg_conn, status="disconnected") + with _service_db(pg_conn): + pause = _pause(_executor(headless=True), _tool(cid)) + assert pause["pause_type"] == "headless_denied" + assert pause["error_type"] == "connection_required" + + def test_connected_tool_does_not_pause(self, pg_conn): + cid = _connection(pg_conn) + with _service_db(pg_conn): + assert _pause(_executor(), _tool(cid)) is None + + def test_mcp_tool_gets_connection_id_not_tokens(self, pg_conn): + cid = _connection(pg_conn, provider="mcp:https://m.example.com", auth_kind="mcp_oauth", + server_url="https://m.example.com", secrets={"tokens": {"access_token": "secret"}}) + tool = {**_tool(cid, name="mcp_tool"), "config": {"server_url": "https://m.example.com/mcp", + "auth_type": "oauth"}} + with _service_db(pg_conn), patch("docsgpt.agents.tool_executor.ToolManager") as manager: + _executor()._get_or_load_tool(tool, "t1", "search") + config = manager.return_value.load_tool.call_args.kwargs["tool_config"] + assert config["connection_id"] == cid + assert "secret" not in str(config) + + +class TestScheduledSync: + def test_connector_sources_with_a_connection_are_dispatched(self, pg_conn): + from docsgpt import worker + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth", + secrets={"token_info": {"access_token": "a"}}) + pg_conn.execute(text( + "INSERT INTO sources (user_id, name, type, sync_frequency, connection_id, remote_data) " + "VALUES ('alice', 'Drive', 'connector:file', 'weekly', CAST(:c AS uuid), '{\"provider\": \"google_drive\"}')" + ), {"c": cid}) + pg_conn.execute(text( + "INSERT INTO sources (user_id, name, type, sync_frequency) VALUES ('alice', 'Old', 'connector:file', 'weekly')" + )) + + @contextmanager + def _yield(): + yield pg_conn + + with patch.object(worker, "db_readonly", _yield), patch( + "docsgpt.api.user.tasks.sync_connector_source.delay" + ) as delay: + counts = worker.sync_worker(MagicMock(), "weekly") + assert delay.call_count == 1 + assert counts["sync_dispatched"] == 1 + assert counts["sync_skipped"] == 1 + + def test_paused_connection_is_not_synced(self, pg_conn): + from docsgpt import worker + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth", status="reconnect_needed", + secrets={"token_info": {}}) + source = pg_conn.execute(text( + "INSERT INTO sources (user_id, name, type, sync_frequency, connection_id) " + "VALUES ('alice', 'Drive', 'connector:file', 'weekly', CAST(:c AS uuid)) RETURNING id" + ), {"c": cid}).scalar() + + @contextmanager + def _yield(): + yield pg_conn + + with patch.object(worker, "db_readonly", _yield), patch.object(worker, "ingest_connector") as ingest: + result = worker.sync_connector_source(MagicMock(), str(source)) + assert result == {"status": "paused"} + ingest.assert_not_called() + + def test_sync_runs_as_the_connection_without_a_browser(self, pg_conn): + from docsgpt import worker + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth", + secrets={"token_info": {"access_token": "a"}}) + source = pg_conn.execute(text( + "INSERT INTO sources (user_id, name, type, sync_frequency, connection_id, remote_data) " + "VALUES ('alice', 'Drive', 'connector:file', 'daily', CAST(:c AS uuid), " + "'{\"provider\": \"google_drive\", \"folder_ids\": [\"f\"], \"recursive\": false}') RETURNING id" + ), {"c": cid}).scalar() + + @contextmanager + def _yield(): + yield pg_conn + + with patch.object(worker, "db_readonly", _yield), patch.object( + worker, "ingest_connector", return_value={"tokens": 1} + ) as ingest: + result = worker.sync_connector_source(MagicMock(), str(source)) + assert result["status"] == "success" + kwargs = ingest.call_args.kwargs + assert kwargs["connection_id"] == cid + assert kwargs["operation_mode"] == "sync" + assert kwargs["folder_ids"] == ["f"] and kwargs["recursive"] is False + assert "session_token" not in kwargs + + +@pytest.mark.parametrize("secret_key", ["token_info", "tokens", "client_info", "encrypted_credentials", + "client_secret", "refresh_token", "access_token"]) +def test_redaction_covers_connection_secrets(secret_key): + from docsgpt.storage.db.redaction import REDACTED, redact_secrets + + assert redact_secrets({secret_key: {"x": "y"}})[secret_key] == REDACTED diff --git a/tests/connectors/test_service_credentials.py b/tests/connectors/test_service_credentials.py new file mode 100644 index 00000000..884cef77 --- /dev/null +++ b/tests/connectors/test_service_credentials.py @@ -0,0 +1,420 @@ +"""Tests for connection credentials: storage, refresh, failure handling.""" + +from __future__ import annotations + +import json +import threading +import time +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +import pytest +from sqlalchemy import text + +from docsgpt.connectors import service +from docsgpt.security.encryption import encrypt_json + + +def _patch_service_db(conn): + @contextmanager + def _yield(): + yield conn + + return patch.multiple("docsgpt.connectors.service", db_session=_yield, db_readonly=_yield) + + +def _connection(conn, *, user="alice", provider="google_drive", secrets=None, status="connected", **extra) -> str: + cols = { + "user_id": user, + "provider": provider, + "connector_key": provider, + "auth_kind": extra.pop("auth_kind", "oauth"), + "status": status, + "account_label": extra.pop("account_label", f"{user}@example.com"), + "encrypted_credentials": encrypt_json(secrets, user) if secrets is not None else None, + **extra, + } + names = ", ".join(cols) + values = ", ".join(f":{k}" for k in cols) + return str( + conn.execute( + text(f"INSERT INTO connector_sessions ({names}) VALUES ({values}) RETURNING id"), cols, + ).scalar() + ) + + +def _source(conn, connection_id, user="alice") -> str: + return str( + conn.execute( + text( + "INSERT INTO sources (user_id, name, type, connection_id) " + "VALUES (:u, 'Docs', 'connector:file', CAST(:c AS uuid)) RETURNING id" + ), + {"u": user, "c": connection_id}, + ).scalar() + ) + + +def _row(conn, connection_id): + from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository + + return ConnectorSessionsRepository(conn).get(connection_id) + + +class _FakeAuth: + """A provider whose refresh rotates the refresh token on every call.""" + + def __init__(self, *, expired=True, fail=None, delay=0.0): + self.expired = expired + self.fail = fail + self.delay = delay + self.calls = [] + self._lock = threading.Lock() + + def is_token_expired(self, token_info): + return self.expired and token_info.get("access_token") == "old-at" + + def refresh_access_token(self, refresh_token): + with self._lock: + self.calls.append(refresh_token) + n = len(self.calls) + if self.delay: + time.sleep(self.delay) + if self.fail: + raise self.fail + if refresh_token != "rt-0": + # A reused rotating refresh token is rejected, like Microsoft's. + raise ValueError("invalid_grant: refresh token already used") + return {"access_token": f"new-at-{n}", "refresh_token": f"rt-{n}", "expiry": None} + + def sanitize_token_info(self, token_info, **extra): + return {k: token_info.get(k) for k in ("access_token", "refresh_token", "expiry", "cloud_id")} + + +def _auth(fake): + return patch("docsgpt.parser.connectors.connector_creator.ConnectorCreator.create_auth", return_value=fake) + + +class TestSecrets: + def test_write_and_read(self, pg_conn): + cid = _connection(pg_conn, secrets=None, status="pending") + service.write_secrets(pg_conn, _row(pg_conn, cid), {"token_info": {"access_token": "a", "refresh_token": "r"}}) + row = _row(pg_conn, cid) + assert row["token_info"] is None + assert row["has_refresh_token"] is True + assert "access_token" not in json.dumps(row["session_data"]) + assert service.read_secrets(row) == {"token_info": {"access_token": "a", "refresh_token": "r"}} + + def test_legacy_plaintext_is_read(self, pg_conn): + from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository + + row = ConnectorSessionsRepository(pg_conn).upsert( + "alice", "share_point", token_info={"access_token": "p"}, status="authorized", + ) + assert service.read_secrets(row)["token_info"]["access_token"] == "p" + + def test_decrypt_failure_flags_reconnect(self, pg_conn): + cid = _connection(pg_conn, secrets={"token_info": {"access_token": "a"}}) + pg_conn.execute( + text("UPDATE connector_sessions SET encrypted_credentials = 'v2:ffffffff:AAAA' WHERE id = CAST(:i AS uuid)"), + {"i": cid}, + ) + with _patch_service_db(pg_conn), patch("docsgpt.events.publisher.publish_user_event"): + assert service.load_secrets(_row(pg_conn, cid)) is None + row = _row(pg_conn, cid) + assert row["status"] == "reconnect_needed" + assert row["last_error"] == service.DECRYPT_ERROR + + +class TestGetValidTokenInfo: + def test_returns_unexpired_token(self, pg_conn): + cid = _connection(pg_conn, secrets={"token_info": {"access_token": "fresh", "refresh_token": "rt-0"}}) + fake = _FakeAuth(expired=True) + with _patch_service_db(pg_conn), _auth(fake): + info = service.get_valid_token_info(cid) + assert info["access_token"] == "fresh" + assert fake.calls == [] + + def test_refreshes_and_persists_rotated_token(self, pg_conn): + cid = _connection(pg_conn, secrets={"token_info": {"access_token": "old-at", "refresh_token": "rt-0"}}) + fake = _FakeAuth() + with _patch_service_db(pg_conn), _auth(fake): + info = service.get_valid_token_info(cid) + assert info["access_token"] == "new-at-1" + stored = service.read_secrets(_row(pg_conn, cid))["token_info"] + assert stored["refresh_token"] == "rt-1" + + def test_rejected_access_token_forces_refresh(self, pg_conn): + cid = _connection(pg_conn, secrets={"token_info": {"access_token": "fresh", "refresh_token": "rt-0"}}) + fake = _FakeAuth(expired=False) + with _patch_service_db(pg_conn), _auth(fake): + info = service.get_valid_token_info(cid, rejected_access_token="fresh") + assert info["access_token"] == "new-at-1" + + def test_rejected_token_already_replaced_is_not_refreshed_again(self, pg_conn): + cid = _connection(pg_conn, secrets={"token_info": {"access_token": "newer", "refresh_token": "rt-0"}}) + fake = _FakeAuth(expired=False) + with _patch_service_db(pg_conn), _auth(fake): + info = service.get_valid_token_info(cid, rejected_access_token="stale") + assert info["access_token"] == "newer" + assert fake.calls == [] + + def test_revoked_grant_pauses_sources_and_notifies(self, pg_conn): + cid = _connection(pg_conn, secrets={"token_info": {"access_token": "old-at", "refresh_token": "rt-0"}}) + source = _source(pg_conn, cid) + fake = _FakeAuth(fail=ValueError("Error refreshing token: invalid_grant")) + with _patch_service_db(pg_conn), _auth(fake), patch( + "docsgpt.events.publisher.publish_user_event" + ) as publish: + with pytest.raises(service.ConnectionUnavailable): + service.get_valid_token_info(cid) + assert _row(pg_conn, cid)["status"] == "reconnect_needed" + meta = pg_conn.execute( + text("SELECT metadata FROM sources WHERE id = CAST(:i AS uuid)"), {"i": source} + ).scalar() + assert meta["sync_state"] == "paused_reconnect" + publish.assert_called_once() + assert publish.call_args.args[1] == "connection.reconnect_needed" + # No account data or secrets in the event. + assert "alice@example.com" not in json.dumps(publish.call_args.args[2]) + + def test_second_failure_does_not_notify_again(self, pg_conn): + cid = _connection(pg_conn, secrets={"token_info": {"access_token": "old-at", "refresh_token": "rt-0"}}) + fake = _FakeAuth(fail=ValueError("invalid_grant")) + with _patch_service_db(pg_conn), _auth(fake), patch( + "docsgpt.events.publisher.publish_user_event" + ) as publish: + for _ in range(2): + with pytest.raises(service.ConnectionUnavailable): + service.get_valid_token_info(cid) + assert publish.call_count == 1 + + def test_transient_failure_keeps_status(self, pg_conn): + import requests + + cid = _connection(pg_conn, secrets={"token_info": {"access_token": "old-at", "refresh_token": "rt-0"}}) + fake = _FakeAuth(fail=requests.exceptions.ConnectionError("network down")) + with _patch_service_db(pg_conn), _auth(fake): + with pytest.raises(service.TransientConnectionError): + service.get_valid_token_info(cid) + assert _row(pg_conn, cid)["status"] == "connected" + + def test_disconnected_connection_is_unavailable(self, pg_conn): + cid = _connection(pg_conn, secrets={}, status="disconnected") + with _patch_service_db(pg_conn): + with pytest.raises(service.ConnectionUnavailable) as exc: + service.get_valid_token_info(cid) + assert exc.value.status == "disconnected" + + def test_concurrent_refresh_spends_the_refresh_token_once(self, pg_engine): + """Two workers with an expiring rotating token: one refresh, both succeed.""" + with pg_engine.begin() as conn: + cid = _connection(conn, secrets={"token_info": {"access_token": "old-at", "refresh_token": "rt-0"}}) + + @contextmanager + def _session(): + with pg_engine.begin() as conn: + yield conn + + fake = _FakeAuth(delay=0.3) + results, errors = [], [] + + def worker(): + try: + results.append(service.get_valid_token_info(cid)["access_token"]) + except Exception as exc: # pragma: no cover - reported below + errors.append(exc) + + with patch.multiple("docsgpt.connectors.service", db_session=_session, db_readonly=_session), _auth(fake): + threads = [threading.Thread(target=worker) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert errors == [] + assert fake.calls == ["rt-0"] + assert results == ["new-at-1", "new-at-1"] + + +class TestApiKeyConnections: + def test_create_and_reuse(self, pg_conn): + from docsgpt.connectors import catalog + + telegram = catalog.get_definition("telegram") + row, created = service.create_api_key_connection(pg_conn, "alice", telegram, {"token": "123456:ABCDEFG"}) + assert created is True + assert row["account_label"] == "…DEFG" + assert row["status"] == "connected" + again, created = service.create_api_key_connection(pg_conn, "alice", telegram, {"token": "123456:ABCDEFG"}) + assert created is False and again["id"] == row["id"] + assert service.get_credentials(again) == {"token": "123456:ABCDEFG"} + + def test_missing_field(self, pg_conn): + from docsgpt.connectors import catalog + + with pytest.raises(ValueError): + service.create_api_key_connection(pg_conn, "alice", catalog.get_definition("telegram"), {}) + + def test_default_key_refused_with_auth(self, pg_conn, monkeypatch): + from docsgpt.connectors import catalog + from docsgpt.core.settings import settings + from docsgpt.security.encryption import DEFAULT_ENCRYPTION_KEY + + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", DEFAULT_ENCRYPTION_KEY) + monkeypatch.setattr(settings, "AUTH_TYPE", "session_jwt") + with pytest.raises(service.EncryptionKeyNotConfigured): + service.create_api_key_connection( + pg_conn, "alice", catalog.get_definition("brave"), {"token": "secret-key-1234"}, + ) + + def test_default_key_allowed_single_user(self, pg_conn, monkeypatch): + from docsgpt.connectors import catalog + from docsgpt.core.settings import settings + from docsgpt.security.encryption import DEFAULT_ENCRYPTION_KEY + + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", DEFAULT_ENCRYPTION_KEY) + monkeypatch.setattr(settings, "AUTH_TYPE", None) + row, _ = service.create_api_key_connection( + pg_conn, "local", catalog.get_definition("brave"), {"token": "secret-key-1234"}, + ) + assert row["encrypted_credentials"].startswith("v2:") + + def test_flagged_connection_credentials_unavailable(self, pg_conn): + cid = _connection( + pg_conn, provider="telegram", auth_kind="api_key", status="reconnect_needed", + secrets={"credentials": {"token": "t"}}, + ) + with pytest.raises(service.ConnectionUnavailable): + service.get_credentials(_row(pg_conn, cid)) + + +class TestOAuthCompletion: + def test_new_account(self, pg_conn): + state_row = service.begin_oauth(pg_conn, "alice", "google_drive") + assert state_row["status"] == "pending" + row = service.complete_oauth(pg_conn, state_row, "google_drive", {"access_token": "a"}, "a@example.com") + assert row["id"] == state_row["id"] + assert row["account_label"] == "a@example.com" + assert row["status"] == "connected" + assert row["session_token"] + + def test_same_account_updates_existing_connection(self, pg_conn): + first = service.complete_oauth( + pg_conn, service.begin_oauth(pg_conn, "alice", "google_drive"), "google_drive", + {"access_token": "a"}, "a@example.com", + ) + source = _source(pg_conn, str(first["id"])) + pg_conn.execute( + text("UPDATE connector_sessions SET status = 'reconnect_needed' WHERE id = CAST(:i AS uuid)"), + {"i": str(first["id"])}, + ) + pg_conn.execute( + text("UPDATE sources SET metadata = '{\"sync_state\": \"paused_reconnect\"}' WHERE id = CAST(:i AS uuid)"), + {"i": source}, + ) + pending = service.begin_oauth(pg_conn, "alice", "google_drive") + again = service.complete_oauth(pg_conn, pending, "google_drive", {"access_token": "b"}, "a@example.com") + assert again["id"] == first["id"] + assert again["status"] == "connected" + assert _row(pg_conn, str(pending["id"])) is None + meta = pg_conn.execute( + text("SELECT metadata FROM sources WHERE id = CAST(:i AS uuid)"), {"i": source} + ).scalar() + assert "sync_state" not in meta + + def test_second_account_is_a_second_connection(self, pg_conn): + first = service.complete_oauth( + pg_conn, service.begin_oauth(pg_conn, "alice", "google_drive"), "google_drive", + {"access_token": "a"}, "a@example.com", + ) + second = service.complete_oauth( + pg_conn, service.begin_oauth(pg_conn, "alice", "google_drive"), "google_drive", + {"access_token": "b"}, "b@example.com", + ) + assert first["id"] != second["id"] + assert len(service.list_connections(pg_conn, "alice")) == 2 + + def test_reconnect_must_name_own_connection(self, pg_conn): + cid = _connection(pg_conn, user="bob", secrets={}) + with pytest.raises(service.ConnectionUnavailable): + service.begin_oauth(pg_conn, "alice", "google_drive", cid) + + +class TestReencrypt: + def test_rewrites_rows_on_previous_key(self, pg_conn, monkeypatch): + from docsgpt.core.settings import settings + from docsgpt.security.encryption import current_key_id, envelope_key_id + + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", "old-key") + cid = _connection(pg_conn, secrets={"token_info": {"access_token": "a"}}) + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", "new-key") + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY_PREVIOUS", "old-key") + with _patch_service_db(pg_conn): + counts = service.reencrypt_all() + assert counts == {"rewritten": 1, "current": 0, "failed": 0} + row = _row(pg_conn, cid) + assert envelope_key_id(row["encrypted_credentials"]) == current_key_id() + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY_PREVIOUS", None) + assert service.read_secrets(row)["token_info"]["access_token"] == "a" + + def test_unreadable_rows_flagged(self, pg_conn, monkeypatch): + from docsgpt.core.settings import settings + + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", "lost-key") + cid = _connection(pg_conn, secrets={"token_info": {"access_token": "a"}}) + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", "new-key") + with _patch_service_db(pg_conn), patch("docsgpt.events.publisher.publish_user_event"): + counts = service.reencrypt_all() + assert counts["failed"] == 1 + assert _row(pg_conn, cid)["status"] == "reconnect_needed" + + +class TestRemove: + def test_keep_sources_delete_tools(self, pg_conn): + cid = _connection(pg_conn, provider="telegram", auth_kind="api_key", secrets={"credentials": {"token": "t"}}) + source = _source(pg_conn, cid) + pg_conn.execute( + text( + "INSERT INTO user_tools (user_id, name, connection_id) VALUES ('alice', 'telegram', CAST(:c AS uuid))" + ), + {"c": cid}, + ) + with patch("docsgpt.connectors.service.revoke_at_provider") as revoke: + to_delete = service.remove_connection(pg_conn, _row(pg_conn, cid)) + assert to_delete == [] + revoke.assert_called_once() + assert _row(pg_conn, cid) is None + assert pg_conn.execute(text("SELECT count(*) FROM user_tools WHERE user_id = 'alice'")).scalar() == 0 + kept = pg_conn.execute( + text("SELECT connection_id, sync_frequency FROM sources WHERE id = CAST(:i AS uuid)"), {"i": source} + ).one() + assert kept.connection_id is None and kept.sync_frequency == "never" + + def test_delete_sources_returns_them(self, pg_conn): + cid = _connection(pg_conn, secrets={}) + source = _source(pg_conn, cid) + with patch("docsgpt.connectors.service.revoke_at_provider"): + to_delete = service.remove_connection(pg_conn, _row(pg_conn, cid), sources="delete") + assert [str(s["id"]) for s in to_delete] == [source] + + +class TestRevoke: + def test_google_revocation_posts_token(self): + response = MagicMock(status_code=200) + with patch("requests.post", return_value=response) as post: + ok = service.revoke_at_provider( + {"provider": "google_drive"}, {"token_info": {"refresh_token": "rt", "access_token": "at"}}, + ) + assert ok + assert post.call_args.kwargs["data"] == {"token": "rt"} + + def test_other_providers_are_local_only(self): + with patch("requests.post") as post: + assert not service.revoke_at_provider({"provider": "share_point"}, {"token_info": {"access_token": "a"}}) + post.assert_not_called() + + def test_failure_never_raises(self): + import requests + + with patch("requests.post", side_effect=requests.exceptions.Timeout()): + assert not service.revoke_at_provider({"provider": "google_drive"}, {"token_info": {"access_token": "a"}}) diff --git a/tests/connectors/test_service_read.py b/tests/connectors/test_service_read.py index 01c8a6d6..2388b6c2 100644 --- a/tests/connectors/test_service_read.py +++ b/tests/connectors/test_service_read.py @@ -242,7 +242,9 @@ class TestDisconnect: ) repo = ConnectorSessionsRepository(pg_conn) service.disconnect(pg_conn, repo.get(mcp)) - assert repo.get(mcp)["session_data"] == {"client_info": {"client_id": "c"}} + row = repo.get(mcp) + assert row["session_data"] == {} + assert service.read_secrets(row) == {"client_info": {"client_id": "c"}} def test_route_rejects_other_users(self, app, pg_conn): from docsgpt.api.connector.connections import ConnectionDisconnect diff --git a/tests/parser/connectors/confluence/test_auth_session.py b/tests/parser/connectors/confluence/test_auth_session.py index c4d11797..49ef5f43 100644 --- a/tests/parser/connectors/confluence/test_auth_session.py +++ b/tests/parser/connectors/confluence/test_auth_session.py @@ -3,7 +3,6 @@ from contextlib import contextmanager from unittest.mock import patch -import pytest @contextmanager @@ -18,82 +17,3 @@ def _patch_db(conn): yield -class TestGetTokenInfoFromSession: - def test_invalid_session_token_raises(self, pg_conn): - from docsgpt.parser.connectors.confluence.auth import ( - ConfluenceAuth, - ) - - auth = ConfluenceAuth.__new__(ConfluenceAuth) - with _patch_db(pg_conn), pytest.raises(ValueError): - auth.get_token_info_from_session("no-such-token") - - def test_missing_token_info_raises(self, pg_conn): - from docsgpt.parser.connectors.confluence.auth import ( - ConfluenceAuth, - ) - from docsgpt.storage.db.repositories.connector_sessions import ( - ConnectorSessionsRepository, - ) - - repo = ConnectorSessionsRepository(pg_conn) - repo.upsert("u", "confluence", status="authorized") - # Set session_token but no token_info - session = repo.get_by_user_provider("u", "confluence") - repo.update(str(session["id"]), {"session_token": "tok-no-info"}) - - auth = ConfluenceAuth.__new__(ConfluenceAuth) - with _patch_db(pg_conn), pytest.raises(ValueError): - auth.get_token_info_from_session("tok-no-info") - - def test_missing_required_fields_raises(self, pg_conn): - from docsgpt.parser.connectors.confluence.auth import ( - ConfluenceAuth, - ) - from docsgpt.storage.db.repositories.connector_sessions import ( - ConnectorSessionsRepository, - ) - - repo = ConnectorSessionsRepository(pg_conn) - repo.upsert("u", "confluence", status="authorized") - session = repo.get_by_user_provider("u", "confluence") - repo.update( - str(session["id"]), - { - "session_token": "tok-partial", - "token_info": {"access_token": "at"}, # missing refresh + cloud_id - }, - ) - - auth = ConfluenceAuth.__new__(ConfluenceAuth) - with _patch_db(pg_conn), pytest.raises(ValueError): - auth.get_token_info_from_session("tok-partial") - - def test_complete_token_info_returned(self, pg_conn): - from docsgpt.parser.connectors.confluence.auth import ( - ConfluenceAuth, - ) - from docsgpt.storage.db.repositories.connector_sessions import ( - ConnectorSessionsRepository, - ) - - repo = ConnectorSessionsRepository(pg_conn) - repo.upsert("u", "confluence", status="authorized") - session = repo.get_by_user_provider("u", "confluence") - repo.update( - str(session["id"]), - { - "session_token": "tok-good", - "token_info": { - "access_token": "at", - "refresh_token": "rt", - "cloud_id": "cid-1", - }, - }, - ) - - auth = ConfluenceAuth.__new__(ConfluenceAuth) - with _patch_db(pg_conn): - got = auth.get_token_info_from_session("tok-good") - assert got["access_token"] == "at" - assert got["cloud_id"] == "cid-1" diff --git a/tests/parser/connectors/confluence/test_loader.py b/tests/parser/connectors/confluence/test_loader.py index ba891d66..61cc3433 100644 --- a/tests/parser/connectors/confluence/test_loader.py +++ b/tests/parser/connectors/confluence/test_loader.py @@ -2,6 +2,7 @@ import os import tempfile +from tests.parser.connectors.token_patch import patch_tokens from unittest.mock import MagicMock, patch import pytest @@ -25,11 +26,12 @@ def _make_loader(token_info=None): with patch("docsgpt.parser.connectors.confluence.loader.ConfluenceAuth") as MockAuth: mock_auth = MagicMock() - mock_auth.get_token_info_from_session.return_value = token_info + token_info = token_info MockAuth.return_value = mock_auth from docsgpt.parser.connectors.confluence.loader import ConfluenceLoader - loader = ConfluenceLoader("session_tok") + with patch_tokens(token_info): + loader = ConfluenceLoader("session_tok") loader.auth = mock_auth return loader @@ -69,7 +71,7 @@ class TestConfluenceLoaderInit: def test_init_sets_attributes(self, loader): assert loader.session_token == "session_tok" assert loader.access_token == "test_at" - assert loader.refresh_token == "test_rt" + assert loader.connection_id == "conn-1" assert loader.cloud_id == "test_cloud" assert loader.next_page_token is None assert "test_cloud" in loader.base_url @@ -834,18 +836,16 @@ class TestRetryOnAuthFailure: # Second call succeeds return _mock_response({"results": [], "_links": {}}) - loader.auth.refresh_access_token = MagicMock(return_value={ - "access_token": "new_at", - "refresh_token": "new_rt", - }) - loader._persist_refreshed_tokens = MagicMock() - - with patch("requests.get", side_effect=flaky_request): + with patch( + "docsgpt.connectors.service.get_valid_token_info", + return_value={"access_token": "new_at", "cloud_id": "test_cloud"}, + ) as tokens, patch("requests.get", side_effect=flaky_request): loader.load_data({}) - assert loader.auth.refresh_access_token.called + # The rejected token goes to the service, which refreshes under the + # connection's row lock and stores the rotated refresh token. + tokens.assert_called_once_with("conn-1", rejected_access_token="test_at") assert loader.access_token == "new_at" - assert loader._persist_refreshed_tokens.called @pytest.mark.unit def test_raises_non_auth_http_error(self, loader): @@ -867,11 +867,9 @@ class TestRetryOnAuthFailure: err = requests.exceptions.HTTPError(response=resp) err.response = resp - loader.auth.refresh_access_token = MagicMock( - side_effect=Exception("refresh failed") - ) - - with patch("requests.get", side_effect=err): + with patch( + "docsgpt.connectors.service.get_valid_token_info", side_effect=Exception("refresh failed"), + ), patch("requests.get", side_effect=err): with pytest.raises(ValueError, match="Authentication failed"): loader.load_data({}) @@ -890,29 +888,12 @@ class TestRetryOnAuthFailure: raise err return _mock_response({"results": [], "_links": {}}) - loader.auth.refresh_access_token = MagicMock(return_value={ - "access_token": "new_at", - "refresh_token": "new_rt", - }) - loader._persist_refreshed_tokens = MagicMock() - - with patch("requests.get", side_effect=flaky): + with patch( + "docsgpt.connectors.service.get_valid_token_info", + return_value={"access_token": "new_at", "cloud_id": "test_cloud"}, + ) as tokens, patch("requests.get", side_effect=flaky): loader.load_data({}) - assert loader.auth.refresh_access_token.called + assert tokens.called -# --------------------------------------------------------------------------- -# _persist_refreshed_tokens -# --------------------------------------------------------------------------- - - -class TestPersistRefreshedTokens: - pass - - @pytest.mark.unit - def test_logs_warning_on_failure(self, loader): - loader.auth.sanitize_token_info = MagicMock(side_effect=Exception("db error")) - - # Should not raise, just log a warning - loader._persist_refreshed_tokens({"access_token": "at"}) diff --git a/tests/parser/connectors/test_auth_token_leakage.py b/tests/parser/connectors/test_auth_token_leakage.py index 98d63aa4..d997620a 100644 --- a/tests/parser/connectors/test_auth_token_leakage.py +++ b/tests/parser/connectors/test_auth_token_leakage.py @@ -51,8 +51,7 @@ def _patches(session_return: Optional[Dict[str, Any]]): ) return ( patch( - "docsgpt.storage.db.repositories.connector_sessions." - "ConnectorSessionsRepository", + "docsgpt.connectors.service.ConnectorSessionsRepository", fake_repo_cls, ), patch( @@ -88,57 +87,18 @@ class TestSessionTokenFingerprint: assert session_token_fingerprint("a") != session_token_fingerprint("b") -class TestConfluenceAuthDoesNotLeakToken: +class TestConnectionLookupDoesNotLeakToken: + """A session token that names no connection is reported by fingerprint only.""" @pytest.mark.unit def test_invalid_session_does_not_interpolate_token(self) -> None: - from docsgpt.parser.connectors.confluence.auth import ConfluenceAuth + from docsgpt.connectors import service - auth = ConfluenceAuth.__new__(ConfluenceAuth) repo_patch, ctx_patch = _patches(None) with repo_patch, ctx_patch: with pytest.raises(ValueError) as excinfo: - auth.get_token_info_from_session(SECRET_TOKEN) + service.connection_id_for_session_token(SECRET_TOKEN) message = str(excinfo.value) assert SECRET_TOKEN not in message assert session_token_fingerprint(SECRET_TOKEN) in message - - -class TestGoogleDriveAuthDoesNotLeakToken: - - @pytest.mark.unit - def test_invalid_session_does_not_interpolate_token(self) -> None: - from docsgpt.parser.connectors.google_drive.auth import GoogleDriveAuth - - auth = GoogleDriveAuth.__new__(GoogleDriveAuth) - repo_patch, ctx_patch = _patches(None) - with repo_patch, ctx_patch: - with pytest.raises(ValueError) as excinfo: - auth.get_token_info_from_session(SECRET_TOKEN) - - # The Google Drive module wraps the inner ValueError in a broad - # ``except Exception as e: raise ValueError(... {str(e)})`` block, - # so the outer message still carries the fingerprint from the - # inner raise but must never carry the raw token. - message = str(excinfo.value) - assert SECRET_TOKEN not in message - assert session_token_fingerprint(SECRET_TOKEN) in message - - -class TestSharePointAuthDoesNotLeakToken: - - @pytest.mark.unit - def test_invalid_session_does_not_interpolate_token(self) -> None: - from docsgpt.parser.connectors.share_point.auth import SharePointAuth - - auth = SharePointAuth.__new__(SharePointAuth) - repo_patch, ctx_patch = _patches(None) - with repo_patch, ctx_patch: - with pytest.raises(ValueError) as excinfo: - auth.get_token_info_from_session(SECRET_TOKEN) - - # SharePoint also wraps the inner ValueError. Same invariants. - message = str(excinfo.value) - assert SECRET_TOKEN not in message - assert session_token_fingerprint(SECRET_TOKEN) in message diff --git a/tests/parser/connectors/test_connector_creator.py b/tests/parser/connectors/test_connector_creator.py index 535441db..321aa016 100644 --- a/tests/parser/connectors/test_connector_creator.py +++ b/tests/parser/connectors/test_connector_creator.py @@ -1,5 +1,6 @@ """Tests for ConnectorCreator factory class.""" +from tests.parser.connectors.token_patch import patch_tokens from unittest.mock import patch, MagicMock import pytest @@ -74,7 +75,7 @@ class TestConnectorCreator: def test_create_connector_google_drive(self): with patch("docsgpt.parser.connectors.google_drive.loader.GoogleDriveAuth") as MockAuth: mock_auth_instance = MagicMock() - mock_auth_instance.get_token_info_from_session.return_value = { + token_info = { "access_token": "at", "refresh_token": "rt" } mock_creds = MagicMock() @@ -84,7 +85,8 @@ class TestConnectorCreator: mock_auth_instance.build_drive_service.return_value = MagicMock() MockAuth.return_value = mock_auth_instance - loader = self.ConnectorCreator.create_connector("google_drive", "session_tok") + with patch_tokens(token_info): + loader = self.ConnectorCreator.create_connector("google_drive", "session_tok") from docsgpt.parser.connectors.google_drive.loader import GoogleDriveLoader assert isinstance(loader, GoogleDriveLoader) @@ -92,11 +94,12 @@ class TestConnectorCreator: def test_create_connector_share_point(self): with patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth") as MockAuth: mock_auth_instance = MagicMock() - mock_auth_instance.get_token_info_from_session.return_value = { + token_info = { "access_token": "at", "refresh_token": "rt" } MockAuth.return_value = mock_auth_instance - loader = self.ConnectorCreator.create_connector("share_point", "session_tok") + with patch_tokens(token_info): + loader = self.ConnectorCreator.create_connector("share_point", "session_tok") from docsgpt.parser.connectors.share_point.loader import SharePointLoader assert isinstance(loader, SharePointLoader) diff --git a/tests/parser/connectors/test_google_drive_auth.py b/tests/parser/connectors/test_google_drive_auth.py index 610e6d11..63c50a4a 100644 --- a/tests/parser/connectors/test_google_drive_auth.py +++ b/tests/parser/connectors/test_google_drive_auth.py @@ -364,71 +364,6 @@ class _FakeReadonlyCtx: return False -class TestGetTokenInfoFromSession: - - def _patches(self, session_return): - fake_repo_cls = type( - "FakeRepo", - (_FakeRepo,), - {"_session": session_return}, - ) - return ( - patch( - "docsgpt.storage.db.repositories.connector_sessions.ConnectorSessionsRepository", - fake_repo_cls, - ), - patch( - "docsgpt.storage.db.session.db_readonly", - lambda: _FakeReadonlyCtx(), - ), - ) - - @pytest.mark.unit - def test_valid_session(self, auth, mock_settings): - repo_patch, ctx_patch = self._patches({ - "session_token": "st", - "token_info": {"access_token": "at", "refresh_token": "rt"}, - }) - with repo_patch, ctx_patch: - result = auth.get_token_info_from_session("st") - assert result["access_token"] == "at" - assert result["token_uri"] == "https://oauth2.googleapis.com/token" - - @pytest.mark.unit - def test_session_not_found_raises(self, auth, mock_settings): - repo_patch, ctx_patch = self._patches(None) - with repo_patch, ctx_patch: - with pytest.raises(ValueError, match="Failed to retrieve Google Drive token"): - auth.get_token_info_from_session("bad_token") - - @pytest.mark.unit - def test_session_missing_token_info_raises(self, auth, mock_settings): - repo_patch, ctx_patch = self._patches({"session_token": "st"}) - with repo_patch, ctx_patch: - with pytest.raises(ValueError, match="Failed to retrieve Google Drive token"): - auth.get_token_info_from_session("st") - - @pytest.mark.unit - def test_missing_required_fields_raises(self, auth, mock_settings): - repo_patch, ctx_patch = self._patches({ - "session_token": "st", - "token_info": {"access_token": "at"}, - }) - with repo_patch, ctx_patch: - with pytest.raises(ValueError, match="Failed to retrieve Google Drive token"): - auth.get_token_info_from_session("st") - - @pytest.mark.unit - def test_empty_token_info_raises(self, auth, mock_settings): - repo_patch, ctx_patch = self._patches({ - "session_token": "st", - "token_info": None, - }) - with repo_patch, ctx_patch: - with pytest.raises(ValueError, match="Failed to retrieve Google Drive token"): - auth.get_token_info_from_session("st") - - class TestValidateCredentials: @pytest.mark.unit diff --git a/tests/parser/connectors/test_google_drive_loader.py b/tests/parser/connectors/test_google_drive_loader.py index ed765e60..3a6dcb67 100644 --- a/tests/parser/connectors/test_google_drive_loader.py +++ b/tests/parser/connectors/test_google_drive_loader.py @@ -1,5 +1,6 @@ """Tests for GoogleDriveLoader.""" +from tests.parser.connectors.token_patch import patch_tokens from unittest.mock import MagicMock, patch import pytest @@ -11,7 +12,7 @@ def _make_loader(service=None): """Create a GoogleDriveLoader with mocked dependencies.""" with patch("docsgpt.parser.connectors.google_drive.loader.GoogleDriveAuth") as MockAuth: mock_auth = MagicMock() - mock_auth.get_token_info_from_session.return_value = { + token_info = { "access_token": "at", "refresh_token": "rt", } @@ -24,7 +25,8 @@ def _make_loader(service=None): MockAuth.return_value = mock_auth from docsgpt.parser.connectors.google_drive.loader import GoogleDriveLoader - loader = GoogleDriveLoader("session_tok") + with patch_tokens(token_info): + loader = GoogleDriveLoader("session_tok") return loader @@ -51,7 +53,7 @@ class TestGoogleDriveLoaderInit: def test_init_service_failure_sets_none(self): with patch("docsgpt.parser.connectors.google_drive.loader.GoogleDriveAuth") as MockAuth: mock_auth = MagicMock() - mock_auth.get_token_info_from_session.return_value = { + token_info = { "access_token": "at", "refresh_token": "rt" } mock_creds = MagicMock() @@ -61,7 +63,8 @@ class TestGoogleDriveLoaderInit: MockAuth.return_value = mock_auth from docsgpt.parser.connectors.google_drive.loader import GoogleDriveLoader - loader = GoogleDriveLoader("st") + with patch_tokens(token_info): + loader = GoogleDriveLoader("st") assert loader.service is None diff --git a/tests/parser/connectors/test_share_point_auth.py b/tests/parser/connectors/test_share_point_auth.py index ecf25a7d..23529ddd 100644 --- a/tests/parser/connectors/test_share_point_auth.py +++ b/tests/parser/connectors/test_share_point_auth.py @@ -285,63 +285,3 @@ class _FakeReadonlyCtx: return False -class TestGetTokenInfoFromSession: - - def _patches(self, session_return): - fake_repo_cls = type( - "FakeRepo", - (_FakeRepo,), - {"_session": session_return}, - ) - return ( - patch( - "docsgpt.storage.db.repositories.connector_sessions.ConnectorSessionsRepository", - fake_repo_cls, - ), - patch( - "docsgpt.storage.db.session.db_readonly", - lambda: _FakeReadonlyCtx(), - ), - ) - - @pytest.mark.unit - def test_valid_session(self, auth, mock_settings): - repo_patch, ctx_patch = self._patches({ - "session_token": "st", - "token_info": {"access_token": "at", "refresh_token": "rt"}, - }) - with repo_patch, ctx_patch: - result = auth.get_token_info_from_session("st") - assert result["access_token"] == "at" - assert "token_uri" in result - - @pytest.mark.unit - def test_session_not_found_raises(self, auth, mock_settings): - repo_patch, ctx_patch = self._patches(None) - with repo_patch, ctx_patch: - with pytest.raises(ValueError, match="Failed to retrieve SharePoint token"): - auth.get_token_info_from_session("bad") - - @pytest.mark.unit - def test_missing_token_info_raises(self, auth, mock_settings): - repo_patch, ctx_patch = self._patches({"session_token": "st"}) - with repo_patch, ctx_patch: - with pytest.raises(ValueError, match="Failed to retrieve SharePoint token"): - auth.get_token_info_from_session("st") - - @pytest.mark.unit - def test_empty_token_info_raises(self, auth, mock_settings): - repo_patch, ctx_patch = self._patches({"session_token": "st", "token_info": None}) - with repo_patch, ctx_patch: - with pytest.raises(ValueError, match="Failed to retrieve SharePoint token"): - auth.get_token_info_from_session("st") - - @pytest.mark.unit - def test_missing_required_fields_raises(self, auth, mock_settings): - repo_patch, ctx_patch = self._patches({ - "session_token": "st", - "token_info": {"access_token": "at"}, - }) - with repo_patch, ctx_patch: - with pytest.raises(ValueError, match="Failed to retrieve SharePoint token"): - auth.get_token_info_from_session("st") diff --git a/tests/parser/connectors/test_share_point_loader.py b/tests/parser/connectors/test_share_point_loader.py index b54eb02b..4390bc2d 100644 --- a/tests/parser/connectors/test_share_point_loader.py +++ b/tests/parser/connectors/test_share_point_loader.py @@ -1,6 +1,7 @@ """Tests for SharePointLoader.""" import os +from tests.parser.connectors.token_patch import patch_tokens from unittest.mock import MagicMock, patch import pytest @@ -13,7 +14,7 @@ def _make_loader(access_token="at", refresh_token="rt", allows_shared=False): """Create a SharePointLoader with mocked dependencies.""" with patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth") as MockAuth: mock_auth = MagicMock() - mock_auth.get_token_info_from_session.return_value = { + token_info = { "access_token": access_token, "refresh_token": refresh_token, "allows_shared_content": allows_shared, @@ -22,7 +23,8 @@ def _make_loader(access_token="at", refresh_token="rt", allows_shared=False): MockAuth.return_value = mock_auth from docsgpt.parser.connectors.share_point.loader import SharePointLoader - loader = SharePointLoader("session_tok") + with patch_tokens(token_info): + loader = SharePointLoader("session_tok") return loader @@ -42,7 +44,7 @@ class TestSharePointLoaderInit: def test_init_sets_attributes(self, loader): assert loader.session_token == "session_tok" assert loader.access_token == "at" - assert loader.refresh_token == "rt" + assert loader.connection_id == "conn-1" assert loader.allows_shared_content is False assert loader.next_page_token is None @@ -50,7 +52,7 @@ class TestSharePointLoaderInit: def test_no_access_token_raises(self): with patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth") as MockAuth: mock_auth = MagicMock() - mock_auth.get_token_info_from_session.return_value = { + token_info = { "access_token": None, "refresh_token": "rt", } @@ -58,7 +60,8 @@ class TestSharePointLoaderInit: from docsgpt.parser.connectors.share_point.loader import SharePointLoader with pytest.raises(ValueError, match="No access token"): - SharePointLoader("st") + with patch_tokens(token_info): + SharePointLoader("st") class TestGetHeaders: @@ -86,16 +89,19 @@ class TestEnsureValidToken: @pytest.mark.unit def test_expired_token_refreshes(self, loader): loader.auth.is_token_expired.return_value = True - loader.auth.refresh_access_token.return_value = {"access_token": "new_at"} - loader._ensure_valid_token() + with patch( + "docsgpt.connectors.service.get_valid_token_info", return_value={"access_token": "new_at"}, + ) as tokens: + loader._ensure_valid_token() assert loader.access_token == "new_at" + tokens.assert_called_once_with("conn-1") @pytest.mark.unit def test_refresh_failure_raises(self, loader): loader.auth.is_token_expired.return_value = True - loader.auth.refresh_access_token.side_effect = Exception("fail") - with pytest.raises(ValueError, match="Failed to refresh"): - loader._ensure_valid_token() + with patch("docsgpt.connectors.service.get_valid_token_info", side_effect=Exception("fail")): + with pytest.raises(ValueError, match="Failed to refresh"): + loader._ensure_valid_token() class TestGetItemUrl: @@ -305,7 +311,7 @@ class TestLoadFileById: return mock_resp_ok loader._process_file = MagicMock(return_value=Document(text="", doc_id="f1", extra_info={})) - loader.auth.refresh_access_token.return_value = {"access_token": "new_at"} + loader._refresh_rejected_token = MagicMock(return_value={"access_token": "new_at"}) with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect): doc = loader._load_file_by_id("f1") @@ -481,7 +487,7 @@ class TestDownloadFileContent: raise http_error return mock_resp_ok - loader.auth.refresh_access_token.return_value = {"access_token": "new_at"} + loader._refresh_rejected_token = MagicMock(return_value={"access_token": "new_at"}) with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect): content = loader._download_file_content("f1") @@ -1004,7 +1010,7 @@ class TestRetryOnAuthFailureDecorator: raise http_error return mock_resp_ok - loader.auth.refresh_access_token.return_value = {"access_token": "new_at"} + loader._refresh_rejected_token = MagicMock(return_value={"access_token": "new_at"}) with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect): content = loader._download_file_content("f1") @@ -1018,7 +1024,7 @@ class TestRetryOnAuthFailureDecorator: mock_resp_403.status_code = 403 http_error = real_requests.exceptions.HTTPError(response=mock_resp_403) - loader.auth.refresh_access_token.side_effect = Exception("refresh fail") + loader._refresh_rejected_token = MagicMock(side_effect=Exception("refresh fail")) with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=http_error): with pytest.raises(ValueError, match="could not be refreshed"): diff --git a/tests/parser/connectors/token_patch.py b/tests/parser/connectors/token_patch.py new file mode 100644 index 00000000..dafcd115 --- /dev/null +++ b/tests/parser/connectors/token_patch.py @@ -0,0 +1,23 @@ +"""Stand-in for the connection service in connector loader tests.""" + +from __future__ import annotations + +from contextlib import ExitStack, contextmanager +from unittest.mock import patch + + +@contextmanager +def patch_tokens(token_info: dict, connection_id: str = "conn-1"): + """Make every loader resolve ``connection_id`` and read ``token_info``. + + Yields the ``get_valid_token_info`` mock so a test can assert on or + change what a refresh returns. + """ + with ExitStack() as stack: + stack.enter_context( + patch("docsgpt.connectors.service.connection_id_for_session_token", return_value=connection_id) + ) + tokens = stack.enter_context( + patch("docsgpt.connectors.service.get_valid_token_info", return_value=token_info) + ) + yield tokens diff --git a/tests/parser/remote/test_share_point_loader.py b/tests/parser/remote/test_share_point_loader.py index 07944ee6..5ee13392 100644 --- a/tests/parser/remote/test_share_point_loader.py +++ b/tests/parser/remote/test_share_point_loader.py @@ -66,7 +66,8 @@ class TestSharePointLoaderLoadFileById: """Test _load_file_by_id method.""" @patch("docsgpt.parser.connectors.share_point.loader.requests.get") - @patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session") + @patch("docsgpt.connectors.service.connection_id_for_session_token", new=lambda token: "conn-1") + @patch("docsgpt.connectors.service.get_valid_token_info") @patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None) @patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token") def test_load_file_by_id_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get): @@ -97,7 +98,8 @@ class TestSharePointLoaderLoadFileById: assert "size" in params["$select"] @patch("docsgpt.parser.connectors.share_point.loader.requests.get") - @patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session") + @patch("docsgpt.connectors.service.connection_id_for_session_token", new=lambda token: "conn-1") + @patch("docsgpt.connectors.service.get_valid_token_info") @patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None) @patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token") def test_load_file_by_id_returns_document_with_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get): @@ -134,7 +136,8 @@ class TestSharePointLoaderListItems: """Test _list_items_in_parent method.""" @patch("docsgpt.parser.connectors.share_point.loader.requests.get") - @patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session") + @patch("docsgpt.connectors.service.connection_id_for_session_token", new=lambda token: "conn-1") + @patch("docsgpt.connectors.service.get_valid_token_info") @patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None) @patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token") def test_list_items_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get): @@ -169,7 +172,8 @@ class TestSharePointLoaderListItems: assert "size" in params["$select"] @patch("docsgpt.parser.connectors.share_point.loader.requests.get") - @patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session") + @patch("docsgpt.connectors.service.connection_id_for_session_token", new=lambda token: "conn-1") + @patch("docsgpt.connectors.service.get_valid_token_info") @patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None) @patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token") def test_list_items_folders_include_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get): diff --git a/tests/storage/db/test_migration_0038.py b/tests/storage/db/test_migration_0038.py index 5bdee6ff..2740f33a 100644 --- a/tests/storage/db/test_migration_0038.py +++ b/tests/storage/db/test_migration_0038.py @@ -152,3 +152,177 @@ class TestMigration0038: with pg_engine.connect() as conn: assert "connector_key" in _columns(conn, "connector_sessions") assert "connection_id" in _columns(conn, "user_tools") + + +def _seed_secrets(conn) -> dict: + """Plaintext tokens and v1 tool secrets, as a pre-0038 install stored them.""" + from docsgpt.security.encryption import encrypt_credentials + + ids = {} + ids["drive"] = conn.execute( + text( + "INSERT INTO connector_sessions (user_id, provider, session_token, user_email, status, token_info) " + "VALUES ('carol', 'google_drive', 'tok-c', 'carol@example.com', 'authorized', CAST(:ti AS jsonb)) " + "RETURNING id" + ), + {"ti": json.dumps({"access_token": "plain-at", "refresh_token": "plain-rt", "scopes": ["drive"]})}, + ).scalar() + ids["mcp"] = conn.execute( + text( + "INSERT INTO connector_sessions (user_id, provider, server_url, session_data) " + "VALUES ('carol', 'mcp:https://mcp.notion.com', 'https://mcp.notion.com', CAST(:sd AS jsonb)) " + "RETURNING id" + ), + {"sd": json.dumps({ + "tokens": {"access_token": "mcp-at", "refresh_token": "mcp-rt"}, + "client_info": {"client_id": "cid", "client_secret": "dcr-secret"}, + "other": 1, + })}, + ).scalar() + ids["mcp_pending"] = conn.execute( + text( + "INSERT INTO connector_sessions (user_id, provider, server_url, session_data) " + "VALUES ('carol', 'mcp:https://half.example.com', 'https://half.example.com', CAST(:sd AS jsonb)) " + "RETURNING id" + ), + {"sd": json.dumps({"client_info": {"client_id": "x"}})}, + ).scalar() + for name, token in (("telegram_a", "111111:AAAAAAAA"), ("telegram_b", "111111:AAAAAAAA")): + ids[name] = conn.execute( + text("INSERT INTO user_tools (user_id, name, config) VALUES ('carol', 'telegram', CAST(:c AS jsonb)) " + "RETURNING id"), + {"c": json.dumps({"encrypted_credentials": encrypt_credentials({"token": token}, "carol")})}, + ).scalar() + ids["mcp_key_tool"] = conn.execute( + text("INSERT INTO user_tools (user_id, name, config) VALUES ('carol', 'mcp_tool', CAST(:c AS jsonb)) " + "RETURNING id"), + {"c": json.dumps({ + "server_url": "https://tools.example.com/mcp", "auth_type": "bearer", + "encrypted_credentials": encrypt_credentials({"bearer_token": "bearer-secret-9999"}, "carol"), + })}, + ).scalar() + ids["mcp_oauth_tool"] = conn.execute( + text("INSERT INTO user_tools (user_id, name, config) VALUES ('carol', 'mcp_tool', CAST(:c AS jsonb)) " + "RETURNING id"), + {"c": json.dumps({"server_url": "https://mcp.notion.com/mcp", "auth_type": "oauth"})}, + ).scalar() + return ids + + +class TestMigration0038Credentials: + def _upgrade_with(self, pg_engine, seed): + url = pg_engine.url.render_as_string(hide_password=False) + _run_alembic(url, "downgrade", _0037) + with pg_engine.begin() as conn: + ids = seed(conn) + _run_alembic(url, "upgrade", "head") + return url, ids + + def test_tokens_are_encrypted_and_plaintext_removed(self, pg_engine): + from docsgpt.connectors.service import read_secrets + + _, ids = self._upgrade_with(pg_engine, _seed_secrets) + with pg_engine.connect() as conn: + rows = { + r.id: dict(r._mapping) + for r in conn.execute(text("SELECT * FROM connector_sessions WHERE user_id = 'carol'")) + } + dump = json.dumps([ + {k: v for k, v in r.items() if k not in ("encrypted_credentials",)} for r in rows.values() + ], default=str) + # Nothing readable is left outside the envelope. + for secret in ("plain-at", "plain-rt", "mcp-at", "mcp-rt", "dcr-secret", "AAAAAAAA", "bearer-secret"): + assert secret not in dump + drive = rows[ids["drive"]] + assert drive["token_info"] is None + assert drive["has_refresh_token"] is True + assert drive["scopes"] == ["drive"] + assert drive["status"] == "connected" + assert read_secrets(drive)["token_info"]["refresh_token"] == "plain-rt" + mcp = rows[ids["mcp"]] + assert mcp["session_data"] == {"other": 1} + assert mcp["status"] == "connected" + assert read_secrets(mcp)["client_info"]["client_secret"] == "dcr-secret" + assert rows[ids["mcp_pending"]]["status"] == "pending" + + def test_api_key_tools_share_one_connection(self, pg_engine): + from docsgpt.connectors.service import read_secrets + + _, ids = self._upgrade_with(pg_engine, _seed_secrets) + with pg_engine.connect() as conn: + links = dict(conn.execute( + text("SELECT id, connection_id FROM user_tools WHERE user_id = 'carol'") + ).fetchall()) + assert links[ids["telegram_a"]] is not None + assert links[ids["telegram_a"]] == links[ids["telegram_b"]] + telegram = dict(conn.execute( + text("SELECT * FROM connector_sessions WHERE id = :i"), {"i": links[ids["telegram_a"]]} + ).one()._mapping) + assert telegram["auth_kind"] == "api_key" + assert telegram["account_label"] == "…AAAA" + assert read_secrets(telegram) == {"credentials": {"token": "111111:AAAAAAAA"}} + bearer = dict(conn.execute( + text("SELECT * FROM connector_sessions WHERE id = :i"), {"i": links[ids["mcp_key_tool"]]} + ).one()._mapping) + assert (bearer["connector_key"], bearer["server_url"]) == ("custom_mcp", "https://tools.example.com") + # Rollback safety: the tool keeps its v1 copy for a release. + config = conn.execute( + text("SELECT config FROM user_tools WHERE id = :i"), {"i": ids["telegram_a"]} + ).scalar() + assert "encrypted_credentials" in config + + def test_oauth_mcp_tools_keep_member_credentials(self, pg_engine): + _, ids = self._upgrade_with(pg_engine, _seed_secrets) + with pg_engine.connect() as conn: + modes = dict(conn.execute( + text("SELECT id, credential_mode FROM user_tools WHERE user_id = 'carol'") + ).fetchall()) + assert modes[ids["mcp_oauth_tool"]] == "member" + assert modes[ids["telegram_a"]] == "owner" + + def test_multiple_accounts_per_provider_allowed(self, pg_engine): + with pg_engine.begin() as conn: + for label in ("a@example.com", "b@example.com"): + conn.execute( + text("INSERT INTO connector_sessions (user_id, provider, account_label, status) " + "VALUES ('dan', 'google_drive', :l, 'connected')"), + {"l": label}, + ) + with pg_engine.connect() as conn: + assert conn.execute( + text("SELECT count(*) FROM connector_sessions WHERE user_id = 'dan'") + ).scalar() == 2 + + def test_downgrade_restores_plaintext(self, pg_engine): + url, ids = self._upgrade_with(pg_engine, _seed_secrets) + _run_alembic(url, "downgrade", _0037) + with pg_engine.connect() as conn: + token_info = conn.execute( + text("SELECT token_info FROM connector_sessions WHERE id = :i"), {"i": ids["drive"]} + ).scalar() + session_data = conn.execute( + text("SELECT session_data FROM connector_sessions WHERE id = :i"), {"i": ids["mcp"]} + ).scalar() + api_rows = conn.execute( + text("SELECT count(*) FROM connector_sessions WHERE provider = 'telegram'") + ).scalar() + assert token_info["refresh_token"] == "plain-rt" + assert session_data["tokens"]["access_token"] == "mcp-at" + assert api_rows == 0 + _run_alembic(url, "upgrade", "head") + + def test_second_upgrade_is_a_no_op(self, pg_engine): + from docsgpt.connectors.service import read_secrets + + url, ids = self._upgrade_with(pg_engine, _seed_secrets) + _run_alembic(url, "downgrade", _0037) + _run_alembic(url, "upgrade", "head") + with pg_engine.connect() as conn: + drive = dict(conn.execute( + text("SELECT * FROM connector_sessions WHERE id = :i"), {"i": ids["drive"]} + ).one()._mapping) + count = conn.execute( + text("SELECT count(*) FROM connector_sessions WHERE provider = 'telegram'") + ).scalar() + assert read_secrets(drive)["token_info"]["access_token"] == "plain-at" + assert count == 1 diff --git a/tests/test_extra_coverage.py b/tests/test_extra_coverage.py index fcb3333f..4731df1c 100644 --- a/tests/test_extra_coverage.py +++ b/tests/test_extra_coverage.py @@ -44,6 +44,10 @@ def _patch_upload_db(conn): "docsgpt.api.user.sources.upload.db_session", _yield ), patch( "docsgpt.api.user.sources.upload.db_readonly", _yield + ), patch( + "docsgpt.connectors.service.db_session", _yield + ), patch( + "docsgpt.connectors.service.db_readonly", _yield ): yield @@ -58,6 +62,10 @@ def _patch_conn_db(conn): "docsgpt.api.connector.routes.db_session", _yield ), patch( "docsgpt.api.connector.routes.db_readonly", _yield + ), patch( + "docsgpt.connectors.service.db_session", _yield + ), patch( + "docsgpt.connectors.service.db_readonly", _yield ): yield @@ -210,7 +218,7 @@ class TestConnectorExtra: yield with patch( - "docsgpt.api.connector.routes.db_session", _broken + "docsgpt.connectors.service.db_readonly", _broken ), app.test_request_context( "/api/connectors/disconnect", method="POST", json={"provider": "x", "session_token": "y"}, diff --git a/tests/worker/test_ingest_connector.py b/tests/worker/test_ingest_connector.py index 998b6d38..2e566bc6 100644 --- a/tests/worker/test_ingest_connector.py +++ b/tests/worker/test_ingest_connector.py @@ -24,6 +24,7 @@ def _mock_connector_pipeline(monkeypatch): from docsgpt import worker fake_connector = MagicMock(name="connector") + fake_connector.connection_id = None fake_connector.download_to_directory.return_value = { "files_downloaded": 1, "empty_result": False, @@ -34,7 +35,7 @@ def _mock_connector_pipeline(monkeypatch): monkeypatch.setattr( worker.ConnectorCreator, "create_connector", - staticmethod(lambda source_type, session_token: fake_connector), + staticmethod(lambda source_type, session_token=None, connection_id=None: fake_connector), ) fake_reader = MagicMock(name="reader") diff --git a/tests/worker/test_remote_worker.py b/tests/worker/test_remote_worker.py index 4f4e0bb6..2eab9b54 100644 --- a/tests/worker/test_remote_worker.py +++ b/tests/worker/test_remote_worker.py @@ -125,7 +125,8 @@ class TestSyncWorker: captured: list[dict] = [] def _fake_sync(self, source_data, name_job, user, loader, - sync_frequency, retriever, doc_id=None, directory="temp"): + sync_frequency, retriever, doc_id=None, directory="temp", + connection_id=None): captured.append({ "name_job": name_job, "user": user, diff --git a/tests/worker/test_worker_sse_publishes.py b/tests/worker/test_worker_sse_publishes.py index f50869b5..e8a1ba1e 100644 --- a/tests/worker/test_worker_sse_publishes.py +++ b/tests/worker/test_worker_sse_publishes.py @@ -470,6 +470,7 @@ def _stub_connector_pipeline( from docsgpt import worker fake_connector = MagicMock(name="connector") + fake_connector.connection_id = None fake_connector.download_to_directory.return_value = { "files_downloaded": files_downloaded, "empty_result": empty_result, @@ -482,7 +483,7 @@ def _stub_connector_pipeline( monkeypatch.setattr( worker.ConnectorCreator, "create_connector", - staticmethod(lambda source_type, session_token: fake_connector), + staticmethod(lambda source_type, session_token=None, connection_id=None: fake_connector), ) fake_reader = MagicMock(name="reader") From 6e2ee439442fcf26323c45a57c44696768cd3757 Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:21:44 +0100 Subject: [PATCH 006/162] Connect services from one wizard, with no tokens in the browser Every entry point (the Connectors page and drawer, Add Source, Add Tool, the chat) opens one connect wizard: sign in or paste the credentials, choose what to sync (content services only, skippable), then a summary with Try it in chat. Tool services create their tools on sign-in with write actions needing approval, and each action's permission (Always allow, Needs approval, Off) can be changed in the wizard or the drawer. The drawer also reconnects, syncs more content, refreshes an MCP server's tools and removes a connection, keeping or deleting what it feeds. The file pickers work from a connection id and fetch a short-lived picker token when they need one; no session token is kept in localStorage any more, and any left by an older build is claimed once and removed. S3 and Reddit reuse the keys stored on a connection. A tool whose account needs signing in pauses on a Connect card in the chat; after connecting, Continue resumes the call. A connection whose sign-in stops working raises a Reconnect toast. The Connectors page is hidden when the backend does not report connectors_enabled. --- docsgpt/agents/tool_executor.py | 2 +- docsgpt/alembic/versions/0038_connections.py | 21 +- frontend/DESIGN.md | 5 +- frontend/src/App.tsx | 2 + frontend/src/api/endpoints.ts | 17 +- .../src/api/services/connectorsService.ts | 91 ++++ frontend/src/api/services/userService.ts | 46 +- frontend/src/components/ConnectorAuth.tsx | 8 +- frontend/src/components/ConnectorTree.tsx | 2 +- frontend/src/components/FilePicker.test.tsx | 135 +++-- frontend/src/components/FilePicker.tsx | 258 +++++----- frontend/src/components/GoogleDrivePicker.tsx | 134 +++-- .../src/connectors/ConnectWizard.test.tsx | 303 +++++++++++ frontend/src/connectors/ConnectWizard.tsx | 484 ++++++++++++++++++ frontend/src/connectors/ConnectionDrawer.tsx | 434 ++++++++++++---- frontend/src/connectors/CredentialForm.tsx | 56 ++ frontend/src/connectors/ToolPermissions.tsx | 118 +++++ frontend/src/connectors/connectorsSlice.ts | 20 +- .../src/connectors/useConnectorLauncher.tsx | 162 +++--- .../conversation/ConnectToolCallBar.test.tsx | 113 ++++ .../src/conversation/ConnectToolCallBar.tsx | 111 ++++ .../src/conversation/ConversationBubble.tsx | 16 +- frontend/src/conversation/types/index.ts | 12 + frontend/src/events/dispatchEvent.ts | 3 + frontend/src/hooks/useDataInitializer.ts | 11 + frontend/src/locale/de.json | 87 +++- frontend/src/locale/en.json | 87 +++- frontend/src/locale/es.json | 87 +++- frontend/src/locale/jp.json | 83 ++- frontend/src/locale/ru.json | 95 +++- frontend/src/locale/zh-TW.json | 83 ++- frontend/src/locale/zh.json | 83 ++- frontend/src/modals/AddToolModal.tsx | 28 +- frontend/src/navigation/SectionIndexPage.tsx | 7 +- frontend/src/navigation/SectionNav.tsx | 8 +- frontend/src/navigation/SectionPills.tsx | 9 +- frontend/src/navigation/sections.ts | 17 +- .../notifications/ConnectionHealthToast.tsx | 116 +++++ frontend/src/settings/Connectors.tsx | 4 +- frontend/src/settings/Sources.tsx | 31 +- frontend/src/upload/Upload.tsx | 73 ++- frontend/src/utils/providerUtils.test.ts | 39 ++ frontend/src/utils/providerUtils.ts | 43 +- 43 files changed, 2989 insertions(+), 555 deletions(-) create mode 100644 frontend/src/connectors/ConnectWizard.test.tsx create mode 100644 frontend/src/connectors/ConnectWizard.tsx create mode 100644 frontend/src/connectors/CredentialForm.tsx create mode 100644 frontend/src/connectors/ToolPermissions.tsx create mode 100644 frontend/src/conversation/ConnectToolCallBar.test.tsx create mode 100644 frontend/src/conversation/ConnectToolCallBar.tsx create mode 100644 frontend/src/notifications/ConnectionHealthToast.tsx create mode 100644 frontend/src/utils/providerUtils.test.ts diff --git a/docsgpt/agents/tool_executor.py b/docsgpt/agents/tool_executor.py index 2e9faf9f..181612ae 100644 --- a/docsgpt/agents/tool_executor.py +++ b/docsgpt/agents/tool_executor.py @@ -1649,7 +1649,7 @@ class ToolExecutor: # Keys the client needs that are not part of the fixed shape below. They are # small and optional, and are copied only when present so an ordinary tool # call does not grow null columns in every persisted row. - _PRESERVED_TOOL_CALL_KEYS = ("artifacts", "device_id") + _PRESERVED_TOOL_CALL_KEYS = ("artifacts", "device_id", "connector_key", "connector_name", "access") def get_truncated_tool_calls(self) -> List[Dict]: """Project tool calls into the shape that is streamed and persisted. diff --git a/docsgpt/alembic/versions/0038_connections.py b/docsgpt/alembic/versions/0038_connections.py index 42586c75..b9da954d 100644 --- a/docsgpt/alembic/versions/0038_connections.py +++ b/docsgpt/alembic/versions/0038_connections.py @@ -347,13 +347,29 @@ def _link_api_key_tools(bind) -> None: def _downgrade_credentials() -> None: """Decrypt the envelopes back into the pre-0038 plaintext columns.""" + + from sqlalchemy import text + + + bind = op.get_bind() + has_envelope = bind.execute( + text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = 'connector_sessions' AND column_name = 'encrypted_credentials'" + ) + ).first() + if has_envelope is not None: + _decrypt_back(bind) + _restore_account_index() + + +def _decrypt_back(bind) -> None: import json from sqlalchemy import text from docsgpt.security.encryption import CredentialDecryptionError, decrypt_json - bind = op.get_bind() # Tools keep their v1 secrets; connections made only for them go away. bind.execute(text("UPDATE user_tools SET connection_id = NULL WHERE connection_id IN " "(SELECT id FROM connector_sessions WHERE auth_kind = 'api_key')")) @@ -384,6 +400,9 @@ def _downgrade_credentials() -> None: "id": row.id, }, ) + + +def _restore_account_index() -> None: # Several accounts per provider cannot survive the old unique index: keep # the most recently updated one. op.execute( diff --git a/frontend/DESIGN.md b/frontend/DESIGN.md index 3584eab5..1298a479 100644 --- a/frontend/DESIGN.md +++ b/frontend/DESIGN.md @@ -600,8 +600,9 @@ overlay. The app has one `ToastViewport`, mounted in `App.tsx`; it is the live region (`role="status"`, `aria-live="polite"`) and the fixed stack, so `Toast` cards carry no role and no toast renders its own rail or positioning. Top -to bottom it holds `TeamNotificationToast`, `ToolApprovalToast`, -`UploadToast` and `ActionToast`, and it moves to the bottom-left while the +to bottom it holds `TeamNotificationToast`, `ConnectionHealthToast` (a +connection that needs reconnecting, with a Reconnect action), +`ToolApprovalToast`, `UploadToast` and `ActionToast`, and it moves to the bottom-left while the workflow Preview drawer is open. A new toast component returns only its `Toast` cards and is added to that viewport. A page that reports the result of an action (the admin Users actions) dispatches diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 20f2ed11..65d84787 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -43,6 +43,7 @@ import Notification from './components/Notification'; import ToolApprovalToast from './notifications/ToolApprovalToast'; import TeamNotificationToast from './notifications/TeamNotificationToast'; import ActionToast from './notifications/ActionToast'; +import ConnectionHealthToast from './notifications/ConnectionHealthToast'; function AuthWrapper({ children }: { children: React.ReactNode }) { const { t } = useTranslation(); @@ -128,6 +129,7 @@ function MainLayout() { onMouseDown={(e) => e.stopPropagation()} > + diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index d621dfea..65a5cb42 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -82,8 +82,10 @@ const endpoints = { DELETE_TOOL: '/api/delete_tool', PARSE_SPEC: '/api/parse_spec', SYNC_CONNECTOR: '/api/connectors/sync', - CONNECTOR_AUTH: (provider: string) => - `/api/connectors/auth?provider=${provider}`, + CONNECTOR_AUTH: (provider: string, connectionId?: string) => + `/api/connectors/auth?provider=${encodeURIComponent(provider)}${ + connectionId ? `&connection_id=${encodeURIComponent(connectionId)}` : '' + }`, CONNECTOR_FILES: '/api/connectors/files', CONNECTOR_VALIDATE_SESSION: '/api/connectors/validate-session', CONNECTOR_DISCONNECT: '/api/connectors/disconnect', @@ -92,6 +94,17 @@ const endpoints = { CONNECTION: (id: string) => `/api/connections/${encodeURIComponent(id)}`, CONNECTION_DISCONNECT: (id: string) => `/api/connections/${encodeURIComponent(id)}/disconnect`, + CONNECTION_SETUP: (id: string) => + `/api/connections/${encodeURIComponent(id)}/setup`, + CONNECTION_RECONNECT: (id: string) => + `/api/connections/${encodeURIComponent(id)}/reconnect`, + CONNECTION_PICKER_TOKEN: (id: string) => + `/api/connections/${encodeURIComponent(id)}/picker-token`, + CONNECTION_REFRESH_TOOLS: (id: string) => + `/api/connections/${encodeURIComponent(id)}/refresh-tools`, + CONNECTION_TOOL_PERMISSIONS: (id: string, toolId: string) => + `/api/connections/${encodeURIComponent(id)}/tools/${encodeURIComponent(toolId)}/permissions`, + CONNECTIONS_CLAIM: '/api/connections/claim', GET_CHUNKS: ( docId: string, page: number, diff --git a/frontend/src/api/services/connectorsService.ts b/frontend/src/api/services/connectorsService.ts index 19640996..985122bc 100644 --- a/frontend/src/api/services/connectorsService.ts +++ b/frontend/src/api/services/connectorsService.ts @@ -11,6 +11,21 @@ const json = async (response: Response) => { } }; +export type ConnectionSetupBody = { + create_tools?: boolean; + tool_permissions?: Record; + sync?: { + items: Record; + frequency?: string; + name?: string; + }; +}; + +export type RemoveConnectionBody = { + sources: 'keep' | 'delete'; + tools: 'keep' | 'delete'; +}; + const connectorsService = { getCatalog: async (token: string | null) => json( @@ -20,10 +35,86 @@ const connectorsService = { json(await throttledApiClient.get(endpoints.USER.CONNECTIONS, token)), getConnection: async (id: string, token: string | null) => json(await apiClient.get(endpoints.USER.CONNECTION(id), token)), + createConnection: async ( + body: { + connector_key: string; + credentials: Record; + label?: string; + }, + token: string | null, + ) => json(await apiClient.post(endpoints.USER.CONNECTIONS, body, token)), + setup: async ( + id: string, + body: ConnectionSetupBody, + token: string | null, + idempotencyKey?: string, + ) => + json( + await apiClient.post( + endpoints.USER.CONNECTION_SETUP(id), + body, + token, + idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : {}, + ), + ), + reconnect: async ( + id: string, + body: { credentials?: Record }, + token: string | null, + ) => + json( + await apiClient.post( + endpoints.USER.CONNECTION_RECONNECT(id), + body, + token, + ), + ), disconnect: async (id: string, token: string | null) => json( await apiClient.post(endpoints.USER.CONNECTION_DISCONNECT(id), {}, token), ), + remove: async ( + id: string, + body: RemoveConnectionBody, + token: string | null, + ) => json(await apiClient.delete(endpoints.USER.CONNECTION(id), token, body)), + pickerToken: async (id: string, token: string | null) => + json( + await apiClient.post( + endpoints.USER.CONNECTION_PICKER_TOKEN(id), + {}, + token, + ), + ), + refreshTools: async (id: string, token: string | null) => + json( + await apiClient.post( + endpoints.USER.CONNECTION_REFRESH_TOOLS(id), + {}, + token, + ), + ), + setToolPermissions: async ( + id: string, + toolId: string, + permissions: Record, + token: string | null, + ) => + json( + await apiClient.put( + endpoints.USER.CONNECTION_TOOL_PERMISSIONS(id, toolId), + { permissions }, + token, + ), + ), + claim: async (provider: string, sessionToken: string, token: string | null) => + json( + await apiClient.post( + endpoints.USER.CONNECTIONS_CLAIM, + { provider, session_token: sessionToken }, + token, + ), + ), }; export default connectorsService; diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index 85253b01..934ec890 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -1,4 +1,3 @@ -import { getSessionToken } from '../../utils/providerUtils'; import apiClient, { throttledApiClient } from '../client'; import endpoints from '../endpoints'; @@ -277,24 +276,15 @@ const userService = { apiClient.post(endpoints.USER.MCP_SAVE_SERVER, data, token), getMCPAuthStatus: (token: string | null): Promise => throttledApiClient.get(endpoints.USER.MCP_AUTH_STATUS, token), - syncConnector: ( - docId: string, + // The source's own connection syncs it; no browser token is involved. + syncConnector: (docId: string, token: string | null): Promise => + apiClient.post(endpoints.USER.SYNC_CONNECTOR, { source_id: docId }, token), + getConnectorAuthUrl: ( provider: string, token: string | null, - ): Promise => { - const sessionToken = getSessionToken(provider); - return apiClient.post( - endpoints.USER.SYNC_CONNECTOR, - { - source_id: docId, - session_token: sessionToken, - provider: provider, - }, - token, - ); - }, - getConnectorAuthUrl: (provider: string, token: string | null): Promise => - apiClient.get(endpoints.USER.CONNECTOR_AUTH(provider), token), + connectionId?: string, + ): Promise => + apiClient.get(endpoints.USER.CONNECTOR_AUTH(provider, connectionId), token), getConnectorFiles: ( data: any, token: string | null, @@ -307,28 +297,6 @@ const userService = { {}, signal, ), - validateConnectorSession: ( - provider: string, - token: string | null, - ): Promise => - apiClient.post( - endpoints.USER.CONNECTOR_VALIDATE_SESSION, - { - provider, - session_token: getSessionToken(provider), - }, - token, - ), - disconnectConnector: ( - provider: string, - sessionToken: string, - token: string | null, - ): Promise => - apiClient.post( - endpoints.USER.CONNECTOR_DISCONNECT, - { provider, session_token: sessionToken }, - token, - ), textToSpeech: ( text: string, token: string | null, diff --git a/frontend/src/components/ConnectorAuth.tsx b/frontend/src/components/ConnectorAuth.tsx index 0f25a5d1..2fad975a 100644 --- a/frontend/src/components/ConnectorAuth.tsx +++ b/frontend/src/components/ConnectorAuth.tsx @@ -11,7 +11,9 @@ import { Button } from './ui/button'; interface ConnectorAuthProps { provider: string; - onSuccess: (data: { session_token: string; user_email: string }) => void; + /** Reconnect this connection (the same account) rather than add one. */ + connectionId?: string; + onSuccess: (data: { connection_id: string; user_email: string }) => void; onError: (error: string) => void; label?: string; isConnected?: boolean; @@ -22,6 +24,7 @@ interface ConnectorAuthProps { const ConnectorAuth: React.FC = ({ provider, + connectionId, onSuccess, onError, label, @@ -76,7 +79,7 @@ const ConnectorAuth: React.FC = ({ cleanup(); authWindowRef.current = null; onSuccess({ - session_token: event.data.session_token, + connection_id: event.data.connection_id, user_email: event.data.user_email || t('modals.uploadDoc.connectors.auth.connectedUser'), @@ -120,6 +123,7 @@ const ConnectorAuth: React.FC = ({ const authResponse = await userService.getConnectorAuthUrl( provider, token, + connectionId, ); if (!mountedRef.current) { authWindow.close(); diff --git a/frontend/src/components/ConnectorTree.tsx b/frontend/src/components/ConnectorTree.tsx index 53538189..0d0ae49a 100644 --- a/frontend/src/components/ConnectorTree.tsx +++ b/frontend/src/components/ConnectorTree.tsx @@ -49,7 +49,7 @@ const ConnectorTree: React.FC = ({ setSyncProgress(0); try { - const response = await userService.syncConnector(docId, provider, token); + const response = await userService.syncConnector(docId, token); const data = await response.json(); if (data.success) { diff --git a/frontend/src/components/FilePicker.test.tsx b/frontend/src/components/FilePicker.test.tsx index 6ac53d29..0fb425d9 100644 --- a/frontend/src/components/FilePicker.test.tsx +++ b/frontend/src/components/FilePicker.test.tsx @@ -1,21 +1,26 @@ +import { configureStore } from '@reduxjs/toolkit'; import { act } from 'react'; import { createRoot, type Root } from 'react-dom/client'; +import { Provider } from 'react-redux'; -const validateConnectorSession = vi.fn(); const getConnectorFiles = vi.fn(); vi.mock('../api/services/userService', () => ({ default: { - validateConnectorSession: (...args: unknown[]) => - validateConnectorSession(...args), getConnectorFiles: (...args: unknown[]) => getConnectorFiles(...args), - disconnectConnector: vi.fn(), }, })); -vi.mock('../utils/providerUtils', () => ({ - getSessionToken: () => 'session-token', - setSessionToken: vi.fn(), - removeSessionToken: vi.fn(), +const pickerToken = vi.fn(); +const disconnect = vi.fn(); +vi.mock('../api/services/connectorsService', () => ({ + default: { + pickerToken: (...args: unknown[]) => pickerToken(...args), + disconnect: (...args: unknown[]) => disconnect(...args), + getCatalog: vi.fn().mockResolvedValue({ success: true, connectors: [] }), + listConnections: vi + .fn() + .mockResolvedValue({ success: true, connections: [] }), + }, })); vi.mock('../components/ConnectorAuth', () => ({ default: () => null })); @@ -24,17 +29,38 @@ vi.mock('react-i18next', () => ({ useTranslation: () => ({ t: (key: string) => key }), })); +import connectorsReducer from '../connectors/connectorsSlice'; +import type { Connection } from '../connectors/types'; import { FilePicker } from './FilePicker'; Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); +const connection = (overrides: Partial = {}): Connection => ({ + id: 'conn-1', + connector_key: 'share_point', + name: 'SharePoint', + display_name: 'SharePoint', + icon: 'sharepoint', + account_label: 'lena@meridian.example', + auth_kind: 'oauth', + status: 'connected', + server_url: null, + last_error: null, + created_at: null, + updated_at: null, + last_used_at: null, + source_count: 0, + tool_count: 0, + ...overrides, +}); + describe('FilePicker', () => { let container: HTMLDivElement; let root: Root; beforeEach(() => { - validateConnectorSession.mockReset(); getConnectorFiles.mockReset(); + pickerToken.mockReset(); container = document.createElement('div'); document.body.appendChild(container); root = createRoot(container); @@ -45,43 +71,70 @@ describe('FilePicker', () => { container.remove(); }); - it('shows an expired session as a destructive alert', async () => { - validateConnectorSession.mockResolvedValue({ ok: false }); + const render = async (provider: string, connections: Connection[]) => { + const store = configureStore({ + reducer: { + connectors: connectorsReducer, + preference: (state = { token: null }) => state, + }, + preloadedState: { + connectors: { + enabled: true, + catalog: [], + connections, + loading: false, + loaded: true, + failed: false, + }, + preference: { token: null }, + }, + }); await act(async () => { root.render( - undefined} - />, + + undefined} + /> + , ); }); + }; + + it('shows a connection that needs signing in as a destructive alert', async () => { + getConnectorFiles.mockResolvedValue({ + json: async () => ({ success: false, reconnect: true }), + }); + await render('google_drive', [ + connection({ connector_key: 'google_drive' }), + ]); const alert = container.querySelector('[role="alert"]'); expect(alert?.textContent).toContain('filePicker.sessionExpiredFor'); expect(alert?.querySelector('svg')).not.toBeNull(); }); + it('lists files with the connection id, never a session token', async () => { + getConnectorFiles.mockResolvedValue({ + json: async () => ({ success: true, files: [], next_page_token: null }), + }); + await render('google_drive', [ + connection({ connector_key: 'google_drive' }), + ]); + const body = getConnectorFiles.mock.calls[0][0]; + expect(body.connection_id).toBe('conn-1'); + expect(body).not.toHaveProperty('session_token'); + }); + async function renderSharePoint() { - validateConnectorSession.mockResolvedValue({ - ok: true, - json: async () => ({ - success: true, - user_email: 'lena@meridian.example', - allows_shared_content: true, - }), + pickerToken.mockResolvedValue({ + success: true, + allows_shared_content: true, }); getConnectorFiles.mockResolvedValue({ json: async () => ({ success: true, files: [], next_page_token: null }), }); - await act(async () => { - root.render( - undefined} - />, - ); - }); + await render('share_point', [connection()]); } it('renders the drive switch as underline tabs with the active one marked', async () => { @@ -112,4 +165,22 @@ describe('FilePicker', () => { expect(page?.getAttribute('title')).toBe('filePicker.myFiles'); expect(trail?.querySelector('button[disabled]')).toBeNull(); }); + + it('offers an account switch when several accounts are connected', async () => { + getConnectorFiles.mockResolvedValue({ + json: async () => ({ success: true, files: [], next_page_token: null }), + }); + pickerToken.mockResolvedValue({ success: true }); + await render('share_point', [ + connection(), + connection({ id: 'conn-2', account_label: 'ops@meridian.example' }), + ]); + expect(container.textContent).toContain('filePicker.account'); + }); + + it('shows nothing to browse without a connection', async () => { + await render('confluence', []); + expect(getConnectorFiles).not.toHaveBeenCalled(); + expect(container.querySelector('[data-slot="tabs-list"]')).toBeNull(); + }); }); diff --git a/frontend/src/components/FilePicker.tsx b/frontend/src/components/FilePicker.tsx index 1e0d1ac1..6f191bcd 100644 --- a/frontend/src/components/FilePicker.tsx +++ b/frontend/src/components/FilePicker.tsx @@ -10,12 +10,22 @@ import { useTranslation } from 'react-i18next'; import userService from '../api/services/userService'; import { formatBytes } from '../utils/stringUtils'; import { formatDateTime } from '../utils/dateTimeUtils'; -import { - getSessionToken, - setSessionToken, - removeSessionToken, -} from '../utils/providerUtils'; +import { useDispatch, useSelector } from 'react-redux'; +import connectorsService from '../api/services/connectorsService'; import ConnectorAuth from '../components/ConnectorAuth'; +import { + loadConnectors, + selectConnections, +} from '../connectors/connectorsSlice'; +import type { AppDispatch } from '../store'; +import { FormField } from './ui/form-field'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from './ui/select'; import SearchInput from './SearchInput'; import { Alert, AlertDescription } from './ui/alert'; import { @@ -77,6 +87,13 @@ interface CloudFilePickerProps { onDisconnect?: () => void; /** Called with the first item's name when the selection goes from empty to one. */ onFirstPickName?: (name: string) => void; + /** + * The connection (signed-in account) to browse. Left out, the picker uses + * the first connected account for ``provider`` and lets the user switch. + */ + connectionId?: string | null; + /** Reports the account the picker is browsing, so the upload can name it. */ + onConnectionChange?: (connectionId: string | null) => void; provider: string; token: string | null; initialSelectedFiles?: string[]; @@ -87,11 +104,14 @@ export const FilePicker: React.FC = ({ onSelectionChange, onDisconnect, onFirstPickName, + connectionId: controlledConnectionId, + onConnectionChange, provider, token, initialSelectedFiles = [], }) => { const { t } = useTranslation(); + const dispatch = useDispatch(); const PROVIDER_CONFIG = { google_drive: { displayName: 'Drive', @@ -116,6 +136,23 @@ export const FilePicker: React.FC = ({ ); }; + const allConnections = useSelector(selectConnections); + const accounts = allConnections.filter( + (connection) => + connection.connector_key === provider && + connection.status === 'connected', + ); + const [chosenConnectionId, setChosenConnectionId] = useState( + null, + ); + const activeConnectionId = + controlledConnectionId ?? + (chosenConnectionId && + accounts.some((account) => account.id === chosenConnectionId) + ? chosenConnectionId + : (accounts[0]?.id ?? null)); + const activeAccount = accounts.find((a) => a.id === activeConnectionId); + const [files, setFiles] = useState([]); const [selectedFiles, setSelectedFiles] = useState(initialSelectedFiles); @@ -134,14 +171,18 @@ export const FilePicker: React.FC = ({ ]); const [searchQuery, setSearchQuery] = useState(''); const [authError, setAuthError] = useState(''); - const [isConnected, setIsConnected] = useState(false); - const [userEmail, setUserEmail] = useState(''); const [allowsSharedContent, setAllowsSharedContent] = useState(false); const [activeTab, setActiveTab] = useState<'my_files' | 'shared'>('my_files'); + const isConnected = !!activeConnectionId; + const scrollContainerRef = useRef(null); const abortControllerRef = useRef(null); + useEffect(() => { + onConnectionChange?.(activeConnectionId); + }, [activeConnectionId]); + const isFolder = (file: CloudFile) => { return ( file.isFolder || @@ -152,7 +193,7 @@ export const FilePicker: React.FC = ({ const loadCloudFiles = useCallback( async ( - sessionToken: string, + connectionId: string, folderId: string | null, pageToken?: string, searchQuery = '', @@ -172,7 +213,7 @@ export const FilePicker: React.FC = ({ try { const body: Record = { provider: provider, - session_token: sessionToken, + connection_id: connectionId, folder_id: folderId, limit: 10, page_token: pageToken, @@ -193,7 +234,14 @@ export const FilePicker: React.FC = ({ setNextPageToken(data.next_page_token); setHasMoreFiles(!!data.next_page_token); } else { - console.error('Error loading files:', data.error); + if (data.reconnect) { + setAuthError( + t('filePicker.sessionExpiredFor', { + provider: getProviderConfig(provider).displayName, + }), + ); + dispatch(loadConnectors({ token })); + } if (!pageToken) { setFiles([]); } @@ -213,72 +261,26 @@ export const FilePicker: React.FC = ({ [token, provider], ); - const validateAndLoadFiles = useCallback(async () => { - const sessionToken = getSessionToken(provider); - if (!sessionToken) { - setIsConnected(false); - return; - } - - try { - const validateResponse = await userService.validateConnectorSession( - provider, - token, - ); - - if (!validateResponse.ok) { - removeSessionToken(provider); - setIsConnected(false); - setAuthError( - t('filePicker.sessionExpiredFor', { - provider: getProviderConfig(provider).displayName, - }), - ); - return; - } - - const validateData = await validateResponse.json(); - if (validateData.success) { - setUserEmail( - validateData.user_email || - t('modals.uploadDoc.connectors.auth.connectedUser'), - ); - setIsConnected(true); - setAuthError(''); - if (provider === 'share_point') { - setAllowsSharedContent(validateData.allows_shared_content ?? false); - } - - setFiles([]); - setNextPageToken(null); - setHasMoreFiles(false); - setCurrentFolderId(null); - setActiveTab('my_files'); - setFolderPath([ - { - id: null, - name: getProviderConfig(provider).rootName, - }, - ]); - loadCloudFiles(sessionToken, null, undefined, ''); - } else { - removeSessionToken(provider); - setIsConnected(false); - setAuthError( - validateData.error || - t('modals.uploadDoc.connectors.googleDrive.sessionExpiredGeneric'), - ); - } - } catch (error) { - console.error('Error validating session:', error); - setAuthError(t('modals.uploadDoc.connectors.googleDrive.validateFailed')); - setIsConnected(false); - } - }, [provider, token, loadCloudFiles]); - + // Browse from the root whenever the account changes. useEffect(() => { - validateAndLoadFiles(); - }, [validateAndLoadFiles]); + setFiles([]); + setNextPageToken(null); + setHasMoreFiles(false); + setCurrentFolderId(null); + setActiveTab('my_files'); + setAllowsSharedContent(false); + setFolderPath([{ id: null, name: getProviderConfig(provider).rootName }]); + if (!activeConnectionId) return; + setAuthError(''); + loadCloudFiles(activeConnectionId, null, undefined, ''); + if (provider === 'share_point') { + // Work and school accounts can browse "Shared with me" too. + connectorsService + .pickerToken(activeConnectionId, token) + .then((data) => setAllowsSharedContent(!!data?.allows_shared_content)) + .catch(() => undefined); + } + }, [activeConnectionId, provider, loadCloudFiles]); const handleScroll = useCallback(() => { const scrollContainer = scrollContainerRef.current; @@ -287,17 +289,20 @@ export const FilePicker: React.FC = ({ const { scrollTop, scrollHeight, clientHeight } = scrollContainer; const isNearBottom = scrollHeight - scrollTop - clientHeight < 50; - if (isNearBottom && hasMoreFiles && !isLoading && nextPageToken) { - const sessionToken = getSessionToken(provider); - if (sessionToken) { - loadCloudFiles( - sessionToken, - currentFolderId, - nextPageToken, - searchQuery, - activeTab === 'shared' && !currentFolderId, - ); - } + if ( + isNearBottom && + hasMoreFiles && + !isLoading && + nextPageToken && + activeConnectionId + ) { + loadCloudFiles( + activeConnectionId, + currentFolderId, + nextPageToken, + searchQuery, + activeTab === 'shared' && !currentFolderId, + ); } }, [ hasMoreFiles, @@ -305,7 +310,7 @@ export const FilePicker: React.FC = ({ nextPageToken, currentFolderId, searchQuery, - provider, + activeConnectionId, loadCloudFiles, activeTab, ]); @@ -325,10 +330,9 @@ export const FilePicker: React.FC = ({ }, []); const debouncedLoadFiles = useDebouncedCallback((query: string) => { - const sessionToken = getSessionToken(provider); - if (sessionToken) { + if (activeConnectionId) { loadCloudFiles( - sessionToken, + activeConnectionId, currentFolderId, undefined, query, @@ -353,9 +357,8 @@ export const FilePicker: React.FC = ({ setFolderPath((prev) => [...prev, { id: folderId, name: folderName }]); setSearchQuery(''); - const sessionToken = getSessionToken(provider); - if (sessionToken) { - loadCloudFiles(sessionToken, folderId, undefined, '', false); + if (activeConnectionId) { + loadCloudFiles(activeConnectionId, folderId, undefined, '', false); } }; @@ -369,10 +372,9 @@ export const FilePicker: React.FC = ({ setCurrentFolderId(newFolderId); setSearchQuery(''); - const sessionToken = getSessionToken(provider); - if (sessionToken) { + if (activeConnectionId) { loadCloudFiles( - sessionToken, + activeConnectionId, newFolderId, undefined, '', @@ -400,9 +402,8 @@ export const FilePicker: React.FC = ({ : getProviderConfig(provider).rootName, }, ]); - const sessionToken = getSessionToken(provider); - if (sessionToken) { - loadCloudFiles(sessionToken, null, undefined, '', tab === 'shared'); + if (activeConnectionId) { + loadCloudFiles(activeConnectionId, null, undefined, '', tab === 'shared'); } }; @@ -430,6 +431,13 @@ export const FilePicker: React.FC = ({ } }; + const switchAccount = (connectionId: string) => { + setChosenConnectionId(connectionId); + setSelectedFiles([]); + setSelectedFolders([]); + onSelectionChange([], []); + }; + return (
{authError && ( @@ -439,45 +447,57 @@ export const FilePicker: React.FC = ({ )} + {accounts.length > 1 && !controlledConnectionId && ( + + + + )} + { - setUserEmail( - data.user_email || - t('modals.uploadDoc.connectors.auth.connectedUser'), - ); - setIsConnected(true); setAuthError(''); - - if (data.session_token) { - setSessionToken(provider, data.session_token); - validateAndLoadFiles(); - } + dispatch(loadConnectors({ token })); + if (data.connection_id) setChosenConnectionId(data.connection_id); }} onError={(error) => { setAuthError(error); - setIsConnected(false); }} isConnected={isConnected} - userEmail={userEmail} + userEmail={ + activeAccount?.account_label || + t('modals.uploadDoc.connectors.auth.connectedUser') + } onDisconnect={() => { - const sessionToken = getSessionToken(provider); - if (sessionToken) { - userService - .disconnectConnector(provider, sessionToken, token) + if (activeConnectionId) { + connectorsService + .disconnect(activeConnectionId, token) .catch((err) => console.error( `Error disconnecting from ${getProviderConfig(provider).displayName}:`, err, ), - ); + ) + .finally(() => dispatch(loadConnectors({ token }))); } - - removeSessionToken(provider); - setIsConnected(false); + setChosenConnectionId(null); setAllowsSharedContent(false); setActiveTab('my_files'); setFiles([]); diff --git a/frontend/src/components/GoogleDrivePicker.tsx b/frontend/src/components/GoogleDrivePicker.tsx index 99f57abc..1c85c326 100644 --- a/frontend/src/components/GoogleDrivePicker.tsx +++ b/frontend/src/components/GoogleDrivePicker.tsx @@ -9,14 +9,15 @@ const useDrivePicker = (( drivePickerImport as unknown as { default?: typeof drivePickerImport } ).default ?? drivePickerImport) as typeof drivePickerImport; -import userService from '../api/services/userService'; -import ConnectorAuth from './ConnectorAuth'; +import { useDispatch, useSelector } from 'react-redux'; + +import connectorsService from '../api/services/connectorsService'; import { - getSessionToken, - setSessionToken, - removeSessionToken, - validateProviderSession, -} from '../utils/providerUtils'; + loadConnectors, + selectConnections, +} from '../connectors/connectorsSlice'; +import type { AppDispatch } from '../store'; +import ConnectorAuth from './ConnectorAuth'; import SkeletonLoader from './SkeletonLoader'; import { Button } from './ui/button'; import { SectionHeader } from './ui/section-header'; @@ -32,6 +33,10 @@ interface PickerFile { interface GoogleDrivePickerProps { token: string | null; + /** The Drive connection to pick from; defaults to the first connected one. */ + connectionId?: string | null; + /** Reports the account the picker uses, so the upload can name it. */ + onConnectionChange?: (connectionId: string | null) => void; onSelectionChange: (fileIds: string[], folderIds?: string[]) => void; /** Called with the first item's name when the selection goes from empty to one. */ onFirstPickName?: (name: string) => void; @@ -39,86 +44,77 @@ interface GoogleDrivePickerProps { const GoogleDrivePicker: React.FC = ({ token, + connectionId: controlledConnectionId, + onConnectionChange, onSelectionChange, onFirstPickName, }) => { const { t } = useTranslation(); + const dispatch = useDispatch(); + const connections = useSelector(selectConnections); + const [chosenConnectionId, setChosenConnectionId] = useState( + null, + ); + const accounts = connections.filter( + (c) => c.connector_key === 'google_drive' && c.status === 'connected', + ); + const activeConnectionId = + controlledConnectionId ?? + (chosenConnectionId && accounts.some((a) => a.id === chosenConnectionId) + ? chosenConnectionId + : (accounts[0]?.id ?? null)); + const activeAccount = accounts.find((a) => a.id === activeConnectionId); + const isConnected = !!activeConnectionId; const [selectedFiles, setSelectedFiles] = useState([]); const [selectedFolders, setSelectedFolders] = useState([]); const [isLoading, setIsLoading] = useState(false); - const [userEmail, setUserEmail] = useState(''); - const [isConnected, setIsConnected] = useState(false); const [authError, setAuthError] = useState(''); - const [accessToken, setAccessToken] = useState(null); const [isValidating, setIsValidating] = useState(false); const [openPicker] = useDrivePicker(); useEffect(() => { - const sessionToken = getSessionToken('google_drive'); - if (sessionToken) { - setIsValidating(true); - setIsConnected(true); // Optimistically set as connected for skeleton - validateSession(sessionToken); - } - }, [token]); + onConnectionChange?.(activeConnectionId); + }, [activeConnectionId]); - const validateSession = async (sessionToken: string) => { + // The Picker runs in the browser and needs an access token. It is fetched + // per use and kept in memory only; the refresh token never leaves the server. + const fetchAccessToken = async (): Promise => { + if (!activeConnectionId) return null; + setIsValidating(true); try { - const validateResponse = await validateProviderSession( + const data = await connectorsService.pickerToken( + activeConnectionId, token, - 'google_drive', ); - - if (!validateResponse.ok) { - setIsConnected(false); + if (!data?.success) { setAuthError( t('modals.uploadDoc.connectors.googleDrive.sessionExpired'), ); - setIsValidating(false); - return false; - } - - const validateData = await validateResponse.json(); - if (validateData.success) { - setUserEmail( - validateData.user_email || - t('modals.uploadDoc.connectors.auth.connectedUser'), - ); - setIsConnected(true); - setAuthError(''); - setAccessToken(validateData.access_token || null); - setIsValidating(false); - return true; - } else { - setIsConnected(false); - setAuthError( - validateData.error || - t('modals.uploadDoc.connectors.googleDrive.sessionExpiredGeneric'), - ); - setIsValidating(false); - return false; + dispatch(loadConnectors({ token })); + return null; } + setAuthError(''); + return data.access_token ?? null; } catch (error) { - console.error('Error validating session:', error); + console.error('Error fetching the picker token:', error); setAuthError(t('modals.uploadDoc.connectors.googleDrive.validateFailed')); - setIsConnected(false); + return null; + } finally { setIsValidating(false); - return false; } }; const handleOpenPicker = async () => { setIsLoading(true); - const sessionToken = getSessionToken('google_drive'); - - if (!sessionToken) { + if (!activeConnectionId) { setAuthError(t('modals.uploadDoc.connectors.googleDrive.noSession')); setIsLoading(false); return; } + const accessToken = await fetchAccessToken(); if (!accessToken) { setAuthError(t('modals.uploadDoc.connectors.googleDrive.noAccessToken')); setIsLoading(false); @@ -216,25 +212,17 @@ const GoogleDrivePicker: React.FC = ({ }; const handleDisconnect = async () => { - const sessionToken = getSessionToken('google_drive'); - if (sessionToken) { + if (activeConnectionId) { try { - await userService.disconnectConnector( - 'google_drive', - sessionToken, - token, - ); + await connectorsService.disconnect(activeConnectionId, token); } catch (err) { console.error('Error disconnecting from Google Drive:', err); } + dispatch(loadConnectors({ token })); } - - removeSessionToken('google_drive'); - setIsConnected(false); + setChosenConnectionId(null); setSelectedFiles([]); setSelectedFolders([]); - setAccessToken(null); - setUserEmail(''); setAuthError(''); onSelectionChange([], []); }; @@ -252,24 +240,18 @@ const GoogleDrivePicker: React.FC = ({ provider="google_drive" label={t('modals.uploadDoc.connectors.googleDrive.connect')} onSuccess={(data) => { - setUserEmail( - data.user_email || - t('modals.uploadDoc.connectors.auth.connectedUser'), - ); - setIsConnected(true); setAuthError(''); - - if (data.session_token) { - setSessionToken('google_drive', data.session_token); - validateSession(data.session_token); - } + dispatch(loadConnectors({ token })); + if (data.connection_id) setChosenConnectionId(data.connection_id); }} onError={(error) => { setAuthError(error); - setIsConnected(false); }} isConnected={isConnected} - userEmail={userEmail} + userEmail={ + activeAccount?.account_label || + t('modals.uploadDoc.connectors.auth.connectedUser') + } onDisconnect={handleDisconnect} errorMessage={authError} /> diff --git a/frontend/src/connectors/ConnectWizard.test.tsx b/frontend/src/connectors/ConnectWizard.test.tsx new file mode 100644 index 00000000..cf04e8e6 --- /dev/null +++ b/frontend/src/connectors/ConnectWizard.test.tsx @@ -0,0 +1,303 @@ +import { configureStore } from '@reduxjs/toolkit'; +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { Provider } from 'react-redux'; +import { MemoryRouter, Route, Routes } from 'react-router-dom'; + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ + t: (key: string, opts?: Record) => + opts && 'count' in opts ? `${key}:${opts.count}` : key, + }), +})); + +const service = vi.hoisted(() => ({ + createConnection: vi.fn(), + setup: vi.fn(), + reconnect: vi.fn(), + getConnection: vi.fn(), + getCatalog: vi.fn(), + listConnections: vi.fn(), + setToolPermissions: vi.fn(), + pickerToken: vi.fn(), +})); +vi.mock('../api/services/connectorsService', () => ({ default: service })); + +// The OAuth popup is covered by ConnectorAuth's own tests; here it only has +// to report a finished sign-in. +vi.mock('../components/ConnectorAuth', () => ({ + default: ({ + onSuccess, + label, + }: { + onSuccess: (data: { connection_id: string; user_email: string }) => void; + label: string; + }) => ( + + ), +})); +vi.mock('../components/FilePicker', () => ({ + FilePicker: ({ + onSelectionChange, + onFirstPickName, + }: { + onSelectionChange: (files: string[], folders?: string[]) => void; + onFirstPickName?: (name: string) => void; + }) => ( + + ), +})); + +import connectorsReducer from './connectorsSlice'; +import ConnectWizard from './ConnectWizard'; +import type { ConnectorDefinition } from './types'; + +Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + +const base: ConnectorDefinition = { + key: 'telegram', + name: 'Telegram', + description: 'Send messages.', + icon: 'tool_telegram', + category: 'messaging', + auth_kind: 'api_key', + capabilities: ['write'], + credential_fields: [ + { key: 'token', label: 'Bot token', secret: true, required: true }, + ], + setup_fields: [], + sync_ingestor: null, + default_sync_frequency: 'weekly', + tool_templates: ['telegram'], + setup: { tools: 'auto', sync: 'off' }, + mcp_url: null, + publisher: 'built_in', + docs_url: null, + oauth_scopes: [], + available: true, + disabled: false, + needs_setup: false, + missing_settings: [], + connected_count: 0, + connection_count: 0, + status: null, + state: 'available', + credential_policy: 'choose', +}; + +const drive: ConnectorDefinition = { + ...base, + key: 'google_drive', + name: 'Google Drive', + icon: 'drive', + category: 'files', + auth_kind: 'oauth', + capabilities: ['sync'], + credential_fields: [], + sync_ingestor: 'google_drive', + tool_templates: [], + setup: { tools: 'off', sync: 'ask' }, +}; + +const TELEGRAM_TOOL = { + id: 'tool-1', + name: 'telegram', + display_name: 'Telegram', + status: true, + credential_mode: 'owner', + actions: [ + { + name: 'telegram_send_message', + description: 'Send', + access: 'write', + permission: 'ask', + }, + ], +}; + +describe('ConnectWizard', () => { + let root: Root; + let container: HTMLDivElement; + + beforeEach(() => { + Object.values(service).forEach((fn) => fn.mockReset()); + service.getCatalog.mockResolvedValue({ success: true, connectors: [] }); + service.listConnections.mockResolvedValue({ + success: true, + connections: [], + }); + container = document.createElement('div'); + document.body.appendChild(container); + root = createRoot(container); + }); + + afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + }); + + const render = async (connector: ConnectorDefinition, onClose = vi.fn()) => { + const store = configureStore({ + reducer: { + connectors: connectorsReducer, + preference: (state = { token: null, selectedDocs: [] }) => state, + conversation: (state = {}) => state, + }, + }); + await act(async () => { + root.render( + + + + + } + /> + NEW_CHAT
} /> + + + , + ); + }); + }; + + const click = async (text: string) => { + const button = Array.from( + document.body.querySelectorAll('button'), + ).find((b) => b.textContent?.trim() === text); + expect(button, `button ${text}`).toBeDefined(); + await act(async () => button!.click()); + }; + + const typeInto = async (input: HTMLInputElement, value: string) => { + const setter = Object.getOwnPropertyDescriptor( + HTMLInputElement.prototype, + 'value', + )!.set!; + await act(async () => { + setter.call(input, value); + input.dispatchEvent(new Event('input', { bubbles: true })); + }); + }; + + it('connects an API-key service and creates its tools with no further questions', async () => { + service.createConnection.mockResolvedValue({ + success: true, + connection: { id: 'conn-1' }, + }); + service.setup.mockResolvedValue({ + success: true, + tools: [TELEGRAM_TOOL], + sources: [], + }); + await render(base); + const input = document.body.querySelector( + 'input[type="password"]', + )!; + await typeInto(input, '123:abc'); + await click('settings.connectors.status.connect'); + expect(service.createConnection).toHaveBeenCalledWith( + { connector_key: 'telegram', credentials: { token: '123:abc' } }, + null, + ); + expect(service.setup).toHaveBeenCalledWith( + 'conn-1', + { create_tools: true }, + null, + ); + // Straight to the summary: no sync step for a tool-only service. + expect(document.body.textContent).toContain( + 'settings.connectors.wizard.doneTitle', + ); + expect(document.body.textContent).toContain( + 'settings.connectors.wizard.toolsHeading:1', + ); + }); + + it('shows the default-key refusal in the form', async () => { + service.createConnection.mockResolvedValue({ + success: false, + code: 'encryption_key_default', + }); + await render(base); + await typeInto( + document.body.querySelector('input[type="password"]')!, + 'x', + ); + await click('settings.connectors.status.connect'); + expect(document.body.textContent).toContain( + 'settings.connectors.error.defaultKey', + ); + }); + + it('lets a content service skip choosing what to sync', async () => { + await render(drive); + await click('settings.connectors.wizard.signIn'); + expect(document.body.textContent).toContain( + 'settings.connectors.wizard.chooseWhatToSync', + ); + await click('settings.connectors.wizard.skip'); + expect(document.body.textContent).toContain( + 'settings.connectors.wizard.doneTitle', + ); + expect(service.setup).not.toHaveBeenCalled(); + }); + + it('syncs the picked folder named after it', async () => { + service.setup.mockResolvedValue({ + success: true, + tools: [], + sources: [{ id: 'src-1', name: 'Handbook' }], + }); + await render(drive); + await click('settings.connectors.wizard.signIn'); + await click('pick-folder'); + await click('modals.uploadDoc.train'); + const [id, body, , key] = service.setup.mock.calls[0]; + expect(id).toBe('conn-drive'); + expect(body.sync).toEqual({ + items: { file_ids: [], folder_ids: ['folder-1'] }, + frequency: 'weekly', + name: 'Handbook', + }); + expect(typeof key).toBe('string'); + expect(document.body.textContent).toContain( + 'settings.connectors.wizard.doneSummary', + ); + }); + + it('opens a new chat from Try it in chat', async () => { + service.createConnection.mockResolvedValue({ + success: true, + connection: { id: 'conn-1' }, + }); + service.setup.mockResolvedValue({ success: true, tools: [], sources: [] }); + const onClose = vi.fn(); + await render(base, onClose); + await typeInto( + document.body.querySelector('input[type="password"]')!, + 't', + ); + await click('settings.connectors.status.connect'); + await click('settings.connectors.wizard.tryInChat'); + expect(onClose).toHaveBeenCalled(); + expect(document.body.textContent).toContain('NEW_CHAT'); + }); +}); diff --git a/frontend/src/connectors/ConnectWizard.tsx b/frontend/src/connectors/ConnectWizard.tsx new file mode 100644 index 00000000..3cc1c23c --- /dev/null +++ b/frontend/src/connectors/ConnectWizard.tsx @@ -0,0 +1,484 @@ +import { CircleAlert } from 'lucide-react'; +import { nanoid } from '@reduxjs/toolkit'; +import { useEffect, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useDispatch, useSelector } from 'react-redux'; +import { useNavigate } from 'react-router-dom'; + +import { envVar } from '@/env'; +import connectorsService from '../api/services/connectorsService'; +import ConnectorAuth from '../components/ConnectorAuth'; +import { FilePicker } from '../components/FilePicker'; +import GoogleDrivePicker from '../components/GoogleDrivePicker'; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from '../components/ui/accordion'; +import { Alert, AlertDescription } from '../components/ui/alert'; +import { FormField } from '../components/ui/form-field'; +import { Input } from '../components/ui/input'; +import { Modal, ModalActions } from '../components/ui/modal'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '../components/ui/select'; +import { + setConversation, + updateConversationId, +} from '../conversation/conversationSlice'; +import { + selectToken, + setSelectedAgent, + setSelectedDocs, +} from '../preferences/preferenceSlice'; +import type { AppDispatch } from '../store'; +import ConnectorIcon from './ConnectorIcon'; +import CredentialForm, { credentialsComplete } from './CredentialForm'; +import { loadConnectors, selectConnections } from './connectorsSlice'; +import { connectorDescription, connectorName } from './i18n'; +import ToolPermissions from './ToolPermissions'; +import type { ConnectionTool, ConnectorDefinition } from './types'; + +export type WizardMode = 'connect' | 'reconnect' | 'sync' | 'done'; + +const FREQUENCIES = ['never', 'daily', 'weekly', 'monthly'] as const; +const PICKER_CONNECTORS = new Set([ + 'google_drive', + 'share_point', + 'confluence', +]); + +type CreatedSource = { id: string; name: string }; + +/** + * The one connect flow every entry point opens: sign in, choose what to + * sync (content connectors only, skippable), then a summary with Try it in + * chat. Tool connectors create their tools on sign-in, writes needing + * approval, so they go from the credentials straight to the summary. + */ +export default function ConnectWizard({ + connector, + mode = 'connect', + connectionId: initialConnectionId = null, + onClose, + onFinished, +}: { + connector: ConnectorDefinition; + mode?: WizardMode; + connectionId?: string | null; + onClose: () => void; + onFinished?: () => void; +}) { + const { t } = useTranslation(); + const dispatch = useDispatch(); + const navigate = useNavigate(); + const token = useSelector(selectToken); + const connections = useSelector(selectConnections); + const name = connectorName(t, connector); + + const [step, setStep] = useState<'signin' | 'setup' | 'done'>( + mode === 'sync' ? 'setup' : mode === 'done' ? 'done' : 'signin', + ); + const [connectionId, setConnectionId] = useState( + initialConnectionId, + ); + const [credentials, setCredentials] = useState>({}); + const [setupValues, setSetupValues] = useState>({}); + const [pending, setPending] = useState(false); + const [error, setError] = useState(''); + const [tools, setTools] = useState([]); + const [sources, setSources] = useState([]); + const [selectedFiles, setSelectedFiles] = useState([]); + const [selectedFolders, setSelectedFolders] = useState([]); + const [sourceName, setSourceName] = useState(''); + const [nameTouched, setNameTouched] = useState(false); + const [frequency, setFrequency] = useState( + connector.default_sync_frequency || 'weekly', + ); + // One key per wizard: a double click on Add source queues one ingest. + const [idempotencyKey] = useState(() => nanoid()); + + const connection = connections.find((c) => c.id === connectionId); + const canSync = connector.setup.sync !== 'off' && !!connector.sync_ingestor; + + // A finished MCP save or a reconnect lands here with only the connection id. + useEffect(() => { + if (mode !== 'done' || !connectionId) return; + connectorsService.getConnection(connectionId, token).then((data) => { + if (data?.success) setTools(data.connection.tools ?? []); + }); + }, [mode, connectionId, token]); + + const refresh = () => dispatch(loadConnectors({ token })); + + const afterSignIn = async (id: string) => { + setConnectionId(id); + refresh(); + if (mode === 'reconnect') { + setStep('done'); + return; + } + if (connector.setup.tools === 'auto' && connector.tool_templates.length) { + const setup = await connectorsService.setup( + id, + { create_tools: true }, + token, + ); + if (setup?.success) setTools(setup.tools ?? []); + } + setStep(canSync ? 'setup' : 'done'); + }; + + const submitCredentials = async () => { + setPending(true); + setError(''); + try { + const data = + mode === 'reconnect' && connectionId + ? await connectorsService.reconnect( + connectionId, + { credentials }, + token, + ) + : await connectorsService.createConnection( + { connector_key: connector.key, credentials }, + token, + ); + if (!data?.success) { + setError( + data?.code === 'encryption_key_default' + ? t('settings.connectors.error.defaultKey') + : t('settings.connectors.wizard.connectFailed'), + ); + return; + } + await afterSignIn(data.connection.id); + } catch { + setError(t('settings.connectors.wizard.connectFailed')); + } finally { + setPending(false); + } + }; + + const prefillName = (picked: string) => { + if (!nameTouched && picked) setSourceName((current) => current || picked); + }; + + const syncItems = (): Record => + PICKER_CONNECTORS.has(connector.key) + ? { file_ids: selectedFiles, folder_ids: selectedFolders } + : setupValues; + + const canAddSource = PICKER_CONNECTORS.has(connector.key) + ? selectedFiles.length + selectedFolders.length > 0 + : credentialsComplete(connector.setup_fields, setupValues); + + const addSource = async () => { + if (!connectionId) return; + setPending(true); + setError(''); + try { + const data = await connectorsService.setup( + connectionId, + { + create_tools: false, + sync: { + items: syncItems(), + frequency, + name: sourceName.trim() || undefined, + }, + }, + token, + idempotencyKey, + ); + if (!data?.success) { + setError(data?.error || t('settings.connectors.wizard.syncFailed')); + return; + } + setSources(data.sources ?? []); + refresh(); + setStep('done'); + } catch { + setError(t('settings.connectors.wizard.syncFailed')); + } finally { + setPending(false); + } + }; + + const tryInChat = () => { + dispatch(setConversation([])); + dispatch(updateConversationId({ query: { conversationId: null } })); + dispatch(setSelectedAgent(null)); + if (sources.length > 0) { + dispatch( + setSelectedDocs( + sources.map((source) => ({ + id: source.id, + name: source.name, + date: new Date().toISOString(), + model: '', + type: 'connector:file', + connectionId, + })), + ), + ); + } + onFinished?.(); + onClose(); + navigate('/c/new'); + }; + + const finish = () => { + onFinished?.(); + onClose(); + }; + + const toolCount = tools.length; + const summary = useMemo(() => { + const account = connection?.account_label ?? ''; + const accountLine = + connection?.auth_kind === 'api_key' + ? t('settings.connectors.wizard.connectedWithKey', { + hint: account, + interpolation: { escapeValue: false }, + }) + : t('settings.connectors.wizard.doneSummaryNone', { + account, + interpolation: { escapeValue: false }, + }); + const sourcesText = t('settings.connectors.wizard.sourcesCount', { + count: sources.length, + }); + const toolsText = t('settings.connectors.wizard.toolsCount', { + count: toolCount, + }); + const countsLine = + sources.length && toolCount + ? t('settings.connectors.wizard.doneCounts', { + sources: sourcesText, + tools: toolsText, + }) + : sources.length + ? t('settings.connectors.wizard.doneSources', { + sources: sourcesText, + }) + : toolCount + ? t('settings.connectors.wizard.doneTools', { tools: toolsText }) + : ''; + return [accountLine, countsLine].filter(Boolean).join(' '); + }, [connection, sources, toolCount, t]); + + const renderSignIn = () => ( +
+
+ + + +

+ {connectorDescription(t, connector)} +

+
+ {error && ( + + + {error} + + )} + {connector.auth_kind === 'oauth' ? ( + afterSignIn(data.connection_id)} + onError={setError} + /> + ) : ( + + )} +
+ ); + + const renderSetup = () => ( +
+ {error && ( + + + {error} + + )} + {connector.key === 'google_drive' && envVar('VITE_GOOGLE_CLIENT_ID') ? ( + { + setSelectedFiles(fileIds); + setSelectedFolders(folderIds); + }} + /> + ) : PICKER_CONNECTORS.has(connector.key) ? ( + { + setSelectedFiles(fileIds); + setSelectedFolders(folderIds); + }} + /> + ) : ( + { + setSetupValues(values); + const first = Object.values(values).find(Boolean); + if (first) prefillName(first); + }} + /> + )} +
+ { + setNameTouched(true); + setSourceName(e.target.value); + }} + /> + + + +
+
+ ); + + const renderDone = () => ( +
+

{summary}

+ {toolCount > 0 && connectionId && ( +
+ + + + {t('settings.connectors.wizard.toolsHeading', { + count: toolCount, + })} + + +
+ {tools.map((tool) => ( + + ))} +
+ + + +
+ )} +
+ ); + + const title = + step === 'signin' + ? mode === 'reconnect' + ? t('settings.connectors.wizard.reconnectTitle', { + name, + interpolation: { escapeValue: false }, + }) + : t('settings.connectors.wizard.connectTitle', { + name, + interpolation: { escapeValue: false }, + }) + : step === 'setup' + ? t('settings.connectors.wizard.chooseWhatToSync') + : t('settings.connectors.wizard.doneTitle', { + name, + interpolation: { escapeValue: false }, + }); + + const footer = + step === 'signin' ? ( + connector.auth_kind === 'oauth' ? undefined : ( + + ) + ) : step === 'setup' ? ( + (mode === 'sync' ? onClose() : setStep('done'))} + submitLabel={t('modals.uploadDoc.train')} + onSubmit={addSource} + pending={pending} + disabled={!canAddSource} + /> + ) : ( + + ); + + return ( + !open && (step === 'done' ? finish() : onClose())} + title={title} + size={step === 'setup' ? 'xl' : 'lg'} + mobileVariant="sheet" + footer={footer} + > + {step === 'signin' && renderSignIn()} + {step === 'setup' && renderSetup()} + {step === 'done' && renderDone()} + + ); +} diff --git a/frontend/src/connectors/ConnectionDrawer.tsx b/frontend/src/connectors/ConnectionDrawer.tsx index e37453a6..e6200cff 100644 --- a/frontend/src/connectors/ConnectionDrawer.tsx +++ b/frontend/src/connectors/ConnectionDrawer.tsx @@ -3,6 +3,8 @@ import { ExternalLink, Plus, RefreshCw, + RotateCw, + Trash2, Unplug, } from 'lucide-react'; import { useCallback, useEffect, useState } from 'react'; @@ -14,26 +16,31 @@ import { Alert, AlertDescription, AlertTitle } from '../components/ui/alert'; import { Badge } from '../components/ui/badge'; import { Button } from '../components/ui/button'; import { Card } from '../components/ui/card'; -import { ActionMenu } from '../components/ui/dropdown-menu'; +import { ActionMenu, type MenuOption } from '../components/ui/dropdown-menu'; import { EmptyState } from '../components/ui/empty-state'; +import { FormField } from '../components/ui/form-field'; import { ListRow, ListRows } from '../components/ui/list-row'; import { LoadingState } from '../components/ui/loading-state'; +import { Modal, ModalActions } from '../components/ui/modal'; import { SectionHeader } from '../components/ui/section-header'; import { Sheet, SheetContent, SheetTitle } from '../components/ui/sheet'; +import { ToggleGroup, ToggleGroupItem } from '../components/ui/toggle-group'; import ConfirmationModal from '../modals/ConfirmationModal'; +import { showActionToast } from '../notifications/actionToastSlice'; import { selectToken } from '../preferences/preferenceSlice'; import type { AppDispatch } from '../store'; import { formatDateTime } from '../utils/dateTimeUtils'; import { CapabilityBadges } from './ConnectorCard'; import ConnectorIcon from './ConnectorIcon'; -import { selectConnections, loadConnectors } from './connectorsSlice'; +import { loadConnectors, selectConnections } from './connectorsSlice'; import { connectorDescription, connectorName } from './i18n'; +import ToolPermissions from './ToolPermissions'; import type { ConnectionDetail, ConnectionStatus, - ConnectionTool, ConnectorDefinition, } from './types'; +import type { LaunchOptions } from './useConnectorLauncher'; const STATUS_VARIANT: Record< ConnectionStatus, @@ -46,82 +53,176 @@ const STATUS_VARIANT: Record< pending: 'neutral', }; -const PERMISSION_VARIANT = { - always: 'success', - ask: 'warning', - off: 'neutral', -} as const; - -function ToolActions({ tool }: { tool: ConnectionTool }) { +/** "Connected as …", or the key hint for pasted credentials. */ +function useAccountTitle() { const { t } = useTranslation(); - const groups = (['read', 'write'] as const) - .map((access) => ({ - access, - actions: tool.actions.filter((action) => action.access === access), - })) - .filter((group) => group.actions.length > 0); + return (detail: ConnectionDetail) => + detail.auth_kind === 'api_key' + ? t('settings.connectors.detail.keyEnding', { + hint: detail.account_label, + interpolation: { escapeValue: false }, + }) + : t('settings.connectors.detail.connectedAs', { + account: detail.account_label, + interpolation: { escapeValue: false }, + }); +} + +function RemoveConnectionModal({ + detail, + name, + onClose, + onRemoved, +}: { + detail: ConnectionDetail; + name: string; + onClose: () => void; + onRemoved: () => void; +}) { + const { t } = useTranslation(); + const token = useSelector(selectToken); + const [sources, setSources] = useState<'keep' | 'delete'>('keep'); + const [tools, setTools] = useState<'keep' | 'delete'>('delete'); + const [pending, setPending] = useState(false); + const [failed, setFailed] = useState(false); + + const remove = () => { + setPending(true); + setFailed(false); + connectorsService + .remove(detail.id, { sources, tools }, token) + .then((data) => { + if (!data?.success) throw new Error('remove failed'); + onRemoved(); + }) + .catch(() => setFailed(true)) + .finally(() => setPending(false)); + }; + return ( - -
- - {tool.display_name} - - - {tool.status - ? t('settings.connectors.detail.toolOn') - : t('settings.connectors.detail.toolOff')} - + !open && onClose()} + title={t('settings.connectors.remove.title', { + name, + interpolation: { escapeValue: false }, + })} + description={t('settings.connectors.remove.description')} + footer={ + + } + > +
+ {failed && ( + + + + {t('settings.connectors.remove.failed')} + + + )} + {detail.sources.length > 0 && ( + + + value && setSources(value as 'keep' | 'delete') + } + > + + {t('settings.connectors.remove.keepSources')} + + + {t('settings.connectors.remove.deleteSources')} + + + + )} + {detail.tools.length > 0 && ( + + + value && setTools(value as 'keep' | 'delete') + } + > + + {t('settings.connectors.remove.deleteTools')} + + + {t('settings.connectors.remove.keepTools')} + + + + )}
- {groups.map((group) => ( -
- -
    - {group.actions.map((action) => ( -
  • - - {action.name} - - - {t(`settings.connectors.permission.${action.permission}`)} - -
    • - ))} -
-
- ))} - + ); } function AccountSection({ + connector, detail, onReconnect, onDisconnect, + onRemove, + onSyncMore, + onRefreshTools, }: { + connector: ConnectorDefinition; detail: ConnectionDetail; - onReconnect: () => void; + onReconnect: (detail: ConnectionDetail) => void; onDisconnect: (detail: ConnectionDetail) => void; + onRemove: (detail: ConnectionDetail) => void; + onSyncMore: (detail: ConnectionDetail) => void; + onRefreshTools: (detail: ConnectionDetail) => Promise; }) { const { t } = useTranslation(); + const accountTitle = useAccountTitle(); + const [refreshing, setRefreshing] = useState(false); + const canSync = connector.setup.sync !== 'off' && !!connector.sync_ingestor; + const isMcp = detail.tools.some((tool) => tool.name === 'mcp_tool'); + const menu: MenuOption[] = []; + if (detail.status !== 'disconnected') { + menu.push({ + icon: Unplug, + label: t('settings.connectors.detail.disconnect'), + onClick: () => onDisconnect(detail), + variant: 'destructive', + }); + } + menu.push({ + icon: Trash2, + label: t('settings.connectors.detail.remove'), + onClick: () => onRemove(detail), + variant: 'destructive', + }); + return (
{detail.last_error} @@ -137,66 +238,77 @@ function AccountSection({ type="button" size="xs" variant="outline" - onClick={onReconnect} + onClick={() => onReconnect(detail)} > {t('settings.connectors.status.reconnect')} )} - {detail.status !== 'disconnected' && ( - onDisconnect(detail), - variant: 'destructive', - }, - ]} - /> - )} +
} /> - {detail.sources.length > 0 && ( + {canSync && (
onSyncMore(detail)} + > + + {t('settings.connectors.detail.syncMore')} + + ) : undefined + } /> - - - {detail.sources.map((source) => ( - - {t('settings.connectors.detail.paused')} - - ) : undefined - } - /> - ))} - - + {detail.sources.length === 0 ? ( +

+ {t('settings.connectors.detail.noSources')} +

+ ) : ( + + + {detail.sources.map((source) => ( + + {t('settings.connectors.detail.paused')} + + ) : undefined + } + /> + ))} + + + )}
)} {detail.tools.length > 0 && ( @@ -205,9 +317,39 @@ function AccountSection({ as="h3" size="xs" title={t('settings.connectors.detail.tools')} + actions={ + isMcp && detail.status === 'connected' ? ( + + ) : undefined + } /> {detail.tools.map((tool) => ( - +
+
+ + {tool.display_name} + + + {tool.status + ? t('settings.connectors.detail.toolOn') + : t('settings.connectors.detail.toolOff')} + +
+ +
))} )} @@ -217,7 +359,8 @@ function AccountSection({ /** * Everything about one connector: its accounts, the sources each syncs and - * the tools each provides. Opens from a Connectors page card. + * the tools each provides, with their permissions. Opens from a Connectors + * page card. */ export default function ConnectionDrawer({ connector, @@ -226,7 +369,7 @@ export default function ConnectionDrawer({ }: { connector: ConnectorDefinition | null; onClose: () => void; - onConnect: (connector: ConnectorDefinition) => void; + onConnect: (connector: ConnectorDefinition, options?: LaunchOptions) => void; }) { const { t } = useTranslation(); const dispatch = useDispatch(); @@ -239,15 +382,18 @@ export default function ConnectionDrawer({ const [toDisconnect, setToDisconnect] = useState( null, ); + const [toRemove, setToRemove] = useState(null); const accountIds = connections .filter((connection) => connection.connector_key === connector?.key) - .map((connection) => connection.id) + .map((connection) => `${connection.id}:${connection.status}`) .join(','); useEffect(() => { if (!connector) return; - const ids = accountIds ? accountIds.split(',') : []; + const ids = accountIds + ? accountIds.split(',').map((entry) => entry.split(':')[0]) + : []; let cancelled = false; setLoading(true); setFailed(false); @@ -280,6 +426,45 @@ export default function ConnectionDrawer({ }); }; + const reconnect = (detail: ConnectionDetail) => { + if (!connector) return; + const mcpTool = detail.tools.find((tool) => tool.name === 'mcp_tool'); + onConnect(connector, { + mode: 'reconnect', + connectionId: detail.id, + mcpServer: + mcpTool && detail.server_url + ? { + id: mcpTool.id, + displayName: mcpTool.display_name, + server_url: detail.server_url, + auth_type: detail.auth_kind === 'mcp_oauth' ? 'oauth' : 'none', + } + : undefined, + }); + }; + + const refreshTools = async (detail: ConnectionDetail) => { + const data = await connectorsService.refreshTools(detail.id, token); + dispatch( + showActionToast( + data?.success + ? { + variant: 'success', + message: t('settings.connectors.detail.refreshed', { + added: data.added?.length ?? 0, + removed: data.removed?.length ?? 0, + }), + } + : { + variant: 'destructive', + message: t('settings.connectors.detail.refreshFailed'), + }, + ), + ); + refresh(); + }; + if (!connector) return null; const name = connectorName(t, connector); @@ -308,6 +493,15 @@ export default function ConnectionDrawer({

+ {connector.publisher === 'custom' && ( + + + + {t('settings.connectors.unverified')} + + + )} + {connector.needs_setup && ( @@ -368,7 +562,7 @@ export default function ConnectionDrawer({ ) : undefined } /> - {loading ? ( + {loading && details.length === 0 ? ( ) : failed ? ( ( onConnect(connector)} + onReconnect={reconnect} onDisconnect={setToDisconnect} + onRemove={setToRemove} + onSyncMore={(d) => + onConnect(connector, { + mode: 'sync', + connectionId: d.id, + }) + } + onRefreshTools={refreshTools} /> ))}
@@ -433,6 +636,17 @@ export default function ConnectionDrawer({ submitLabel={t('settings.connectors.detail.disconnect')} variant="destructive" /> + {toRemove && ( + setToRemove(null)} + onRemoved={() => { + setToRemove(null); + refresh(); + }} + /> + )} ); } diff --git a/frontend/src/connectors/CredentialForm.tsx b/frontend/src/connectors/CredentialForm.tsx new file mode 100644 index 00000000..94353c41 --- /dev/null +++ b/frontend/src/connectors/CredentialForm.tsx @@ -0,0 +1,56 @@ +import { useTranslation } from 'react-i18next'; + +import { Input } from '../components/ui/input'; +import type { CredentialField } from './types'; + +/** + * The short form an API-key connector asks for, generated from its catalog + * `credential_fields` (or `setup_fields`). Secrets are masked. Labels come + * from `settings.connectors.fields._`, then + * `settings.connectors.fields.`, then the catalog label. + */ +export default function CredentialForm({ + connectorKey, + fields, + values, + onChange, + idPrefix, + labelSurface = 'card', +}: { + /** Picks the connector's own label for a shared key (Telegram's "Bot token"). */ + connectorKey: string; + fields: CredentialField[]; + values: Record; + onChange: (values: Record) => void; + idPrefix: string; + labelSurface?: 'card' | 'background' | 'muted'; +}) { + const { t } = useTranslation(); + return ( +
+ {fields.map((field) => ( + onChange({ ...values, [field.key]: e.target.value })} + /> + ))} +
+ ); +} + +/** Whether every required field has a value. */ +export const credentialsComplete = ( + fields: CredentialField[], + values: Record, +) => fields.every((field) => !field.required || !!values[field.key]?.trim()); diff --git a/frontend/src/connectors/ToolPermissions.tsx b/frontend/src/connectors/ToolPermissions.tsx new file mode 100644 index 00000000..c7b05cff --- /dev/null +++ b/frontend/src/connectors/ToolPermissions.tsx @@ -0,0 +1,118 @@ +import { useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useSelector } from 'react-redux'; + +import connectorsService from '../api/services/connectorsService'; +import { Card } from '../components/ui/card'; +import { SectionHeader } from '../components/ui/section-header'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '../components/ui/select'; +import { selectToken } from '../preferences/preferenceSlice'; +import type { + ActionPermission, + ConnectionTool, + ConnectionToolAction, +} from './types'; + +const PERMISSIONS: ActionPermission[] = ['always', 'ask', 'off']; + +/** + * One tool's actions, grouped Read and Write, each with Always allow / + * Needs approval / Off. Changes save immediately; a failed save puts the + * previous choice back. + */ +export default function ToolPermissions({ + connectionId, + tool, + onChange, + readOnly = false, +}: { + connectionId: string; + tool: ConnectionTool; + onChange?: (tool: ConnectionTool) => void; + readOnly?: boolean; +}) { + const { t } = useTranslation(); + const token = useSelector(selectToken); + const [actions, setActions] = useState(tool.actions); + + const setPermission = (name: string, permission: ActionPermission) => { + const previous = actions; + const next = actions.map((action) => + action.name === name ? { ...action, permission } : action, + ); + setActions(next); + connectorsService + .setToolPermissions(connectionId, tool.id, { [name]: permission }, token) + .then((data) => { + if (!data?.success) throw new Error('save failed'); + onChange?.(data.tool); + }) + .catch(() => setActions(previous)); + }; + + const groups = (['read', 'write'] as const) + .map((access) => ({ + access, + actions: actions.filter((action) => action.access === access), + })) + .filter((group) => group.actions.length > 0); + + return ( + + {groups.map((group) => ( +
+ +
    + {group.actions.map((action) => ( +
  • + + {action.name} + + +
    • + ))} +
+
+ ))} + + ); +} diff --git a/frontend/src/connectors/connectorsSlice.ts b/frontend/src/connectors/connectorsSlice.ts index cc26674b..e6771ab2 100644 --- a/frontend/src/connectors/connectorsSlice.ts +++ b/frontend/src/connectors/connectorsSlice.ts @@ -1,9 +1,15 @@ -import { createAsyncThunk, createSlice } from '@reduxjs/toolkit'; +import { + createAsyncThunk, + createSlice, + type PayloadAction, +} from '@reduxjs/toolkit'; import connectorsService from '../api/services/connectorsService'; import type { Connection, ConnectorDefinition } from './types'; export type ConnectorsState = { + /** False when the backend predates connectors (`/api/config`). */ + enabled: boolean; catalog: ConnectorDefinition[]; connections: Connection[]; loading: boolean; @@ -12,6 +18,7 @@ export type ConnectorsState = { }; const initialState: ConnectorsState = { + enabled: true, catalog: [], connections: [], loading: false, @@ -44,7 +51,11 @@ export const loadConnectors = createAsyncThunk< const connectorsSlice = createSlice({ name: 'connectors', initialState, - reducers: {}, + reducers: { + setConnectorsEnabled(state, action: PayloadAction) { + state.enabled = action.payload; + }, + }, extraReducers: (builder) => { builder .addCase(loadConnectors.pending, (state) => { @@ -64,8 +75,13 @@ const connectorsSlice = createSlice({ }, }); +export const { setConnectorsEnabled } = connectorsSlice.actions; + type RootLike = { connectors: ConnectorsState }; +export const selectConnectorsEnabled = (state: RootLike) => + state.connectors?.enabled !== false; + export const selectConnectorCatalog = (state: RootLike) => state.connectors.catalog; export const selectConnections = (state: RootLike) => diff --git a/frontend/src/connectors/useConnectorLauncher.tsx b/frontend/src/connectors/useConnectorLauncher.tsx index df8d81d1..bc1b033d 100644 --- a/frontend/src/connectors/useConnectorLauncher.tsx +++ b/frontend/src/connectors/useConnectorLauncher.tsx @@ -2,28 +2,46 @@ import { useCallback, useState, type ReactNode } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import { useNavigate } from 'react-router-dom'; +import connectorsService from '../api/services/connectorsService'; import userService from '../api/services/userService'; -import ConfigToolModal from '../modals/ConfigToolModal'; import MCPServerModal from '../modals/MCPServerModal'; import type { AvailableToolType } from '../modals/types'; import type { ActiveState } from '../models/misc'; import { selectToken } from '../preferences/preferenceSlice'; import type { AppDispatch } from '../store'; -import Upload from '../upload/Upload'; -import type { IngestorType } from '../upload/types/ingestor'; +import ConnectWizard, { type WizardMode } from './ConnectWizard'; import { loadConnectors } from './connectorsSlice'; import type { ConnectorDefinition } from './types'; -type Launch = - | { kind: 'source'; ingestor: IngestorType } - | { kind: 'tool'; tool: AvailableToolType } - | { kind: 'mcp'; server?: Record } +export type LaunchOptions = { + /** `connect` a new account, `reconnect` one, or `sync` more from one. */ + mode?: Exclude; + connectionId?: string; + /** An existing MCP tool to reconnect through the MCP server form. */ + mcpServer?: Record; +}; + +type Active = + | { + kind: 'wizard'; + connector: ConnectorDefinition; + mode: WizardMode; + connectionId?: string; + } + | { + kind: 'mcp'; + connector: ConnectorDefinition; + server?: Record; + } | null; +const isMcp = (connector: ConnectorDefinition) => + connector.auth_kind === 'mcp' || connector.auth_kind === 'mcp_oauth'; + /** - * One way to start connecting any catalog entry, used by every entry point - * (the Connectors page, Add Source, Add Tool). Returns `launch` and the - * modals it drives; render `modals` once where the hook is used. + * The one way to start connecting a catalog entry, used by every entry point + * (the Connectors page and drawer, Add Source, Add Tool, the chat's Connect + * card). Returns `launch` and the modals it drives; render `modals` once. */ export default function useConnectorLauncher({ onConnected, @@ -31,52 +49,43 @@ export default function useConnectorLauncher({ const dispatch = useDispatch(); const navigate = useNavigate(); const token = useSelector(selectToken); - const [active, setActive] = useState(null); + const [active, setActive] = useState(null); - const finish = useCallback(() => { - setActive(null); + const refresh = useCallback(() => { dispatch(loadConnectors({ token })); onConnected?.(); }, [dispatch, token, onConnected]); const launch = useCallback( - async (connector: ConnectorDefinition) => { - if (connector.sync_ingestor) { - setActive({ - kind: 'source', - ingestor: connector.sync_ingestor as IngestorType, - }); - return; - } - if ( - connector.auth_kind === 'mcp' || - connector.auth_kind === 'mcp_oauth' - ) { + async (connector: ConnectorDefinition, options: LaunchOptions = {}) => { + if (isMcp(connector)) { setActive({ kind: 'mcp', - server: connector.mcp_url - ? { - displayName: connector.name, - server_url: connector.mcp_url, - auth_type: - connector.auth_kind === 'mcp_oauth' ? 'oauth' : 'none', - preset: true, - } - : undefined, + connector, + server: + options.mcpServer ?? + (connector.mcp_url + ? { + displayName: connector.name, + server_url: connector.mcp_url, + auth_type: + connector.auth_kind === 'mcp_oauth' ? 'oauth' : 'none', + oauth_scopes: connector.oauth_scopes.join(', '), + preset: true, + } + : undefined), }); return; } - const templateName = connector.tool_templates[0]; - if (!templateName) return; - const response = await userService.getAvailableTools(token); - const data = await response.json(); - const tool = (data.data as AvailableToolType[] | undefined)?.find( - (candidate) => candidate.name === templateName, - ); - if (!tool) return; - if (Object.keys(tool.configRequirements ?? {}).length === 0) { - // The OpenAPI connector: create the empty API tool and open it so - // the user can import a spec (today's API Tool flow). + if (connector.key === 'custom_openapi') { + // An OpenAPI tool starts empty; its spec import is today's API Tool + // screen on the Tools page. + const response = await userService.getAvailableTools(token); + const data = await response.json(); + const tool = (data.data as AvailableToolType[] | undefined)?.find( + (candidate) => candidate.name === 'api_tool', + ); + if (!tool) return; const created = await userService.createTool( { name: tool.name, @@ -94,43 +103,62 @@ export default function useConnectorLauncher({ } return; } - setActive({ kind: 'tool', tool }); + setActive({ + kind: 'wizard', + connector, + mode: options.mode ?? 'connect', + connectionId: options.connectionId, + }); }, [navigate, token], ); - const close = (state: ActiveState) => { + const afterMcpSave = async () => { + if (active?.kind !== 'mcp') return; + const connector = active.connector; + const list = await connectorsService.listConnections(token); + refresh(); + const connections = (list?.connections ?? []) as { + id: string; + connector_key: string; + updated_at: string | null; + }[]; + // The newest connection for this connector is the one just saved. + const saved = connections + .filter((c) => c.connector_key === connector.key) + .sort((a, b) => + (b.updated_at ?? '').localeCompare(a.updated_at ?? ''), + )[0]; + setActive( + saved + ? { kind: 'wizard', connector, mode: 'done', connectionId: saved.id } + : null, + ); + }; + + const closeMcp = (state: ActiveState) => { if (state === 'INACTIVE') setActive(null); }; const modals: ReactNode = ( <> - {active?.kind === 'source' && ( - setActive(null)} - initialIngestor={active.ingestor} - onSuccessfulUpload={finish} - selectUploadedDoc={false} - /> - )} - {active?.kind === 'tool' && ( - { + setActive(null); + refresh(); + }} /> )} {active?.kind === 'mcp' && ( )} diff --git a/frontend/src/conversation/ConnectToolCallBar.test.tsx b/frontend/src/conversation/ConnectToolCallBar.test.tsx new file mode 100644 index 00000000..2175639a --- /dev/null +++ b/frontend/src/conversation/ConnectToolCallBar.test.tsx @@ -0,0 +1,113 @@ +import { configureStore } from '@reduxjs/toolkit'; +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { Provider } from 'react-redux'; +import { MemoryRouter, Route, Routes } from 'react-router-dom'; + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ + t: (key: string, opts?: { name?: string }) => + opts?.name ? `${key}:${opts.name}` : key, + }), +})); + +vi.mock('../api/services/connectorsService', () => ({ + default: { + getCatalog: vi.fn().mockResolvedValue({ success: true, connectors: [] }), + listConnections: vi + .fn() + .mockResolvedValue({ success: true, connections: [] }), + }, +})); + +import connectorsReducer from '../connectors/connectorsSlice'; +import ConnectToolCallBar from './ConnectToolCallBar'; +import type { ToolCallsType } from './types'; + +Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + +const call = (status: string): ToolCallsType => ({ + tool_name: 'mcp_tool', + action_name: 'search_pages', + call_id: 'call-1', + arguments: {}, + status: 'awaiting_approval', + connection_required: { + connector_key: 'mcp:notion', + connector_name: 'Notion', + status, + }, +}); + +describe('ConnectToolCallBar', () => { + let root: Root; + let container: HTMLDivElement; + + beforeEach(() => { + container = document.createElement('div'); + document.body.appendChild(container); + root = createRoot(container); + }); + + afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + }); + + const render = async (toolCall: ToolCallsType, onToolAction = vi.fn()) => { + const store = configureStore({ + reducer: { + connectors: connectorsReducer, + preference: (state = { token: null }) => state, + }, + }); + await act(async () => { + root.render( + + + + + } + /> + DRAWER
} /> + + + , + ); + }); + return onToolAction; + }; + + const button = (text: string) => + Array.from(container.querySelectorAll('button')).find( + (b) => b.textContent === text, + )!; + + it('names the service it needs', async () => { + await render(call('missing')); + expect(container.textContent).toContain( + 'conversation.toolApproval.connectTitle:Notion', + ); + expect(button('conversation.toolApproval.connect:Notion')).toBeDefined(); + }); + + it('skipping denies the pending call', async () => { + const onToolAction = await render(call('missing')); + await act(async () => button('conversation.toolApproval.skip').click()); + expect(onToolAction).toHaveBeenCalledWith('call-1', 'denied'); + }); + + it('an account that needs signing in again is healed from its drawer', async () => { + await render(call('reconnect_needed')); + await act(async () => + button('conversation.toolApproval.connect:Notion').click(), + ); + expect(document.body.textContent).toContain('DRAWER'); + }); +}); diff --git a/frontend/src/conversation/ConnectToolCallBar.tsx b/frontend/src/conversation/ConnectToolCallBar.tsx new file mode 100644 index 00000000..462fb372 --- /dev/null +++ b/frontend/src/conversation/ConnectToolCallBar.tsx @@ -0,0 +1,111 @@ +import { Plug } from 'lucide-react'; +import { useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useDispatch, useSelector } from 'react-redux'; +import { useNavigate } from 'react-router-dom'; + +import { Button } from '../components/ui/button'; +import ConnectorIcon from '../connectors/ConnectorIcon'; +import { + loadConnectors, + selectConnectorCatalog, + selectConnectorsLoaded, +} from '../connectors/connectorsSlice'; +import useConnectorLauncher from '../connectors/useConnectorLauncher'; +import { selectToken } from '../preferences/preferenceSlice'; +import type { AppDispatch } from '../store'; +import type { ToolCallsType } from './types'; + +/** + * The approval card's "Connect to continue" variant: a paused call to a tool + * whose account needs signing in. Connecting keeps the call pending; Continue + * resumes it with the new connection, Skip denies it. + */ +export default function ConnectToolCallBar({ + toolCall, + onToolAction, +}: { + toolCall: ToolCallsType; + onToolAction?: (callId: string, decision: 'approved' | 'denied') => void; +}) { + const { t } = useTranslation(); + const dispatch = useDispatch(); + const navigate = useNavigate(); + const token = useSelector(selectToken); + const catalog = useSelector(selectConnectorCatalog); + const loaded = useSelector(selectConnectorsLoaded); + const [connected, setConnected] = useState(false); + const { launch, modals } = useConnectorLauncher({ + onConnected: () => setConnected(true), + }); + const required = toolCall.connection_required; + const connector = catalog.find((c) => c.key === required?.connector_key); + const name = + required?.connector_name || + connector?.name || + t('conversation.toolApproval.thisService'); + + useEffect(() => { + if (!loaded) dispatch(loadConnectors({ token })); + }, [loaded, dispatch, token]); + + const connect = () => { + // A member who has no account for the service connects one. An owner's + // existing account that needs signing in again is healed from its drawer, + // which reconnects that same connection for every tool and source. + if (connector && required?.status === 'missing') { + launch(connector); + return; + } + navigate( + `/settings/connectors${connector ? `?connector=${encodeURIComponent(connector.key)}` : ''}`, + ); + }; + + return ( +
+
+ {connector ? ( + + ) : ( + + )} + + {t('conversation.toolApproval.connectTitle', { + name, + interpolation: { escapeValue: false }, + })} + +
+
+ {connected ? ( + + ) : ( + + )} + +
+ {modals} +
+ ); +} diff --git a/frontend/src/conversation/ConversationBubble.tsx b/frontend/src/conversation/ConversationBubble.tsx index fba3e819..623e70dd 100644 --- a/frontend/src/conversation/ConversationBubble.tsx +++ b/frontend/src/conversation/ConversationBubble.tsx @@ -47,6 +47,7 @@ import { } from '../preferences/preferenceSlice'; import { isToolCallRunning } from '../utils/streamingStatusUtils'; import AnswerFlow from './AnswerFlow'; +import ConnectToolCallBar from './ConnectToolCallBar'; import { AnswerSegment } from './answerSegments'; import { deriveArtifactChips } from './artifactChips'; import { FEEDBACK, MESSAGE_TYPE, ResearchState } from './conversationModels'; @@ -473,10 +474,17 @@ const ConversationBubble = forwardRef< onOpenArtifact={onOpenArtifact} renderApproval={(toolCall: ToolCallsType) => (
- + {toolCall.connection_required ? ( + + ) : ( + + )}
)} renderWikiWrite={(toolCall: ToolCallsType, isLive: boolean) => ( diff --git a/frontend/src/conversation/types/index.ts b/frontend/src/conversation/types/index.ts index 730ca286..3d75137a 100644 --- a/frontend/src/conversation/types/index.ts +++ b/frontend/src/conversation/types/index.ts @@ -22,4 +22,16 @@ export type ToolCallsType = { // Remote-device tool calls carry the device id so the approval UI can // offer a "don't ask again" sticky-pattern action without a lookup. device_id?: string; + // A connection-backed tool whose account needs signing in pauses on a + // Connect card instead of an approval. Never carries an account or secret. + connection_required?: { + connector_key: string | null; + connector_name: string | null; + status: 'missing' | 'reconnect_needed' | 'disconnected' | 'error' | string; + }; + // Which connection a tool call used, for the connector's logo and name on + // its chip (never an account or a secret). + connector_key?: string | null; + connector_name?: string | null; + access?: 'read' | 'write' | null; }; diff --git a/frontend/src/events/dispatchEvent.ts b/frontend/src/events/dispatchEvent.ts index 3d8fc151..bcb0dc01 100644 --- a/frontend/src/events/dispatchEvent.ts +++ b/frontend/src/events/dispatchEvent.ts @@ -44,6 +44,9 @@ const KNOWN_TYPES: ReadonlySet = new Set([ 'graph.extract.progress', 'graph.extract.completed', 'graph.extract.failed', + // A connection's sign-in stopped working (connectors/service.py); + // consumed by ConnectionHealthToast via selectRecentEvents. + 'connection.reconnect_needed', ]); /** diff --git a/frontend/src/hooks/useDataInitializer.ts b/frontend/src/hooks/useDataInitializer.ts index 8cb7a3cb..b78501b3 100644 --- a/frontend/src/hooks/useDataInitializer.ts +++ b/frontend/src/hooks/useDataInitializer.ts @@ -2,6 +2,8 @@ import { useEffect } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import userService from '../api/services/userService'; +import { setConnectorsEnabled } from '../connectors/connectorsSlice'; +import { claimLegacySessionTokens } from '../utils/providerUtils'; import { getDocs, getConversations, @@ -45,10 +47,19 @@ export default function useDataInitializer(isAuthLoading: boolean) { stt: config?.stt_available !== false, }), ); + // A backend from before connectors has no flag: hide the page. + dispatch(setConnectorsEnabled(config?.connectors_enabled === true)); }) .catch(() => undefined); }, [dispatch]); + // Connector sign-ins used to leave a session token in localStorage. Link + // each one to its server-side connection once, then forget it. + useEffect(() => { + if (isAuthLoading) return; + claimLegacySessionTokens(token); + }, [isAuthLoading, token]); + // Initialize documents useEffect(() => { // Skip if auth is still loading diff --git a/frontend/src/locale/de.json b/frontend/src/locale/de.json index ea2373ff..501d5ac0 100644 --- a/frontend/src/locale/de.json +++ b/frontend/src/locale/de.json @@ -1088,18 +1088,88 @@ "toolOn": "An", "toolOff": "Aus", "accountMenu": "Kontoaktionen", - "disconnect": "Trennen" + "disconnect": "Trennen", + "keyEnding": "Schlüssel endet auf {{hint}}", + "syncMore": "Weitere Inhalte synchronisieren", + "refreshTools": "Werkzeuge aktualisieren", + "noSources": "Aus dieser Verbindung wurde noch nichts synchronisiert.", + "remove": "Entfernen", + "refreshed": "Werkzeuge aktualisiert: {{added}} hinzugefügt, {{removed}} entfernt.", + "refreshFailed": "Die Werkzeuge konnten nicht aktualisiert werden." }, "permission": { "always": "Immer erlauben", "ask": "Freigabe nötig", - "off": "Aus" + "off": "Aus", + "label": "Berechtigung für {{action}}" }, "disconnect": { "title": "{{name}} trennen?", "body_zero": "Werkzeuge aus dieser Verbindung funktionieren erst wieder, wenn du neu verbindest.", "body_one": "Deine Quelle behält ihren Inhalt, wird aber nicht mehr synchronisiert. Werkzeuge aus dieser Verbindung funktionieren erst wieder, wenn du neu verbindest.", "body_other": "Deine {{count}} Quellen behalten ihren Inhalt, werden aber nicht mehr synchronisiert. Werkzeuge aus dieser Verbindung funktionieren erst wieder, wenn du neu verbindest." + }, + "fields": { + "token": "API-Schlüssel", + "aws_access_key_id": "Zugriffsschlüssel-ID", + "aws_secret_access_key": "Geheimer Zugriffsschlüssel", + "region": "Region", + "endpoint_url": "Eigene Endpunkt-URL", + "client_id": "Client-ID", + "client_secret": "Client-Secret", + "user_agent": "User-Agent", + "bucket": "Bucket", + "prefix": "Pfadpräfix", + "search_queries": "Suchanfragen", + "number_posts": "Anzahl der Beiträge", + "telegram_token": "Bot-Token", + "ntfy_token": "Zugriffstoken", + "postgres_token": "Verbindungszeichenfolge", + "brave_token": "API-Schlüssel" + }, + "wizard": { + "connectTitle": "{{name}} verbinden", + "reconnectTitle": "{{name}} neu verbinden", + "signIn": "Bei {{name}} anmelden", + "chooseWhatToSync": "Auswählen, was synchronisiert wird", + "skip": "Vorerst überspringen", + "syncFrequency": "Sync", + "doneTitle": "{{name}} ist verbunden", + "doneSummary": "Verbunden als {{account}}. {{sources}} werden synchronisiert, {{tools}} bereit.", + "doneSummaryNone": "Verbunden als {{account}}.", + "sourcesCount_one": "{{count}} Quelle", + "sourcesCount_other": "{{count}} Quellen", + "toolsCount_one": "{{count}} Werkzeug", + "toolsCount_other": "{{count}} Werkzeuge", + "toolsHeading": "Werkzeuge ({{count}})", + "tryInChat": "Im Chat ausprobieren", + "done": "Fertig", + "connectFailed": "Verbindung fehlgeschlagen. Prüfe die Angaben und versuche es erneut.", + "syncFailed": "Die Synchronisierung konnte nicht gestartet werden. Versuche es erneut.", + "connectedWithKey": "Verbunden mit dem Schlüssel, der auf {{hint}} endet.", + "doneCounts": "{{sources}} werden synchronisiert, {{tools}} bereit.", + "doneSources": "{{sources}} werden synchronisiert.", + "doneTools": "{{tools}} bereit." + }, + "remove": { + "title": "{{name}} entfernen?", + "description": "Die gespeicherte Anmeldung wird gelöscht. Wähle, was mit den Inhalten dieser Verbindung passiert.", + "sourcesLabel_one": "Ihre Quelle", + "sourcesLabel_other": "Ihre {{count}} Quellen", + "keepSources": "Inhalt behalten (Sync stoppen)", + "deleteSources": "Löschen", + "toolsLabel_one": "Ihr Werkzeug", + "toolsLabel_other": "Ihre {{count}} Werkzeuge", + "deleteTools": "Löschen", + "keepTools": "Behalten, ausgeschaltet", + "failed": "Die Verbindung konnte nicht entfernt werden. Versuche es erneut." + }, + "health": { + "reconnect": "Verbinde {{name}} neu, um weiter zu synchronisieren." + }, + "error": { + "decrypt": "Die gespeicherten Zugangsdaten konnten nicht entschlüsselt werden. Verbinde neu, um fortzufahren.", + "defaultKey": "Ein Admin muss ENCRYPTION_SECRET_KEY setzen, bevor Dienste verbunden werden können." } } }, @@ -1256,7 +1326,8 @@ }, "groupUploadWeb": "Upload und Web", "groupConnection": "Aus einer Verbindung", - "tileConnectedAs": "Verbunden als {{account}}" + "tileConnectedAs": "Verbunden als {{account}}", + "newCredentials": "Neue Zugangsdaten eingeben" }, "createAPIKey": { "label": "Neuen API-Schlüssel erstellen", @@ -1572,7 +1643,12 @@ "approveAlways": "Genehmigen, nicht mehr fragen", "deny": "Ablehnen", "details": "Details", - "denyReasonPlaceholder": "Optionaler Grund für die Ablehnung…" + "denyReasonPlaceholder": "Optionaler Grund für die Ablehnung…", + "connectTitle": "Dieses Werkzeug braucht dein {{name}}-Konto.", + "connect": "{{name}} verbinden", + "continue": "Weiter", + "skip": "Überspringen", + "thisService": "diesen Dienst" }, "research": { "title": "Recherche", @@ -2265,7 +2341,8 @@ "shared": "Geteilt", "connectTo": "Mit {{provider}} verbinden", "sessionExpiredFor": "Sitzung abgelaufen. Bitte verbinde dich erneut mit {{provider}}.", - "selectFilesFrom": "Dateien aus {{provider}} auswählen" + "selectFilesFrom": "Dateien aus {{provider}} auswählen", + "account": "Konto" }, "actionButtons": { "openNewChat": "Neuen Chat öffnen", diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index af1443a8..66f638a7 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -1094,18 +1094,88 @@ "toolOn": "On", "toolOff": "Off", "accountMenu": "Account actions", - "disconnect": "Disconnect" + "disconnect": "Disconnect", + "keyEnding": "Key ending {{hint}}", + "syncMore": "Sync more content", + "refreshTools": "Refresh tools", + "noSources": "Nothing synced from this connection yet.", + "remove": "Remove", + "refreshed": "Tools refreshed: {{added}} added, {{removed}} removed.", + "refreshFailed": "Could not refresh the tools." }, "permission": { "always": "Always allow", "ask": "Needs approval", - "off": "Off" + "off": "Off", + "label": "Permission for {{action}}" }, "disconnect": { "title": "Disconnect {{name}}?", "body_zero": "Tools from this connection stop working until you reconnect.", "body_one": "Your source keeps its content but stops syncing. Tools from this connection stop working until you reconnect.", "body_other": "Your {{count}} sources keep their content but stop syncing. Tools from this connection stop working until you reconnect." + }, + "fields": { + "token": "API key", + "aws_access_key_id": "Access key ID", + "aws_secret_access_key": "Secret access key", + "region": "Region", + "endpoint_url": "Custom endpoint URL", + "client_id": "Client ID", + "client_secret": "Client secret", + "user_agent": "User agent", + "bucket": "Bucket", + "prefix": "Path prefix", + "search_queries": "Search queries", + "number_posts": "Number of posts", + "telegram_token": "Bot token", + "ntfy_token": "Access token", + "postgres_token": "Connection string", + "brave_token": "API key" + }, + "wizard": { + "connectTitle": "Connect {{name}}", + "reconnectTitle": "Reconnect {{name}}", + "signIn": "Sign in to {{name}}", + "chooseWhatToSync": "Choose what to sync", + "skip": "Skip for now", + "syncFrequency": "Sync", + "doneTitle": "{{name}} is connected", + "doneSummary": "Connected as {{account}}. {{sources}} syncing, {{tools}} ready.", + "doneSummaryNone": "Connected as {{account}}.", + "sourcesCount_one": "{{count}} source", + "sourcesCount_other": "{{count}} sources", + "toolsCount_one": "{{count}} tool", + "toolsCount_other": "{{count}} tools", + "toolsHeading": "Tools ({{count}})", + "tryInChat": "Try it in chat", + "done": "Done", + "connectFailed": "Could not connect. Check the details and try again.", + "syncFailed": "Could not start the sync. Try again.", + "connectedWithKey": "Connected with the key ending {{hint}}.", + "doneCounts": "{{sources}} syncing, {{tools}} ready.", + "doneSources": "{{sources}} syncing.", + "doneTools": "{{tools}} ready." + }, + "remove": { + "title": "Remove {{name}}?", + "description": "The stored sign-in is deleted. Choose what happens to what this connection feeds.", + "sourcesLabel_one": "Its source", + "sourcesLabel_other": "Its {{count}} sources", + "keepSources": "Keep content (stop syncing)", + "deleteSources": "Delete", + "toolsLabel_one": "Its tool", + "toolsLabel_other": "Its {{count}} tools", + "deleteTools": "Delete", + "keepTools": "Keep, turned off", + "failed": "Could not remove the connection. Try again." + }, + "health": { + "reconnect": "Reconnect {{name}} to keep syncing." + }, + "error": { + "decrypt": "Stored credentials could not be decrypted. Reconnect to continue.", + "defaultKey": "An admin must set ENCRYPTION_SECRET_KEY before services can be connected." } } }, @@ -1262,7 +1332,8 @@ }, "groupUploadWeb": "Upload and web", "groupConnection": "From a connection", - "tileConnectedAs": "Connected as {{account}}" + "tileConnectedAs": "Connected as {{account}}", + "newCredentials": "Enter new credentials" }, "createAPIKey": { "label": "Create New API Key", @@ -1588,7 +1659,12 @@ "approveAlways": "Approve, don't ask again", "deny": "Deny", "details": "Details", - "denyReasonPlaceholder": "Optional reason for denying…" + "denyReasonPlaceholder": "Optional reason for denying…", + "connectTitle": "This tool needs your {{name}} account.", + "connect": "Connect {{name}}", + "continue": "Continue", + "skip": "Skip", + "thisService": "this service" }, "research": { "title": "Research", @@ -2300,7 +2376,8 @@ "shared": "Shared", "connectTo": "Connect to {{provider}}", "sessionExpiredFor": "Session expired. Please reconnect to {{provider}}.", - "selectFilesFrom": "Select Files from {{provider}}" + "selectFilesFrom": "Select Files from {{provider}}", + "account": "Account" }, "actionButtons": { "openNewChat": "Open New Chat", diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index 4f009258..d1c250ea 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -1088,18 +1088,88 @@ "toolOn": "Activada", "toolOff": "Desactivada", "accountMenu": "Acciones de la cuenta", - "disconnect": "Desconectar" + "disconnect": "Desconectar", + "keyEnding": "Clave que termina en {{hint}}", + "syncMore": "Sincronizar más contenido", + "refreshTools": "Actualizar herramientas", + "noSources": "Aún no se ha sincronizado nada desde esta conexión.", + "remove": "Eliminar", + "refreshed": "Herramientas actualizadas: {{added}} añadidas, {{removed}} eliminadas.", + "refreshFailed": "No se pudieron actualizar las herramientas." }, "permission": { "always": "Permitir siempre", "ask": "Requiere aprobación", - "off": "Desactivada" + "off": "Desactivada", + "label": "Permiso para {{action}}" }, "disconnect": { "title": "¿Desconectar {{name}}?", "body_zero": "Las herramientas de esta conexión dejarán de funcionar hasta que vuelvas a conectarla.", "body_one": "Tu fuente conserva su contenido pero deja de sincronizarse. Las herramientas de esta conexión dejarán de funcionar hasta que vuelvas a conectarla.", "body_other": "Tus {{count}} fuentes conservan su contenido pero dejan de sincronizarse. Las herramientas de esta conexión dejarán de funcionar hasta que vuelvas a conectarla." + }, + "fields": { + "token": "Clave de API", + "aws_access_key_id": "ID de clave de acceso", + "aws_secret_access_key": "Clave de acceso secreta", + "region": "Región", + "endpoint_url": "URL de endpoint personalizada", + "client_id": "ID de cliente", + "client_secret": "Secreto de cliente", + "user_agent": "User agent", + "bucket": "Bucket", + "prefix": "Prefijo de ruta", + "search_queries": "Consultas de búsqueda", + "number_posts": "Número de publicaciones", + "telegram_token": "Token del bot", + "ntfy_token": "Token de acceso", + "postgres_token": "Cadena de conexión", + "brave_token": "Clave de API" + }, + "wizard": { + "connectTitle": "Conectar {{name}}", + "reconnectTitle": "Reconectar {{name}}", + "signIn": "Iniciar sesión en {{name}}", + "chooseWhatToSync": "Elige qué sincronizar", + "skip": "Omitir por ahora", + "syncFrequency": "Sincronizar", + "doneTitle": "{{name}} está conectado", + "doneSummary": "Conectado como {{account}}. {{sources}} sincronizando, {{tools}} listas.", + "doneSummaryNone": "Conectado como {{account}}.", + "sourcesCount_one": "{{count}} fuente", + "sourcesCount_other": "{{count}} fuentes", + "toolsCount_one": "{{count}} herramienta", + "toolsCount_other": "{{count}} herramientas", + "toolsHeading": "Herramientas ({{count}})", + "tryInChat": "Probar en el chat", + "done": "Listo", + "connectFailed": "No se pudo conectar. Revisa los datos e inténtalo de nuevo.", + "syncFailed": "No se pudo iniciar la sincronización. Inténtalo de nuevo.", + "connectedWithKey": "Conectado con la clave que termina en {{hint}}.", + "doneCounts": "{{sources}} sincronizando, {{tools}} listas.", + "doneSources": "{{sources}} sincronizando.", + "doneTools": "{{tools}} listas." + }, + "remove": { + "title": "¿Eliminar {{name}}?", + "description": "Se borra el inicio de sesión guardado. Elige qué pasa con lo que alimenta esta conexión.", + "sourcesLabel_one": "Su fuente", + "sourcesLabel_other": "Sus {{count}} fuentes", + "keepSources": "Conservar el contenido (dejar de sincronizar)", + "deleteSources": "Eliminar", + "toolsLabel_one": "Su herramienta", + "toolsLabel_other": "Sus {{count}} herramientas", + "deleteTools": "Eliminar", + "keepTools": "Conservar, desactivadas", + "failed": "No se pudo eliminar la conexión. Inténtalo de nuevo." + }, + "health": { + "reconnect": "Reconecta {{name}} para seguir sincronizando." + }, + "error": { + "decrypt": "No se pudieron descifrar las credenciales guardadas. Reconecta para continuar.", + "defaultKey": "Un administrador debe definir ENCRYPTION_SECRET_KEY antes de poder conectar servicios." } } }, @@ -1256,7 +1326,8 @@ }, "groupUploadWeb": "Subida y web", "groupConnection": "Desde una conexión", - "tileConnectedAs": "Conectado como {{account}}" + "tileConnectedAs": "Conectado como {{account}}", + "newCredentials": "Introducir credenciales nuevas" }, "createAPIKey": { "label": "Crear Nueva Clave de API", @@ -1572,7 +1643,12 @@ "approveAlways": "Aprobar y no volver a preguntar", "deny": "Denegar", "details": "Detalles", - "denyReasonPlaceholder": "Motivo del rechazo (opcional)…" + "denyReasonPlaceholder": "Motivo del rechazo (opcional)…", + "connectTitle": "Esta herramienta necesita tu cuenta de {{name}}.", + "connect": "Conectar {{name}}", + "continue": "Continuar", + "skip": "Omitir", + "thisService": "este servicio" }, "research": { "title": "Investigación", @@ -2265,7 +2341,8 @@ "shared": "Compartido", "connectTo": "Conectar a {{provider}}", "sessionExpiredFor": "Sesión expirada. Por favor, reconecte a {{provider}}.", - "selectFilesFrom": "Seleccionar archivos de {{provider}}" + "selectFilesFrom": "Seleccionar archivos de {{provider}}", + "account": "Cuenta" }, "actionButtons": { "openNewChat": "Abrir nuevo chat", diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index 2595f2f3..9b408cda 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -1087,17 +1087,83 @@ "toolOn": "オン", "toolOff": "オフ", "accountMenu": "アカウントの操作", - "disconnect": "切断" + "disconnect": "切断", + "keyEnding": "末尾が {{hint}} のキー", + "syncMore": "さらに同期する", + "refreshTools": "ツールを更新", + "noSources": "この接続からはまだ何も同期されていません。", + "remove": "削除", + "refreshed": "ツールを更新しました: {{added}} 件追加、{{removed}} 件削除。", + "refreshFailed": "ツールを更新できませんでした。" }, "permission": { "always": "常に許可", "ask": "承認が必要", - "off": "オフ" + "off": "オフ", + "label": "{{action}} の権限" }, "disconnect": { "title": "{{name}} を切断しますか？", "body_zero": "この接続のツールは、再接続するまで使えなくなります。", "body_other": "{{count}} 件のソースはコンテンツを保持しますが、同期は停止します。この接続のツールは、再接続するまで使えなくなります。" + }, + "fields": { + "token": "API キー", + "aws_access_key_id": "アクセスキー ID", + "aws_secret_access_key": "シークレットアクセスキー", + "region": "リージョン", + "endpoint_url": "カスタムエンドポイント URL", + "client_id": "クライアント ID", + "client_secret": "クライアントシークレット", + "user_agent": "ユーザーエージェント", + "bucket": "バケット", + "prefix": "パスのプレフィックス", + "search_queries": "検索クエリ", + "number_posts": "投稿数", + "telegram_token": "ボットトークン", + "ntfy_token": "アクセストークン", + "postgres_token": "接続文字列", + "brave_token": "API キー" + }, + "wizard": { + "connectTitle": "{{name}} に接続", + "reconnectTitle": "{{name}} に再接続", + "signIn": "{{name}} にサインイン", + "chooseWhatToSync": "同期する内容を選択", + "skip": "今はスキップ", + "syncFrequency": "同期", + "doneTitle": "{{name}} に接続しました", + "doneSummary": "{{account}} として接続しました。{{sources}}を同期中、{{tools}}を利用できます。", + "doneSummaryNone": "{{account}} として接続しました。", + "sourcesCount_other": "{{count}} 件のソース", + "toolsCount_other": "{{count}} 個のツール", + "toolsHeading": "ツール ({{count}})", + "tryInChat": "チャットで試す", + "done": "完了", + "connectFailed": "接続できませんでした。入力内容を確認して、もう一度お試しください。", + "syncFailed": "同期を開始できませんでした。もう一度お試しください。", + "connectedWithKey": "末尾が {{hint}} のキーで接続しました。", + "doneCounts": "{{sources}}を同期中、{{tools}}を利用できます。", + "doneSources": "{{sources}}を同期中です。", + "doneTools": "{{tools}}を利用できます。" + }, + "remove": { + "title": "{{name}} を削除しますか？", + "description": "保存されたサインイン情報は削除されます。この接続から提供されているものをどうするか選んでください。", + "sourcesLabel_other": "{{count}} 件のソース", + "keepSources": "コンテンツを保持（同期を停止）", + "deleteSources": "削除", + "toolsLabel_other": "{{count}} 個のツール", + "deleteTools": "削除", + "keepTools": "保持してオフにする", + "failed": "接続を削除できませんでした。もう一度お試しください。" + }, + "health": { + "reconnect": "同期を続けるには {{name}} を再接続してください。" + }, + "error": { + "decrypt": "保存された認証情報を復号できませんでした。続行するには再接続してください。", + "defaultKey": "サービスを接続する前に、管理者が ENCRYPTION_SECRET_KEY を設定する必要があります。" } } }, @@ -1254,7 +1320,8 @@ }, "groupUploadWeb": "アップロードとウェブ", "groupConnection": "接続から", - "tileConnectedAs": "{{account}} として接続中" + "tileConnectedAs": "{{account}} として接続中", + "newCredentials": "新しい認証情報を入力" }, "createAPIKey": { "label": "新しいAPIキーを作成", @@ -1568,7 +1635,12 @@ "approveAlways": "承認し、今後は確認しない", "deny": "拒否", "details": "詳細", - "denyReasonPlaceholder": "拒否の理由（任意）…" + "denyReasonPlaceholder": "拒否の理由（任意）…", + "connectTitle": "このツールには {{name}} アカウントが必要です。", + "connect": "{{name}} に接続", + "continue": "続行", + "skip": "スキップ", + "thisService": "このサービス" }, "research": { "title": "リサーチ", @@ -2258,7 +2330,8 @@ "shared": "共有", "connectTo": "{{provider}}に接続", "sessionExpiredFor": "セッションが期限切れです。{{provider}}に再接続してください。", - "selectFilesFrom": "{{provider}}からファイルを選択" + "selectFilesFrom": "{{provider}}からファイルを選択", + "account": "アカウント" }, "actionButtons": { "openNewChat": "新しいチャットを開く", diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index b3cb6d3d..b18c6471 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -1128,12 +1128,20 @@ "toolOn": "Вкл.", "toolOff": "Выкл.", "accountMenu": "Действия с аккаунтом", - "disconnect": "Отключить" + "disconnect": "Отключить", + "keyEnding": "Ключ, оканчивающийся на {{hint}}", + "syncMore": "Синхронизировать ещё", + "refreshTools": "Обновить инструменты", + "noSources": "Из этого подключения пока ничего не синхронизировано.", + "remove": "Удалить", + "refreshed": "Инструменты обновлены: добавлено {{added}}, удалено {{removed}}.", + "refreshFailed": "Не удалось обновить инструменты." }, "permission": { "always": "Всегда разрешать", "ask": "Нужно одобрение", - "off": "Выкл." + "off": "Выкл.", + "label": "Разрешение для {{action}}" }, "disconnect": { "title": "Отключить {{name}}?", @@ -1142,6 +1150,76 @@ "body_few": "Ваши {{count}} источника сохранят контент, но перестанут синхронизироваться. Инструменты из этого подключения перестанут работать, пока вы не переподключитесь.", "body_many": "Ваши {{count}} источников сохранят контент, но перестанут синхронизироваться. Инструменты из этого подключения перестанут работать, пока вы не переподключитесь.", "body_other": "Ваши {{count}} источника сохранят контент, но перестанут синхронизироваться. Инструменты из этого подключения перестанут работать, пока вы не переподключитесь." + }, + "fields": { + "token": "API-ключ", + "aws_access_key_id": "ID ключа доступа", + "aws_secret_access_key": "Секретный ключ доступа", + "region": "Регион", + "endpoint_url": "Свой URL эндпоинта", + "client_id": "ID клиента", + "client_secret": "Секрет клиента", + "user_agent": "User agent", + "bucket": "Бакет", + "prefix": "Префикс пути", + "search_queries": "Поисковые запросы", + "number_posts": "Количество постов", + "telegram_token": "Токен бота", + "ntfy_token": "Токен доступа", + "postgres_token": "Строка подключения", + "brave_token": "API-ключ" + }, + "wizard": { + "connectTitle": "Подключить {{name}}", + "reconnectTitle": "Переподключить {{name}}", + "signIn": "Войти в {{name}}", + "chooseWhatToSync": "Выберите, что синхронизировать", + "skip": "Пока пропустить", + "syncFrequency": "Синхронизация", + "doneTitle": "{{name}} подключён", + "doneSummary": "Подключено как {{account}}. Синхронизируется: {{sources}}, готово: {{tools}}.", + "doneSummaryNone": "Подключено как {{account}}.", + "sourcesCount_one": "{{count}} источник", + "sourcesCount_few": "{{count}} источника", + "sourcesCount_many": "{{count}} источников", + "sourcesCount_other": "{{count}} источника", + "toolsCount_one": "{{count}} инструмент", + "toolsCount_few": "{{count}} инструмента", + "toolsCount_many": "{{count}} инструментов", + "toolsCount_other": "{{count}} инструмента", + "toolsHeading": "Инструменты ({{count}})", + "tryInChat": "Попробовать в чате", + "done": "Готово", + "connectFailed": "Не удалось подключиться. Проверьте данные и попробуйте ещё раз.", + "syncFailed": "Не удалось запустить синхронизацию. Попробуйте ещё раз.", + "connectedWithKey": "Подключено ключом, оканчивающимся на {{hint}}.", + "doneCounts": "Синхронизируется: {{sources}}, готово: {{tools}}.", + "doneSources": "Синхронизируется: {{sources}}.", + "doneTools": "Готово: {{tools}}." + }, + "remove": { + "title": "Удалить {{name}}?", + "description": "Сохранённый вход будет удалён. Выберите, что станет с тем, что даёт это подключение.", + "sourcesLabel_one": "Его источник", + "sourcesLabel_few": "Его {{count}} источника", + "sourcesLabel_many": "Его {{count}} источников", + "sourcesLabel_other": "Его {{count}} источника", + "keepSources": "Оставить контент (остановить синхронизацию)", + "deleteSources": "Удалить", + "toolsLabel_one": "Его инструмент", + "toolsLabel_few": "Его {{count}} инструмента", + "toolsLabel_many": "Его {{count}} инструментов", + "toolsLabel_other": "Его {{count}} инструмента", + "deleteTools": "Удалить", + "keepTools": "Оставить выключенными", + "failed": "Не удалось удалить подключение. Попробуйте ещё раз." + }, + "health": { + "reconnect": "Переподключите {{name}}, чтобы синхронизация продолжилась." + }, + "error": { + "decrypt": "Не удалось расшифровать сохранённые учётные данные. Переподключитесь, чтобы продолжить.", + "defaultKey": "Администратор должен задать ENCRYPTION_SECRET_KEY, прежде чем можно будет подключать сервисы." } } }, @@ -1298,7 +1376,8 @@ }, "groupUploadWeb": "Загрузка и веб", "groupConnection": "Из подключения", - "tileConnectedAs": "Подключено как {{account}}" + "tileConnectedAs": "Подключено как {{account}}", + "newCredentials": "Ввести новые учётные данные" }, "createAPIKey": { "label": "Создать новый API ключ", @@ -1618,7 +1697,12 @@ "approveAlways": "Одобрить и больше не спрашивать", "deny": "Отклонить", "details": "Подробности", - "denyReasonPlaceholder": "Причина отказа (необязательно)…" + "denyReasonPlaceholder": "Причина отказа (необязательно)…", + "connectTitle": "Этому инструменту нужен ваш аккаунт {{name}}.", + "connect": "Подключить {{name}}", + "continue": "Продолжить", + "skip": "Пропустить", + "thisService": "этот сервис" }, "research": { "title": "Исследование", @@ -2325,7 +2409,8 @@ "shared": "Общие", "connectTo": "Подключиться к {{provider}}", "sessionExpiredFor": "Сеанс истек. Пожалуйста, переподключитесь к {{provider}}.", - "selectFilesFrom": "Выберите файлы из {{provider}}" + "selectFilesFrom": "Выберите файлы из {{provider}}", + "account": "Аккаунт" }, "actionButtons": { "openNewChat": "Открыть новый чат", diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index 183deb66..67ea0cfc 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -1087,17 +1087,83 @@ "toolOn": "開啟", "toolOff": "關閉", "accountMenu": "帳號操作", - "disconnect": "中斷連線" + "disconnect": "中斷連線", + "keyEnding": "結尾為 {{hint}} 的金鑰", + "syncMore": "同步更多內容", + "refreshTools": "重新整理工具", + "noSources": "尚未從此連線同步任何內容。", + "remove": "移除", + "refreshed": "工具已重新整理：新增 {{added}} 個，移除 {{removed}} 個。", + "refreshFailed": "無法重新整理工具。" }, "permission": { "always": "一律允許", "ask": "需要核准", - "off": "關閉" + "off": "關閉", + "label": "{{action}} 的權限" }, "disconnect": { "title": "要中斷 {{name}} 的連線嗎？", "body_zero": "在重新連線之前，此連線的工具將無法使用。", "body_other": "你的 {{count}} 個來源會保留內容，但會停止同步。在重新連線之前，此連線的工具將無法使用。" + }, + "fields": { + "token": "API 金鑰", + "aws_access_key_id": "存取金鑰 ID", + "aws_secret_access_key": "秘密存取金鑰", + "region": "區域", + "endpoint_url": "自訂端點 URL", + "client_id": "用戶端 ID", + "client_secret": "用戶端密鑰", + "user_agent": "User agent", + "bucket": "儲存貯體", + "prefix": "路徑前綴", + "search_queries": "搜尋詞", + "number_posts": "貼文數量", + "telegram_token": "機器人權杖", + "ntfy_token": "存取權杖", + "postgres_token": "連線字串", + "brave_token": "API 金鑰" + }, + "wizard": { + "connectTitle": "連線 {{name}}", + "reconnectTitle": "重新連線 {{name}}", + "signIn": "登入 {{name}}", + "chooseWhatToSync": "選擇要同步的內容", + "skip": "暫時略過", + "syncFrequency": "同步", + "doneTitle": "{{name}} 已連線", + "doneSummary": "已以 {{account}} 連線。{{sources}}同步中，{{tools}}已就緒。", + "doneSummaryNone": "已以 {{account}} 連線。", + "sourcesCount_other": "{{count}} 個來源", + "toolsCount_other": "{{count}} 個工具", + "toolsHeading": "工具（{{count}}）", + "tryInChat": "在聊天中試試", + "done": "完成", + "connectFailed": "無法連線。請檢查資料後再試一次。", + "syncFailed": "無法開始同步。請再試一次。", + "connectedWithKey": "已使用結尾為 {{hint}} 的金鑰連線。", + "doneCounts": "{{sources}}同步中，{{tools}}已就緒。", + "doneSources": "{{sources}}同步中。", + "doneTools": "{{tools}}已就緒。" + }, + "remove": { + "title": "要移除 {{name}} 嗎？", + "description": "儲存的登入資訊將被刪除。請選擇如何處理此連線提供的內容。", + "sourcesLabel_other": "它的 {{count}} 個來源", + "keepSources": "保留內容（停止同步）", + "deleteSources": "刪除", + "toolsLabel_other": "它的 {{count}} 個工具", + "deleteTools": "刪除", + "keepTools": "保留並關閉", + "failed": "無法移除此連線。請再試一次。" + }, + "health": { + "reconnect": "重新連線 {{name}} 以繼續同步。" + }, + "error": { + "decrypt": "無法解密已儲存的憑證。請重新連線以繼續。", + "defaultKey": "管理員必須先設定 ENCRYPTION_SECRET_KEY，才能連線服務。" } } }, @@ -1254,7 +1320,8 @@ }, "groupUploadWeb": "上傳和網頁", "groupConnection": "來自連線", - "tileConnectedAs": "已以 {{account}} 連線" + "tileConnectedAs": "已以 {{account}} 連線", + "newCredentials": "輸入新的憑證" }, "createAPIKey": { "label": "建立新的 API 金鑰", @@ -1568,7 +1635,12 @@ "approveAlways": "核准，不再詢問", "deny": "拒絕", "details": "詳細資料", - "denyReasonPlaceholder": "拒絕原因（選填）…" + "denyReasonPlaceholder": "拒絕原因（選填）…", + "connectTitle": "此工具需要你的 {{name}} 帳號。", + "connect": "連線 {{name}}", + "continue": "繼續", + "skip": "略過", + "thisService": "此服務" }, "research": { "title": "研究", @@ -2258,7 +2330,8 @@ "shared": "共用", "connectTo": "連接到 {{provider}}", "sessionExpiredFor": "工作階段已過期。請重新連接到 {{provider}}。", - "selectFilesFrom": "從 {{provider}} 選擇檔案" + "selectFilesFrom": "從 {{provider}} 選擇檔案", + "account": "帳號" }, "actionButtons": { "openNewChat": "開啟新聊天", diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index 29b4880b..6fd94305 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -1087,17 +1087,83 @@ "toolOn": "开启", "toolOff": "关闭", "accountMenu": "账号操作", - "disconnect": "断开连接" + "disconnect": "断开连接", + "keyEnding": "以 {{hint}} 结尾的密钥", + "syncMore": "同步更多内容", + "refreshTools": "刷新工具", + "noSources": "尚未从此连接同步任何内容。", + "remove": "移除", + "refreshed": "工具已刷新：新增 {{added}} 个，移除 {{removed}} 个。", + "refreshFailed": "无法刷新工具。" }, "permission": { "always": "始终允许", "ask": "需要批准", - "off": "关闭" + "off": "关闭", + "label": "{{action}} 的权限" }, "disconnect": { "title": "断开 {{name}}？", "body_zero": "在重新连接之前，此连接的工具将无法使用。", "body_other": "你的 {{count}} 个来源会保留内容，但将停止同步。在重新连接之前，此连接的工具将无法使用。" + }, + "fields": { + "token": "API 密钥", + "aws_access_key_id": "访问密钥 ID", + "aws_secret_access_key": "秘密访问密钥", + "region": "区域", + "endpoint_url": "自定义端点 URL", + "client_id": "客户端 ID", + "client_secret": "客户端密钥", + "user_agent": "User agent", + "bucket": "存储桶", + "prefix": "路径前缀", + "search_queries": "搜索词", + "number_posts": "帖子数量", + "telegram_token": "机器人令牌", + "ntfy_token": "访问令牌", + "postgres_token": "连接字符串", + "brave_token": "API 密钥" + }, + "wizard": { + "connectTitle": "连接 {{name}}", + "reconnectTitle": "重新连接 {{name}}", + "signIn": "登录 {{name}}", + "chooseWhatToSync": "选择要同步的内容", + "skip": "暂时跳过", + "syncFrequency": "同步", + "doneTitle": "{{name}} 已连接", + "doneSummary": "已作为 {{account}} 连接。{{sources}}正在同步，{{tools}}已就绪。", + "doneSummaryNone": "已作为 {{account}} 连接。", + "sourcesCount_other": "{{count}} 个来源", + "toolsCount_other": "{{count}} 个工具", + "toolsHeading": "工具（{{count}}）", + "tryInChat": "在聊天中试试", + "done": "完成", + "connectFailed": "无法连接。请检查信息后重试。", + "syncFailed": "无法开始同步。请重试。", + "connectedWithKey": "已使用以 {{hint}} 结尾的密钥连接。", + "doneCounts": "{{sources}}正在同步，{{tools}}已就绪。", + "doneSources": "{{sources}}正在同步。", + "doneTools": "{{tools}}已就绪。" + }, + "remove": { + "title": "移除 {{name}}？", + "description": "保存的登录信息将被删除。请选择如何处理此连接提供的内容。", + "sourcesLabel_other": "它的 {{count}} 个来源", + "keepSources": "保留内容（停止同步）", + "deleteSources": "删除", + "toolsLabel_other": "它的 {{count}} 个工具", + "deleteTools": "删除", + "keepTools": "保留并关闭", + "failed": "无法移除此连接。请重试。" + }, + "health": { + "reconnect": "重新连接 {{name}} 以继续同步。" + }, + "error": { + "decrypt": "无法解密保存的凭据。请重新连接以继续。", + "defaultKey": "管理员必须先设置 ENCRYPTION_SECRET_KEY，才能连接服务。" } } }, @@ -1254,7 +1320,8 @@ }, "groupUploadWeb": "上传和网页", "groupConnection": "来自连接", - "tileConnectedAs": "已作为 {{account}} 连接" + "tileConnectedAs": "已作为 {{account}} 连接", + "newCredentials": "输入新的凭据" }, "createAPIKey": { "label": "创建新的 API 密钥", @@ -1568,7 +1635,12 @@ "approveAlways": "批准，不再询问", "deny": "拒绝", "details": "详情", - "denyReasonPlaceholder": "拒绝原因（可选）…" + "denyReasonPlaceholder": "拒绝原因（可选）…", + "connectTitle": "此工具需要你的 {{name}} 账号。", + "connect": "连接 {{name}}", + "continue": "继续", + "skip": "跳过", + "thisService": "此服务" }, "research": { "title": "研究", @@ -2258,7 +2330,8 @@ "shared": "共享", "connectTo": "连接到 {{provider}}", "sessionExpiredFor": "会话已过期。请重新连接到 {{provider}}。", - "selectFilesFrom": "从 {{provider}} 选择文件" + "selectFilesFrom": "从 {{provider}} 选择文件", + "account": "账号" }, "actionButtons": { "openNewChat": "打开新聊天", diff --git a/frontend/src/modals/AddToolModal.tsx b/frontend/src/modals/AddToolModal.tsx index 421831fe..0e79c97c 100644 --- a/frontend/src/modals/AddToolModal.tsx +++ b/frontend/src/modals/AddToolModal.tsx @@ -1,7 +1,7 @@ import { ArrowRight, Plus } from 'lucide-react'; import React from 'react'; import { useTranslation } from 'react-i18next'; -import { useSelector } from 'react-redux'; +import { useDispatch, useSelector } from 'react-redux'; import { Link } from 'react-router-dom'; import userService from '../api/services/userService'; @@ -12,7 +12,13 @@ import { Card, CardDescription, CardTitle } from '../components/ui/card'; import { Modal } from '../components/ui/modal'; import { SectionHeader } from '../components/ui/section-header'; import { useLoaderState } from '../hooks'; +import { + loadConnectors, + selectConnectorCatalog, +} from '../connectors/connectorsSlice'; +import useConnectorLauncher from '../connectors/useConnectorLauncher'; import PairDeviceModal from '../settings/PairDeviceModal'; +import type { AppDispatch } from '../store'; import { ActiveState } from '../models/misc'; import { selectToken } from '../preferences/preferenceSlice'; import ConfigToolModal from './ConfigToolModal'; @@ -35,6 +41,11 @@ export default function AddToolModal({ }) { const { t } = useTranslation(); const token = useSelector(selectToken); + const dispatch = useDispatch(); + const catalog = useSelector(selectConnectorCatalog); + const { launch, modals: connectModals } = useConnectorLauncher({ + onConnected: getUserTools, + }); const [availableTools, setAvailableTools] = React.useState< AvailableToolType[] >([]); @@ -71,6 +82,15 @@ export default function AddToolModal({ setPairModalState('ACTIVE'); return; } + // A service's tool is added by connecting the service: the key goes on + // a connection and the tool is created with write actions needing + // approval. + const connector = catalog.find((c) => c.key === tool.connector_key); + if (tool.group === 'service' && connector) { + setModalState('INACTIVE'); + launch(connector); + return; + } if (Object.keys(tool.configRequirements).length === 0) { userService .createTool( @@ -108,7 +128,10 @@ export default function AddToolModal({ }; React.useEffect(() => { - if (modalState === 'ACTIVE') getAvailableTools(); + if (modalState === 'ACTIVE') { + getAvailableTools(); + dispatch(loadConnectors({ token })); + } }, [modalState]); return ( @@ -210,6 +233,7 @@ export default function AddToolModal({ tool={selectedTool} getUserTools={getUserTools} /> + {connectModals} diff --git a/frontend/src/navigation/SectionNav.tsx b/frontend/src/navigation/SectionNav.tsx index 1b2ef85a..71abcfe0 100644 --- a/frontend/src/navigation/SectionNav.tsx +++ b/frontend/src/navigation/SectionNav.tsx @@ -1,9 +1,11 @@ import { ArrowLeft } from 'lucide-react'; import { useTranslation } from 'react-i18next'; +import { useSelector } from 'react-redux'; import { Link } from 'react-router-dom'; import { Button } from '@/components/ui/button'; +import { selectConnectorsEnabled } from '../connectors/connectorsSlice'; import { getVisibleGroups, type Section, type SectionItem } from './sections'; import { useSidebarLevel } from './SidebarLevelProvider'; @@ -35,7 +37,11 @@ export default function SectionNav({ }: SectionNavProps) { const { t } = useTranslation(); const { goToLevel } = useSidebarLevel(); - const groups = getVisibleGroups(section, { isAdmin }); + const connectorsEnabled = useSelector(selectConnectorsEnabled); + const groups = getVisibleGroups(section, { + isAdmin, + features: { connectors: connectorsEnabled }, + }); const sectionTitle = section.title ?? t(section.titleKey); const renderItem = (item: SectionItem) => { diff --git a/frontend/src/navigation/SectionPills.tsx b/frontend/src/navigation/SectionPills.tsx index 4a4d10f0..29e6acaa 100644 --- a/frontend/src/navigation/SectionPills.tsx +++ b/frontend/src/navigation/SectionPills.tsx @@ -6,6 +6,7 @@ import { Button } from '@/components/ui/button'; import { cn } from '@/lib/utils'; import { selectIsAdmin } from '@/preferences/preferenceSlice'; +import { selectConnectorsEnabled } from '../connectors/connectorsSlice'; import { getVisibleGroups } from './sections'; import { useSectionContext } from './useSectionContext'; @@ -23,11 +24,13 @@ export default function SectionPills({ className }: { className?: string }) { const { t } = useTranslation(); const { section, item } = useSectionContext(); const isAdmin = useSelector(selectIsAdmin); + const connectorsEnabled = useSelector(selectConnectorsEnabled); if (!section) return null; - const items = getVisibleGroups(section, { isAdmin }).flatMap( - (group) => group.items, - ); + const items = getVisibleGroups(section, { + isAdmin, + features: { connectors: connectorsEnabled }, + }).flatMap((group) => group.items); if (items.length < 2) return null; return ( diff --git a/frontend/src/navigation/sections.ts b/frontend/src/navigation/sections.ts index 0334f18c..6f5c680b 100644 --- a/frontend/src/navigation/sections.ts +++ b/frontend/src/navigation/sections.ts @@ -42,6 +42,8 @@ export type SectionItem = { adminOnly?: boolean; /** Jumps to a different section rather than navigating within this one. */ leavesSection?: boolean; + /** Hidden when the backend reports this feature off (`/api/config`). */ + feature?: 'connectors'; }; /** Items sharing a heading in the nav. */ @@ -121,6 +123,7 @@ export const SETTINGS_SECTION: Section = { path: '/settings/connectors', labelKey: 'settings.connectors.label', icon: Plug, + feature: 'connectors', }, { key: 'tools', @@ -355,12 +358,22 @@ export function getSectionItems( /** Groups with admin-only entries removed, dropping any group left empty. */ export function getVisibleGroups( section: Section, - { isAdmin = true }: { isAdmin?: boolean } = {}, + { + isAdmin = true, + features = {}, + }: { + isAdmin?: boolean; + features?: Partial, boolean>>; + } = {}, ): SectionGroup[] { return section.groups .map((group) => ({ ...group, - items: group.items.filter((item) => !item.adminOnly || isAdmin), + items: group.items.filter( + (item) => + (!item.adminOnly || isAdmin) && + (!item.feature || features[item.feature] !== false), + ), })) .filter((group) => group.items.length > 0); } diff --git a/frontend/src/notifications/ConnectionHealthToast.tsx b/frontend/src/notifications/ConnectionHealthToast.tsx new file mode 100644 index 00000000..987ef7e1 --- /dev/null +++ b/frontend/src/notifications/ConnectionHealthToast.tsx @@ -0,0 +1,116 @@ +import { X } from 'lucide-react'; +import { useCallback, useEffect, useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useDispatch, useSelector } from 'react-redux'; +import { Link } from 'react-router-dom'; + +import { Button } from '../components/ui/button'; +import { + Toast, + ToastActions, + ToastFooter, + ToastHeader, + ToastTitle, +} from '../components/ui/toast'; +import { loadConnectors } from '../connectors/connectorsSlice'; +import { selectToken } from '../preferences/preferenceSlice'; +import type { AppDispatch } from '../store'; +import { + dismissShareNotification, + selectDismissedShareNotifications, + selectRecentEvents, + type SSEEvent, +} from './notificationsSlice'; + +// Backlog replay re-delivers up to a day of events; an old one is noise. +const MAX_AGE_MS = 24 * 60 * 60 * 1000; +const MAX_VISIBLE = 2; + +/** + * "Reconnect Google Drive to keep syncing": a connection whose sign-in + * stopped working (``connection.reconnect_needed``). Stays until the user + * reconnects or dismisses it, since syncing is paused until then. Shares + * the team notifications' persisted dismissals, so a reload does not pop it + * again. + */ +export default function ConnectionHealthToast() { + const dispatch = useDispatch(); + const { t } = useTranslation(); + const token = useSelector(selectToken); + const events = useSelector(selectRecentEvents); + const dismissed = useSelector(selectDismissedShareNotifications); + const dismissedSet = useMemo(() => new Set(dismissed), [dismissed]); + + const onDismiss = useCallback( + (id: string) => dispatch(dismissShareNotification(id)), + [dispatch], + ); + + const now = Date.now(); + const visible: SSEEvent[] = []; + const seenConnections = new Set(); + for (const event of events) { + if (event.type !== 'connection.reconnect_needed') continue; + if (!event.id || dismissedSet.has(event.id)) continue; + if (event.ts) { + const age = now - Date.parse(event.ts); + if (Number.isFinite(age) && age > MAX_AGE_MS) continue; + } + const connectionId = String(event.scope?.id ?? event.id); + if (seenConnections.has(connectionId)) continue; + seenConnections.add(connectionId); + visible.push(event); + if (visible.length >= MAX_VISIBLE) break; + } + + // Refresh the badges on the Connectors, Sources and Tools pages. + const newest = visible[0]?.id; + useEffect(() => { + if (newest) dispatch(loadConnectors({ token })); + }, [newest, dispatch, token]); + + if (visible.length === 0) return null; + + return ( + <> + {visible.map((event) => { + const payload = (event.payload ?? {}) as Record; + const name = String(payload.name ?? ''); + const key = String(payload.connector_key ?? ''); + return ( + + + + {t('settings.connectors.health.reconnect', { + name, + interpolation: { escapeValue: false }, + })} + + + + + + + + + + ); + })} + + ); +} diff --git a/frontend/src/settings/Connectors.tsx b/frontend/src/settings/Connectors.tsx index cf2bdfd2..8656327b 100644 --- a/frontend/src/settings/Connectors.tsx +++ b/frontend/src/settings/Connectors.tsx @@ -209,9 +209,9 @@ export default function Connectors() { { + onConnect={(connector, options) => { closeDrawer(); - launch(connector); + launch(connector, options); }} /> {modals} diff --git a/frontend/src/settings/Sources.tsx b/frontend/src/settings/Sources.tsx index a95c53ed..2289195b 100644 --- a/frontend/src/settings/Sources.tsx +++ b/frontend/src/settings/Sources.tsx @@ -218,42 +218,13 @@ export default function Sources({ }); }; - const getConnectorProvider = async (doc: Doc): Promise => { - if (doc.provider) { - return doc.provider; - } - if (!doc.id) { - return null; - } - try { - const directoryResponse = await userService.getDirectoryStructure( - doc.id, - token, - ); - const directoryData = await directoryResponse.json(); - return directoryData?.provider ?? null; - } catch (error) { - console.error('Error fetching connector provider:', error); - return null; - } - }; - const handleSyncNow = async (doc: Doc) => { if (!doc.id) { return; } try { if (doc.type?.startsWith('connector')) { - const provider = await getConnectorProvider(doc); - if (!provider) { - console.error('Sync now failed: provider not found'); - return; - } - const response = await userService.syncConnector( - doc.id, - provider, - token, - ); + const response = await userService.syncConnector(doc.id, token); const data = await response.json(); if (!data.success) { console.error('Sync now failed:', data.error || data.message); diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 9c02bacb..41c95e86 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -11,7 +11,6 @@ import type { RootState } from '../store'; import userService from '../api/services/userService'; import modelService from '../api/services/modelService'; import type { Model } from '../models/types'; -import { getSessionToken } from '../utils/providerUtils'; import { Button } from '../components/ui/button'; import { Input } from '../components/ui/input'; import { FormField as UiFormField } from '../components/ui/form-field'; @@ -113,6 +112,9 @@ function Upload({ // File picker state const [selectedFiles, setSelectedFiles] = useState([]); const [selectedFolders, setSelectedFolders] = useState([]); + // The connection (account) the source syncs from. Pickers report it; S3 + // and Reddit pick it here ('' means "enter new credentials"). + const [connectionId, setConnectionId] = useState(null); useEffect(() => { let cancelled = false; @@ -155,11 +157,39 @@ function Upload({ if (!ingestorSchema) return null; const schema: FormField[] = ingestorSchema.fields; - const generalFields = schema.filter((field: FormField) => !field.advanced); + const generalFields = schema.filter( + (field: FormField) => + !field.advanced && !(usingSavedKeys && credentialKeys.has(field.name)), + ); const advancedFields = schema.filter((field: FormField) => field.advanced); return (
+ {keyAccounts.length > 0 && ( + + + + )}
{generalFields.map((field: FormField) => renderField(field))}
@@ -353,6 +383,7 @@ function Upload({ setSelectedFolders(selectedFolderIds); }} onFirstPickName={prefillName} + onConnectionChange={setConnectionId} provider={ingestor.type as unknown as string} token={token} initialSelectedFiles={selectedFiles} @@ -373,6 +404,7 @@ function Upload({ setSelectedFolders(selectedFolderIds); }} onFirstPickName={prefillName} + onConnectionChange={setConnectionId} token={token} /> ) : ( @@ -386,6 +418,7 @@ function Upload({ setSelectedFolders(selectedFolderIds); }} onFirstPickName={prefillName} + onConnectionChange={setConnectionId} provider="google_drive" token={token} initialSelectedFiles={selectedFiles} @@ -404,6 +437,7 @@ function Upload({ setSelectedFolders(selectedFolderIds); }} onFirstPickName={prefillName} + onConnectionChange={setConnectionId} provider="share_point" token={token} initialSelectedFiles={selectedFiles} @@ -422,6 +456,7 @@ function Upload({ setSelectedFolders(selectedFolderIds); }} onFirstPickName={prefillName} + onConnectionChange={setConnectionId} provider="confluence" token={token} initialSelectedFiles={selectedFiles} @@ -466,6 +501,30 @@ function Upload({ : undefined; const selectedConnector = connectorFor(ingestor.type); const needsSetup = !!selectedConnector && !selectedConnector.available; + // S3 and Reddit keep their keys on a connection: once an account is + // chosen the key fields go away ("enter secrets once"). + const keyAccounts = + selectedConnector?.auth_kind === 'api_key' + ? connections.filter( + (c) => + c.connector_key === selectedConnector.key && + c.status === 'connected', + ) + : []; + const credentialKeys = new Set( + (selectedConnector?.credential_fields ?? []).map((f) => f.key), + ); + const usingSavedKeys = + keyAccounts.length > 0 && !!connectionId && connectionId !== 'new'; + + useEffect(() => { + if (selectedConnector?.auth_kind === 'api_key') { + setConnectionId(keyAccounts[0]?.id ?? 'new'); + } else { + setConnectionId(null); + } + // Reset only when the source type changes. + }, [ingestor.type]); const ingestorOptions: IngestorOption[] = IngestorFormSchemas.map( (schema) => ({ @@ -798,15 +857,20 @@ function Upload({ hasSharePointPicker || hasConfluencePicker ) { - const sessionToken = getSessionToken(ingestor.type as string); configData = { provider: ingestor.type as string, - session_token: sessionToken, + connection_id: connectionId, file_ids: selectedFiles, folder_ids: selectedFolders, }; } + if (usingSavedKeys) { + configData = Object.fromEntries( + Object.entries(configData).filter(([key]) => !credentialKeys.has(key)), + ); + configData.connection_id = connectionId; + } formData.append('data', JSON.stringify(configData)); const apiHost: string = envVar('VITE_API_HOST'); @@ -1028,6 +1092,7 @@ function Upload({ if (!ingestorSchemaForFields) return false; const formFields: FormField[] = ingestorSchemaForFields.fields; for (const field of formFields) { + if (usingSavedKeys && credentialKeys.has(field.name)) continue; if (field.required) { // Validate only required fields const value = diff --git a/frontend/src/utils/providerUtils.test.ts b/frontend/src/utils/providerUtils.test.ts new file mode 100644 index 00000000..b13ddabf --- /dev/null +++ b/frontend/src/utils/providerUtils.test.ts @@ -0,0 +1,39 @@ +const claim = vi.fn(); +vi.mock('../api/services/connectorsService', () => ({ + default: { claim: (...args: unknown[]) => claim(...args) }, +})); + +import { claimLegacySessionTokens } from './providerUtils'; + +describe('claimLegacySessionTokens', () => { + beforeEach(() => { + claim.mockReset(); + localStorage.clear(); + }); + + it('claims every stored session token once and removes it', async () => { + localStorage.setItem('google_drive_session_token', 'g-token'); + localStorage.setItem('confluence_session_token', 'c-token'); + claim.mockResolvedValue({ success: true, connection_id: 'x' }); + + await claimLegacySessionTokens('jwt'); + + expect(claim).toHaveBeenCalledWith('google_drive', 'g-token', 'jwt'); + expect(claim).toHaveBeenCalledWith('confluence', 'c-token', 'jwt'); + expect(claim).toHaveBeenCalledTimes(2); + expect(localStorage.getItem('google_drive_session_token')).toBeNull(); + expect(localStorage.getItem('confluence_session_token')).toBeNull(); + }); + + it('drops the token even when the claim fails', async () => { + localStorage.setItem('share_point_session_token', 's-token'); + claim.mockRejectedValue(new Error('offline')); + await claimLegacySessionTokens(null); + expect(localStorage.getItem('share_point_session_token')).toBeNull(); + }); + + it('does nothing without stored tokens', async () => { + await claimLegacySessionTokens(null); + expect(claim).not.toHaveBeenCalled(); + }); +}); diff --git a/frontend/src/utils/providerUtils.ts b/frontend/src/utils/providerUtils.ts index 837df1c7..da4625a5 100644 --- a/frontend/src/utils/providerUtils.ts +++ b/frontend/src/utils/providerUtils.ts @@ -1,25 +1,34 @@ /** - * Utility functions for managing session tokens for different cloud service providers. - * Follows the convention: {provider}_session_token + * Connector sign-ins used to keep a random session handle per provider in + * localStorage (``_session_token``). Credentials now live only on + * the server and the browser refers to a connection by its id, so the old + * handles are claimed once (linked to the signed-in user's connection) and + * removed. */ -import userService from '../api/services/userService'; +import connectorsService from '../api/services/connectorsService'; -export const getSessionToken = (provider: string): string | null => { - return localStorage.getItem(`${provider}_session_token`); -}; +const LEGACY_PROVIDERS = ['google_drive', 'share_point', 'confluence']; -export const setSessionToken = (provider: string, token: string): void => { - localStorage.setItem(`${provider}_session_token`, token); -}; +const legacyKey = (provider: string) => `${provider}_session_token`; -export const removeSessionToken = (provider: string): void => { - localStorage.removeItem(`${provider}_session_token`); -}; - -export const validateProviderSession = async ( +export const claimLegacySessionTokens = async ( token: string | null, - provider: string, -) => { - return await userService.validateConnectorSession(provider, token); +): Promise => { + for (const provider of LEGACY_PROVIDERS) { + let value: string | null = null; + try { + value = localStorage.getItem(legacyKey(provider)); + } catch { + return; + } + if (!value) continue; + try { + await connectorsService.claim(provider, value, token); + } catch { + // The handle is useless to this frontend either way; drop it. + } finally { + localStorage.removeItem(legacyKey(provider)); + } + } }; From b692eaf7adedb095440a9323bdcc4966415f3654 Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:30:53 +0100 Subject: [PATCH 007/162] Add MCP presets, admin connector policies and connector attribution Six vendor-run MCP servers (Notion, Linear, Atlassian, Sentry, Asana, Stripe) join the catalog as presets from docsgpt/connectors/presets/ mcp.yaml; existing connections to those servers show under them. connector_policies lets an admin turn a connector off or force whose account every share uses, and app_metadata's connectors.allow_custom_mcp turns custom MCP servers off. Both are enforced on the server: new connections, OAuth sign-ins, MCP test and save, and the tools of a disabled connector stop resolving. GET/PUT /api/admin/connectors reads and changes them. A shared tool's owner picks owner or member credentials with PUT /api/connections/tools//credential-mode, and a member running the owner's account always confirms write actions. Tool-call events and retrieved chunks name the connector they came from (key and display name, never the account), and tool calls keep those fields when the conversation is reloaded. --- docsgpt/agents/tool_executor.py | 19 ++ docsgpt/alembic/versions/0038_connections.py | 19 ++ docsgpt/api/admin/__init__.py | 1 + docsgpt/api/admin/connectors.py | 130 ++++++++ docsgpt/api/connector/connections.py | 35 ++ docsgpt/api/connector/routes.py | 2 + docsgpt/api/user/sources/upload.py | 2 + docsgpt/api/user/tools/mcp.py | 31 ++ docsgpt/api/user/tools/routes.py | 2 + docsgpt/connectors/attribution.py | 68 ++++ docsgpt/connectors/presets/mcp.yaml | 71 ++++ docsgpt/connectors/resolve.py | 11 +- docsgpt/connectors/service.py | 53 ++- docsgpt/retriever/base.py | 11 + docsgpt/retriever/classic_rag.py | 6 +- docsgpt/retriever/graph_rag.py | 2 +- docsgpt/storage/db/models.py | 11 + .../db/repositories/connector_policies.py | 58 ++++ .../test_policies_and_attribution.py | 315 ++++++++++++++++++ tests/storage/db/test_migration_0038.py | 1 + 20 files changed, 844 insertions(+), 4 deletions(-) create mode 100644 docsgpt/api/admin/connectors.py create mode 100644 docsgpt/connectors/attribution.py create mode 100644 docsgpt/connectors/presets/mcp.yaml create mode 100644 docsgpt/storage/db/repositories/connector_policies.py create mode 100644 tests/connectors/test_policies_and_attribution.py diff --git a/docsgpt/agents/tool_executor.py b/docsgpt/agents/tool_executor.py index 181612ae..778f4a1a 100644 --- a/docsgpt/agents/tool_executor.py +++ b/docsgpt/agents/tool_executor.py @@ -932,6 +932,14 @@ class ToolExecutor: or require_approval ) + # A member running someone else's account (a shared tool in owner + # mode) always confirms write actions, whatever the owner chose for + # themselves. + if not require_approval and resolved is not None and resolved.delegated: + from docsgpt.connectors.permissions import ACCESS_WRITE, action_access + + require_approval = action_access(tool_data.get("name"), action_data) == ACCESS_WRITE + if require_approval: if self.headless: tool_row_id = str(tool_data.get("id") or tool_id) @@ -1286,6 +1294,12 @@ class ToolExecutor: "arguments": call_args, } tool_data = tools_dict[tool_id] + # Name the service a connection-backed tool used, so the chip can show + # its logo ("Searched Notion"). Never the account behind it. + resolved = self._resolve_connection(tool_data) + if resolved is not None and resolved.connector_key: + tool_call_data["connector_key"] = resolved.connector_key + tool_call_data["connector_name"] = resolved.connector_name # Surface the device id on remote_device tool-call events so the # approval UI can wire up the sticky "don't ask again" button. if tool_data.get("name") == "remote_device": @@ -1331,6 +1345,11 @@ class ToolExecutor: else next(action for action in tool_data["actions"] if action["name"] == action_name) ) + if "connector_key" in tool_call_data: + from docsgpt.connectors.permissions import action_access + + tool_call_data["access"] = action_access(tool_data.get("name"), action_data) + query_params, headers, body, parameters = {}, {}, {}, {} param_types = { "query_params": query_params, diff --git a/docsgpt/alembic/versions/0038_connections.py b/docsgpt/alembic/versions/0038_connections.py index b9da954d..962e5c4a 100644 --- a/docsgpt/alembic/versions/0038_connections.py +++ b/docsgpt/alembic/versions/0038_connections.py @@ -40,6 +40,11 @@ Backfill (idempotent, only fills NULLs or unconverted rows): resolved each invoking member's own token before this migration and keep doing so (``member``); owners can switch them in the share dialog. +``connector_policies`` holds the admin's per-connector switches (enabled, +forced credential mode). The instance-wide "Allow custom MCP servers" switch +is the ``connectors.allow_custom_mcp`` key in ``app_metadata`` (absent means +allowed). + Revision ID: 0038_connections Revises: 0037_request_traces """ @@ -75,6 +80,19 @@ _BATCH = 500 def upgrade() -> None: _upgrade_links() _upgrade_credentials() + op.execute( + """ + CREATE TABLE IF NOT EXISTS connector_policies ( + connector_key TEXT PRIMARY KEY, + enabled BOOLEAN NOT NULL DEFAULT true, + credential_mode TEXT NOT NULL DEFAULT 'choose' + CONSTRAINT connector_policies_credential_mode_chk + CHECK (credential_mode IN ('choose', 'owner', 'member')), + updated_by TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ); + """ + ) def _upgrade_links() -> None: @@ -433,6 +451,7 @@ def _restore_account_index() -> None: def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS connector_policies;") _downgrade_credentials() op.execute("DROP INDEX IF EXISTS user_tools_connection_idx;") op.execute("DROP INDEX IF EXISTS sources_connection_idx;") diff --git a/docsgpt/api/admin/__init__.py b/docsgpt/api/admin/__init__.py index 1dcdd686..bf47f250 100644 --- a/docsgpt/api/admin/__init__.py +++ b/docsgpt/api/admin/__init__.py @@ -1,5 +1,6 @@ from .routes import admin_ns from . import quotas # noqa: F401 (registers the quota resources on admin_ns) from . import activity # noqa: F401 (registers the activity resources on admin_ns) +from . import connectors # noqa: F401 (registers the connector policy resources on admin_ns) __all__ = ["admin_ns"] diff --git a/docsgpt/api/admin/connectors.py b/docsgpt/api/admin/connectors.py new file mode 100644 index 00000000..5b2689f0 --- /dev/null +++ b/docsgpt/api/admin/connectors.py @@ -0,0 +1,130 @@ +"""Admin > Connectors: turn connectors on or off and set sharing policy. + +Every resource here is behind ``@admin_required`` (the frontend guard is +cosmetic). The page also shows what each OAuth connector still needs from +the server (settings and the redirect URI to register), never their values. +""" + +from __future__ import annotations + +import logging + +from flask import jsonify, make_response, request +from flask_restx import Resource +from sqlalchemy import text + +from docsgpt.api.admin.routes import _actor, admin_ns +from docsgpt.api.user.authz import admin_required +from docsgpt.connectors import catalog, service +from docsgpt.core.settings import settings +from docsgpt.security.encryption import is_default_encryption_key +from docsgpt.storage.db.repositories.app_metadata import AppMetadataRepository +from docsgpt.storage.db.repositories.connector_policies import ( + ALLOW_CUSTOM_MCP_KEY, + CREDENTIAL_POLICIES, + ConnectorPoliciesRepository, +) +from docsgpt.storage.db.session import db_readonly, db_session + +logger = logging.getLogger(__name__) + + +def _connection_counts(conn) -> dict[str, int]: + rows = conn.execute( + text( + "SELECT connector_key, provider, server_url, count(*) AS n FROM connector_sessions " + "WHERE COALESCE(status, '') <> 'pending' GROUP BY connector_key, provider, server_url" + ) + ).fetchall() + counts: dict[str, int] = {} + for row in rows: + key = catalog.connector_key_for_row(dict(row._mapping)) + if key: + counts[key] = counts.get(key, 0) + int(row.n) + return counts + + +def _mcp_redirect_uri() -> str: + from docsgpt.agents.tools.mcp_tool import MCPTool + + return MCPTool._resolve_redirect_uri(MCPTool.__new__(MCPTool), None) + + +@admin_ns.route("/admin/connectors") +class AdminConnectorsResource(Resource): + @admin_required + def get(self): + """Every connector with its policy, setup state and connection count.""" + with db_readonly() as conn: + policies = ConnectorPoliciesRepository(conn).all() + allow_custom = service.custom_mcp_allowed(conn) + counts = _connection_counts(conn) + connectors = [] + for definition in catalog.all_definitions(): + policy = policies.get(definition.key) or {} + connectors.append( + { + "key": definition.key, + "name": definition.name, + "icon": definition.icon, + "category": definition.category, + "publisher": definition.publisher, + "auth_kind": definition.auth_kind, + "enabled": policy.get("enabled", True), + "credential_mode": policy.get("credential_mode", "choose"), + "configured": definition.configured, + "required_settings": [ + {"name": name, "set": bool(getattr(settings, name, None))} + for name in definition.required_settings + ], + "connection_count": counts.get(definition.key, 0), + "docs_url": definition.docs_url, + "mcp_url": definition.mcp_url, + } + ) + return make_response( + jsonify( + { + "success": True, + "connectors": connectors, + "allow_custom_mcp": allow_custom, + "default_encryption_key": is_default_encryption_key(), + "oauth_redirect_uri": settings.CONNECTOR_REDIRECT_BASE_URI, + "mcp_redirect_uri": _mcp_redirect_uri(), + } + ), + 200, + ) + + @admin_required + def put(self): + """Update policies: {policies: {key: {enabled?, credential_mode?}}, allow_custom_mcp?}.""" + body = request.get_json(silent=True) or {} + updates = body.get("policies") or {} + if not isinstance(updates, dict): + return make_response(jsonify({"success": False, "message": "policies must be an object"}), 400) + for key, change in updates.items(): + if catalog.get_definition(key) is None or not isinstance(change, dict): + return make_response(jsonify({"success": False, "message": f"Unknown connector: {key}"}), 400) + mode = change.get("credential_mode") + if mode is not None and mode not in CREDENTIAL_POLICIES: + return make_response(jsonify({"success": False, "message": "Unknown credential mode"}), 400) + if "enabled" in change and not isinstance(change["enabled"], bool): + return make_response(jsonify({"success": False, "message": "enabled must be true or false"}), 400) + allow_custom = body.get("allow_custom_mcp") + if allow_custom is not None and not isinstance(allow_custom, bool): + return make_response(jsonify({"success": False, "message": "allow_custom_mcp must be a boolean"}), 400) + actor = _actor() + with db_session() as conn: + repo = ConnectorPoliciesRepository(conn) + for key, change in updates.items(): + repo.upsert( + key, + enabled=change.get("enabled"), + credential_mode=change.get("credential_mode"), + updated_by=actor, + ) + if allow_custom is not None: + AppMetadataRepository(conn).set(ALLOW_CUSTOM_MCP_KEY, "true" if allow_custom else "false") + logger.info("connector_policies_updated", extra={"admin": actor, "connectors": sorted(updates)}) + return self.get() diff --git a/docsgpt/api/connector/connections.py b/docsgpt/api/connector/connections.py index 32bbd0af..daf1e06e 100644 --- a/docsgpt/api/connector/connections.py +++ b/docsgpt/api/connector/connections.py @@ -97,6 +97,8 @@ class ConnectionsList(Resource): ) except service.EncryptionKeyNotConfigured as err: return _error(str(err), 400, code="encryption_key_default") + except service.ConnectorDisabled as err: + return _error(str(err), 403, code="disabled") except ValueError as err: return _error(str(err), 400) except Exception as err: @@ -448,3 +450,36 @@ class ConnectionRefreshTools(Resource): current_app.logger.error(f"Error refreshing MCP tools: {err}", exc_info=True) return _error("Failed to refresh tools", 502) return make_response(jsonify({"success": True, **diff}), 200) + + +@connections_ns.route("/connections/tools//credential-mode") +class ToolCredentialMode(Resource): + @api.doc( + description=( + "Whose account a shared connection-backed tool uses: {mode: owner | member}. " + "Owner only; refused when an admin forces a mode for the connector." + ) + ) + def put(self, tool_id: str): + from docsgpt.connectors.resolve import MODE_MEMBER, MODE_OWNER + from docsgpt.storage.db.repositories.user_tools import UserToolsRepository + + user_id = _user_id() + if not user_id: + return _unauthorized() + mode = _json_body().get("mode") + if mode not in (MODE_OWNER, MODE_MEMBER): + return _error("mode must be owner or member", 400) + with db_session() as conn: + tools = UserToolsRepository(conn) + tool = tools.get_any(tool_id, user_id) + if tool is None or tool.get("user_id") != user_id or not tool.get("connection_id"): + return _error("Tool not found", 404) + connection = ConnectorSessionsRepository(conn).get(str(tool["connection_id"])) + forced = service.forced_credential_mode( + conn, catalog.connector_key_for_row(connection) if connection else None, + ) + if forced and forced != mode: + return _error("An admin sets this for every share", 409, code="forced", mode=forced) + tools.update(str(tool["id"]), user_id, {"credential_mode": mode}) + return make_response(jsonify({"success": True, "mode": mode}), 200) diff --git a/docsgpt/api/connector/routes.py b/docsgpt/api/connector/routes.py index 03822102..2d124df8 100644 --- a/docsgpt/api/connector/routes.py +++ b/docsgpt/api/connector/routes.py @@ -222,6 +222,8 @@ class ConnectorAuth(Resource): ) except service.ConnectionUnavailable: return make_response(jsonify({"success": False, "error": "Connection not found"}), 404) + except service.ConnectorDisabled as err: + return make_response(jsonify({"success": False, "error": str(err), "code": "disabled"}), 403) # The popup drops results for origins outside the allowlist, which the # user only sees as a cancelled sign-in; name the missing origin here. request_origin = _origin_of(request.headers.get("Origin")) diff --git a/docsgpt/api/user/sources/upload.py b/docsgpt/api/user/sources/upload.py index 52b62fd1..9465f4de 100644 --- a/docsgpt/api/user/sources/upload.py +++ b/docsgpt/api/user/sources/upload.py @@ -482,6 +482,8 @@ def _remote_credentials(user, source, config): try: with db_session() as conn: row, _ = service.create_api_key_connection(conn, user, definition, provided) + except service.ConnectorDisabled as err: + return None, None, make_response(jsonify({"success": False, "error": str(err)}), 403) except (service.EncryptionKeyNotConfigured, ValueError): return config, None, None return public, str(row["id"]), None diff --git a/docsgpt/api/user/tools/mcp.py b/docsgpt/api/user/tools/mcp.py index cd69bf75..6b2567eb 100644 --- a/docsgpt/api/user/tools/mcp.py +++ b/docsgpt/api/user/tools/mcp.py @@ -112,6 +112,31 @@ def _mcp_connection(user, config, auth_type, auth_credentials, display_name): return str(row["id"]) if row else None +def _mcp_policy_error(config: dict): + """A 403 when an admin turned this MCP server's connector off, else None. + + A preset's own switch applies to its server; any other server is a + custom connector and needs "Allow custom MCP servers". + """ + from docsgpt.connectors import catalog, service + + preset = catalog.preset_for_url(config.get("server_url")) + key = preset.key if preset else "custom_mcp" + try: + with db_readonly() as conn: + service.ensure_connector_allowed(conn, key) + except service.ConnectorDisabled: + return make_response( + jsonify({"success": False, "error": "This MCP server is turned off by an admin", "code": "disabled"}), + 403, + ) + except Exception: + # The switch is an admin preference; when the policy table cannot be + # read, saving the tool (which needs the database) fails on its own. + current_app.logger.warning("Could not read connector policies", exc_info=True) + return None + + @tools_mcp_ns.route("/mcp_server/test") class TestMCPServerConfig(Resource): @api.expect( @@ -147,6 +172,9 @@ class TestMCPServerConfig(Resource): ) _validate_mcp_server_url(config) + policy_error = _mcp_policy_error(config) + if policy_error is not None: + return policy_error auth_credentials = _extract_auth_credentials(config) test_config = config.copy() @@ -243,6 +271,9 @@ class MCPServerSave(Resource): ) _validate_mcp_server_url(config) + policy_error = _mcp_policy_error(config) + if policy_error is not None: + return policy_error auth_credentials = _extract_auth_credentials(config) auth_type = config.get("auth_type", "none") diff --git a/docsgpt/api/user/tools/routes.py b/docsgpt/api/user/tools/routes.py index db72a847..93c26e5b 100644 --- a/docsgpt/api/user/tools/routes.py +++ b/docsgpt/api/user/tools/routes.py @@ -470,6 +470,8 @@ def _create_connected_tool(user, data, definition, tool_instance): ) except connection_service.EncryptionKeyNotConfigured: return None + except connection_service.ConnectorDisabled as err: + return make_response(jsonify({"success": False, "message": str(err)}), 403) except ValueError as err: return make_response(jsonify({"success": False, "message": str(err)}), 400) return make_response(jsonify({"id": str(created["id"]), "connection_id": str(connection["id"])}), 200) diff --git a/docsgpt/connectors/attribution.py b/docsgpt/connectors/attribution.py new file mode 100644 index 00000000..9719a8a6 --- /dev/null +++ b/docsgpt/connectors/attribution.py @@ -0,0 +1,68 @@ +"""Which connector a retrieved chunk came from, for "From Google Drive" citations. + +Only the connector's key and display name are attached, never the account +behind the connection, so shared conversations show the same attribution +without revealing whose Drive it was. +""" + +from __future__ import annotations + +import logging +from typing import Optional + +from sqlalchemy import text + +from docsgpt.storage.db.base_repository import looks_like_uuid +from docsgpt.storage.db.session import db_readonly + +logger = logging.getLogger(__name__) + + +def connector_labels(source_ids: list[str]) -> dict[str, dict]: + """``{source_id: {"connector_key", "connector_name"}}`` for synced sources. + + Sources that do not come from a connection are left out. Never raises: + attribution is decoration, and a lookup failure must not break an answer. + """ + from docsgpt.connectors import service + + ids = [str(i) for i in source_ids if i and looks_like_uuid(str(i))] + if not ids: + return {} + try: + with db_readonly() as conn: + rows = conn.execute( + text( + "SELECT s.id AS source_id, cs.* FROM sources s " + "JOIN connector_sessions cs ON cs.id = s.connection_id " + "WHERE s.id = ANY(CAST(:ids AS uuid[]))" + ), + {"ids": ids}, + ).fetchall() + except Exception: + logger.warning("connector attribution lookup failed", exc_info=True) + return {} + labels = {} + for row in rows: + data = dict(row._mapping) + public = service.serialize_connection({**data, "id": data["id"]}) + labels[str(data["source_id"])] = { + "connector_key": public["connector_key"], + "connector_name": public["name"], + } + return labels + + +class ConnectorLabelCache: + """Per-search memo so each source is looked up once.""" + + def __init__(self) -> None: + self._labels: dict[str, dict] = {} + + def for_source(self, source_id: Optional[str]) -> dict: + if not source_id: + return {} + key = str(source_id) + if key not in self._labels: + self._labels[key] = connector_labels([key]).get(key, {}) + return self._labels[key] diff --git a/docsgpt/connectors/presets/mcp.yaml b/docsgpt/connectors/presets/mcp.yaml new file mode 100644 index 00000000..c9642c80 --- /dev/null +++ b/docsgpt/connectors/presets/mcp.yaml @@ -0,0 +1,71 @@ +# Curated remote MCP servers shown as Connectors catalog cards. +# +# Inclusion rules: the vendor operates the server, it has a stable public URL, +# and it signs in with OAuth plus dynamic client registration (or needs no +# auth), so connecting takes no admin setup. Each entry was checked for OAuth +# authorization-server metadata with a registration endpoint and for a 401 +# challenge on the MCP endpoint. Keep the list short; anything else is a +# custom connector. +# +# Keys look like ``mcp:``. The frontend reads ``settings.connectors +# .descriptions.mcp_`` for a translated description. + +- key: mcp:notion + name: Notion + description: Search, read and update Notion pages and databases. + icon: notion + category: knowledge + mcp_url: https://mcp.notion.com/mcp + auth_kind: mcp_oauth + capabilities: [read, write] + docs_url: https://developers.notion.com/docs/mcp + +- key: mcp:linear + name: Linear + description: Find, create and update Linear issues and projects. + icon: linear + category: dev + mcp_url: https://mcp.linear.app/mcp + auth_kind: mcp_oauth + capabilities: [read, write] + docs_url: https://linear.app/docs/mcp + +- key: mcp:atlassian + name: Atlassian (Jira and Confluence) + description: Search and update Jira issues and Confluence pages. + icon: atlassian + category: dev + mcp_url: https://mcp.atlassian.com/v1/mcp + auth_kind: mcp_oauth + capabilities: [read, write] + docs_url: https://support.atlassian.com/rovo/docs/getting-started-with-the-atlassian-remote-mcp-server/ + +- key: mcp:sentry + name: Sentry + description: Look up Sentry issues, events and releases. + icon: sentry + category: dev + mcp_url: https://mcp.sentry.dev/mcp + auth_kind: mcp_oauth + capabilities: [read, write] + docs_url: https://docs.sentry.io/product/sentry-mcp/ + +- key: mcp:asana + name: Asana + description: Find and update Asana tasks and projects. + icon: asana + category: knowledge + mcp_url: https://mcp.asana.com/v2/mcp + auth_kind: mcp_oauth + capabilities: [read, write] + docs_url: https://developers.asana.com/docs/using-asanas-mcp-server + +- key: mcp:stripe + name: Stripe + description: Look up customers, payments and subscriptions in Stripe. + icon: stripe + category: dev + mcp_url: https://mcp.stripe.com + auth_kind: mcp_oauth + capabilities: [read, write] + docs_url: https://docs.stripe.com/mcp diff --git a/docsgpt/connectors/resolve.py b/docsgpt/connectors/resolve.py index 8580b842..490d7c7f 100644 --- a/docsgpt/connectors/resolve.py +++ b/docsgpt/connectors/resolve.py @@ -76,6 +76,11 @@ def resolve_connection(resource: dict, invoker_user_id: Optional[str]) -> Option with db_readonly() as conn: repo = ConnectorSessionsRepository(conn) owned = repo.get(str(connection_id)) + owned_key = catalog.connector_key_for_row(owned) if owned else None + policy = service.load_policies(conn).get(owned_key) or {} if owned_key else {} + if policy.get("credential_mode") in (MODE_OWNER, MODE_MEMBER): + # An admin forces whose account every share of this connector uses. + mode = policy["credential_mode"] if owned is not None and owner and owned.get("user_id") != owner: # A resource may only point at its own owner's connection. logger.warning( @@ -86,7 +91,11 @@ def resolve_connection(resource: dict, invoker_user_id: Optional[str]) -> Option if mode == MODE_MEMBER and invoker_user_id and invoker_user_id != owner: row = _member_connection(repo, owned, invoker_user_id) key = catalog.connector_key_for_row(row or owned or {}) - available = row is not None and service.normalize_status(row) == service.STATUS_CONNECTED + available = ( + row is not None + and service.normalize_status(row) == service.STATUS_CONNECTED + and policy.get("enabled") is not False + ) return ResolvedConnection( row=row, available=available, diff --git a/docsgpt/connectors/service.py b/docsgpt/connectors/service.py index 52c9f31d..5e00d795 100644 --- a/docsgpt/connectors/service.py +++ b/docsgpt/connectors/service.py @@ -214,7 +214,8 @@ def catalog_for_user(conn, user_id: str, *, is_admin: bool, policies: Optional[d else only learns that setup is needed. policies: ``connector_key`` to policy row, when admin policies exist. """ - policies = policies or {} + if policies is None: + policies = load_policies(conn) by_key: dict[str, list[str]] = {} for connection in list_connections(conn, user_id): by_key.setdefault(connection["connector_key"], []).append(connection["status"]) @@ -686,6 +687,7 @@ def create_api_key_connection( else: credentials = {k: v for k, v in credentials.items() if v not in (None, "")} ensure_can_store_credentials() + ensure_connector_allowed(conn, definition.key) secret_values = {k: v for k, v in credentials.items() if (fields.get(k).secret if fields.get(k) else True)} account_label = label or credential_hint(secret_values or credentials) repo = ConnectorSessionsRepository(conn) @@ -736,6 +738,7 @@ def begin_oauth(conn, user_id: str, provider: str, connection_id: Optional[str] if row is None or row.get("provider") != provider: raise ConnectionUnavailable("Connection not found", connection_id=connection_id, status="missing") return row + ensure_connector_allowed(conn, provider) definition = catalog.get_definition(provider) result = conn.execute( text( @@ -1168,3 +1171,51 @@ def reencrypt_all(batch_size: int = 500) -> dict: for connection_id in failed_ids: mark_reconnect_needed(connection_id, DECRYPT_ERROR) return counts + + +# --------------------------------------------------------------------------- +# Admin policy +# --------------------------------------------------------------------------- + + +class ConnectorDisabled(Exception): + """An admin turned this connector (or custom MCP servers) off.""" + + +def custom_mcp_allowed(conn) -> bool: + """The instance-wide "Allow custom MCP servers" switch (on unless turned off).""" + from docsgpt.storage.db.repositories.app_metadata import AppMetadataRepository + from docsgpt.storage.db.repositories.connector_policies import ALLOW_CUSTOM_MCP_KEY + + return AppMetadataRepository(conn).get(ALLOW_CUSTOM_MCP_KEY) != "false" + + +def load_policies(conn) -> dict[str, dict]: + """Every connector's policy, with custom MCP folded in from its switch.""" + from docsgpt.storage.db.repositories.connector_policies import ConnectorPoliciesRepository + + policies = dict(ConnectorPoliciesRepository(conn).all()) + if not custom_mcp_allowed(conn): + policies["custom_mcp"] = {**policies.get("custom_mcp", {}), "enabled": False} + return policies + + +def ensure_connector_allowed(conn, connector_key: Optional[str]) -> None: + """Refuse a new connection to a connector an admin turned off. + + Raises: + ConnectorDisabled: The connector, or custom MCP servers, are off. + """ + if not connector_key: + return + policy = load_policies(conn).get(connector_key) or {} + if policy.get("enabled") is False: + raise ConnectorDisabled(f"{connector_key} is turned off by an admin") + + +def forced_credential_mode(conn, connector_key: Optional[str]) -> Optional[str]: + """``owner`` or ``member`` when an admin forces one for this connector.""" + if not connector_key: + return None + mode = (load_policies(conn).get(connector_key) or {}).get("credential_mode") + return mode if mode in ("owner", "member") else None diff --git a/docsgpt/retriever/base.py b/docsgpt/retriever/base.py index a1b49d9c..897ab622 100644 --- a/docsgpt/retriever/base.py +++ b/docsgpt/retriever/base.py @@ -2,6 +2,17 @@ from abc import ABC, abstractmethod class BaseRetriever(ABC): + @property + def _connector_labels(self): + """Per-retriever memo of each source's connector ("From Google Drive").""" + cache = self.__dict__.get("_connector_label_cache") + if cache is None: + from docsgpt.connectors.attribution import ConnectorLabelCache + + cache = ConnectorLabelCache() + self.__dict__["_connector_label_cache"] = cache + return cache + def __init__(self): pass diff --git a/docsgpt/retriever/classic_rag.py b/docsgpt/retriever/classic_rag.py index 76f343c3..3e8779be 100644 --- a/docsgpt/retriever/classic_rag.py +++ b/docsgpt/retriever/classic_rag.py @@ -385,7 +385,11 @@ class ClassicRAG(BaseRetriever): doc_tokens = num_tokens_from_string(doc_text_with_header) if cumulative_tokens + doc_tokens < token_budget: - entry = {"text": page_content, **labels} + entry = { + "text": page_content, + **labels, + **self._connector_labels.for_source(vectorstore_id), + } if self.include_scores: entry["score"] = score entry["score_kind"] = score_kind diff --git a/docsgpt/retriever/graph_rag.py b/docsgpt/retriever/graph_rag.py index 4b12e1b3..48fa0fd5 100644 --- a/docsgpt/retriever/graph_rag.py +++ b/docsgpt/retriever/graph_rag.py @@ -584,7 +584,7 @@ class GraphRAGRetriever(BaseRetriever): doc_tokens = num_tokens_from_string(f"{labels['filename']}\n{text}") if cumulative_tokens + doc_tokens >= token_budget: break - docs.append({"text": text, **labels}) + docs.append({"text": text, **labels, **self._connector_labels.for_source(source_id)}) cumulative_tokens += doc_tokens return docs diff --git a/docsgpt/storage/db/models.py b/docsgpt/storage/db/models.py index 4e8d052a..59568d22 100644 --- a/docsgpt/storage/db/models.py +++ b/docsgpt/storage/db/models.py @@ -611,6 +611,17 @@ connector_sessions_table = Table( ) +connector_policies_table = Table( + "connector_policies", + metadata, + Column("connector_key", Text, primary_key=True), + Column("enabled", Boolean, nullable=False, server_default="true"), + Column("credential_mode", Text, nullable=False, server_default="choose"), + Column("updated_by", Text), + Column("updated_at", DateTime(timezone=True), nullable=False, server_default=func.now()), +) + + # --- Conversations, messages, workflows ------------------------------------- conversations_table = Table( diff --git a/docsgpt/storage/db/repositories/connector_policies.py b/docsgpt/storage/db/repositories/connector_policies.py new file mode 100644 index 00000000..2373b8cd --- /dev/null +++ b/docsgpt/storage/db/repositories/connector_policies.py @@ -0,0 +1,58 @@ +"""Repository for ``connector_policies``: the admin's per-connector switches.""" + +from __future__ import annotations + +from typing import Optional + +from sqlalchemy import Connection, text + +from docsgpt.storage.db.base_repository import row_to_dict + +CREDENTIAL_POLICIES = ("choose", "owner", "member") +ALLOW_CUSTOM_MCP_KEY = "connectors.allow_custom_mcp" + + +class ConnectorPoliciesRepository: + """Whether a connector is enabled and which credential mode it forces.""" + + def __init__(self, conn: Connection) -> None: + self._conn = conn + + def all(self) -> dict[str, dict]: + """Every stored policy, by connector key. Missing keys use the defaults.""" + result = self._conn.execute(text("SELECT * FROM connector_policies")) + return {row["connector_key"]: row for row in (row_to_dict(r) for r in result.fetchall())} + + def get(self, connector_key: str) -> Optional[dict]: + row = self._conn.execute( + text("SELECT * FROM connector_policies WHERE connector_key = :key"), {"key": connector_key}, + ).fetchone() + return row_to_dict(row) if row is not None else None + + def upsert( + self, + connector_key: str, + *, + enabled: Optional[bool] = None, + credential_mode: Optional[str] = None, + updated_by: Optional[str] = None, + ) -> dict: + """Set one connector's policy; fields left None keep their value.""" + if credential_mode is not None and credential_mode not in CREDENTIAL_POLICIES: + raise ValueError(f"unknown credential mode: {credential_mode!r}") + row = self._conn.execute( + text( + """ + INSERT INTO connector_policies (connector_key, enabled, credential_mode, updated_by) + VALUES (:key, COALESCE(:enabled, true), COALESCE(:mode, 'choose'), :by) + ON CONFLICT (connector_key) DO UPDATE SET + enabled = COALESCE(:enabled, connector_policies.enabled), + credential_mode = COALESCE(:mode, connector_policies.credential_mode), + updated_by = :by, + updated_at = now() + RETURNING * + """ + ), + {"key": connector_key, "enabled": enabled, "mode": credential_mode, "by": updated_by}, + ).fetchone() + return row_to_dict(row) diff --git a/tests/connectors/test_policies_and_attribution.py b/tests/connectors/test_policies_and_attribution.py new file mode 100644 index 00000000..a3a5f1a7 --- /dev/null +++ b/tests/connectors/test_policies_and_attribution.py @@ -0,0 +1,315 @@ +"""Admin connector policies, sharing modes and connector attribution.""" + +from __future__ import annotations + +import json +from contextlib import contextmanager +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from flask import Flask +from sqlalchemy import text + +from docsgpt.connectors import catalog, service +from docsgpt.security.encryption import encrypt_json + + +@pytest.fixture +def app(): + return Flask(__name__) + + +@contextmanager +def _db(conn): + @contextmanager + def _yield(): + yield conn + + modules = ( + "docsgpt.connectors.service", + "docsgpt.connectors.resolve", + "docsgpt.connectors.attribution", + "docsgpt.api.connector.connections", + "docsgpt.api.admin.connectors", + "docsgpt.api.user.tools.mcp", + ) + patches = [] + for module in modules: + for name in ("db_session", "db_readonly"): + target = f"{module}.{name}" + try: + patches.append(patch(target, _yield)) + patches[-1].start() + except AttributeError: + patches.pop() + try: + yield + finally: + for p in patches: + p.stop() + + +def _connection(conn, user="alice", provider="telegram", status="connected", auth_kind="api_key", + server_url=None, secrets=None) -> str: + return str(conn.execute( + text( + "INSERT INTO connector_sessions (user_id, provider, connector_key, auth_kind, status, account_label, " + "server_url, encrypted_credentials) VALUES (:u, :p, :p, :a, :s, :l, :url, :e) RETURNING id" + ), + {"u": user, "p": provider, "a": auth_kind, "s": status, "l": f"{user}@example.com", "url": server_url, + "e": encrypt_json(secrets or {"credentials": {"token": "tok"}}, user)}, + ).scalar()) + + +def _call(app, resource, method, path, *, user="alice", body=None, roles=None, args=()): + with app.test_request_context(path, method=method.upper(), json=body): + from flask import request + + request.decoded_token = {"sub": user, "roles": roles or ["user"]} if user else None + return getattr(resource(), method)(*args) + + +class TestPresets: + def test_presets_are_in_the_catalog(self): + presets = [d for d in catalog.all_definitions() if d.publisher == "preset"] + assert {p.key for p in presets} >= {"mcp:notion", "mcp:linear"} + for preset in presets: + assert preset.mcp_url.startswith("https://") + assert preset.auth_kind in ("mcp_oauth", "none") + assert preset.tool_templates == ("mcp_tool",) + + def test_existing_mcp_connection_maps_to_its_preset(self): + row = {"provider": "mcp:https://mcp.linear.app", "server_url": "https://mcp.linear.app", + "connector_key": "custom_mcp"} + assert catalog.connector_key_for_row(row) == "mcp:linear" + + +class TestAdminPolicies: + def test_requires_admin(self, app, pg_conn): + from docsgpt.api.admin.connectors import AdminConnectorsResource + + with _db(pg_conn): + resp = _call(app, AdminConnectorsResource, "get", "/api/admin/connectors") + assert resp.status_code == 403 + + def test_lists_setup_state_without_values(self, app, pg_conn, monkeypatch): + from docsgpt.api.admin.connectors import AdminConnectorsResource + from docsgpt.core.settings import settings + + monkeypatch.setattr(settings, "GOOGLE_CLIENT_ID", "secret-client-id") + monkeypatch.setattr(settings, "GOOGLE_CLIENT_SECRET", None) + _connection(pg_conn, provider="google_drive", auth_kind="oauth") + with _db(pg_conn): + resp = _call(app, AdminConnectorsResource, "get", "/api/admin/connectors", roles=["admin"]) + payload = resp.get_json() + drive = next(c for c in payload["connectors"] if c["key"] == "google_drive") + assert drive["required_settings"] == [ + {"name": "GOOGLE_CLIENT_ID", "set": True}, {"name": "GOOGLE_CLIENT_SECRET", "set": False}, + ] + assert drive["connection_count"] == 1 + assert "secret-client-id" not in json.dumps(payload) + assert payload["allow_custom_mcp"] is True + + def test_disable_connector_and_custom_mcp(self, app, pg_conn): + from docsgpt.api.admin.connectors import AdminConnectorsResource + + with _db(pg_conn): + resp = _call(app, AdminConnectorsResource, "put", "/api/admin/connectors", roles=["admin"], body={ + "policies": {"telegram": {"enabled": False, "credential_mode": "member"}}, + "allow_custom_mcp": False, + }) + assert resp.status_code == 200 + entries = {e["key"]: e for e in service.catalog_for_user(pg_conn, "bob", is_admin=False)} + assert entries["telegram"]["state"] == "disabled" + assert entries["custom_mcp"]["state"] == "disabled" + with pytest.raises(service.ConnectorDisabled): + service.create_api_key_connection( + pg_conn, "bob", catalog.get_definition("telegram"), {"token": "long-enough-token"}, + ) + + def test_rejects_unknown_connector(self, app, pg_conn): + from docsgpt.api.admin.connectors import AdminConnectorsResource + + with _db(pg_conn): + resp = _call(app, AdminConnectorsResource, "put", "/api/admin/connectors", roles=["admin"], + body={"policies": {"nope": {"enabled": False}}}) + assert resp.status_code == 400 + + def test_custom_mcp_switch_is_enforced_server_side(self, app, pg_conn): + from docsgpt.api.user.tools.mcp import TestMCPServerConfig + from docsgpt.storage.db.repositories.app_metadata import AppMetadataRepository + + AppMetadataRepository(pg_conn).set("connectors.allow_custom_mcp", "false") + with _db(pg_conn), patch("docsgpt.api.user.tools.mcp._validate_mcp_server_url"): + custom = _call(app, TestMCPServerConfig, "post", "/api/mcp_server/test", + body={"config": {"server_url": "https://tools.example.com/mcp"}}) + assert custom.status_code == 403 + + def test_disabled_connector_stops_existing_tools(self, pg_conn): + from docsgpt.connectors.resolve import resolve_connection + from docsgpt.storage.db.repositories.connector_policies import ConnectorPoliciesRepository + + cid = _connection(pg_conn) + ConnectorPoliciesRepository(pg_conn).upsert("telegram", enabled=False) + with _db(pg_conn): + resolved = resolve_connection({"connection_id": cid, "user_id": "alice"}, "alice") + assert resolved.available is False + + def test_forced_member_mode_overrides_the_share(self, pg_conn): + from docsgpt.connectors.resolve import resolve_connection + from docsgpt.storage.db.repositories.connector_policies import ConnectorPoliciesRepository + + owner = _connection(pg_conn) + bobs = _connection(pg_conn, user="bob") + ConnectorPoliciesRepository(pg_conn).upsert("telegram", credential_mode="member") + with _db(pg_conn): + resolved = resolve_connection( + {"connection_id": owner, "user_id": "alice", "credential_mode": "owner"}, "bob", + ) + assert resolved.connection_id == bobs + + +class TestCredentialMode: + def _tool(self, conn, cid, user="alice"): + return str(conn.execute( + text("INSERT INTO user_tools (user_id, name, connection_id) VALUES (:u, 'telegram', CAST(:c AS uuid)) " + "RETURNING id"), + {"u": user, "c": cid}, + ).scalar()) + + def test_owner_sets_member_mode(self, app, pg_conn): + from docsgpt.api.connector.connections import ToolCredentialMode + + tool = self._tool(pg_conn, _connection(pg_conn)) + with _db(pg_conn): + resp = _call(app, ToolCredentialMode, "put", "/x", body={"mode": "member"}, args=[tool]) + assert resp.status_code == 200 + mode = pg_conn.execute(text("SELECT credential_mode FROM user_tools WHERE id = CAST(:i AS uuid)"), + {"i": tool}).scalar() + assert mode == "member" + + def test_other_users_cannot(self, app, pg_conn): + from docsgpt.api.connector.connections import ToolCredentialMode + + tool = self._tool(pg_conn, _connection(pg_conn)) + with _db(pg_conn): + resp = _call(app, ToolCredentialMode, "put", "/x", user="bob", body={"mode": "member"}, args=[tool]) + assert resp.status_code == 404 + + def test_forced_policy_wins(self, app, pg_conn): + from docsgpt.api.connector.connections import ToolCredentialMode + from docsgpt.storage.db.repositories.connector_policies import ConnectorPoliciesRepository + + tool = self._tool(pg_conn, _connection(pg_conn)) + ConnectorPoliciesRepository(pg_conn).upsert("telegram", credential_mode="owner") + with _db(pg_conn): + resp = _call(app, ToolCredentialMode, "put", "/x", body={"mode": "member"}, args=[tool]) + assert resp.status_code == 409 + assert resp.get_json()["mode"] == "owner" + + +def _executor(user): + from docsgpt.agents.tool_executor import ToolExecutor + + return ToolExecutor(user=user) + + +def _pause(executor, tool, action_name): + with patch("docsgpt.agents.tool_executor.ToolActionParser") as parser: + parser.return_value.parse_args.return_value = ("t1", action_name, {}) + return executor.check_pause( + {"t1": tool}, SimpleNamespace(id="c1", name=action_name, thought_signature=None), "OpenAILLM", + ) + + +class TestSharedWrites: + def _tool(self, cid): + return { + "id": "tool-1", + "user_id": "alice", + "name": "postgres", + "config": {}, + "connection_id": cid, + "credential_mode": "owner", + "actions": [ + {"name": "postgres_execute_sql", "access": "write", "require_approval": False, "active": True}, + {"name": "postgres_get_schema", "access": "read", "require_approval": False, "active": True}, + ], + } + + def test_member_on_owners_account_must_approve_writes(self, pg_conn): + cid = _connection(pg_conn, provider="postgres") + with _db(pg_conn): + pause = _pause(_executor("bob"), self._tool(cid), "postgres_execute_sql") + assert pause["pause_type"] == "awaiting_approval" + + def test_member_reads_are_not_gated(self, pg_conn): + cid = _connection(pg_conn, provider="postgres") + with _db(pg_conn): + assert _pause(_executor("bob"), self._tool(cid), "postgres_get_schema") is None + + def test_owner_keeps_their_own_choice(self, pg_conn): + cid = _connection(pg_conn, provider="postgres") + with _db(pg_conn): + assert _pause(_executor("alice"), self._tool(cid), "postgres_execute_sql") is None + + +class TestAttribution: + def test_tool_call_events_name_the_connector_not_the_account(self, pg_conn): + cid = _connection(pg_conn, provider="mcp:https://mcp.notion.com", auth_kind="mcp_oauth", + server_url="https://mcp.notion.com", secrets={"tokens": {"access_token": "t"}}) + # MCP rows are stored as custom_mcp and named after their preset. + pg_conn.execute(text("UPDATE connector_sessions SET connector_key = 'custom_mcp' WHERE id = CAST(:i AS uuid)"), + {"i": cid}) + executor = _executor("alice") + tool = { + "id": "tool-1", "user_id": "alice", "name": "mcp_tool", "connection_id": cid, + "config": {"server_url": "https://mcp.notion.com/mcp", "auth_type": "oauth"}, + "actions": [{"name": "search", "active": True, "parameters": {"properties": {}}, + "annotations": {"readOnlyHint": True}}], + } + fake_tool = SimpleNamespace(execute_action=lambda *a, **k: "ok", config={}) + call = SimpleNamespace(id="c1", name="search", arguments="{}", thought_signature=None) + with _db(pg_conn), patch("docsgpt.agents.tool_executor.ToolActionParser") as parser, patch.object( + type(executor), "_get_or_load_tool", return_value=fake_tool + ), patch("docsgpt.agents.tool_executor._record_proposed", return_value=True), patch( + "docsgpt.agents.tool_executor._mark_executed" + ): + parser.return_value.parse_args.return_value = ("t1", "search", {}) + events = list(executor._execute({"t1": tool}, call, "OpenAILLM")) + final = events[-1]["data"] + assert final["connector_key"] == "mcp:notion" + assert final["connector_name"] == "Notion" + assert final["access"] == "read" + assert "alice@example.com" not in json.dumps(events) + + def test_built_in_tools_carry_no_connector(self): + executor = _executor("alice") + assert executor._resolve_connection({"id": "x", "name": "memory"}) is None + + def test_citations_name_the_connector(self, pg_conn): + from docsgpt.connectors.attribution import connector_labels + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth") + source = str(pg_conn.execute(text( + "INSERT INTO sources (user_id, name, connection_id) VALUES ('alice', 'Handbook', CAST(:c AS uuid)) " + "RETURNING id" + ), {"c": cid}).scalar()) + upload = str(pg_conn.execute(text( + "INSERT INTO sources (user_id, name) VALUES ('alice', 'Upload') RETURNING id" + )).scalar()) + with _db(pg_conn): + labels = connector_labels([source, upload, "not-a-uuid"]) + assert labels == {source: {"connector_key": "google_drive", "connector_name": "Google Drive"}} + + def test_classic_retriever_stamps_connector(self, pg_conn): + from docsgpt.retriever.classic_rag import ClassicRAG + + retriever = ClassicRAG.__new__(ClassicRAG) + with patch("docsgpt.connectors.attribution.connector_labels", + return_value={"s1": {"connector_key": "google_drive", "connector_name": "Google Drive"}}) as lookup: + first = retriever._connector_labels.for_source("s1") + again = retriever._connector_labels.for_source("s1") + assert first == again == {"connector_key": "google_drive", "connector_name": "Google Drive"} + assert lookup.call_count == 1 diff --git a/tests/storage/db/test_migration_0038.py b/tests/storage/db/test_migration_0038.py index 2740f33a..5b84179b 100644 --- a/tests/storage/db/test_migration_0038.py +++ b/tests/storage/db/test_migration_0038.py @@ -95,6 +95,7 @@ class TestMigration0038: ) assert "connection_id" in _columns(conn, "sources") assert "connection_id" in _columns(conn, "user_tools") + assert {"connector_key", "enabled", "credential_mode"} <= _columns(conn, "connector_policies") def test_backfill_links_legacy_rows(self, pg_engine): url = pg_engine.url.render_as_string(hide_password=False) From cefb117277bd86fc046aaf36ef95c2fb6b1d0679 Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:55:56 +0100 Subject: [PATCH 008/162] Keep tool keys when rolling back 0038 Downgrade removes API-key connections and assumed every tool still had its v1 copy. Tools added through the wizard after the upgrade never had one, and a reconnect can change the key after the backfill, so write the connection's current credentials back to each linked tool first. --- docsgpt/alembic/versions/0038_connections.py | 31 ++++++++++++++++++-- tests/storage/db/test_migration_0038.py | 31 ++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/docsgpt/alembic/versions/0038_connections.py b/docsgpt/alembic/versions/0038_connections.py index 962e5c4a..052a40db 100644 --- a/docsgpt/alembic/versions/0038_connections.py +++ b/docsgpt/alembic/versions/0038_connections.py @@ -31,7 +31,9 @@ Backfill (idempotent, only fills NULLs or unconverted rows): 5. API-key tools (Brave, Telegram, ntfy, PostgreSQL, custom MCP with a key, bearer token or basic auth) get one connection per distinct credential, re-encrypted into v2. The tool keeps its v1 copy for one release so a - rollback still works; the executor prefers the connection. + rollback still works; the executor prefers the connection. Downgrade + writes a v1 copy back to every tool on an API-key connection, including + tools added after the upgrade. 6. ``token_info`` and the secret parts of ``session_data`` (``tokens``, ``client_info``) are encrypted into ``encrypted_credentials`` and removed from the plaintext columns. This needs ``ENCRYPTION_SECRET_KEY`` set to @@ -386,9 +388,32 @@ def _decrypt_back(bind) -> None: from sqlalchemy import text - from docsgpt.security.encryption import CredentialDecryptionError, decrypt_json + from docsgpt.security.encryption import CredentialDecryptionError, decrypt_json, encrypt_credentials - # Tools keep their v1 secrets; connections made only for them go away. + # API-key connections go away, so their tools get a v1 copy back: tools + # added after the upgrade never had one, and a reconnect may have changed + # the key since the backfill. + linked = bind.execute( + text( + "SELECT t.id, t.user_id, c.encrypted_credentials FROM user_tools t " + "JOIN connector_sessions c ON c.id = t.connection_id " + "WHERE c.auth_kind = 'api_key' AND c.encrypted_credentials IS NOT NULL" + ) + ).fetchall() + for row in linked: + try: + credentials = decrypt_json(row.encrypted_credentials, row.user_id).get("credentials") + except CredentialDecryptionError: + continue + if not credentials: + continue + bind.execute( + text( + "UPDATE user_tools SET config = COALESCE(config, '{}'::jsonb) " + "|| jsonb_build_object('encrypted_credentials', CAST(:blob AS text)) WHERE id = :id" + ), + {"blob": encrypt_credentials(credentials, row.user_id), "id": row.id}, + ) bind.execute(text("UPDATE user_tools SET connection_id = NULL WHERE connection_id IN " "(SELECT id FROM connector_sessions WHERE auth_kind = 'api_key')")) bind.execute(text("DELETE FROM connector_sessions WHERE auth_kind = 'api_key'")) diff --git a/tests/storage/db/test_migration_0038.py b/tests/storage/db/test_migration_0038.py index 5b84179b..c99884c3 100644 --- a/tests/storage/db/test_migration_0038.py +++ b/tests/storage/db/test_migration_0038.py @@ -312,6 +312,37 @@ class TestMigration0038Credentials: assert api_rows == 0 _run_alembic(url, "upgrade", "head") + def test_downgrade_keeps_secrets_of_tools_made_after_upgrade(self, pg_engine): + from docsgpt.security.encryption import decrypt_credentials, encrypt_json + + url, _ = self._upgrade_with(pg_engine, _seed_secrets) + # A tool added through the wizard keeps its key only on the connection. + with pg_engine.begin() as conn: + connection_id = conn.execute( + text( + "INSERT INTO connector_sessions (user_id, provider, connector_key, auth_kind, status, " + "account_label, encrypted_credentials, session_data) VALUES ('erin', 'ntfy', 'ntfy', " + "'api_key', 'connected', '…9999', :blob, '{}'::jsonb) RETURNING id" + ), + {"blob": encrypt_json({"credentials": {"token": "ntfy-secret-9999"}}, "erin")}, + ).scalar() + tool_id = conn.execute( + text("INSERT INTO user_tools (user_id, name, config, connection_id) " + "VALUES ('erin', 'ntfy', CAST(:c AS jsonb), :cid) RETURNING id"), + {"c": json.dumps({"server_url": "https://ntfy.sh"}), "cid": connection_id}, + ).scalar() + _run_alembic(url, "downgrade", _0037) + with pg_engine.connect() as conn: + config = conn.execute(text("SELECT config FROM user_tools WHERE id = :i"), {"i": tool_id}).scalar() + assert config["server_url"] == "https://ntfy.sh" + assert decrypt_credentials(config["encrypted_credentials"], "erin") == {"token": "ntfy-secret-9999"} + _run_alembic(url, "upgrade", "head") + with pg_engine.connect() as conn: + relinked = conn.execute( + text("SELECT connection_id FROM user_tools WHERE id = :i"), {"i": tool_id} + ).scalar() + assert relinked is not None + def test_second_upgrade_is_a_no_op(self, pg_engine): from docsgpt.connectors.service import read_secrets From 241111804836c5d0c857b9411297ce8d21ad0ca1 Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:55:56 +0100 Subject: [PATCH 009/162] Show which service did what, and let owners pick whose account a share uses - Tool chips and the streaming status name the service: "Searched Notion", "Used Linear: create issue". Citations from a connector source read "From Google Drive". - The share dialog for a connection-backed tool asks whose account team members use. Owner mode warns which account they act as and needs an explicit OK when the tool can write; an admin-forced mode is locked. - Admin > Connectors: enable or disable each connector, force a sharing mode, allow custom MCP servers, and a setup guide per OAuth connector with its redirect URI and the server settings still missing. Warns when credentials use the default encryption key. - The composer and the agent builder group tools by connection, and the Sources and Tools popovers link to Connect more. - MCP presets open the server form titled after the service. --- frontend/src/admin/Connectors.test.tsx | 158 ++++++++ frontend/src/admin/Connectors.tsx | 375 ++++++++++++++++++ frontend/src/admin/index.tsx | 2 + frontend/src/agents/NewAgent.test.tsx | 6 + frontend/src/agents/NewAgent.tsx | 58 ++- frontend/src/api/endpoints.ts | 3 + .../src/api/services/connectorsService.ts | 16 + frontend/src/components/MessageInput.test.tsx | 2 + frontend/src/components/MessageInput.tsx | 45 ++- .../components/SourcesPopoverFooter.test.tsx | 5 +- .../src/components/SourcesPopoverFooter.tsx | 25 +- .../components/message-input/ToolsTrigger.tsx | 20 +- frontend/src/connectors/i18n.ts | 24 ++ frontend/src/conversation/AnswerFlow.tsx | 17 +- .../src/conversation/ConversationBubble.tsx | 49 ++- .../src/conversation/conversationModels.ts | 9 +- frontend/src/locale/de.json | 37 +- frontend/src/locale/en.json | 43 +- frontend/src/locale/es.json | 37 +- frontend/src/locale/jp.json | 37 +- frontend/src/locale/ru.json | 37 +- frontend/src/locale/zh-TW.json | 37 +- frontend/src/locale/zh.json | 37 +- frontend/src/modals/MCPServerModal.tsx | 7 +- frontend/src/navigation/sections.ts | 7 + frontend/src/settings/Connectors.tsx | 13 +- frontend/src/settings/Tools.tsx | 36 +- frontend/src/settings/types/index.ts | 4 + frontend/src/teams/ShareToTeamModal.test.tsx | 137 +++++++ frontend/src/teams/ShareToTeamModal.tsx | 115 +++++- .../src/utils/streamingStatusUtils.test.ts | 60 +++ frontend/src/utils/streamingStatusUtils.ts | 53 ++- 32 files changed, 1427 insertions(+), 84 deletions(-) create mode 100644 frontend/src/admin/Connectors.test.tsx create mode 100644 frontend/src/admin/Connectors.tsx create mode 100644 frontend/src/teams/ShareToTeamModal.test.tsx diff --git a/frontend/src/admin/Connectors.test.tsx b/frontend/src/admin/Connectors.test.tsx new file mode 100644 index 00000000..265a4315 --- /dev/null +++ b/frontend/src/admin/Connectors.test.tsx @@ -0,0 +1,158 @@ +import { configureStore } from '@reduxjs/toolkit'; +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { Provider } from 'react-redux'; + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ t: (key: string) => key }), +})); + +const getAdmin = vi.fn(); +const updateAdmin = vi.fn(); +vi.mock('../api/services/connectorsService', () => ({ + default: { + getAdmin: (...args: unknown[]) => getAdmin(...args), + updateAdmin: (...args: unknown[]) => updateAdmin(...args), + }, +})); + +import { prefSlice } from '../preferences/preferenceSlice'; +import Connectors from './Connectors'; + +Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + +const connector = (overrides: Record = {}) => ({ + key: 'google_drive', + name: 'Google Drive', + icon: 'google-drive', + publisher: 'built_in', + auth_kind: 'oauth', + enabled: true, + credential_mode: 'choose', + configured: false, + required_settings: [ + { name: 'GOOGLE_CLIENT_ID', set: true }, + { name: 'GOOGLE_CLIENT_SECRET', set: false }, + ], + connection_count: 3, + docs_url: null, + mcp_url: null, + ...overrides, +}); + +const payload = (overrides: Record = {}) => ({ + success: true, + connectors: [ + connector(), + connector({ + key: 'mcp_notion', + name: 'Notion', + icon: 'notion', + publisher: 'preset', + auth_kind: 'mcp_oauth', + configured: true, + required_settings: [], + connection_count: 0, + }), + ], + allow_custom_mcp: true, + default_encryption_key: false, + oauth_redirect_uri: 'https://docs.example/api/connectors/callback', + mcp_redirect_uri: 'https://docs.example/api/mcp_server/callback', + ...overrides, +}); + +describe('Admin Connectors', () => { + let container: HTMLDivElement; + let root: Root; + + beforeEach(() => { + getAdmin.mockReset(); + updateAdmin.mockReset(); + container = document.createElement('div'); + document.body.appendChild(container); + root = createRoot(container); + }); + + afterEach(() => { + act(() => root.unmount()); + container.remove(); + }); + + const render = async () => { + const store = configureStore({ + reducer: { preference: prefSlice.reducer }, + }); + await act(async () => { + root.render( + + + , + ); + }); + }; + + it('lists each connector with its setup state and redirect URIs', async () => { + getAdmin.mockResolvedValue(payload()); + await render(); + const rows = Array.from(container.querySelectorAll('tbody tr')); + expect(rows).toHaveLength(2); + expect(rows[0].textContent).toContain('Google Drive'); + expect(rows[0].textContent).toContain('Needs setup'); + expect(rows[0].textContent).toContain('Setup guide'); + expect(rows[1].textContent).toContain('Preset'); + expect(rows[1].textContent).toContain('Ready'); + expect(rows[1].textContent).not.toContain('Setup guide'); + expect(container.textContent).toContain( + 'https://docs.example/api/connectors/callback', + ); + }); + + it('warns when credentials use the default encryption key', async () => { + getAdmin.mockResolvedValue(payload({ default_encryption_key: true })); + await render(); + expect(container.textContent).toContain('admin.connectors.defaultKey'); + expect(container.textContent).toContain('docsgpt connectors reencrypt'); + }); + + it('saves a connector toggle as a policy', async () => { + getAdmin.mockResolvedValue(payload()); + updateAdmin.mockResolvedValue( + payload({ + connectors: [connector({ enabled: false })], + }), + ); + await render(); + const toggle = container.querySelector( + '[aria-label="Google Drive enabled"]', + )!; + await act(async () => toggle.click()); + expect(updateAdmin).toHaveBeenCalledWith( + { policies: { google_drive: { enabled: false } } }, + null, + ); + expect( + container + .querySelector('[aria-label="Google Drive enabled"]') + ?.getAttribute('aria-checked'), + ).toBe('false'); + }); + + it('shows an error and keeps the page when a save fails', async () => { + getAdmin.mockResolvedValue(payload()); + updateAdmin.mockResolvedValue({ success: false }); + await render(); + await act(async () => + container.querySelector('#allow-custom-mcp')!.click(), + ); + expect(updateAdmin).toHaveBeenCalledWith({ allow_custom_mcp: false }, null); + expect(container.textContent).toContain('Could not save the change.'); + expect(container.querySelectorAll('tbody tr')).toHaveLength(2); + }); + + it('offers a retry when loading fails', async () => { + getAdmin.mockRejectedValue(new Error('offline')); + await render(); + expect(container.textContent).toContain('Failed to load connectors.'); + }); +}); diff --git a/frontend/src/admin/Connectors.tsx b/frontend/src/admin/Connectors.tsx new file mode 100644 index 00000000..b00d4d14 --- /dev/null +++ b/frontend/src/admin/Connectors.tsx @@ -0,0 +1,375 @@ +import { ExternalLink, TriangleAlert } from 'lucide-react'; +import { useCallback, useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useSelector } from 'react-redux'; + +import connectorsService from '../api/services/connectorsService'; +import CopyButton from '../components/CopyButton'; +import { Alert, AlertDescription, AlertTitle } from '../components/ui/alert'; +import { Badge } from '../components/ui/badge'; +import { Button } from '../components/ui/button'; +import { Card } from '../components/ui/card'; +import { LoadingState } from '../components/ui/loading-state'; +import { Modal } from '../components/ui/modal'; +import { SectionHeader } from '../components/ui/section-header'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '../components/ui/select'; +import { SettingRow, SettingRows } from '../components/ui/setting-row'; +import { Switch } from '../components/ui/switch'; +import { + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableHeader, + TableRow, +} from '../components/ui/table'; +import ConnectorIcon from '../connectors/ConnectorIcon'; +import { selectToken } from '../preferences/preferenceSlice'; +import { LoadError, fmtNumber } from './AdminUI'; + +type Policy = 'choose' | 'owner' | 'member'; + +type AdminConnector = { + key: string; + name: string; + icon: string; + publisher: 'built_in' | 'preset' | 'custom'; + auth_kind: string; + enabled: boolean; + credential_mode: Policy; + configured: boolean; + required_settings: { name: string; set: boolean }[]; + connection_count: number; + docs_url: string | null; + mcp_url: string | null; +}; + +type AdminConnectorsData = { + success: boolean; + connectors: AdminConnector[]; + allow_custom_mcp: boolean; + default_encryption_key: boolean; + oauth_redirect_uri: string; + mcp_redirect_uri: string; +}; + +const POLICY_LABELS: Record = { + choose: 'Let owners choose', + member: "Always each member's own account", + owner: "Always the owner's account", +}; + +function CodeRow({ value }: { value: string }) { + return ( + + 
+        {value}
+
+ + + ); +} + +function SetupGuide({ + connector, + redirectUri, + onClose, +}: { + connector: AdminConnector; + redirectUri: string; + onClose: () => void; +}) { + return ( + !open && onClose()} + title={`Set up ${connector.name}`} + description="Register DocsGPT as an OAuth app with the provider, then set these server settings and restart the API and the worker." + footer={ + + } + > +
+
+ + +
+
+ +
    + {connector.required_settings.map((setting) => ( +
  • + {setting.name} + + {setting.set ? 'Set' : 'Missing'} + +
    • + ))} +
+
+ {connector.key === 'google_drive' && ( + + + + Publish the Google OAuth app (or use an internal Workspace app). + Apps left in Testing get refresh tokens that expire after seven + days, which stops background sync. + + + )} + {connector.docs_url && ( + + )} +
+ + ); +} + +/** + * Admin > Connectors: which connectors members may use, whose account a + * shared tool runs with, and what each OAuth connector still needs from the + * server. English only, like the rest of the admin pages. + */ +export default function Connectors() { + const { t } = useTranslation(); + const token = useSelector(selectToken); + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [saveError, setSaveError] = useState(false); + const [guide, setGuide] = useState(null); + + const load = useCallback(async () => { + setLoading(true); + try { + setData(await connectorsService.getAdmin(token)); + } catch { + setData({ success: false } as AdminConnectorsData); + } finally { + setLoading(false); + } + }, [token]); + + useEffect(() => { + load(); + }, [load]); + + const save = async (body: { + policies?: Record; + allow_custom_mcp?: boolean; + }) => { + setSaveError(false); + try { + const next = await connectorsService.updateAdmin(body, token); + if (next?.success) setData(next); + else setSaveError(true); + } catch { + setSaveError(true); + } + }; + + if (data === null && loading) return ; + if (!data?.success) + return ; + + return ( +
+

+ Choose which connectors members can use and whose account a shared tool + runs with. Connectors that need server settings stay listed for members + as Needs admin setup until you add them. +

+ + {data.default_encryption_key && ( + + + {t('admin.connectors.defaultKey')} + + Stored credentials are encrypted with ENCRYPTION_SECRET_KEY, which + still has its public default value. Set your own, keep the old one + in ENCRYPTION_SECRET_KEY_PREVIOUS and run{' '} + + docsgpt connectors reencrypt + + . + + + )} + {saveError && ( + + + Could not save the change. + + )} + + + + + save({ allow_custom_mcp: checked === true }) + } + /> + + + +
+ +

+ Register these with each provider's OAuth app. +

+
+
+ + Google Drive, SharePoint and Confluence + + +
+
+ MCP servers + +
+
+
+ +
+ + + + + + Connector + Status + Connections + Enabled + Shared tools use + + + + + {data.connectors.map((connector) => ( + + + + + {connector.name} + {connector.publisher !== 'built_in' && ( + + {connector.publisher === 'preset' + ? 'Preset' + : 'Custom'} + + )} + + + + + {connector.configured ? 'Ready' : 'Needs setup'} + + + + {fmtNumber(connector.connection_count)} + + + + save({ + policies: { + [connector.key]: { enabled: checked === true }, + }, + }) + } + /> + + + + + + {connector.required_settings.length > 0 && ( + + )} + + + ))} + +
+ +
+ + {guide && ( + setGuide(null)} + /> + )} +
+ ); +} diff --git a/frontend/src/admin/index.tsx b/frontend/src/admin/index.tsx index 5a39bb09..296ebb51 100644 --- a/frontend/src/admin/index.tsx +++ b/frontend/src/admin/index.tsx @@ -8,6 +8,7 @@ import Admins from './Admins'; import Activity from './Activity'; import Overview from './Overview'; import Quotas from './Quotas'; +import Connectors from './Connectors'; import Usage from './Usage'; import Users from './Users'; @@ -39,6 +40,7 @@ export default function Admin() { } /> } /> } /> + } /> } /> } /> diff --git a/frontend/src/agents/NewAgent.test.tsx b/frontend/src/agents/NewAgent.test.tsx index 55d0b937..306ad6b7 100644 --- a/frontend/src/agents/NewAgent.test.tsx +++ b/frontend/src/agents/NewAgent.test.tsx @@ -57,6 +57,12 @@ vi.mock('../api/services/devicesService', () => ({ }, })); +vi.mock('../api/services/connectorsService', () => ({ + default: { + listConnections: () => Promise.resolve({ connections: [] }), + }, +})); + vi.mock('../api/services/modelService', () => ({ default: { getModels: () => jsonResponse({ models: [] }), diff --git a/frontend/src/agents/NewAgent.tsx b/frontend/src/agents/NewAgent.tsx index a565e58c..dabd555d 100644 --- a/frontend/src/agents/NewAgent.tsx +++ b/frontend/src/agents/NewAgent.tsx @@ -45,6 +45,9 @@ import { } from '../components/MultiSelectPopover'; import SourcesPopoverFooter from '../components/SourcesPopoverFooter'; import ToolIcon from '../components/ToolIcon'; +import connectorsService from '../api/services/connectorsService'; +import ConnectorIcon from '../connectors/ConnectorIcon'; +import type { Connection } from '../connectors/types'; import AgentDetailsModal from '../modals/AgentDetailsModal'; import ShareToTeamModal from '../teams/ShareToTeamModal'; import ConfirmationModal from '../modals/ConfirmationModal'; @@ -552,12 +555,22 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { useEffect(() => { const getTools = async () => { - const [toolsResponse, devicesResult] = await Promise.all([ - userService.getUserTools(token), - // Tolerate failures here: the picker should still render the - // tool list even if /api/devices returns an error or 401. - devicesService.list(token).catch(() => ({ devices: [] })), - ]); + const [toolsResponse, devicesResult, connectionsResult] = + await Promise.all([ + userService.getUserTools(token), + // Tolerate failures here: the picker should still render the + // tool list even if /api/devices returns an error or 401. + devicesService.list(token).catch(() => ({ devices: [] })), + connectorsService + .listConnections(token) + .catch(() => ({ connections: [] })), + ]); + const connectionsById = new Map( + ((connectionsResult?.connections ?? []) as Connection[]).map((c) => [ + c.id, + c, + ]), + ); if (!toolsResponse.ok) throw new Error('Failed to fetch tools'); const data = await toolsResponse.json(); // Hide workflow-only builtins (e.g. read_document) from the classic @@ -576,19 +589,39 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { !Number.isNaN(seen) && Date.now() - seen < onlineWindowMs; devicesById.set(d.id, { online, last_seen_at: d.last_seen_at }); }); - // Group ordering: builtins -> defaults -> user tools (sorted via the - // MultiSelectPopover first-appearance grouping). + // Group ordering: builtins -> defaults -> one group per connection + // (the service and its account) -> custom tools, via the + // MultiSelectPopover first-appearance grouping. + const connectionOf = (tool: UserToolType) => + tool.connection_id + ? connectionsById.get(tool.connection_id) + : undefined; + const rank = (tool: UserToolType) => + tool.builtin ? 0 : tool.default ? 1 : connectionOf(tool) ? 2 : 3; const groupFor = (tool: UserToolType): string => { if (tool.builtin) return t('agents.form.toolsPopup.groupBuiltin'); if (tool.default) return t('agents.form.toolsPopup.groupDefault'); + const connection = connectionOf(tool); + if (connection) + return t('agents.form.toolsPopup.groupConnection', { + name: connection.name, + account: connection.account_label, + interpolation: { escapeValue: false }, + }); return t('agents.form.toolsPopup.groupCustom'); }; - const tools: MultiSelectPopoverItem[] = visibleTools.map( - (tool: UserToolType) => { + const tools: MultiSelectPopoverItem[] = [...visibleTools] + .sort((a, b) => rank(a) - rank(b)) + .map((tool: UserToolType) => { + const connection = connectionOf(tool); const base: MultiSelectPopoverItem = { id: tool.id, label: getToolDisplayName(tool), - icon: , + icon: connection ? ( + + ) : ( + + ), group: groupFor(tool), }; if (tool.name === 'remote_device') { @@ -607,8 +640,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { ); } return base; - }, - ); + }); const groupOrder = [ t('agents.form.toolsPopup.groupBuiltin'), t('agents.form.toolsPopup.groupDefault'), diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index 65a5cb42..26604a31 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -105,6 +105,9 @@ const endpoints = { CONNECTION_TOOL_PERMISSIONS: (id: string, toolId: string) => `/api/connections/${encodeURIComponent(id)}/tools/${encodeURIComponent(toolId)}/permissions`, CONNECTIONS_CLAIM: '/api/connections/claim', + TOOL_CREDENTIAL_MODE: (toolId: string) => + `/api/connections/tools/${encodeURIComponent(toolId)}/credential-mode`, + ADMIN_CONNECTORS: '/api/admin/connectors', GET_CHUNKS: ( docId: string, page: number, diff --git a/frontend/src/api/services/connectorsService.ts b/frontend/src/api/services/connectorsService.ts index 985122bc..ebe7c7bd 100644 --- a/frontend/src/api/services/connectorsService.ts +++ b/frontend/src/api/services/connectorsService.ts @@ -107,6 +107,22 @@ const connectorsService = { token, ), ), + setCredentialMode: async ( + toolId: string, + mode: 'owner' | 'member', + token: string | null, + ) => + json( + await apiClient.put( + endpoints.USER.TOOL_CREDENTIAL_MODE(toolId), + { mode }, + token, + ), + ), + getAdmin: async (token: string | null) => + json(await apiClient.get(endpoints.USER.ADMIN_CONNECTORS, token)), + updateAdmin: async (body: Record, token: string | null) => + json(await apiClient.put(endpoints.USER.ADMIN_CONNECTORS, body, token)), claim: async (provider: string, sessionToken: string, token: string | null) => json( await apiClient.post( diff --git a/frontend/src/components/MessageInput.test.tsx b/frontend/src/components/MessageInput.test.tsx index ff658ed7..ef3d832a 100644 --- a/frontend/src/components/MessageInput.test.tsx +++ b/frontend/src/components/MessageInput.test.tsx @@ -9,6 +9,7 @@ vi.mock('react-i18next', () => ({ // The upload modal pulls in the whole ingest UI; the composer never opens it here. vi.mock('../upload/Upload', () => ({ default: () => null })); +import connectorsReducer from '../connectors/connectorsSlice'; import notificationsReducer from '../notifications/notificationsSlice'; import { prefSlice } from '../preferences/preferenceSlice'; import type { RootState } from '../store'; @@ -28,6 +29,7 @@ const makeStore = () => preference: prefSlice.reducer, upload: uploadReducer, notifications: notificationsReducer, + connectors: connectorsReducer, }, }); diff --git a/frontend/src/components/MessageInput.tsx b/frontend/src/components/MessageInput.tsx index 1fd733df..a7c25d5d 100644 --- a/frontend/src/components/MessageInput.tsx +++ b/frontend/src/components/MessageInput.tsx @@ -35,13 +35,18 @@ import { selectToken, setSelectedDocs, } from '../preferences/preferenceSlice'; -import type { RootState } from '../store'; +import type { AppDispatch, RootState } from '../store'; import Upload from '../upload/Upload'; import { isTouchDevice } from '../utils/browserUtils'; import { Button } from './ui/button'; import { IconButton } from './ui/icon-button'; import { type MultiSelectPopoverItem } from './MultiSelectPopover'; import ToolIcon from './ToolIcon'; +import ConnectorIcon from '../connectors/ConnectorIcon'; +import { + loadConnectors, + selectConnections, +} from '../connectors/connectorsSlice'; import { AttachFileButton, AttachmentChipList, @@ -338,6 +343,7 @@ export default function MessageInput({ const [isSourcesPopupOpen, setIsSourcesPopupOpen] = useState(false); const [isToolsPopupOpen, setIsToolsPopupOpen] = useState(false); const [userTools, setUserTools] = useState([]); + const connections = useSelector(selectConnections); const [toolsLoading, setToolsLoading] = useState(false); const [uploadModalState, setUploadModalState] = useState('INACTIVE'); @@ -1561,14 +1567,39 @@ export default function MessageInput({ }, [token]); useEffect(() => { - if (isToolsPopupOpen) fetchUserTools(); + if (isToolsPopupOpen) { + fetchUserTools(); + (dispatch as AppDispatch)(loadConnectors({ token })); + } }, [isToolsPopupOpen, fetchUserTools]); - const toolItems: MultiSelectPopoverItem[] = userTools.map((tool) => ({ - id: tool.id, - label: tool.customName || tool.displayName, - icon: , - })); + // Tools from a connected service sit under that service; the rest are + // built in. Groups only appear once some tool comes from a connection. + const toolConnection = (tool: UserToolType) => + tool.connection_id + ? connections.find((c) => c.id === tool.connection_id) + : undefined; + const anyConnectedTool = userTools.some((tool) => toolConnection(tool)); + const toolItems: MultiSelectPopoverItem[] = [...userTools] + .sort( + (a, b) => + Number(Boolean(toolConnection(a))) - Number(Boolean(toolConnection(b))), + ) + .map((tool) => { + const connection = toolConnection(tool); + return { + id: tool.id, + label: tool.customName || tool.displayName, + icon: connection ? ( + + ) : ( + + ), + group: anyConnectedTool + ? (connection?.name ?? t('settings.tools.groupBuiltIn')) + : undefined, + }; + }); const selectedToolIds = userTools .filter((tool) => tool.status) diff --git a/frontend/src/components/SourcesPopoverFooter.test.tsx b/frontend/src/components/SourcesPopoverFooter.test.tsx index 8e5f6aae..4ed7e827 100644 --- a/frontend/src/components/SourcesPopoverFooter.test.tsx +++ b/frontend/src/components/SourcesPopoverFooter.test.tsx @@ -53,8 +53,11 @@ describe('SourcesPopoverFooter', () => { it('renders the sources link as an inline link Button', async () => { await renderFooter(); - const link = container.querySelector('a')!; + const [link, connect] = Array.from(container.querySelectorAll('a')); expect(link.getAttribute('href')).toBe('/settings/sources'); + expect(connect.getAttribute('href')).toBe( + '/settings/connectors?capability=sync', + ); expect(link.dataset.slot).toBe('button'); expect(link.className).toContain('text-primary'); expect(link.className).toContain('text-sm'); diff --git a/frontend/src/components/SourcesPopoverFooter.tsx b/frontend/src/components/SourcesPopoverFooter.tsx index 0c7a3dcb..27378bad 100644 --- a/frontend/src/components/SourcesPopoverFooter.tsx +++ b/frontend/src/components/SourcesPopoverFooter.tsx @@ -10,7 +10,10 @@ type SourcesPopoverFooterProps = { onUploadClick: () => void; }; -/** Shared footer for source pickers: a link to the sources page and an upload shortcut. */ +/** + * Shared footer for source pickers: links to the sources page and to the + * connectors that can sync content, and an upload shortcut. + */ export default function SourcesPopoverFooter({ onNavigate, onUploadClick, @@ -21,12 +24,20 @@ export default function SourcesPopoverFooter({ // One row when it fits (link left, upload right); on a narrow sheet or a // long locale the button wraps under the link.
- +
+ + +
+ } trigger={ + ) : source.connector_name ? ( +
+ +

+ {t('conversation.sources.fromConnectorTitle', { + name: source.connector_name, + title: source.title, + interpolation: { escapeValue: false }, + })} +

+
) : (
@@ -665,7 +690,13 @@ function onActivateKey( } type AllSourcesProps = { - sources: { title: string; text: string; link?: string }[]; + sources: { + title: string; + text: string; + link?: string; + connector_key?: string | null; + connector_name?: string | null; + }[]; }; function AllSources(sources: AllSourcesProps) { @@ -718,6 +749,20 @@ function AllSources(sources: AllSourcesProps) { )}

+ {source.connector_name && ( +

+ + + {t('conversation.sources.fromConnector', { + name: source.connector_name, + interpolation: { escapeValue: false }, + })} + +

+ )}

{source.text}

diff --git a/frontend/src/conversation/conversationModels.ts b/frontend/src/conversation/conversationModels.ts index e8ba8713..fb0f2360 100644 --- a/frontend/src/conversation/conversationModels.ts +++ b/frontend/src/conversation/conversationModels.ts @@ -61,7 +61,14 @@ export interface Query { conversationId?: string | null; title?: string | null; thought?: string; - sources?: { title: string; text: string; link: string }[]; + sources?: { + title: string; + text: string; + link: string; + // A chunk synced from a connection names the service, never the account. + connector_key?: string | null; + connector_name?: string | null; + }[]; tool_calls?: ToolCallsType[]; // Arrival-ordered layout of the fields above, so reasoning and tool calls // render where they happened. Live-stream only; absent on reload, where diff --git a/frontend/src/locale/de.json b/frontend/src/locale/de.json index 501d5ac0..e18e9030 100644 --- a/frontend/src/locale/de.json +++ b/frontend/src/locale/de.json @@ -1066,7 +1066,13 @@ "ntfy": "Push-Benachrichtigungen über einen ntfy-Server senden.", "postgres": "Schema lesen und SQL auf einer Postgres-Datenbank ausführen.", "custom_mcp": "Einen beliebigen Remote-Server für das Model Context Protocol verbinden.", - "custom_openapi": "Eine OpenAPI-Spezifikation importieren und ihre Endpunkte als Werkzeuge aufrufen." + "custom_openapi": "Eine OpenAPI-Spezifikation importieren und ihre Endpunkte als Werkzeuge aufrufen.", + "mcp_notion": "Notion-Seiten und -Datenbanken durchsuchen, lesen und aktualisieren.", + "mcp_linear": "Linear-Issues und -Projekte finden, anlegen und aktualisieren.", + "mcp_atlassian": "Jira-Issues und Confluence-Seiten durchsuchen und aktualisieren.", + "mcp_sentry": "Sentry-Issues, -Events und -Releases nachschlagen.", + "mcp_asana": "Asana-Aufgaben und -Projekte finden und aktualisieren.", + "mcp_stripe": "Kunden, Zahlungen und Abonnements in Stripe nachschlagen." }, "empty": "Noch nichts verbunden. Verbinde einen Dienst, um seine Inhalte zu synchronisieren oder Agenten neue Werkzeuge zu geben.", "noMatches": "Keine Konnektoren passen zu deiner Suche.", @@ -1170,6 +1176,18 @@ "error": { "decrypt": "Die gespeicherten Zugangsdaten konnten nicht entschlüsselt werden. Verbinde neu, um fortzufahren.", "defaultKey": "Ein Admin muss ENCRYPTION_SECRET_KEY setzen, bevor Dienste verbunden werden können." + }, + "sharing": { + "owner": "Teammitglieder nutzen mein Konto", + "member": "Jedes Mitglied verbindet sein eigenes Konto" + }, + "share": { + "heading": "Welches Konto Teammitglieder nutzen", + "ownerWarning": "Teammitglieder handeln in {{name}} als {{account}}.", + "confirmWrite": "Ich verstehe, dass Teammitglieder mit meinem Konto Aktionen ausführen können.", + "memberNote": "Jedes Mitglied verbindet sein eigenes {{name}}-Konto, bevor es dieses Werkzeug nutzen kann.", + "forced": "Ein Admin hat das für alle Freigaben dieses Konnektors festgelegt.", + "saveFailed": "Die Änderung konnte nicht gespeichert werden. Versuche es erneut." } } }, @@ -1541,7 +1559,10 @@ "noSourcesAvailable": "Keine Quellen verfügbar", "selectedCount": "{{count}} Quellen ausgewählt", "groupOwn": "Deine Quellen", - "groupTeam": "Mit dem Team geteilt" + "groupTeam": "Mit dem Team geteilt", + "connectMore": "Weitere verbinden", + "fromConnector": "Aus {{name}}", + "fromConnectorTitle": "Aus {{name}}: {{title}}" }, "attachments": { "attach": "Anhängen", @@ -1582,7 +1603,9 @@ "accessingMemory": "Greife auf das Gedächtnis zu…", "runningCode": "Führe Code aus…", "creatingArtifact": "Erstelle ein Artefakt…", - "usingTool": "Verwende {{tool}}…" + "usingTool": "Verwende {{tool}}…", + "searchingConnector": "Durchsuche {{name}}...", + "usingConnector": "Nutze {{name}}..." }, "toolChip": { "searchingWeb": "Im Web gesucht · „{{query}}“", @@ -1593,7 +1616,10 @@ "accessingMemory": "Auf Gedächtnis zugegriffen", "runningCode": "Code ausgeführt", "creatingArtifact": "Artefakt erstellt", - "usingTool": "{{tool}} verwendet" + "usingTool": "{{tool}} verwendet", + "searchedConnector": "{{name}} durchsucht", + "readConnector": "Aus {{name}} gelesen", + "usedConnector": "{{name}} genutzt: {{action}}" }, "inlineSteps": { "arguments": "Argumente", @@ -1783,7 +1809,8 @@ "noOptionsMessage": "Keine Werkzeuge verfügbar", "groupBuiltin": "Integriert", "groupDefault": "Standard", - "groupCustom": "Benutzerdefiniert" + "groupCustom": "Benutzerdefiniert", + "groupConnection": "{{name}} · {{account}}" }, "modelsPopup": { "title": "Modelle auswählen", diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index 66f638a7..36e5a1ee 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -1072,7 +1072,13 @@ "ntfy": "Send push notifications through an ntfy server.", "postgres": "Read the schema and run SQL against a Postgres database.", "custom_mcp": "Connect any remote Model Context Protocol server.", - "custom_openapi": "Import an OpenAPI spec and call its endpoints as tools." + "custom_openapi": "Import an OpenAPI spec and call its endpoints as tools.", + "mcp_notion": "Search, read and update Notion pages and databases.", + "mcp_linear": "Find, create and update Linear issues and projects.", + "mcp_atlassian": "Search and update Jira issues and Confluence pages.", + "mcp_sentry": "Look up Sentry issues, events and releases.", + "mcp_asana": "Find and update Asana tasks and projects.", + "mcp_stripe": "Look up customers, payments and subscriptions in Stripe." }, "empty": "Nothing connected yet. Connect a service to sync its content or give agents new tools.", "noMatches": "No connectors match your search.", @@ -1176,6 +1182,18 @@ "error": { "decrypt": "Stored credentials could not be decrypted. Reconnect to continue.", "defaultKey": "An admin must set ENCRYPTION_SECRET_KEY before services can be connected." + }, + "sharing": { + "owner": "Team members use my account", + "member": "Each member connects their own account" + }, + "share": { + "heading": "Whose account team members use", + "ownerWarning": "Team members will act as {{account}} on {{name}}.", + "confirmWrite": "I understand teammates can take actions with my account.", + "memberNote": "Each member connects their own {{name}} account before they can use this tool.", + "forced": "An admin chose this for every share of this connector.", + "saveFailed": "Could not save the change. Try again." } } }, @@ -1547,7 +1565,10 @@ "noSourcesAvailable": "No sources available", "selectedCount": "{{count}} sources selected", "groupOwn": "Your sources", - "groupTeam": "Shared with team" + "groupTeam": "Shared with team", + "connectMore": "Connect more", + "fromConnector": "From {{name}}", + "fromConnectorTitle": "From {{name}}: {{title}}" }, "attachments": { "attach": "Attach", @@ -1593,7 +1614,9 @@ "accessingMemory": "Accessing memory…", "runningCode": "Running code…", "creatingArtifact": "Creating an artifact…", - "usingTool": "Using {{tool}}…" + "usingTool": "Using {{tool}}…", + "searchingConnector": "Searching {{name}}...", + "usingConnector": "Using {{name}}..." }, "toolChip": { "searchingWeb": "Searched the web · “{{query}}”", @@ -1604,7 +1627,10 @@ "accessingMemory": "Accessed memory", "runningCode": "Ran code", "creatingArtifact": "Created an artifact", - "usingTool": "Used {{tool}}" + "usingTool": "Used {{tool}}", + "searchedConnector": "Searched {{name}}", + "readConnector": "Read from {{name}}", + "usedConnector": "Used {{name}}: {{action}}" }, "inlineSteps": { "arguments": "Arguments", @@ -1799,7 +1825,8 @@ "noOptionsMessage": "No tools available", "groupBuiltin": "Built-in", "groupDefault": "Default", - "groupCustom": "Custom" + "groupCustom": "Custom", + "groupConnection": "{{name}} · {{account}}" }, "modelsPopup": { "title": "Select Models", @@ -2444,7 +2471,8 @@ "admins": "Admins", "usage": "Usage", "quotas": "Quotas", - "audit": "Activity" + "audit": "Activity", + "connectors": "Connectors" }, "quotas": { "unpriced": { @@ -2452,6 +2480,9 @@ "body": "These were used this {{period}} and recorded at $0: {{models}}. Use a token limit for them, or declare their rates in the model catalog.", "model": "{{model}} ({{tokens}} tokens)" } + }, + "connectors": { + "defaultKey": "Set ENCRYPTION_SECRET_KEY before connecting services." } } } diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index d1c250ea..8c76df28 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -1066,7 +1066,13 @@ "ntfy": "Envía notificaciones push mediante un servidor ntfy.", "postgres": "Lee el esquema y ejecuta SQL en una base de datos Postgres.", "custom_mcp": "Conecta cualquier servidor remoto de Model Context Protocol.", - "custom_openapi": "Importa una especificación OpenAPI y usa sus endpoints como herramientas." + "custom_openapi": "Importa una especificación OpenAPI y usa sus endpoints como herramientas.", + "mcp_notion": "Busca, lee y actualiza páginas y bases de datos de Notion.", + "mcp_linear": "Encuentra, crea y actualiza incidencias y proyectos de Linear.", + "mcp_atlassian": "Busca y actualiza incidencias de Jira y páginas de Confluence.", + "mcp_sentry": "Consulta incidencias, eventos y versiones de Sentry.", + "mcp_asana": "Encuentra y actualiza tareas y proyectos de Asana.", + "mcp_stripe": "Consulta clientes, pagos y suscripciones en Stripe." }, "empty": "Aún no hay nada conectado. Conecta un servicio para sincronizar su contenido o dar nuevas herramientas a los agentes.", "noMatches": "Ningún conector coincide con tu búsqueda.", @@ -1170,6 +1176,18 @@ "error": { "decrypt": "No se pudieron descifrar las credenciales guardadas. Reconecta para continuar.", "defaultKey": "Un administrador debe definir ENCRYPTION_SECRET_KEY antes de poder conectar servicios." + }, + "sharing": { + "owner": "Los miembros usan mi cuenta", + "member": "Cada miembro conecta su propia cuenta" + }, + "share": { + "heading": "Qué cuenta usan los miembros del equipo", + "ownerWarning": "Los miembros del equipo actuarán como {{account}} en {{name}}.", + "confirmWrite": "Entiendo que los miembros del equipo pueden realizar acciones con mi cuenta.", + "memberNote": "Cada miembro conecta su propia cuenta de {{name}} antes de poder usar esta herramienta.", + "forced": "Un administrador eligió esto para todos los recursos compartidos de este conector.", + "saveFailed": "No se pudo guardar el cambio. Inténtalo de nuevo." } } }, @@ -1541,7 +1559,10 @@ "noSourcesAvailable": "No hay fuentes disponibles", "selectedCount": "{{count}} fuentes seleccionadas", "groupOwn": "Tus fuentes", - "groupTeam": "Compartidas con el equipo" + "groupTeam": "Compartidas con el equipo", + "connectMore": "Conectar más", + "fromConnector": "De {{name}}", + "fromConnectorTitle": "De {{name}}: {{title}}" }, "attachments": { "attach": "Adjuntar", @@ -1582,7 +1603,9 @@ "accessingMemory": "Accediendo a la memoria…", "runningCode": "Ejecutando código…", "creatingArtifact": "Creando un artefacto…", - "usingTool": "Usando {{tool}}…" + "usingTool": "Usando {{tool}}…", + "searchingConnector": "Buscando en {{name}}...", + "usingConnector": "Usando {{name}}..." }, "toolChip": { "searchingWeb": "Buscado en la web · “{{query}}”", @@ -1593,7 +1616,10 @@ "accessingMemory": "Memoria consultada", "runningCode": "Código ejecutado", "creatingArtifact": "Artefacto creado", - "usingTool": "Usado {{tool}}" + "usingTool": "Usado {{tool}}", + "searchedConnector": "Buscó en {{name}}", + "readConnector": "Leyó de {{name}}", + "usedConnector": "Usó {{name}}: {{action}}" }, "inlineSteps": { "arguments": "Argumentos", @@ -1783,7 +1809,8 @@ "noOptionsMessage": "No hay herramientas disponibles", "groupBuiltin": "Integradas", "groupDefault": "Predeterminadas", - "groupCustom": "Personalizadas" + "groupCustom": "Personalizadas", + "groupConnection": "{{name}} · {{account}}" }, "modelsPopup": { "title": "Seleccionar Modelos", diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index 9b408cda..0f1eb4cb 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -1065,7 +1065,13 @@ "ntfy": "ntfy サーバー経由でプッシュ通知を送信します。", "postgres": "Postgres データベースのスキーマを読み取り、SQL を実行します。", "custom_mcp": "任意のリモート Model Context Protocol サーバーに接続します。", - "custom_openapi": "OpenAPI 仕様をインポートし、そのエンドポイントをツールとして呼び出します。" + "custom_openapi": "OpenAPI 仕様をインポートし、そのエンドポイントをツールとして呼び出します。", + "mcp_notion": "Notion のページやデータベースを検索、閲覧、更新します。", + "mcp_linear": "Linear の課題やプロジェクトを検索、作成、更新します。", + "mcp_atlassian": "Jira の課題と Confluence のページを検索、更新します。", + "mcp_sentry": "Sentry の課題、イベント、リリースを調べます。", + "mcp_asana": "Asana のタスクやプロジェクトを検索、更新します。", + "mcp_stripe": "Stripe の顧客、支払い、サブスクリプションを調べます。" }, "empty": "まだ何も接続されていません。サービスを接続すると、コンテンツを同期したり、エージェントに新しいツールを提供したりできます。", "noMatches": "検索に一致するコネクタはありません。", @@ -1164,6 +1170,18 @@ "error": { "decrypt": "保存された認証情報を復号できませんでした。続行するには再接続してください。", "defaultKey": "サービスを接続する前に、管理者が ENCRYPTION_SECRET_KEY を設定する必要があります。" + }, + "sharing": { + "owner": "メンバーは自分のアカウントを使う", + "member": "各メンバーが自分のアカウントを接続する" + }, + "share": { + "heading": "チームメンバーが使うアカウント", + "ownerWarning": "チームメンバーは {{name}} で {{account}} として操作します。", + "confirmWrite": "チームメンバーが自分のアカウントで操作を行えることを理解しました。", + "memberNote": "各メンバーは、このツールを使う前に自分の {{name}} アカウントを接続します。", + "forced": "このコネクタのすべての共有に対して管理者が設定しています。", + "saveFailed": "変更を保存できませんでした。もう一度お試しください。" } } }, @@ -1533,7 +1551,10 @@ "noSourcesAvailable": "利用可能なソースがありません", "selectedCount": "{{count}} 件のソースを選択中", "groupOwn": "自分のソース", - "groupTeam": "チームと共有" + "groupTeam": "チームと共有", + "connectMore": "さらに接続", + "fromConnector": "{{name}} から", + "fromConnectorTitle": "{{name}} から: {{title}}" }, "attachments": { "attach": "添付", @@ -1574,7 +1595,9 @@ "accessingMemory": "メモリにアクセス中…", "runningCode": "コードを実行中…", "creatingArtifact": "アーティファクトを作成中…", - "usingTool": "{{tool}} を使用中…" + "usingTool": "{{tool}} を使用中…", + "searchingConnector": "{{name}} を検索中...", + "usingConnector": "{{name}} を使用中..." }, "toolChip": { "searchingWeb": "ウェブ検索済み · 「{{query}}」", @@ -1585,7 +1608,10 @@ "accessingMemory": "メモリにアクセス済み", "runningCode": "コード実行済み", "creatingArtifact": "アーティファクト作成済み", - "usingTool": "{{tool}} を使用済み" + "usingTool": "{{tool}} を使用済み", + "searchedConnector": "{{name}} を検索しました", + "readConnector": "{{name}} から読み取りました", + "usedConnector": "{{name}} を使用: {{action}}" }, "inlineSteps": { "arguments": "引数", @@ -1774,7 +1800,8 @@ "noOptionsMessage": "利用可能なツールがありません", "groupBuiltin": "ビルトイン", "groupDefault": "デフォルト", - "groupCustom": "カスタム" + "groupCustom": "カスタム", + "groupConnection": "{{name}} · {{account}}" }, "modelsPopup": { "title": "モデルを選択", diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index b18c6471..515a2fc8 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -1106,7 +1106,13 @@ "ntfy": "Отправляет push-уведомления через сервер ntfy.", "postgres": "Читает схему и выполняет SQL в базе данных Postgres.", "custom_mcp": "Подключает любой удалённый сервер Model Context Protocol.", - "custom_openapi": "Импортирует спецификацию OpenAPI и вызывает её эндпоинты как инструменты." + "custom_openapi": "Импортирует спецификацию OpenAPI и вызывает её эндпоинты как инструменты.", + "mcp_notion": "Поиск, чтение и обновление страниц и баз данных Notion.", + "mcp_linear": "Поиск, создание и обновление задач и проектов Linear.", + "mcp_atlassian": "Поиск и обновление задач Jira и страниц Confluence.", + "mcp_sentry": "Просмотр проблем, событий и релизов Sentry.", + "mcp_asana": "Поиск и обновление задач и проектов Asana.", + "mcp_stripe": "Просмотр клиентов, платежей и подписок в Stripe." }, "empty": "Пока ничего не подключено. Подключите сервис, чтобы синхронизировать его контент или дать агентам новые инструменты.", "noMatches": "Нет коннекторов, подходящих под запрос.", @@ -1220,6 +1226,18 @@ "error": { "decrypt": "Не удалось расшифровать сохранённые учётные данные. Переподключитесь, чтобы продолжить.", "defaultKey": "Администратор должен задать ENCRYPTION_SECRET_KEY, прежде чем можно будет подключать сервисы." + }, + "sharing": { + "owner": "Участники используют мой аккаунт", + "member": "Каждый участник подключает свой аккаунт" + }, + "share": { + "heading": "Чей аккаунт используют участники команды", + "ownerWarning": "Участники команды будут действовать в {{name}} как {{account}}.", + "confirmWrite": "Я понимаю, что участники команды могут выполнять действия от имени моего аккаунта.", + "memberNote": "Каждый участник подключает свой аккаунт {{name}}, прежде чем пользоваться этим инструментом.", + "forced": "Администратор задал это для всех общих доступов к этому коннектору.", + "saveFailed": "Не удалось сохранить изменение. Попробуйте ещё раз." } } }, @@ -1595,7 +1613,10 @@ "noSourcesAvailable": "Нет доступных источников", "selectedCount": "Выбрано источников: {{count}}", "groupOwn": "Ваши источники", - "groupTeam": "Общие с командой" + "groupTeam": "Общие с командой", + "connectMore": "Подключить ещё", + "fromConnector": "Из {{name}}", + "fromConnectorTitle": "Из {{name}}: {{title}}" }, "attachments": { "attach": "Прикрепить", @@ -1636,7 +1657,9 @@ "accessingMemory": "Обращаюсь к памяти…", "runningCode": "Выполняю код…", "creatingArtifact": "Создаю артефакт…", - "usingTool": "Использую {{tool}}…" + "usingTool": "Использую {{tool}}…", + "searchingConnector": "Поиск в {{name}}...", + "usingConnector": "Использую {{name}}..." }, "toolChip": { "searchingWeb": "Поиск в интернете · «{{query}}»", @@ -1647,7 +1670,10 @@ "accessingMemory": "Обращение к памяти выполнено", "runningCode": "Код выполнен", "creatingArtifact": "Артефакт создан", - "usingTool": "Использован {{tool}}" + "usingTool": "Использован {{tool}}", + "searchedConnector": "Поиск в {{name}}", + "readConnector": "Прочитано из {{name}}", + "usedConnector": "Использован {{name}}: {{action}}" }, "inlineSteps": { "arguments": "Аргументы", @@ -1839,7 +1865,8 @@ "noOptionsMessage": "Нет доступных инструментов", "groupBuiltin": "Встроенные", "groupDefault": "По умолчанию", - "groupCustom": "Пользовательские" + "groupCustom": "Пользовательские", + "groupConnection": "{{name}} · {{account}}" }, "modelsPopup": { "title": "Выберите Модели", diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index 67ea0cfc..89513c4a 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -1065,7 +1065,13 @@ "ntfy": "透過 ntfy 伺服器傳送推播通知。", "postgres": "讀取 Postgres 資料庫的結構並執行 SQL。", "custom_mcp": "連接任何遠端 Model Context Protocol 伺服器。", - "custom_openapi": "匯入 OpenAPI 規格，並將其端點作為工具呼叫。" + "custom_openapi": "匯入 OpenAPI 規格，並將其端點作為工具呼叫。", + "mcp_notion": "搜尋、閱讀和更新 Notion 頁面與資料庫。", + "mcp_linear": "尋找、建立和更新 Linear 議題與專案。", + "mcp_atlassian": "搜尋和更新 Jira 議題與 Confluence 頁面。", + "mcp_sentry": "查看 Sentry 問題、事件和發行版本。", + "mcp_asana": "尋找和更新 Asana 任務與專案。", + "mcp_stripe": "查看 Stripe 中的客戶、付款和訂閱。" }, "empty": "尚未連線任何服務。連線一個服務以同步其內容，或為代理提供新工具。", "noMatches": "沒有符合搜尋的連接器。", @@ -1164,6 +1170,18 @@ "error": { "decrypt": "無法解密已儲存的憑證。請重新連線以繼續。", "defaultKey": "管理員必須先設定 ENCRYPTION_SECRET_KEY，才能連線服務。" + }, + "sharing": { + "owner": "團隊成員使用我的帳號", + "member": "每位成員連線自己的帳號" + }, + "share": { + "heading": "團隊成員使用誰的帳號", + "ownerWarning": "團隊成員將在 {{name}} 中以 {{account}} 的身分操作。", + "confirmWrite": "我了解團隊成員可以使用我的帳號執行操作。", + "memberNote": "每位成員須先連線自己的 {{name}} 帳號，才能使用此工具。", + "forced": "管理員已為此連接器的所有共用設定此項。", + "saveFailed": "無法儲存變更。請再試一次。" } } }, @@ -1533,7 +1551,10 @@ "noSourcesAvailable": "沒有可用的來源", "selectedCount": "已選擇 {{count}} 個來源", "groupOwn": "我的來源", - "groupTeam": "團隊共享" + "groupTeam": "團隊共享", + "connectMore": "連線更多", + "fromConnector": "來自 {{name}}", + "fromConnectorTitle": "來自 {{name}}：{{title}}" }, "attachments": { "attach": "附件", @@ -1574,7 +1595,9 @@ "accessingMemory": "正在存取記憶…", "runningCode": "正在執行程式碼…", "creatingArtifact": "正在建立 Artifact…", - "usingTool": "正在使用 {{tool}}…" + "usingTool": "正在使用 {{tool}}…", + "searchingConnector": "正在搜尋 {{name}}...", + "usingConnector": "正在使用 {{name}}..." }, "toolChip": { "searchingWeb": "已搜尋網路 · 「{{query}}」", @@ -1585,7 +1608,10 @@ "accessingMemory": "已存取記憶", "runningCode": "已執行程式碼", "creatingArtifact": "已建立 Artifact", - "usingTool": "已使用 {{tool}}" + "usingTool": "已使用 {{tool}}", + "searchedConnector": "已搜尋 {{name}}", + "readConnector": "已從 {{name}} 讀取", + "usedConnector": "已使用 {{name}}：{{action}}" }, "inlineSteps": { "arguments": "參數", @@ -1774,7 +1800,8 @@ "noOptionsMessage": "沒有可用的工具", "groupBuiltin": "內建", "groupDefault": "預設", - "groupCustom": "自訂" + "groupCustom": "自訂", + "groupConnection": "{{name}} · {{account}}" }, "modelsPopup": { "title": "選擇模型", diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index 6fd94305..714c7d8a 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -1065,7 +1065,13 @@ "ntfy": "通过 ntfy 服务器发送推送通知。", "postgres": "读取 Postgres 数据库的结构并执行 SQL。", "custom_mcp": "连接任意远程 Model Context Protocol 服务器。", - "custom_openapi": "导入 OpenAPI 规范，并将其端点作为工具调用。" + "custom_openapi": "导入 OpenAPI 规范，并将其端点作为工具调用。", + "mcp_notion": "搜索、阅读和更新 Notion 页面与数据库。", + "mcp_linear": "查找、创建和更新 Linear 问题与项目。", + "mcp_atlassian": "搜索和更新 Jira 问题与 Confluence 页面。", + "mcp_sentry": "查看 Sentry 问题、事件和发布版本。", + "mcp_asana": "查找和更新 Asana 任务与项目。", + "mcp_stripe": "查看 Stripe 中的客户、付款和订阅。" }, "empty": "尚未连接任何服务。连接一个服务以同步其内容，或为智能体提供新工具。", "noMatches": "没有与搜索匹配的连接器。", @@ -1164,6 +1170,18 @@ "error": { "decrypt": "无法解密保存的凭据。请重新连接以继续。", "defaultKey": "管理员必须先设置 ENCRYPTION_SECRET_KEY，才能连接服务。" + }, + "sharing": { + "owner": "团队成员使用我的账号", + "member": "每位成员连接自己的账号" + }, + "share": { + "heading": "团队成员使用谁的账号", + "ownerWarning": "团队成员将在 {{name}} 中以 {{account}} 的身份操作。", + "confirmWrite": "我了解团队成员可以使用我的账号执行操作。", + "memberNote": "每位成员需先连接自己的 {{name}} 账号才能使用此工具。", + "forced": "管理员已为此连接器的所有共享设定了此项。", + "saveFailed": "无法保存更改。请重试。" } } }, @@ -1533,7 +1551,10 @@ "noSourcesAvailable": "没有可用的来源", "selectedCount": "已选择 {{count}} 个来源", "groupOwn": "我的来源", - "groupTeam": "团队共享" + "groupTeam": "团队共享", + "connectMore": "连接更多", + "fromConnector": "来自 {{name}}", + "fromConnectorTitle": "来自 {{name}}：{{title}}" }, "attachments": { "attach": "附件", @@ -1574,7 +1595,9 @@ "accessingMemory": "正在访问记忆…", "runningCode": "正在运行代码…", "creatingArtifact": "正在创建工件…", - "usingTool": "正在使用 {{tool}}…" + "usingTool": "正在使用 {{tool}}…", + "searchingConnector": "正在搜索 {{name}}...", + "usingConnector": "正在使用 {{name}}..." }, "toolChip": { "searchingWeb": "已搜索网络 · “{{query}}”", @@ -1585,7 +1608,10 @@ "accessingMemory": "已访问记忆", "runningCode": "已运行代码", "creatingArtifact": "已创建工件", - "usingTool": "已使用 {{tool}}" + "usingTool": "已使用 {{tool}}", + "searchedConnector": "已搜索 {{name}}", + "readConnector": "已从 {{name}} 读取", + "usedConnector": "已使用 {{name}}：{{action}}" }, "inlineSteps": { "arguments": "参数", @@ -1774,7 +1800,8 @@ "noOptionsMessage": "没有可用的工具", "groupBuiltin": "内置", "groupDefault": "默认", - "groupCustom": "自定义" + "groupCustom": "自定义", + "groupConnection": "{{name}} · {{account}}" }, "modelsPopup": { "title": "选择模型", diff --git a/frontend/src/modals/MCPServerModal.tsx b/frontend/src/modals/MCPServerModal.tsx index 7fd62f6c..d8a38ba8 100644 --- a/frontend/src/modals/MCPServerModal.tsx +++ b/frontend/src/modals/MCPServerModal.tsx @@ -558,7 +558,12 @@ export default function MCPServerModal({ ? t('settings.tools.mcp.reconnectServer', { defaultValue: 'Reconnect Server', }) - : t('settings.tools.mcp.addServer') + : server?.preset + ? t('settings.connectors.wizard.connectTitle', { + name: server.displayName, + interpolation: { escapeValue: false }, + }) + : t('settings.tools.mcp.addServer') } size="lg" mobileVariant="sheet" diff --git a/frontend/src/navigation/sections.ts b/frontend/src/navigation/sections.ts index 6f5c680b..b2eb6e66 100644 --- a/frontend/src/navigation/sections.ts +++ b/frontend/src/navigation/sections.ts @@ -220,6 +220,13 @@ export const ADMIN_SECTION: Section = { labelKey: 'admin.tabs.quotas', icon: Gauge, }, + { + key: 'connectors', + path: '/admin/connectors', + labelKey: 'admin.tabs.connectors', + icon: Plug, + feature: 'connectors', + }, { key: 'audit', path: '/admin/audit', diff --git a/frontend/src/settings/Connectors.tsx b/frontend/src/settings/Connectors.tsx index 8656327b..843bdfbf 100644 --- a/frontend/src/settings/Connectors.tsx +++ b/frontend/src/settings/Connectors.tsx @@ -83,9 +83,20 @@ export default function Connectors() { const custom = catalog.filter((c) => c.publisher === 'custom'); const openConnector = catalog.find((c) => c.key === openKey) ?? null; + // "Connect more" in the composer opens the connectors that can do what the + // picker is for: sync content, or give tools. + const capability = searchParams.get('capability'); + const visible = useMemo(() => { const query = search.trim().toLowerCase(); return catalog + .filter((connector) => + capability === 'sync' + ? connector.capabilities.includes('sync') + : capability === 'tools' + ? connector.capabilities.some((c) => c !== 'sync') + : true, + ) .filter((connector) => { if (filter === 'connected') return isConnected(connector); if (filter !== 'all') return connector.category === filter; @@ -98,7 +109,7 @@ export default function Connectors() { connectorDescription(t, connector).toLowerCase().includes(query), ) .sort((a, b) => STATE_ORDER[a.state] - STATE_ORDER[b.state]); - }, [catalog, filter, search, t]); + }, [catalog, filter, search, t, capability]); const open = (connector: ConnectorDefinition) => { if (connector.state === 'available' || connector.state === 'custom') { diff --git a/frontend/src/settings/Tools.tsx b/frontend/src/settings/Tools.tsx index 1e7c6731..aea97ce1 100644 --- a/frontend/src/settings/Tools.tsx +++ b/frontend/src/settings/Tools.tsx @@ -20,6 +20,7 @@ import ConnectorIcon from '../connectors/ConnectorIcon'; import { loadConnectors, selectConnections, + selectConnectorCatalog, } from '../connectors/connectorsSlice'; import { useLoaderState } from '../hooks'; import AddToolModal from '../modals/AddToolModal'; @@ -28,7 +29,9 @@ import MCPServerModal from '../modals/MCPServerModal'; import { ActiveState } from '../models/misc'; import { selectToken } from '../preferences/preferenceSlice'; import type { AppDispatch } from '../store'; -import ShareToTeamModal from '../teams/ShareToTeamModal'; +import ShareToTeamModal, { + type ShareCredentials, +} from '../teams/ShareToTeamModal'; import RemoteDeviceConfig from './RemoteDeviceConfig'; import ToolConfig from './ToolConfig'; import { APIToolType, UserToolType } from './types'; @@ -38,6 +41,7 @@ export default function Tools() { const token = useSelector(selectToken); const dispatch = useDispatch(); const connections = useSelector(selectConnections); + const catalog = useSelector(selectConnectorCatalog); const location = useLocation(); const navigate = useNavigate(); @@ -64,6 +68,30 @@ export default function Tools() { [toolId: string]: string; }>({}); + // A connection-backed tool shares its connection's account or asks each + // member to connect their own; the share dialog shows that choice. + const shareCredentials = ( + tool: UserToolType, + ): ShareCredentials | undefined => { + const connection = tool.connection_id + ? connections.find((c) => c.id === tool.connection_id) + : undefined; + if (!connection) return undefined; + const policy = catalog.find( + (c) => c.key === connection.connector_key, + )?.credential_policy; + return { + toolId: tool.id, + connectorName: connection.name, + account: connection.account_label, + mode: tool.credential_mode === 'member' ? 'member' : 'owner', + forcedMode: policy === 'owner' || policy === 'member' ? policy : null, + hasWrites: (tool.actions ?? []).some( + (action) => action.access === 'write', + ), + }; + }; + const handleDeleteTool = (tool: UserToolType) => { setToolToDelete(tool); setDeleteModalState('ACTIVE'); @@ -478,7 +506,11 @@ export default function Tools() { resourceType="tool" resourceId={toolToShare.id} resourceName={toolToShare.customName || toolToShare.displayName} - onClose={() => setToolToShare(null)} + credentials={shareCredentials(toolToShare)} + onClose={() => { + setToolToShare(null); + getUserTools(); + }} /> )}
diff --git a/frontend/src/settings/types/index.ts b/frontend/src/settings/types/index.ts index cc325908..4073eb8d 100644 --- a/frontend/src/settings/types/index.ts +++ b/frontend/src/settings/types/index.ts @@ -170,6 +170,8 @@ export type UserToolType = { team_access?: 'viewer' | 'editor' | null; // The connection whose account or credentials the tool runs with. connection_id?: string | null; + // Whether team members use the owner's account or their own. + credential_mode?: 'owner' | 'member'; config: { [key: string]: any; }; @@ -193,6 +195,8 @@ export type UserToolType = { }; active: boolean; require_approval?: boolean; + // Read or write, set on tools that come from a connection. + access?: 'read' | 'write'; }[]; }; diff --git a/frontend/src/teams/ShareToTeamModal.test.tsx b/frontend/src/teams/ShareToTeamModal.test.tsx new file mode 100644 index 00000000..f62cc7e5 --- /dev/null +++ b/frontend/src/teams/ShareToTeamModal.test.tsx @@ -0,0 +1,137 @@ +import { configureStore } from '@reduxjs/toolkit'; +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { Provider } from 'react-redux'; + +vi.mock('react-i18next', () => ({ + useTranslation: () => ({ + t: (key: string, opts?: Record) => + opts?.account ? `${key}|${opts.account}` : key, + }), +})); + +vi.mock('../api/services/teamsService', () => ({ + default: { + list: vi.fn().mockResolvedValue({ + teams: [{ id: 't1', name: 'Support', slug: 'support', owner_id: 'me' }], + }), + listResourceShares: vi.fn().mockResolvedValue({ shares: [] }), + listMembers: vi.fn().mockResolvedValue({ members: [] }), + }, +})); + +const setCredentialMode = vi.fn(); +vi.mock('../api/services/connectorsService', () => ({ + default: { + setCredentialMode: (...args: unknown[]) => setCredentialMode(...args), + }, +})); + +import { prefSlice } from '../preferences/preferenceSlice'; +import ShareToTeamModal, { type ShareCredentials } from './ShareToTeamModal'; +import teamsReducer from './teamsSlice'; + +Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + +const credentials = ( + overrides: Partial = {}, +): ShareCredentials => ({ + toolId: 'tool-1', + connectorName: 'Linear', + account: 'lena@meridian.example', + mode: 'owner', + hasWrites: false, + ...overrides, +}); + +describe('ShareToTeamModal credentials', () => { + let container: HTMLDivElement; + let root: Root; + + beforeEach(() => { + setCredentialMode.mockReset(); + container = document.createElement('div'); + document.body.appendChild(container); + root = createRoot(container); + }); + + afterEach(() => { + act(() => root.unmount()); + container.remove(); + }); + + const render = async (creds?: ShareCredentials) => { + const store = configureStore({ + reducer: { preference: prefSlice.reducer, teams: teamsReducer }, + }); + await act(async () => { + root.render( + + undefined} + /> + , + ); + }); + }; + + const toggle = (value: 'owner' | 'member') => + Array.from( + document.body.querySelectorAll( + '[data-slot="option-card"]', + ), + ).find((item) => + item.textContent?.includes(`settings.connectors.sharing.${value}`), + )!; + const picker = () => + document.body.querySelector('[role="combobox"]')!; + + it('shows nothing about accounts for a resource without a connection', async () => { + await render(); + expect(document.body.textContent).not.toContain( + 'settings.connectors.share.heading', + ); + }); + + it('warns that members act as the owner in owner mode', async () => { + await render(credentials()); + expect(toggle('owner').getAttribute('aria-checked')).toBe('true'); + expect(document.body.textContent).toContain( + 'settings.connectors.share.ownerWarning|lena@meridian.example', + ); + }); + + it('blocks sharing an owner-mode tool with writes until confirmed', async () => { + await render(credentials({ hasWrites: true })); + expect(picker().disabled).toBe(true); + const box = document.body.querySelector( + '#share-confirm-writes', + )!; + await act(async () => box.click()); + expect(picker().disabled).toBe(false); + }); + + it('saves a mode change and rolls back when the save fails', async () => { + setCredentialMode.mockResolvedValue({ success: false }); + await render(credentials({ hasWrites: true })); + await act(async () => toggle('member').click()); + expect(setCredentialMode).toHaveBeenCalledWith('tool-1', 'member', null); + expect(toggle('owner').getAttribute('aria-checked')).toBe('true'); + expect(document.body.textContent).toContain( + 'settings.connectors.share.saveFailed', + ); + }); + + it('locks the other mode when an admin forces one', async () => { + await render(credentials({ forcedMode: 'member' })); + expect(toggle('member').getAttribute('aria-checked')).toBe('true'); + expect(toggle('owner').disabled).toBe(true); + expect(document.body.textContent).toContain( + 'settings.connectors.share.forced', + ); + }); +}); diff --git a/frontend/src/teams/ShareToTeamModal.tsx b/frontend/src/teams/ShareToTeamModal.tsx index 54f4c9d7..da19ff81 100644 --- a/frontend/src/teams/ShareToTeamModal.tsx +++ b/frontend/src/teams/ShareToTeamModal.tsx @@ -1,4 +1,4 @@ -import { CircleAlert, Trash2 } from 'lucide-react'; +import { CircleAlert, Trash2, UserRound, UsersRound } from 'lucide-react'; import { useEffect, useMemo, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useDispatch, useSelector } from 'react-redux'; @@ -9,7 +9,11 @@ import teamsService, { ResourceType, TeamMember, } from '../api/services/teamsService'; +import connectorsService from '../api/services/connectorsService'; import { Alert, AlertDescription } from '../components/ui/alert'; +import { Checkbox } from '../components/ui/checkbox'; +import { Label } from '../components/ui/label'; +import { OptionCard } from '../components/ui/option-card'; import { Avatar } from '../components/ui/avatar'; import { Button } from '../components/ui/button'; import { @@ -41,10 +45,23 @@ import { AppDispatch } from '../store'; import { decodeJwtPayload } from '../utils/jwtUtils'; import { loadTeams, selectTeams } from './teamsSlice'; +/** A connection-backed tool: whose account shares of it run with. */ +export type ShareCredentials = { + toolId: string; + connectorName: string; + account: string; + mode: 'owner' | 'member'; + /** Set when an admin forces one mode for every share of this connector. */ + forcedMode?: 'owner' | 'member' | null; + /** Owner-mode shares of a tool with write actions need an explicit OK. */ + hasWrites: boolean; +}; + type Props = { resourceType: ResourceType; resourceId: string; resourceName?: string; + credentials?: ShareCredentials; onClose: () => void; }; @@ -84,6 +101,7 @@ export default function ShareToTeamModal({ resourceType, resourceId, resourceName, + credentials, onClose, }: Props) { const { t } = useTranslation(); @@ -101,6 +119,27 @@ export default function ShareToTeamModal({ const [loadError, setLoadError] = useState(false); const [actionError, setActionError] = useState(null); + const [credentialMode, setCredentialMode] = useState<'owner' | 'member'>( + credentials?.forcedMode ?? credentials?.mode ?? 'owner', + ); + const [writesConfirmed, setWritesConfirmed] = useState(false); + const needsWriteConfirm = + !!credentials && credentialMode === 'owner' && credentials.hasWrites; + const changeCredentialMode = (mode: 'owner' | 'member') => { + if (!credentials || mode === credentialMode) return; + const previous = credentialMode; + setCredentialMode(mode); + connectorsService + .setCredentialMode(credentials.toolId, mode, token) + .then((data) => { + if (!data?.success) throw new Error('save failed'); + }) + .catch(() => { + setCredentialMode(previous); + setActionError(t('settings.connectors.share.saveFailed')); + }); + }; + // The access level applied to the next suggestion picked from the combobox. const [accessLevel, setAccessLevel] = useState('viewer'); @@ -508,6 +547,76 @@ export default function ShareToTeamModal({

{t('settings.teams.share.noTeams')}

) : ( <> + {credentials && ( +
+ +
+ {(['owner', 'member'] as const).map((mode) => ( + : } + title={t(`settings.connectors.sharing.${mode}`)} + selected={credentialMode === mode} + disabled={ + !!credentials.forcedMode && + credentials.forcedMode !== mode + } + onClick={() => changeCredentialMode(mode)} + /> + ))} +
+ {credentials.forcedMode && ( +

+ {t('settings.connectors.share.forced')} +

+ )} + {credentialMode === 'owner' ? ( + + + ) : ( +

+ {t('settings.connectors.share.memberNote', { + name: credentials.connectorName, + interpolation: { escapeValue: false }, + })} +

+ )} + {needsWriteConfirm && ( +
+ + setWritesConfirmed(checked === true) + } + /> + +
+ )} +
+ )} + {/* Add row: type-ahead combobox + access level select. */}
@@ -517,7 +626,9 @@ export default function ShareToTeamModal({ variant="combobox" role="combobox" aria-expanded={pickerOpen} - disabled={committing} + disabled={ + committing || (needsWriteConfirm && !writesConfirmed) + } data-placeholder="" className="min-w-0 flex-1 justify-start" > diff --git a/frontend/src/utils/streamingStatusUtils.test.ts b/frontend/src/utils/streamingStatusUtils.test.ts index 1b30823c..5973bdc2 100644 --- a/frontend/src/utils/streamingStatusUtils.test.ts +++ b/frontend/src/utils/streamingStatusUtils.test.ts @@ -125,4 +125,64 @@ describe('getToolChipLabel', () => { ), ).toBe('conversation.toolChip.usingTool|Mcp Tool'); }); + it('names a connection-backed call after its service', () => { + const notion = { tool_name: 'mcp_tool', connector_name: 'Notion' }; + expect( + getToolChipLabel( + call({ ...notion, action_name: 'notion-search', access: 'read' }), + t, + ), + ).toBe('conversation.toolChip.searchedConnector|Notion'); + expect( + getToolChipLabel( + call({ + ...notion, + action_name: 'notion-fetch', + access: 'read', + status: 'pending', + }), + t, + ), + ).toBe('conversation.streamingStatus.usingConnector|Notion'); + expect( + getToolChipLabel( + call({ ...notion, action_name: 'notion-fetch', access: 'read' }), + t, + ), + ).toBe('conversation.toolChip.readConnector|Notion'); + }); + + it('names the action of a write without the service prefix', () => { + expect( + getToolChipLabel( + call({ + tool_name: 'mcp_tool', + connector_name: 'Linear', + action_name: 'linear_create_issue', + access: 'write', + }), + t, + ), + ).toBe('conversation.toolChip.usedConnector|Linear,create issue'); + expect( + getToolChipLabel( + call({ + tool_name: 'telegram', + connector_name: 'Telegram', + action_name: 'send_message', + access: 'write', + }), + t, + ), + ).toBe('conversation.toolChip.usedConnector|Telegram,send message'); + }); + + it('keeps built-in labels for web search even with a connector name', () => { + expect( + getToolChipLabel( + call({ connector_name: 'Brave', arguments: { query: 'docsgpt' } }), + t, + ), + ).toBe('conversation.toolChip.searchingWeb|docsgpt'); + }); }); diff --git a/frontend/src/utils/streamingStatusUtils.ts b/frontend/src/utils/streamingStatusUtils.ts index f1925447..dc0a466d 100644 --- a/frontend/src/utils/streamingStatusUtils.ts +++ b/frontend/src/utils/streamingStatusUtils.ts @@ -41,6 +41,53 @@ const NO_ESCAPE = { interpolation: { escapeValue: false } } as const; */ export type ToolActivity = { key: string; values?: Record }; +const CONNECTOR_SEARCH_WORDS = /search|query|find|list/i; + +/** + * A call to a connection-backed tool, named after its service: "Searched + * Notion", "Read from Google Drive", "Used Linear: create issue". Reads that + * look up something search; other reads read; writes name their action. + */ +function describeConnectorCall(toolCall: ToolCallsType): ToolActivity | null { + const name = toolCall.connector_name; + if (!name) return null; + const action = toolCall.action_name ?? ''; + if (toolCall.access === 'write') { + const readable = action + .replace(/^[a-z0-9]+[_-](?=[a-z])/i, (prefix) => + // "linear_create_issue" and "notion-create-pages" drop the service + // prefix; a bare verb ("search") keeps its whole name. + name.toLowerCase().startsWith(prefix.slice(0, -1).toLowerCase()) + ? '' + : prefix, + ) + .replace(/[_-]+/g, ' ') + .trim() + .toLowerCase(); + return { key: 'connectorWrite', values: { name, action: readable } }; + } + if (CONNECTOR_SEARCH_WORDS.test(action)) + return { key: 'connectorSearch', values: { name } }; + return { key: 'connectorRead', values: { name } }; +} + +// Connector activities read differently while running and once done. +const CONNECTOR_KEYS: Record< + 'streamingStatus' | 'toolChip', + Record +> = { + streamingStatus: { + connectorSearch: 'searchingConnector', + connectorRead: 'usingConnector', + connectorWrite: 'usingConnector', + }, + toolChip: { + connectorSearch: 'searchedConnector', + connectorRead: 'readConnector', + connectorWrite: 'usedConnector', + }, +}; + export function describeToolCall(toolCall: ToolCallsType): ToolActivity { const { tool_name, action_name, arguments: args } = toolCall; const query = typeof args?.query === 'string' ? args.query : undefined; @@ -63,6 +110,8 @@ export function describeToolCall(toolCall: ToolCallsType): ToolActivity { return { key: 'readingPage', values: { target } }; } } + const connector = describeConnectorCall(toolCall); + if (connector) return connector; if (action_name === 'run_code') return { key: 'runningCode' }; if (ARTIFACT_ACTIONS.has(action_name)) return { key: 'creatingArtifact' }; if (tool_name === 'internal_search') return { key: 'searchingKnowledge' }; @@ -84,7 +133,9 @@ function activityLabel( t: TFunction, ): string { const key = - activity.key === 'web' ? GENERIC_SEARCH_KEY[namespace] : activity.key; + activity.key === 'web' + ? GENERIC_SEARCH_KEY[namespace] + : (CONNECTOR_KEYS[namespace][activity.key] ?? activity.key); return t( `conversation.${namespace}.${key}`, activity.values ? { ...activity.values, ...NO_ESCAPE } : undefined, From d9f823d601e783ddb9aae8d62a44ee6a5bccfd4d Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:59:36 +0100 Subject: [PATCH 010/162] Stop syncing sources of a connector an admin turned off Scheduled syncs checked the connection's status but not the admin policy, so a disabled connector kept pulling data. Connector sources now return "disabled" and S3/Reddit syncs get no credentials until the connector is turned back on. --- docsgpt/connectors/service.py | 14 ++++++++++++ docsgpt/worker.py | 13 +++++++---- tests/connectors/test_runtime.py | 38 ++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/docsgpt/connectors/service.py b/docsgpt/connectors/service.py index 5e00d795..1bcb1352 100644 --- a/docsgpt/connectors/service.py +++ b/docsgpt/connectors/service.py @@ -1200,6 +1200,20 @@ def load_policies(conn) -> dict[str, dict]: return policies +def connector_enabled(conn, row: dict) -> bool: + """Whether an admin left on the connector a connection row belongs to. + + Args: + conn: Open database connection. + row: A ``connector_sessions`` row. + + Returns: + False when the connector (or custom MCP servers) is turned off. + """ + key = catalog.connector_key_for_row(row) + return (load_policies(conn).get(key) or {}).get("enabled") is not False + + def ensure_connector_allowed(conn, connector_key: Optional[str]) -> None: """Refuse a new connection to a connector an admin turned off. diff --git a/docsgpt/worker.py b/docsgpt/worker.py index 83f7abc0..6dede185 100755 --- a/docsgpt/worker.py +++ b/docsgpt/worker.py @@ -2186,14 +2186,15 @@ def _with_connection_credentials(source_data, connection_id: str): S3 and Reddit sources made from a connection keep their keys on the connection only, never in ``sources.remote_data``. Returns None when the - connection is gone or needs reconnecting. + connection is gone, needs reconnecting, or its connector is turned off. """ from docsgpt.connectors import service from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository with db_readonly() as conn: row = ConnectorSessionsRepository(conn).get(str(connection_id)) - if row is None: + enabled = row is not None and service.connector_enabled(conn, row) + if row is None or not enabled: return None try: credentials = service.get_credentials(row) @@ -2235,9 +2236,10 @@ def sync_connector_source(self, source_id: str) -> Dict[str, Any]: source_id: The source to sync. Returns: - ``{"status": "success" | "paused" | "skipped"}`` plus the ingest result. + ``{"status": "success" | "paused" | "disabled" | "skipped"}`` plus + the ingest result. """ - from docsgpt.connectors.service import ConnectionUnavailable, normalize_status + from docsgpt.connectors.service import ConnectionUnavailable, connector_enabled, normalize_status from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository with db_readonly() as conn: @@ -2256,8 +2258,11 @@ def sync_connector_source(self, source_id: str) -> Dict[str, Any]: if source and source.get("connection_id") else None ) + enabled = connection is not None and connector_enabled(conn, connection) if not source or not connection: return {"status": "skipped"} + if not enabled: + return {"status": "disabled"} if normalize_status(connection) != "connected": return {"status": "paused"} remote_data = source.get("remote_data") or {} diff --git a/tests/connectors/test_runtime.py b/tests/connectors/test_runtime.py index b02f2442..a58d622a 100644 --- a/tests/connectors/test_runtime.py +++ b/tests/connectors/test_runtime.py @@ -219,6 +219,44 @@ class TestScheduledSync: assert result == {"status": "paused"} ingest.assert_not_called() + def test_disabled_connector_is_not_synced(self, pg_conn): + from docsgpt import worker + from docsgpt.storage.db.repositories.connector_policies import ConnectorPoliciesRepository + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth", + secrets={"token_info": {"access_token": "a"}}) + source = pg_conn.execute(text( + "INSERT INTO sources (user_id, name, type, sync_frequency, connection_id, remote_data) " + "VALUES ('alice', 'Drive', 'connector:file', 'weekly', CAST(:c AS uuid), " + "'{\"provider\": \"google_drive\"}') RETURNING id" + ), {"c": cid}).scalar() + ConnectorPoliciesRepository(pg_conn).upsert("google_drive", enabled=False) + + @contextmanager + def _yield(): + yield pg_conn + + with patch.object(worker, "db_readonly", _yield), patch.object(worker, "ingest_connector") as ingest: + result = worker.sync_connector_source(MagicMock(), str(source)) + assert result == {"status": "disabled"} + ingest.assert_not_called() + + def test_disabled_connector_gives_remote_sync_no_credentials(self, pg_conn): + from docsgpt import worker + from docsgpt.storage.db.repositories.connector_policies import ConnectorPoliciesRepository + + cid = _connection(pg_conn, provider="s3", auth_kind="api_key", + secrets={"credentials": {"aws_access_key_id": "AKIA", "aws_secret_access_key": "s"}}) + + @contextmanager + def _yield(): + yield pg_conn + + with patch.object(worker, "db_readonly", _yield): + assert worker._with_connection_credentials({"bucket": "b"}, cid)["aws_access_key_id"] == "AKIA" + ConnectorPoliciesRepository(pg_conn).upsert("s3", enabled=False) + assert worker._with_connection_credentials({"bucket": "b"}, cid) is None + def test_sync_runs_as_the_connection_without_a_browser(self, pg_conn): from docsgpt import worker From 8f00118afc97ba6e52dfd9b85744738709ca30a0 Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:59:36 +0100 Subject: [PATCH 011/162] Document connectors and the upgrade to encrypted connections - Guides > Connectors: using connections, sharing modes, attribution, and admin setup (encryption key and rotation, redirect URIs, Google Drive, SharePoint, Confluence, S3, MCP presets). - Integration pages register CONNECTOR_REDIRECT_BASE_URI as-is; the ?provider= suffix never matched what the backend sends. VITE_GOOGLE_CLIENT_ID is optional. - Upgrading: set ENCRYPTION_SECRET_KEY before migrating multi-user installs. --- docs/content/Guides/Connectors.mdx | 137 ++++++++++++++++++ .../Integrations/confluence-connector.mdx | 12 +- .../Integrations/google-drive-connector.mdx | 20 ++- .../Integrations/mcp-tool-integration.mdx | 6 +- .../Integrations/sharepoint-connector.mdx | 12 +- docs/content/Guides/_meta.js | 4 + docs/content/Sources/_meta.js | 4 + docs/content/Tools/basics.mdx | 3 +- docs/content/upgrading.mdx | 17 +++ 9 files changed, 196 insertions(+), 19 deletions(-) create mode 100644 docs/content/Guides/Connectors.mdx diff --git a/docs/content/Guides/Connectors.mdx b/docs/content/Guides/Connectors.mdx new file mode 100644 index 00000000..26e6ffb9 --- /dev/null +++ b/docs/content/Guides/Connectors.mdx @@ -0,0 +1,137 @@ +--- +title: Connectors +description: Connect DocsGPT to the services your team uses. One connection can sync content into Sources and give agents tools, with credentials encrypted on the server. +--- + +import { Callout } from 'nextra/components' +import { Steps } from 'nextra/components' + +# Connectors + +A connector is a service DocsGPT can connect to: Google Drive, SharePoint, Confluence, Amazon S3, Reddit, Brave Search, Telegram, ntfy, PostgreSQL, curated MCP servers (Notion, Linear, Atlassian, Sentry, Asana, Stripe), and any MCP server or OpenAPI spec you add yourself. A **connection** is one signed-in account or one saved API key for a connector. + +Connections are the single place credentials live. A synced source and an agent tool both point at a connection, so you sign in once, reconnect once when a token expires, and disconnect in one place. + +## Using connectors + +Open **Settings > Connectors**. Filters narrow the list by category, and **Connected** shows what you already use. + +1. Pick a connector. Services with OAuth open the provider's sign-in page in a pop-up; API-key services ask for the key. +2. Choose what the connection sets up. File services offer a synced source (pick files or folders, then how often to sync: never, daily, weekly or monthly). Tool services create their tools right away. +3. Adjust what the tools may do. Each action is **Read** or **Write**, and each can be **Always allow**, **Needs approval** or **Off**. Writes default to needing approval. + +The **Add Source** and **Add Tool** dialogs, the composer's Sources and Tools menus, and the agent builder all start the same flow, and list tools grouped by the connection they use. + +Click a connected card to see its account, the sources and tools that use it, and to **Reconnect** or **Disconnect**. Disconnecting keeps a source's indexed content but stops its sync. + +### When a connection needs attention + +If a provider rejects a token that DocsGPT cannot refresh, the connection is marked **Reconnect needed**, its synced sources pause, and you get a notification. Reconnecting the same account resumes them. In a chat, a tool whose connection is missing or expired shows a **Connect** prompt instead of failing; the answer continues once you connect. + +### Sharing a tool with a team + +Sharing a tool that uses a connection asks whose account team members use: + +- **Team members use my account.** Everyone acts as your account on that service. When the tool can write, you confirm this before sharing, and a member's write actions always need approval. +- **Each member connects their own account.** Members see a **Connect** prompt the first time they use the tool. + +Tools from OAuth MCP servers default to each member's own account. An admin can force one mode for a connector. + +### What answers show + +Tool calls name the service that ran them: "Searched Notion", "Read from Google Drive", "Used Linear: create issue". Citations from a synced source read "From Google Drive". Answers never show which account was used. + +## Admin setup + +Admins manage connectors in **Admin > Connectors**. It lists every connector with its status, how many connections use it, and: + +- **Enabled** turns a connector off for everyone. Members see it as turned off and cannot connect it, its tools stop working, and its sources stop syncing until you turn it back on. Existing connections are kept. +- **Shared tools use** is **Let owners choose** (default), **Always the owner's account** or **Always each member's own account**. +- **Allow custom MCP servers** controls whether members can add their own MCP servers. Presets are enabled one by one. +- **Setup guide** on an OAuth connector shows the redirect URI to register and which server settings are still missing. + +A connector that needs server settings stays visible to members as **Needs admin setup** until the settings are present. + +### Encryption key + +Credentials are encrypted with a key derived from `ENCRYPTION_SECRET_KEY` and bound to the connection's owner. **Set your own value before anyone connects a service.** + +```env +ENCRYPTION_SECRET_KEY=a-long-random-value +``` + +When authentication is on (`AUTH_TYPE` set), DocsGPT refuses to store new credentials while the key is the public default, and Admin > Connectors shows a warning. A single-user local install keeps working and logs a warning at startup. + +To rotate the key, move the old value to `ENCRYPTION_SECRET_KEY_PREVIOUS`, set the new one, restart the API and worker, then run: + +```bash +docsgpt connectors reencrypt +``` + +The command prints how many connections it rewrote. Connections it cannot decrypt with either key are marked **Reconnect needed** for their owners. Once it has run, you can remove `ENCRYPTION_SECRET_KEY_PREVIOUS`. + +### Redirect URIs + +OAuth connectors return to one callback, set by `CONNECTOR_REDIRECT_BASE_URI` (default `http://127.0.0.1:7091/api/connectors/callback`). Register it with each provider exactly as set, without query parameters. MCP servers use `MCP_OAUTH_REDIRECT_URI`, which is derived from the same base when unset. Admin > Connectors shows both values with copy buttons. + +If the frontend runs on a different origin from the API, list it in `CONNECTOR_ALLOWED_ORIGINS` so the sign-in pop-up can hand the result back. + +### Google Drive + + + +### Create OAuth credentials + +In the [Google Cloud Console](https://console.cloud.google.com/), enable the **Google Drive API**, then create an **OAuth client ID** of type **Web application**. Add the redirect URI from Admin > Connectors under **Authorized redirect URIs**. + +### Set the server settings + +```env +GOOGLE_CLIENT_ID=your-client-id +GOOGLE_CLIENT_SECRET=your-client-secret +``` + +To offer Google's own file picker, also build the frontend with `VITE_GOOGLE_CLIENT_ID` (and optionally `VITE_GOOGLE_PICKER_API_KEY`). Without them, members browse files in DocsGPT's picker. + +### Publish the app + +Publish the OAuth consent screen, or make it an **Internal** Workspace app. + + +Apps left in **Testing** get refresh tokens that expire after seven days, which stops background sync. + + + + +### SharePoint and OneDrive + +Register an app in [Microsoft Entra ID](https://entra.microsoft.com/) with a **Web** redirect URI from Admin > Connectors, create a client secret, and grant the delegated Microsoft Graph permissions `Files.Read`, `Sites.Read.All` and `User.Read`. + +```env +MICROSOFT_CLIENT_ID=your-application-id +MICROSOFT_CLIENT_SECRET=your-client-secret +MICROSOFT_TENANT_ID=common # or your tenant id for a single-tenant app +``` + +See [SharePoint / OneDrive](/Guides/Integrations/sharepoint-connector) for tenant options. + +### Confluence + +Create an **OAuth 2.0 (3LO)** app in the [Atlassian developer console](https://developer.atlassian.com/console/myapps/), add the redirect URI from Admin > Connectors as its callback URL, and add the scopes `read:page:confluence`, `read:space:confluence`, `read:attachment:confluence` and `read:me`. See [Confluence](/Guides/Integrations/confluence-connector) for details. + +```env +CONFLUENCE_CLIENT_ID=your-client-id +CONFLUENCE_CLIENT_SECRET=your-client-secret +``` + +### Amazon S3 + +S3 needs no server settings. Each member connects with an access key that can list and read the bucket (`s3:ListBucket`, `s3:GetObject`), then picks a bucket and optional path prefix. A custom endpoint URL connects S3-compatible storage such as MinIO or Cloudflare R2. + +### MCP presets + +Notion, Linear, Atlassian, Sentry, Asana and Stripe are remote MCP servers that support OAuth with dynamic client registration, so they need no server settings. Members sign in with their own account. The presets ship in `docsgpt/connectors/presets/mcp.yaml`. + + +Background sync runs on the Celery worker and beat. Keep both running, as in the bundled Compose and Kubernetes files. + diff --git a/docs/content/Guides/Integrations/confluence-connector.mdx b/docs/content/Guides/Integrations/confluence-connector.mdx index d9702fd2..6b81f2a7 100644 --- a/docs/content/Guides/Integrations/confluence-connector.mdx +++ b/docs/content/Guides/Integrations/confluence-connector.mdx @@ -10,6 +10,10 @@ import { Steps } from 'nextra/components' Connect your Confluence Cloud workspace to upload and process pages directly as an external knowledge base. Supports page content and attachments (PDFs, Office files, text files, images, and more). Authentication is handled via Atlassian OAuth 2.0 with automatic token refresh. + +Members connect this service from **Settings > Connectors**, and one sign-in serves every source that uses it. Admins can check what is still missing in **Admin > Connectors**. See [Connectors](/Guides/Connectors). + + ## Setup @@ -18,8 +22,8 @@ Connect your Confluence Cloud workspace to upload and process pages directly as 1. Go to [developer.atlassian.com/console/myapps](https://developer.atlassian.com/console/myapps/) and click **Create** > **OAuth 2.0 integration** 2. Under **Authorization**, add a callback URL: - - Local: `http://localhost:7091/api/connectors/callback?provider=confluence` - - Production: `https://yourdomain.com/api/connectors/callback?provider=confluence` + - Local: `http://localhost:7091/api/connectors/callback` + - Production: `https://yourdomain.com/api/connectors/callback` (the value of `CONNECTOR_REDIRECT_BASE_URI`, registered as-is) ### Step 2: Configure Permissions @@ -56,7 +60,7 @@ VITE_CONFLUENCE_CLIENT_ID=your-atlassian-client-id ### Step 5: Restart and Use -Restart your application, then go to the upload section in DocsGPT and select **Confluence** as the source. You'll be redirected to Atlassian to sign in, then can browse spaces and select pages to process. +Restart your application, then go to **Settings > Connectors** and pick **Confluence**. You'll be redirected to Atlassian to sign in, then can browse spaces and select pages to process. @@ -64,6 +68,6 @@ Restart your application, then go to the upload section in DocsGPT and select ** - **Option not appearing** — Verify `VITE_CONFLUENCE_CLIENT_ID` is set in the frontend `.env`, then restart. - **Sign-in popup closes but the account never connects** — The frontend origin is not allowed to receive the result. Add it to `CONNECTOR_ALLOWED_ORIGINS` in the backend `.env`. -- **Authentication failed** — Check that the callback URL matches exactly, including `?provider=confluence`. +- **Authentication failed** — Check that the callback URL matches exactly and equals `CONNECTOR_REDIRECT_BASE_URI`, with no query parameters. - **No accessible sites** — Ensure the authenticating user has access to at least one Confluence Cloud site. - **Permission denied** — Verify that the Confluence API scopes are enabled in your Atlassian app settings. diff --git a/docs/content/Guides/Integrations/google-drive-connector.mdx b/docs/content/Guides/Integrations/google-drive-connector.mdx index 3014c784..53af5056 100644 --- a/docs/content/Guides/Integrations/google-drive-connector.mdx +++ b/docs/content/Guides/Integrations/google-drive-connector.mdx @@ -10,6 +10,10 @@ import { Steps } from 'nextra/components' Connect your Google Drive account to upload and process files directly as an external knowledge base. Supports Google Workspace files (Docs, Sheets, Slides), Office files, PDFs, text files, CSVs, images, and more. Authentication is handled via Google OAuth 2.0 with automatic token refresh. + +Members connect this service from **Settings > Connectors**, and one sign-in serves every source that uses it. Admins can check what is still missing in **Admin > Connectors**. See [Connectors](/Guides/Connectors). + + ## Setup @@ -26,8 +30,8 @@ Connect your Google Drive account to upload and process files directly as an ext 3. Select **Web application** as the application type 4. Add your DocsGPT URL to **Authorized JavaScript origins** (e.g. `http://localhost:3000`) 5. Add your callback URL to **Authorized redirect URIs**: - - Local: `http://localhost:7091/api/connectors/callback?provider=google_drive` - - Production: `https://yourdomain.com/api/connectors/callback?provider=google_drive` + - Local: `http://localhost:7091/api/connectors/callback` + - Production: `https://yourdomain.com/api/connectors/callback` (the value of `CONNECTOR_REDIRECT_BASE_URI`, registered as-is) 6. Click **Create** and copy the **Client ID** and **Client Secret** ### Step 3: Configure Environment Variables @@ -39,7 +43,7 @@ GOOGLE_CLIENT_ID=your-google-client-id GOOGLE_CLIENT_SECRET=your-google-client-secret ``` -Add to your frontend `.env` file: +Optionally, to use Google's own file picker instead of DocsGPT's, add to your frontend `.env` file: ```env VITE_GOOGLE_CLIENT_ID=your-google-client-id @@ -49,23 +53,23 @@ VITE_GOOGLE_CLIENT_ID=your-google-client-id |----------|-------------|----------| | `GOOGLE_CLIENT_ID` | OAuth Client ID from GCP Credentials | Yes | | `GOOGLE_CLIENT_SECRET` | OAuth Client Secret from GCP Credentials | Yes | -| `VITE_GOOGLE_CLIENT_ID` | Same Client ID, used by the frontend to show the Google Drive option | Yes | +| `VITE_GOOGLE_CLIENT_ID` | Same Client ID, used by the frontend for Google's file picker | No | | `CONNECTOR_ALLOWED_ORIGINS` | Comma-separated frontend origins allowed to receive the sign-in result, e.g. `https://docsgpt.example.com`. Not needed when the frontend shares the API origin, or in local dev when the callback is on `localhost`/`127.0.0.1` and the frontend runs on port 5173 | When the frontend is on its own origin | -Make sure to use the same Google Client ID in both backend and frontend configurations. +If you set `VITE_GOOGLE_CLIENT_ID`, use the same Client ID as the backend. Publish the OAuth consent screen (or use an internal Workspace app): apps left in Testing get refresh tokens that expire after seven days, which stops background sync. ### Step 4: Restart and Use -Restart your application, then go to the upload section in DocsGPT and select **Google Drive** as the source. You'll be redirected to Google to sign in, then can browse and select files to process. +Restart your application, then go to **Settings > Connectors** and pick **Google Drive**. You'll be redirected to Google to sign in, then can browse and select files to process. ## Troubleshooting -- **Option not appearing** — Verify `VITE_GOOGLE_CLIENT_ID` is set in the frontend `.env`, then restart. -- **Authentication failed** — Check that the redirect URI matches exactly, including `?provider=google_drive`. Ensure the Google Drive API is enabled. +- **Shows Needs admin setup** — `GOOGLE_CLIENT_ID` or `GOOGLE_CLIENT_SECRET` is missing from the backend `.env`. Admin > Connectors lists which one. +- **Authentication failed** — Check that the redirect URI matches exactly and equals `CONNECTOR_REDIRECT_BASE_URI`, with no query parameters. Ensure the Google Drive API is enabled. - **Sign-in popup closes but the account never connects** — The frontend origin is not allowed to receive the result. Add it to `CONNECTOR_ALLOWED_ORIGINS` in the backend `.env`. - **Permission denied** — Verify the OAuth consent screen is configured and the user has access to the target files. - **Files not processing** — Check backend logs and verify that backend environment variables are correctly set. diff --git a/docs/content/Guides/Integrations/mcp-tool-integration.mdx b/docs/content/Guides/Integrations/mcp-tool-integration.mdx index d33e1b94..4d89fafe 100644 --- a/docs/content/Guides/Integrations/mcp-tool-integration.mdx +++ b/docs/content/Guides/Integrations/mcp-tool-integration.mdx @@ -22,11 +22,11 @@ Only needed if your MCP servers use OAuth authentication: MCP_OAUTH_REDIRECT_URI=https://yourdomain.com/api/mcp_server/callback ``` -If not set, falls back to `API_URL/api/mcp_server/callback`. +If not set, it is derived from the host of `CONNECTOR_REDIRECT_BASE_URI`, then from `API_URL`. ### Step 2: Add an MCP Server -Go to **Settings** > **Tools** > **Add Tool** > **MCP Server**. Enter the server URL, select an auth type, and click **Test Connection** to verify, then **Save**. +Go to **Settings** > **Connectors** and pick a preset (Notion, Linear, Atlassian, Sentry, Asana, Stripe), or choose **Add custom connector** > **MCP server**. The same form opens from **Settings** > **Tools** > **Add Tool**. Enter the server URL, select an auth type, and click **Test Connection** to verify, then **Save**. ### Step 3: Enable for Your Agent @@ -34,6 +34,8 @@ In your agent configuration, enable the MCP tools you want the agent to use. +Presets fill in the server URL and sign in with OAuth. Admins can turn presets off one by one, and turn off custom MCP servers, in **Admin > Connectors**. See [Connectors](/Guides/Connectors). + ## Authentication Types | Auth Type | Config Fields | diff --git a/docs/content/Guides/Integrations/sharepoint-connector.mdx b/docs/content/Guides/Integrations/sharepoint-connector.mdx index 58829f38..626b9788 100644 --- a/docs/content/Guides/Integrations/sharepoint-connector.mdx +++ b/docs/content/Guides/Integrations/sharepoint-connector.mdx @@ -10,6 +10,10 @@ import { Steps } from 'nextra/components' Connect your SharePoint or OneDrive account to upload and process files directly as an external knowledge base. Supports Office files, PDFs, text files, CSVs, images, and more. Authentication is handled via Microsoft Entra ID (Azure AD) with automatic token refresh. + +Members connect this service from **Settings > Connectors**, and one sign-in serves every source that uses it. Admins can check what is still missing in **Admin > Connectors**. See [Connectors](/Guides/Connectors). + + ## Setup @@ -18,8 +22,8 @@ Connect your SharePoint or OneDrive account to upload and process files directly 1. Go to the [Azure Portal](https://portal.azure.com/) > **Microsoft Entra ID** > **App registrations** > **New registration** 2. Set **Redirect URI** (Web) to: - - Local: `http://localhost:7091/api/connectors/callback?provider=share_point` - - Production: `https://yourdomain.com/api/connectors/callback?provider=share_point` + - Local: `http://localhost:7091/api/connectors/callback` + - Production: `https://yourdomain.com/api/connectors/callback` (the value of `CONNECTOR_REDIRECT_BASE_URI`, registered as-is) ### Step 2: Configure API Permissions @@ -53,13 +57,13 @@ MICROSOFT_TENANT_ID=your-azure-ad-tenant-id ### Step 5: Restart and Use -Restart your application, then go to the upload section in DocsGPT and select **SharePoint / OneDrive** as the source. You'll be redirected to Microsoft to sign in, then can browse and select files to process. +Restart your application, then go to **Settings > Connectors** and pick **SharePoint**. You'll be redirected to Microsoft to sign in, then can browse and select files to process. ## Troubleshooting - **Option not appearing** — Verify `MICROSOFT_CLIENT_ID` and `MICROSOFT_CLIENT_SECRET` are set, then restart. -- **Authentication failed** — Check that the redirect URI matches exactly, including `?provider=share_point`. +- **Authentication failed** — Check that the redirect URI matches exactly and equals `CONNECTOR_REDIRECT_BASE_URI`, with no query parameters. - **Sign-in popup closes but the account never connects** — The frontend origin is not allowed to receive the result. Add it to `CONNECTOR_ALLOWED_ORIGINS` in the backend `.env`. - **Permission denied** — Ensure admin consent is granted and the user has access to the target files. diff --git a/docs/content/Guides/_meta.js b/docs/content/Guides/_meta.js index 33cca51f..a9122c32 100644 --- a/docs/content/Guides/_meta.js +++ b/docs/content/Guides/_meta.js @@ -1,4 +1,8 @@ export default { + "Connectors": { + "title": "🔌 Connectors", + "href": "/Guides/Connectors" + }, "Customising-prompts": { "title": "️💻 Customising Prompts", "href": "/Guides/Customising-prompts" diff --git a/docs/content/Sources/_meta.js b/docs/content/Sources/_meta.js index 06a1d99a..cc800b43 100644 --- a/docs/content/Sources/_meta.js +++ b/docs/content/Sources/_meta.js @@ -1,4 +1,8 @@ export default { + "Connectors": { + "title": "🔌 Synced Sources (Connectors)", + "href": "/Guides/Connectors" + }, "Per-source-configuration": { "title": "🎛️ Per-Source Configuration", "href": "/Sources/Per-source-configuration" diff --git a/docs/content/Tools/basics.mdx b/docs/content/Tools/basics.mdx index b578eb8c..03581680 100644 --- a/docs/content/Tools/basics.mdx +++ b/docs/content/Tools/basics.mdx @@ -121,7 +121,8 @@ Interacting with tools in DocsGPT is designed to be intuitive: 2. **Configuration in UI:** * Tools are generally managed and configured within the DocsGPT application's settings, found under a "Tools" section in the GUI. - * For tools that interact with external services (like Brave Search, Telegram, or any service via the API Tool), you might need to provide authentication credentials (e.g., API keys, tokens) or specific endpoint information during the tool's setup in the UI. + * Tools for external services (Brave Search, Telegram, ntfy, PostgreSQL, MCP servers) are set up from **Settings > Connectors**. You connect the service once; its credentials stay encrypted on the server and never reach the browser, and the tools it provides are grouped under that connection. Each action is marked **Read** or **Write** and can be set to **Always allow**, **Needs approval** or **Off**. See [Connectors](/Guides/Connectors). + * When a tool's connection is missing or expired, the chat shows a **Connect** prompt and continues once you connect. 3. **Prompt Engineering for Tools:** While the LLM aims to intelligently use tools, for more complex or reliable agent-like behaviors, you might need to customize the system prompts. Modifying the prompt can guide the LLM on when and how to prioritize or chain tools to achieve specific outcomes, especially if you're building an agent designed to perform a certain sequence of actions every time. For more on this, see [Customising Prompts](/Guides/Customising-prompts). diff --git a/docs/content/upgrading.mdx b/docs/content/upgrading.mdx index c5397390..2d7d1589 100644 --- a/docs/content/upgrading.mdx +++ b/docs/content/upgrading.mdx @@ -11,6 +11,23 @@ import { Callout } from 'nextra/components' **Upgrading from 0.16.x?** User data moved from MongoDB to Postgres in 0.17.0. Follow the [Postgres Migration guide](/Deploying/Postgres-Migration) before running `docker compose pull` or `git pull` — existing deployments will not start cleanly without it. +## Connectors: set ENCRYPTION_SECRET_KEY first + +Service credentials (OAuth tokens for Google Drive, SharePoint, Confluence and MCP servers, and API keys for tools) now live on **connections** and are encrypted with a key derived from `ENCRYPTION_SECRET_KEY`. Migration `0038_connections` runs on startup, encrypts the stored tokens and removes their plaintext copies. + + + **Multi-user installs** (any `AUTH_TYPE`): set `ENCRYPTION_SECRET_KEY` to your own value **before** you upgrade, in the environment of the API, the worker and anything that runs migrations. The migration encrypts with the key it sees; changing it afterwards makes every connection ask its owner to reconnect. While the key is still the public default, DocsGPT refuses to store new credentials and Admin > Connectors shows a warning. + + +If you already used a key and want to change it, see [rotating the key](/Guides/Connectors#encryption-key). Single-user local installs keep working with the default and log a warning at startup. + +Other changes: + +- Register `CONNECTOR_REDIRECT_BASE_URI` exactly as set (no `?provider=` query) as the redirect URI of each OAuth app. Admin > Connectors shows it. +- `VITE_GOOGLE_CLIENT_ID` is optional now; without it, members pick Drive files in DocsGPT's own picker. +- Synced sources run as their connection, without a browser session. Keep Celery beat running for scheduled syncs. +- Downgrading to `0037_request_traces` decrypts the tokens back and gives each tool its key again. + ## pip installs: data home moved An installed package (`pip install docsgpt`, pipx, `uv tool`) used to keep its data home, meaning `.env`, `inputs/`, `indexes/` and `models/`, in the directory you ran `docsgpt api` and `docsgpt worker` from. It is now `~/.docsgpt/server` (`/opt/docsgpt` for root on Linux). Either move those files there, or set `DOCSGPT_HOME` to the old directory in the environment of both commands. Both commands print the data home they use, and point out a `.env` in the working directory that they no longer read. Source checkouts and the Docker images are not affected. From 6b4220d401a444d1ba61dc10b81263b585395a09 Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:45:27 +0100 Subject: [PATCH 012/162] Address review: MCP token scope, API-key identity, reconnect lock, setup retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - An MCP connection's tokens only go to its own server, and a connection_id can no longer come from a client: the MCP test/save routes drop it and the tool executor uses only the resolved one. - API-key connections are reused only when they hold the same credentials. Two keys that share a hint or a label get separate connections ("…abcd (2)"), at runtime and in the 0038 backfill. - Reconnecting a connection whose secrets cannot be decrypted no longer flags it from a second transaction while holding its row lock. - Connection setup validates the sync request before claiming its Idempotency-Key, so a corrected retry is queued. - A request that names no connection resolves to none without opening a database connection; fixes two tests that reached a real database. --- docsgpt/agents/tool_executor.py | 3 + docsgpt/alembic/versions/0038_connections.py | 45 +++++++--- docsgpt/api/connector/connections.py | 84 ++++++++++--------- docsgpt/api/user/tools/mcp.py | 5 +- docsgpt/connectors/service.py | 53 ++++++++++-- tests/connectors/test_connections_api.py | 37 ++++++++ tests/connectors/test_runtime.py | 11 +++ tests/connectors/test_service_credentials.py | 69 +++++++++++++++ .../connectors/test_auth_token_leakage.py | 2 +- tests/storage/db/test_migration_0038.py | 26 ++++++ 10 files changed, 274 insertions(+), 61 deletions(-) diff --git a/docsgpt/agents/tool_executor.py b/docsgpt/agents/tool_executor.py index 778f4a1a..a3b7a88a 100644 --- a/docsgpt/agents/tool_executor.py +++ b/docsgpt/agents/tool_executor.py @@ -1560,6 +1560,9 @@ class ToolExecutor: tool_config["body_encoding_rules"] = action_config.get("body_encoding_rules", {}) else: tool_config = tool_data["config"].copy() if tool_data["config"] else {} + # Whose MCP tokens a tool uses is decided by resolving its + # connection below, never by a value stored in its config. + tool_config.pop("connection_id", None) # Credentials are PBKDF2-bound to the tool OWNER's sub, not the # invoker's. Decrypt with the tool row's user_id so a team member # running an owner's shared tool authenticates with the owner's diff --git a/docsgpt/alembic/versions/0038_connections.py b/docsgpt/alembic/versions/0038_connections.py index 052a40db..e2205929 100644 --- a/docsgpt/alembic/versions/0038_connections.py +++ b/docsgpt/alembic/versions/0038_connections.py @@ -292,7 +292,12 @@ def _link_api_key_tools(bind) -> None: from sqlalchemy import text - from docsgpt.security.encryption import decrypt_credentials, encrypt_json + from docsgpt.security.encryption import ( + CredentialDecryptionError, + decrypt_credentials, + decrypt_json, + encrypt_json, + ) rows = bind.execute( text( @@ -323,18 +328,32 @@ def _link_api_key_tools(bind) -> None: server_url = None connector_key = row.name display_name = _TOOL_CONNECTORS[row.name] - label = _credential_hint(credentials) - existing = bind.execute( - text( - """ - SELECT id, encrypted_credentials FROM connector_sessions - WHERE user_id = :user_id AND provider = :provider - AND COALESCE(server_url, '') = COALESCE(:server_url, '') - AND COALESCE(account_label, '') = :label - """ - ), - {"user_id": row.user_id, "provider": connector_key, "server_url": server_url, "label": label}, - ).fetchone() + # The hint is not an identity: reuse a connection only when it holds + # the same credentials, and give a different key its own label. + hint = _credential_hint(credentials) + label, suffix = hint, 1 + while True: + existing = bind.execute( + text( + """ + SELECT id, encrypted_credentials FROM connector_sessions + WHERE user_id = :user_id AND provider = :provider + AND COALESCE(server_url, '') = COALESCE(:server_url, '') + AND COALESCE(account_label, '') = :label + """ + ), + {"user_id": row.user_id, "provider": connector_key, "server_url": server_url, "label": label}, + ).fetchone() + if existing is None: + break + try: + stored = decrypt_json(existing.encrypted_credentials or "", row.user_id).get("credentials") + except CredentialDecryptionError: + stored = None + if stored == credentials: + break + suffix += 1 + label = f"{hint} ({suffix})" if existing is None: connection_id = bind.execute( text( diff --git a/docsgpt/api/connector/connections.py b/docsgpt/api/connector/connections.py index daf1e06e..934faa3a 100644 --- a/docsgpt/api/connector/connections.py +++ b/docsgpt/api/connector/connections.py @@ -18,6 +18,7 @@ from docsgpt.api.user.authz import ROLE_ADMIN, has_role from docsgpt.connectors import catalog, service from docsgpt.storage.db.repositories.connector_sessions import ConnectorSessionsRepository from docsgpt.storage.db.session import db_readonly, db_session +from docsgpt.security.encryption import CredentialDecryptionError _FREQUENCIES = ("never", "daily", "weekly", "monthly") @@ -258,6 +259,39 @@ def _start_sync(user_id: str, row: dict, sync: dict): if frequency not in _FREQUENCIES: return ("Unknown sync frequency", 400) name = (sync.get("name") or "").strip() or definition.name + # Validate before claiming the idempotency key: a rejected request must + # leave the key free for the corrected retry. + if definition.auth_kind == "oauth": + file_ids = [str(i) for i in items.get("file_ids") or [] if i] + folder_ids = [str(i) for i in items.get("folder_ids") or [] if i] + if not file_ids and not folder_ids: + return ("Pick at least one file or folder", 400) + task_fn = ingest_connector_task + kwargs = { + "job_name": name, + "user": user_id, + "source_type": definition.sync_ingestor, + "connection_id": str(row["id"]), + "file_ids": file_ids, + "folder_ids": folder_ids, + "recursive": bool(items.get("recursive", True)), + "sync_frequency": frequency, + } + else: + fields = {f.key for f in definition.setup_fields} + source_data = {k: v for k, v in items.items() if k in fields and v not in (None, "")} + missing = [f.label for f in definition.setup_fields if f.required and f.key not in source_data] + if missing: + return (f"Missing: {', '.join(missing)}", 400) + task_fn = ingest_remote + kwargs = { + "source_data": source_data, + "job_name": name, + "user": user_id, + "loader": definition.sync_ingestor, + "connection_id": str(row["id"]), + "sync_frequency": frequency, + } idempotency_key, _ = _read_idempotency_key() scoped_key = _scoped_idempotency_key(idempotency_key, user_id) task_id = None @@ -267,45 +301,9 @@ def _start_sync(user_id: str, row: dict, sync: dict): return {"id": cached.get("source_id"), "task_id": cached.get("task_id"), "name": name} source_id = str(_derive_source_id(scoped_key)) if scoped_key else str(uuid.uuid4()) options = {"task_id": task_id} if task_id else {} - if definition.auth_kind == "oauth": - file_ids = [str(i) for i in items.get("file_ids") or [] if i] - folder_ids = [str(i) for i in items.get("folder_ids") or [] if i] - if not file_ids and not folder_ids: - return ("Pick at least one file or folder", 400) - task = ingest_connector_task.apply_async( - kwargs={ - "job_name": name, - "user": user_id, - "source_type": definition.sync_ingestor, - "connection_id": str(row["id"]), - "file_ids": file_ids, - "folder_ids": folder_ids, - "recursive": bool(items.get("recursive", True)), - "sync_frequency": frequency, - "idempotency_key": scoped_key, - "source_id": source_id, - }, - **options, - ) - else: - fields = {f.key for f in definition.setup_fields} - source_data = {k: v for k, v in items.items() if k in fields and v not in (None, "")} - missing = [f.label for f in definition.setup_fields if f.required and f.key not in source_data] - if missing: - return (f"Missing: {', '.join(missing)}", 400) - task = ingest_remote.apply_async( - kwargs={ - "source_data": source_data, - "job_name": name, - "user": user_id, - "loader": definition.sync_ingestor, - "connection_id": str(row["id"]), - "sync_frequency": frequency, - "idempotency_key": scoped_key, - "source_id": source_id, - }, - **options, - ) + task = task_fn.apply_async( + kwargs={**kwargs, "idempotency_key": scoped_key, "source_id": source_id}, **options, + ) return {"id": source_id, "task_id": task_id or task.id, "name": name, "sync_frequency": frequency} @@ -347,7 +345,13 @@ class ConnectionReconnect(Resource): service.ensure_can_store_credentials() with db_session() as conn: locked = ConnectorSessionsRepository(conn).get_for_update(connection_id) - stored = service.load_secrets(locked) or {} + # read_secrets, not load_secrets: flagging an unreadable row + # would write it from a second transaction while this one + # holds its lock. The new credentials replace it anyway. + try: + stored = service.read_secrets(locked) + except CredentialDecryptionError: + stored = {} merged = {**(stored.get("credentials") or {}), **{k: v for k, v in credentials.items() if v}} service.write_secrets( conn, locked, {**stored, "credentials": merged}, diff --git a/docsgpt/api/user/tools/mcp.py b/docsgpt/api/user/tools/mcp.py index 6b2567eb..bfe6fb09 100644 --- a/docsgpt/api/user/tools/mcp.py +++ b/docsgpt/api/user/tools/mcp.py @@ -27,13 +27,16 @@ def _sanitize_mcp_transport(config): """Normalise and validate the transport_type field. Strips ``command`` / ``args`` keys that are only valid for local STDIO - transports and returns the cleaned transport type string. + transports, and ``connection_id``, which only the tool executor sets + (it picks whose MCP tokens the tool uses). Returns the cleaned + transport type string. """ transport_type = (config.get("transport_type") or "auto").lower() if transport_type not in _ALLOWED_TRANSPORTS: raise ValueError(f"Unsupported transport_type: {transport_type}") config.pop("command", None) config.pop("args", None) + config.pop("connection_id", None) config["transport_type"] = transport_type return transport_type diff --git a/docsgpt/connectors/service.py b/docsgpt/connectors/service.py index 1bcb1352..4054e432 100644 --- a/docsgpt/connectors/service.py +++ b/docsgpt/connectors/service.py @@ -647,6 +647,40 @@ def get_credentials(row: dict) -> dict: return dict(secrets.get("credentials") or {}) +def _api_key_account( + repo: ConnectorSessionsRepository, + user_id: str, + connector_key: str, + server_url: Optional[str], + label: str, + credentials: dict, +) -> tuple[Optional[dict], str]: + """The connection holding exactly ``credentials``, or a free label for a new one. + + A label (a hint of the key, or what the user typed) is not an identity: + two different keys can share it. A row under the label is reused only + when it holds the same credentials; otherwise the label gets a + ``(2)``, ``(3)`` suffix until it names a matching row or no row. + + Returns: + ``(row, label)``: the row to reuse (None to create one) and its label. + """ + from docsgpt.security.encryption import CredentialDecryptionError + + candidate, suffix = label, 1 + while True: + existing = repo.find_account(user_id, connector_key, server_url=server_url, account_label=candidate) + if existing is None: + return None, candidate + try: + if (read_secrets(existing).get("credentials") or {}) == credentials: + return existing, candidate + except CredentialDecryptionError: + pass + suffix += 1 + candidate = f"{label} ({suffix})" + + def create_api_key_connection( conn, user_id: str, @@ -689,9 +723,10 @@ def create_api_key_connection( ensure_can_store_credentials() ensure_connector_allowed(conn, definition.key) secret_values = {k: v for k, v in credentials.items() if (fields.get(k).secret if fields.get(k) else True)} - account_label = label or credential_hint(secret_values or credentials) repo = ConnectorSessionsRepository(conn) - existing = repo.find_account(user_id, definition.key, server_url=server_url, account_label=account_label) + existing, account_label = _api_key_account( + repo, user_id, definition.key, server_url, label or credential_hint(secret_values or credentials), credentials, + ) if existing is not None: write_secrets(conn, existing, {"credentials": credentials}, status=STATUS_CONNECTED, last_error=None) resume_sources(conn, str(existing["id"])) @@ -833,16 +868,16 @@ def resolve_request_connection(user_id: str, provider: Optional[str], data: dict """ connection_id = data.get("connection_id") session_token = data.get("session_token") + if not connection_id and not session_token: + return None with db_readonly() as conn: repo = ConnectorSessionsRepository(conn) if connection_id: row = repo.get_for_user(str(connection_id), user_id) - elif session_token: + else: row = repo.get_by_session_token(session_token) if not owns_connector_session(row, user_id, provider): return None - else: - return None if row is None: return None if provider and (row.get("provider") or "").lower() != provider.lower(): @@ -867,7 +902,13 @@ def _mcp_row(conn, user_id: str, base_url: str, connection_id: Optional[str], *, repo = ConnectorSessionsRepository(conn) if connection_id: - return repo.get_for_update(connection_id) if lock else repo.get(connection_id) + row = repo.get_for_update(connection_id) if lock else repo.get(connection_id) + # A connection's tokens only ever go to its own server. Ownership is + # checked where the id is chosen (the tool executor); clients cannot + # supply one (see ``_sanitize_mcp_transport``). + if row is None or row.get("provider") != mcp_provider(base_url): + return None + return row result = conn.execute( text( "SELECT * FROM connector_sessions WHERE user_id = :user_id AND provider = :provider " diff --git a/tests/connectors/test_connections_api.py b/tests/connectors/test_connections_api.py index e02144c8..410bdef9 100644 --- a/tests/connectors/test_connections_api.py +++ b/tests/connectors/test_connections_api.py @@ -141,6 +141,26 @@ class TestSetupSync: body={"sync": {"items": {}}}, args=[cid]) assert resp.status_code == 400 + def test_rejected_request_does_not_claim_its_idempotency_key(self, app, pg_conn): + from docsgpt.api.connector.connections import ConnectionSetup + + @contextmanager + def _yield(): + yield pg_conn + + cid = _connection(pg_conn, provider="google_drive", auth_kind="oauth", secrets={"token_info": {}}) + headers = {"Idempotency-Key": "setup-retry-1"} + with _db(pg_conn), patch("docsgpt.api.user.sources.upload.db_session", _yield), patch( + "docsgpt.api.user.tasks.ingest_connector_task.apply_async", return_value=MagicMock(id="t"), + ) as apply: + bad = _call(app, ConnectionSetup, "post", f"/api/connections/{cid}/setup", + body={"sync": {"items": {}}}, headers=headers, args=[cid]) + fixed = _call(app, ConnectionSetup, "post", f"/api/connections/{cid}/setup", + body={"sync": {"items": {"folder_ids": ["f1"]}}}, headers=headers, args=[cid]) + assert bad.status_code == 400 + assert fixed.status_code == 200 + apply.assert_called_once() + def test_s3_keys_stay_on_the_connection(self, app, pg_conn): from docsgpt.api.connector.connections import ConnectionSetup @@ -186,6 +206,23 @@ class TestReconnect: meta = pg_conn.execute(text("SELECT metadata FROM sources WHERE user_id = 'alice'")).scalar() assert "sync_state" not in meta + def test_reconnect_recovers_undecryptable_credentials_without_a_second_write(self, app, pg_conn): + """After a lost key the reconnect replaces the blob; it must not flag the row it holds locked.""" + from docsgpt.api.connector.connections import ConnectionReconnect + + cid = _connection(pg_conn, status="reconnect_needed") + pg_conn.execute(text( + "UPDATE connector_sessions SET encrypted_credentials = :blob WHERE id = CAST(:c AS uuid)" + ), {"c": cid, "blob": encrypt_json({"credentials": {"token": "old"}}, "someone-else")}) + with _db(pg_conn), patch.object(service, "mark_reconnect_needed") as flag: + resp = _call(app, ConnectionReconnect, "post", f"/api/connections/{cid}/reconnect", + body={"credentials": {"token": "new"}}, args=[cid]) + assert resp.status_code == 200 + flag.assert_not_called() + row = pg_conn.execute(text("SELECT * FROM connector_sessions WHERE id = CAST(:c AS uuid)"), + {"c": cid}).one()._mapping + assert service.read_secrets(dict(row))["credentials"] == {"token": "new"} + def test_oauth_reconnect_returns_authorization_url(self, app, pg_conn): from docsgpt.api.connector.connections import ConnectionReconnect diff --git a/tests/connectors/test_runtime.py b/tests/connectors/test_runtime.py index a58d622a..2649ef7b 100644 --- a/tests/connectors/test_runtime.py +++ b/tests/connectors/test_runtime.py @@ -173,6 +173,17 @@ class TestExecutor: assert config["connection_id"] == cid assert "secret" not in str(config) + def test_stored_connection_id_in_config_is_ignored(self, pg_conn): + """Only a resolved connection reaches the tool; a config value never does.""" + victim = _connection(pg_conn, user="victim", provider="mcp:https://m.example.com", auth_kind="mcp_oauth", + server_url="https://m.example.com", secrets={"tokens": {"access_token": "v"}}) + tool = {**_tool(None, name="mcp_tool"), "config": {"server_url": "https://m.example.com/mcp", + "auth_type": "oauth", "connection_id": victim}} + with _service_db(pg_conn), patch("docsgpt.agents.tool_executor.ToolManager") as manager: + _executor()._get_or_load_tool(tool, "t1", "search") + config = manager.return_value.load_tool.call_args.kwargs["tool_config"] + assert "connection_id" not in config + class TestScheduledSync: def test_connector_sources_with_a_connection_are_dispatched(self, pg_conn): diff --git a/tests/connectors/test_service_credentials.py b/tests/connectors/test_service_credentials.py index 884cef77..986cecaa 100644 --- a/tests/connectors/test_service_credentials.py +++ b/tests/connectors/test_service_credentials.py @@ -249,6 +249,36 @@ class TestApiKeyConnections: assert created is False and again["id"] == row["id"] assert service.get_credentials(again) == {"token": "123456:ABCDEFG"} + def test_different_keys_with_the_same_hint_stay_apart(self, pg_conn): + """Two keys ending in the same four characters are two accounts, not one.""" + from docsgpt.connectors import catalog + + postgres = catalog.get_definition("postgres") + first, _ = service.create_api_key_connection( + pg_conn, "alice", postgres, {"token": "postgresql://ro@db-a/app"}, + ) + second, created = service.create_api_key_connection( + pg_conn, "alice", postgres, {"token": "postgresql://rw@db-b/app"}, + ) + assert created is True and second["id"] != first["id"] + assert second["account_label"] != first["account_label"] + assert service.get_credentials(_row(pg_conn, str(first["id"]))) == {"token": "postgresql://ro@db-a/app"} + again, created = service.create_api_key_connection( + pg_conn, "alice", postgres, {"token": "postgresql://rw@db-b/app"}, + ) + assert created is False and again["id"] == second["id"] + + def test_same_label_different_key_does_not_overwrite(self, pg_conn): + from docsgpt.connectors import catalog + + brave = catalog.get_definition("brave") + first, _ = service.create_api_key_connection(pg_conn, "alice", brave, {"token": "key-one-111111"}, label="Team") + second, created = service.create_api_key_connection( + pg_conn, "alice", brave, {"token": "key-two-222222"}, label="Team", + ) + assert created is True and second["id"] != first["id"] + assert service.get_credentials(_row(pg_conn, str(first["id"]))) == {"token": "key-one-111111"} + def test_missing_field(self, pg_conn): from docsgpt.connectors import catalog @@ -418,3 +448,42 @@ class TestRevoke: with patch("requests.post", side_effect=requests.exceptions.Timeout()): assert not service.revoke_at_provider({"provider": "google_drive"}, {"token_info": {"access_token": "a"}}) + + +class TestMcpConnectionScope: + def _mcp(self, conn, user="alice", base="https://mcp.notion.com"): + return _connection( + conn, user=user, provider=f"mcp:{base}", auth_kind="mcp_oauth", server_url=base, + secrets={"tokens": {"access_token": f"{user}-mcp-token"}}, + ) + + def test_reads_tokens_of_the_named_connection(self, pg_conn): + cid = self._mcp(pg_conn) + with _patch_service_db(pg_conn): + data = service.read_mcp_secrets("alice", "https://mcp.notion.com", cid) + assert data["tokens"]["access_token"] == "alice-mcp-token" + + def test_connection_for_another_server_yields_nothing(self, pg_conn): + """A connection id never sends its tokens to a different server.""" + cid = self._mcp(pg_conn) + with _patch_service_db(pg_conn): + assert service.read_mcp_secrets("alice", "https://attacker.example", cid) == {} + + def test_writes_never_land_on_another_servers_connection(self, pg_conn): + cid = self._mcp(pg_conn) + with _patch_service_db(pg_conn): + row = service.update_mcp_secrets( + "alice", "https://attacker.example", {"tokens": {"access_token": "planted"}}, connection_id=cid, + ) + assert str(row["id"]) != cid + assert service.read_mcp_secrets("alice", "https://mcp.notion.com", cid)["tokens"]["access_token"] == ( + "alice-mcp-token" + ) + + def test_mcp_routes_drop_client_supplied_connection_id(self): + """Only the tool executor may pick the connection whose tokens a tool uses.""" + from docsgpt.api.user.tools.mcp import _sanitize_mcp_transport + + config = {"transport_type": "http", "connection_id": "someone-elses-connection"} + _sanitize_mcp_transport(config) + assert "connection_id" not in config diff --git a/tests/parser/connectors/test_auth_token_leakage.py b/tests/parser/connectors/test_auth_token_leakage.py index d997620a..217b437e 100644 --- a/tests/parser/connectors/test_auth_token_leakage.py +++ b/tests/parser/connectors/test_auth_token_leakage.py @@ -55,7 +55,7 @@ def _patches(session_return: Optional[Dict[str, Any]]): fake_repo_cls, ), patch( - "docsgpt.storage.db.session.db_readonly", + "docsgpt.connectors.service.db_readonly", lambda: _FakeReadonlyCtx(), ), ) diff --git a/tests/storage/db/test_migration_0038.py b/tests/storage/db/test_migration_0038.py index c99884c3..66123837 100644 --- a/tests/storage/db/test_migration_0038.py +++ b/tests/storage/db/test_migration_0038.py @@ -272,6 +272,32 @@ class TestMigration0038Credentials: ).scalar() assert "encrypted_credentials" in config + def test_different_keys_with_the_same_hint_get_their_own_connections(self, pg_engine): + from docsgpt.connectors.service import read_secrets + from docsgpt.security.encryption import encrypt_credentials + + def seed(conn): + ids = {} + for name, token in (("a", "111111:SAMEEND1"), ("b", "222222:SAMEEND1")): + ids[name] = conn.execute( + text("INSERT INTO user_tools (user_id, name, config) VALUES ('frank', 'telegram', " + "CAST(:c AS jsonb)) RETURNING id"), + {"c": json.dumps({"encrypted_credentials": encrypt_credentials({"token": token}, "frank")})}, + ).scalar() + return ids + + _, ids = self._upgrade_with(pg_engine, seed) + with pg_engine.connect() as conn: + links = dict(conn.execute( + text("SELECT id, connection_id FROM user_tools WHERE user_id = 'frank'") + ).fetchall()) + assert links[ids["a"]] != links[ids["b"]] + for name, token in (("a", "111111:SAMEEND1"), ("b", "222222:SAMEEND1")): + row = dict(conn.execute( + text("SELECT * FROM connector_sessions WHERE id = :i"), {"i": links[ids[name]]} + ).one()._mapping) + assert read_secrets(row) == {"credentials": {"token": token}} + def test_oauth_mcp_tools_keep_member_credentials(self, pg_engine): _, ids = self._upgrade_with(pg_engine, _seed_secrets) with pg_engine.connect() as conn: From 269aa13bba321c6ba9e367ef8a54335ef83a5fc5 Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:45:27 +0100 Subject: [PATCH 013/162] Address review: tool group order in the agent builder, saved S3/Reddit account - Connection groups sit between the default and custom tools; a second sort that did not know connection labels moved them to the top. - Upload picks the first saved account when connections load after the modal opened on S3 or Reddit, without overriding the user's choice. --- frontend/src/agents/NewAgent.test.tsx | 52 +++++++++++++++++++++++++-- frontend/src/agents/NewAgent.tsx | 16 ++++----- frontend/src/upload/Upload.test.tsx | 42 ++++++++++++++++++++++ frontend/src/upload/Upload.tsx | 25 ++++++++----- 4 files changed, 114 insertions(+), 21 deletions(-) diff --git a/frontend/src/agents/NewAgent.test.tsx b/frontend/src/agents/NewAgent.test.tsx index 5bd59f13..d6ffa3b8 100644 --- a/frontend/src/agents/NewAgent.test.tsx +++ b/frontend/src/agents/NewAgent.test.tsx @@ -27,6 +27,8 @@ const mocks = vi.hoisted(() => { dispatch: vi.fn(), getAgent: vi.fn(() => jsonResponse({})), createAgent: vi.fn(() => jsonResponse({ message: 'Name is taken' }, false)), + tools: null as unknown[] | null, + connections: [] as unknown[], }; }); const { jsonResponse } = mocks; @@ -40,7 +42,7 @@ vi.mock('../api/services/userService', () => ({ default: { getUserTools: () => jsonResponse({ - tools: [ + tools: mocks.tools ?? [ { id: 'tool-1', name: 'remote_device', @@ -69,7 +71,7 @@ vi.mock('../api/services/devicesService', () => ({ vi.mock('../api/services/connectorsService', () => ({ default: { - listConnections: () => Promise.resolve({ connections: [] }), + listConnections: () => Promise.resolve({ connections: mocks.connections }), }, })); @@ -93,7 +95,9 @@ vi.mock('../components/MultiSelectPopover', () => ({
{trigger} {items.map((item) => ( -
{item.descriptionNode}
+
+ {item.descriptionNode} +
))}
), @@ -164,6 +168,8 @@ describe('NewAgent form', () => { mocks.dispatch.mockClear(); mocks.getAgent.mockClear(); mocks.createAgent.mockClear(); + mocks.tools = null; + mocks.connections = []; }); const render = async () => { @@ -251,6 +257,46 @@ describe('NewAgent form', () => { expect(description.className).toContain('sm:col-start-2'); }); + it('lists tool groups as built-in, default, one per connection, then custom', async () => { + mocks.tools = [ + { id: 'custom', name: 'api_tool', display_name: 'My API' }, + { + id: 'linear', + name: 'mcp_tool', + display_name: 'Linear', + connection_id: 'c-lin', + }, + { id: 'memory', name: 'memory', display_name: 'Memory', builtin: true }, + { + id: 'notion', + name: 'mcp_tool', + display_name: 'Notion', + connection_id: 'c-not', + }, + { + id: 'reader', + name: 'read_webpage', + display_name: 'Reader', + default: true, + }, + ]; + mocks.connections = [ + { id: 'c-lin', name: 'Linear', account_label: 'a@x', icon: 'linear' }, + { id: 'c-not', name: 'Notion', account_label: 'b@x', icon: 'notion' }, + ]; + await render(); + const groups = Array.from( + container.querySelectorAll('[data-testid="picker"] [data-group]'), + ).map((item) => item.getAttribute('data-group')); + const order = groups.filter((g, i) => groups.indexOf(g) === i); + expect(order).toEqual([ + 'agents.form.toolsPopup.groupBuiltin', + 'agents.form.toolsPopup.groupDefault', + 'agents.form.toolsPopup.groupConnection', + 'agents.form.toolsPopup.groupCustom', + ]); + }); + it('labels every picker with a floating label', async () => { await render(); const labels = Array.from( diff --git a/frontend/src/agents/NewAgent.tsx b/frontend/src/agents/NewAgent.tsx index ba2719a0..92b68320 100644 --- a/frontend/src/agents/NewAgent.tsx +++ b/frontend/src/agents/NewAgent.tsx @@ -636,7 +636,12 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { return t('agents.form.toolsPopup.groupCustom'); }; const tools: MultiSelectPopoverItem[] = [...visibleTools] - .sort((a, b) => rank(a) - rank(b)) + .sort( + (a, b) => + rank(a) - rank(b) || + // Keeps each connection's tools together. + (rank(a) === 2 ? groupFor(a).localeCompare(groupFor(b)) : 0), + ) .map((tool: UserToolType) => { const connection = connectionOf(tool); const base: MultiSelectPopoverItem = { @@ -666,15 +671,6 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) { } return base; }); - const groupOrder = [ - t('agents.form.toolsPopup.groupBuiltin'), - t('agents.form.toolsPopup.groupDefault'), - t('agents.form.toolsPopup.groupCustom'), - ]; - tools.sort( - (a, b) => - groupOrder.indexOf(a.group || '') - groupOrder.indexOf(b.group || ''), - ); setUserTools(tools); setRawUserTools(visibleTools); }; diff --git a/frontend/src/upload/Upload.test.tsx b/frontend/src/upload/Upload.test.tsx index 457a8777..1373ef84 100644 --- a/frontend/src/upload/Upload.test.tsx +++ b/frontend/src/upload/Upload.test.tsx @@ -169,6 +169,48 @@ describe('Upload source-type tiles', () => { connectorsState.connections = []; }); + it('picks the saved account once connections finish loading', async () => { + connectorsState.catalog = [ + { + key: 's3', + icon: 's3', + sync_ingestor: 's3', + auth_kind: 'api_key', + available: true, + missing_settings: [], + credential_fields: [ + { key: 'aws_access_key_id', label: 'Access key ID', secret: false }, + ], + }, + ]; + const props = { + receivedFile: [], + setModalState: vi.fn(), + isOnboarding: false, + renderTab: null, + close: vi.fn(), + initialIngestor: 's3' as const, + }; + await act(async () => root.render()); + const triggers = () => + Array.from( + document.body.querySelectorAll('[data-slot="select-trigger"]'), + ).map((el) => el.textContent); + expect(triggers()).not.toContain('modals.uploadDoc.newCredentials'); + connectorsState.connections = [ + { + id: 'k1', + connector_key: 's3', + status: 'connected', + account_label: '…WXYZ', + }, + ]; + await act(async () => root.render()); + expect(triggers()).toContain('settings.connectors.detail.keyEnding'); + connectorsState.catalog = []; + connectorsState.connections = []; + }); + it('leaves the disabled Train button on the default variant', async () => { await render(); const crawler = tiles().find((tile) => diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 418004c0..22c5e4e5 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -1,7 +1,7 @@ import { ChevronLeft, CircleAlert, FileText } from 'lucide-react'; import { envVar } from '@/env'; import { cn } from '@/lib/utils'; -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { nanoid } from '@reduxjs/toolkit'; import type { FileRejection } from 'react-dropzone'; import { useTranslation } from 'react-i18next'; @@ -179,7 +179,10 @@ function Upload({ setFilter(value as Filter)} + > + + + + + {filters.map((key) => ( + + {t(`settings.connectors.categories.${key}`)} + + ))} + + +
+ {(capability === 'sync' || capability === 'tools') && ( +

+ {t(`settings.connectors.capabilityFilter.${capability}`)}{' '} + +

+ )} +
{!loaded && !failed ? ( @@ -191,6 +251,7 @@ export default function Connectors() { - ); - })()}
+ {(() => { + const connection = tool.connection_id + ? connections.find( + (c) => c.id === tool.connection_id, + ) + : undefined; + if (!connection) return null; + // The same meta line as a Sources tile; mr-12 keeps + // it clear of the status switch. + return ( + + + + + {t('settings.tools.viaConnection', { + name: connection.name, + interpolation: { escapeValue: false }, + })} + + + + ); + })()}
-
diff --git a/tests/connectors/test_runtime.py b/tests/connectors/test_runtime.py index bdaca042..137afd76 100644 --- a/tests/connectors/test_runtime.py +++ b/tests/connectors/test_runtime.py @@ -351,3 +351,96 @@ class TestMcpServerMismatch: # Someone else's connection, or one that signs in another way, is not kept. assert _previous_connection(existing, same, "bob") is None assert _previous_connection(existing, {**same, "auth_type": "oauth"}, "alice") is None + + +class TestExternalApiCallers: + """An agent called with its API key runs as the owner, and nobody can approve there.""" + + def _external(self, allowlist=None): + from docsgpt.agents.tool_executor import ToolExecutor + + return ToolExecutor(user="alice", external_caller=True, api_write_allowlist=allowlist) + + def test_owner_account_write_is_denied(self, pg_conn): + cid = _connection(pg_conn) + with _service_db(pg_conn): + pause = _pause(self._external(), _tool(cid)) + assert pause["pause_type"] == "headless_denied" + assert pause["error_type"] == "tool_not_allowed" + assert "Access details" in pause["deny_reason"] + + def test_even_always_allow_writes_are_denied(self, pg_conn): + cid = _connection(pg_conn) + tool = _tool(cid) + tool["actions"][0]["require_approval"] = False + with _service_db(pg_conn): + assert _pause(self._external(), tool)["pause_type"] == "headless_denied" + + def test_allowlisted_write_runs(self, pg_conn): + cid = _connection(pg_conn) + with _service_db(pg_conn): + pause = _pause(self._external(["tool-1:telegram_send_message"]), _tool(cid)) + assert pause is None + + def test_allowlist_does_not_cover_other_actions(self, pg_conn): + cid = _connection(pg_conn) + with _service_db(pg_conn): + pause = _pause(self._external(["tool-1:telegram_send_image"]), _tool(cid)) + assert pause["pause_type"] == "headless_denied" + + def test_missing_connection_is_denied_not_paused(self, pg_conn): + """The widget cannot show a Connect card.""" + cid = _connection(pg_conn, status="reconnect_needed") + with _service_db(pg_conn): + pause = _pause(self._external(), _tool(cid)) + assert pause["pause_type"] == "headless_denied" + assert pause["error_type"] == "connection_required" + + def test_the_owner_in_the_app_is_not_external(self, pg_conn): + cid = _connection(pg_conn) + with _service_db(pg_conn): + assert _pause(_executor(), _tool(cid)) is None + + +class TestExternalCallerDetection: + def test_api_key_request_from_someone_else_is_external(self): + from docsgpt.api.answer.services.stream_processor import is_external_api_caller + + assert is_external_api_caller({"api_key": "k"}, {"sub": "visitor"}, "alice") is True + assert is_external_api_caller({"api_key": "k"}, None, "alice") is True + + def test_owner_previewing_their_agent_is_not_external(self): + from docsgpt.api.answer.services.stream_processor import is_external_api_caller + + assert is_external_api_caller({"api_key": "k"}, {"sub": "alice"}, "alice") is False + assert is_external_api_caller({}, {"sub": "visitor"}, "alice") is False + + +class TestApiWriteAllowlistConfig: + def test_accepts_tool_action_pairs(self): + from docsgpt.guardrails.config import AgentConfig + + config = AgentConfig.model_validate({"api_write_allowlist": ["tool-1:telegram_send_message"]}) + assert config.api_write_allowlist == ["tool-1:telegram_send_message"] + + def test_rejects_malformed_entries(self): + from docsgpt.guardrails.config import AgentConfig + + with pytest.raises(Exception): + AgentConfig.model_validate({"api_write_allowlist": ["no-action-part"]}) + + def test_old_configs_still_parse(self): + from docsgpt.guardrails.config import AgentConfig + + assert AgentConfig.parse({"guardrails": {}}).api_write_allowlist == [] + + +class TestAllowlistOwnership: + def test_team_editor_cannot_change_the_allowlist(self): + from docsgpt.api.user.agents.routes import keep_owner_only_config + + existing = {"config": {"api_write_allowlist": ["t:a"]}} + sent = {"guardrails": {"controls": []}, "api_write_allowlist": ["t:a", "t:b"]} + assert keep_owner_only_config(sent, existing, True)["api_write_allowlist"] == ["t:a"] + assert keep_owner_only_config(sent, existing, False)["api_write_allowlist"] == ["t:a", "t:b"] + assert keep_owner_only_config({}, {"config": None}, True) == {"api_write_allowlist": []} From f6c89503d89e8b9c9f937accd16020074f4982fe Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:37:19 +0100 Subject: [PATCH 022/162] Reconnect in place from the chat bar and the health toast - The Connect card's payload carries connection_id for the caller's own connection (never an owner's), so "Reconnect" in chat signs in again right there instead of sending the user to Settings. The bar offers Continue as soon as the connection works again, also after leaving and coming back to the chat. - The reconnect event carries source and tool counts, and the toast says what stopped: "3 sources are paused", "so agents can use it again", or both. Reconnect signs in again in place and the toast closes itself once the connection works; MCP servers still reconnect from their drawer. --- docsgpt/agents/tool_executor.py | 13 +- docsgpt/connectors/service.py | 3 + .../conversation/ConnectToolCallBar.test.tsx | 68 ++++++++- .../src/conversation/ConnectToolCallBar.tsx | 24 ++- frontend/src/conversation/types/index.ts | 4 + frontend/src/locale/de.json | 7 +- frontend/src/locale/en.json | 7 +- frontend/src/locale/es.json | 7 +- frontend/src/locale/jp.json | 5 +- frontend/src/locale/ru.json | 11 +- frontend/src/locale/zh-TW.json | 5 +- frontend/src/locale/zh.json | 5 +- .../ConnectionHealthToast.test.tsx | 138 ++++++++++++++++++ .../notifications/ConnectionHealthToast.tsx | 73 +++++++-- tests/connectors/test_runtime.py | 10 ++ tests/connectors/test_service_credentials.py | 3 + 16 files changed, 353 insertions(+), 30 deletions(-) create mode 100644 frontend/src/notifications/ConnectionHealthToast.test.tsx diff --git a/docsgpt/agents/tool_executor.py b/docsgpt/agents/tool_executor.py index 12b36c5b..f4da8263 100644 --- a/docsgpt/agents/tool_executor.py +++ b/docsgpt/agents/tool_executor.py @@ -812,15 +812,24 @@ class ToolExecutor: @staticmethod def _connection_payload(resolved) -> Dict: - """What the chat's Connect card needs; never an account or a secret.""" - return { + """What the chat's Connect card needs; never an account or a secret. + + ``connection_id`` is only the caller's own connection, which the card + reconnects in place; an owner's account (``owner_account``) is not + the caller's to reconnect. + """ + payload = { "connector_key": resolved.connector_key, "connector_name": resolved.connector_name, "status": ( "missing" if resolved.row is None else (resolved.row.get("status") or "reconnect_needed") ), + "owner_account": bool(resolved.delegated), } + if resolved.row is not None and not resolved.delegated: + payload["connection_id"] = str(resolved.row["id"]) + return payload def check_pause(self, tools_dict: Dict, call, llm_class_name: str) -> Optional[Dict]: """Return a pending-action dict (approval / client / headless_denied) or None. diff --git a/docsgpt/connectors/service.py b/docsgpt/connectors/service.py index 7d223b63..b801f1c3 100644 --- a/docsgpt/connectors/service.py +++ b/docsgpt/connectors/service.py @@ -464,6 +464,7 @@ def mark_reconnect_needed(connection_id: str, error: str) -> None: if row is None: return already = normalize_status(row) == STATUS_RECONNECT + counts = repo.resource_counts([connection_id]).get(str(connection_id), {}) repo.update(connection_id, {"status": STATUS_RECONNECT, "last_error": error[:500]}) conn.execute( text( @@ -480,6 +481,8 @@ def mark_reconnect_needed(connection_id: str, error: str) -> None: "connection_id": connection_id, "connector_key": catalog.connector_key_for_row(row), "name": serialize_connection(row)["name"], + "source_count": counts.get("sources", 0), + "tool_count": counts.get("tools", 0), }, scope={"kind": "connection", "id": connection_id}, ) diff --git a/frontend/src/conversation/ConnectToolCallBar.test.tsx b/frontend/src/conversation/ConnectToolCallBar.test.tsx index 2175639a..e5043680 100644 --- a/frontend/src/conversation/ConnectToolCallBar.test.tsx +++ b/frontend/src/conversation/ConnectToolCallBar.test.tsx @@ -20,6 +20,11 @@ vi.mock('../api/services/connectorsService', () => ({ }, })); +const launch = vi.fn(); +vi.mock('../connectors/useConnectorLauncher', () => ({ + default: () => ({ launch, modals: null }), +})); + import connectorsReducer from '../connectors/connectorsSlice'; import ConnectToolCallBar from './ConnectToolCallBar'; import type { ToolCallsType } from './types'; @@ -54,13 +59,31 @@ describe('ConnectToolCallBar', () => { container.remove(); }); - const render = async (toolCall: ToolCallsType, onToolAction = vi.fn()) => { + const render = async ( + toolCall: ToolCallsType, + onToolAction = vi.fn(), + connectors?: Record, + ) => { const store = configureStore({ reducer: { connectors: connectorsReducer, preference: (state = { token: null }) => state, }, - }); + preloadedState: connectors + ? { + connectors: { + enabled: true, + loading: false, + loaded: true, + failed: false, + catalog: [], + connections: [], + ...connectors, + }, + preference: { token: null }, + } + : undefined, + } as Parameters[0]); await act(async () => { root.render( @@ -103,6 +126,47 @@ describe('ConnectToolCallBar', () => { expect(onToolAction).toHaveBeenCalledWith('call-1', 'denied'); }); + const TELEGRAM = { + key: 'telegram', + name: 'Telegram', + icon: 'tool_telegram', + auth_kind: 'api_key', + }; + const ownCall = (status: string): ToolCallsType => ({ + ...call(status), + connection_required: { + connector_key: 'telegram', + connector_name: 'Telegram', + status, + connection_id: 'conn-1', + owner_account: false, + }, + }); + + it('reconnects your own account right here', async () => { + launch.mockClear(); + await render(ownCall('reconnect_needed'), vi.fn(), { + catalog: [TELEGRAM], + connections: [{ id: 'conn-1', status: 'reconnect_needed' }], + }); + await act(async () => + button('conversation.toolApproval.connect:Telegram').click(), + ); + expect(launch).toHaveBeenCalledWith(TELEGRAM, { + mode: 'reconnect', + connectionId: 'conn-1', + }); + expect(document.body.textContent).not.toContain('DRAWER'); + }); + + it('offers Continue once the connection works again', async () => { + await render(ownCall('reconnect_needed'), vi.fn(), { + catalog: [TELEGRAM], + connections: [{ id: 'conn-1', status: 'connected' }], + }); + expect(button('conversation.toolApproval.continue')).toBeDefined(); + }); + it('an account that needs signing in again is healed from its drawer', async () => { await render(call('reconnect_needed')); await act(async () => diff --git a/frontend/src/conversation/ConnectToolCallBar.tsx b/frontend/src/conversation/ConnectToolCallBar.tsx index 462fb372..eb12bd0e 100644 --- a/frontend/src/conversation/ConnectToolCallBar.tsx +++ b/frontend/src/conversation/ConnectToolCallBar.tsx @@ -8,6 +8,7 @@ import { Button } from '../components/ui/button'; import ConnectorIcon from '../connectors/ConnectorIcon'; import { loadConnectors, + selectConnections, selectConnectorCatalog, selectConnectorsLoaded, } from '../connectors/connectorsSlice'; @@ -34,12 +35,18 @@ export default function ConnectToolCallBar({ const token = useSelector(selectToken); const catalog = useSelector(selectConnectorCatalog); const loaded = useSelector(selectConnectorsLoaded); + const connections = useSelector(selectConnections); const [connected, setConnected] = useState(false); const { launch, modals } = useConnectorLauncher({ onConnected: () => setConnected(true), }); const required = toolCall.connection_required; const connector = catalog.find((c) => c.key === required?.connector_key); + const ownConnection = required?.connection_id + ? connections.find((c) => c.id === required.connection_id) + : undefined; + // Connected here, or healthy again when the user comes back to the chat. + const ready = connected || ownConnection?.status === 'connected'; const name = required?.connector_name || connector?.name || @@ -50,13 +57,22 @@ export default function ConnectToolCallBar({ }, [loaded, dispatch, token]); const connect = () => { - // A member who has no account for the service connects one. An owner's - // existing account that needs signing in again is healed from its drawer, - // which reconnects that same connection for every tool and source. + // No account yet: connect one. The caller's own account that needs + // signing in again is reconnected right here (every tool and source on + // it heals). MCP servers and anyone else's account go to the drawer. if (connector && required?.status === 'missing') { launch(connector); return; } + const inPlace = + connector?.auth_kind === 'oauth' || connector?.auth_kind === 'api_key'; + if (connector && required?.connection_id && inPlace) { + launch(connector, { + mode: 'reconnect', + connectionId: required.connection_id, + }); + return; + } navigate( `/settings/connectors${connector ? `?connector=${encodeURIComponent(connector.key)}` : ''}`, ); @@ -78,7 +94,7 @@ export default function ConnectToolCallBar({
- {connected ? ( + {ready ? ( + )} + {connector.auth_kind === 'oauth' || isMcpPreset ? null : ( { + mcp.cancel(); + onClose(); + }} submitLabel={t('settings.connectors.wizard.signIn', { name, interpolation: { escapeValue: false }, })} - onSubmit={startSignIn} + onSubmit={isMcpPreset ? startMcpSignIn : startSignIn} + pending={isMcpPreset && (mcp.pending || pending)} /> ) : ( + connector.auth_kind === 'mcp_oauth' && !!connector.mcp_url; + +/** Connectors whose sign-in the wizard redoes in place, with no form. */ +export const reconnectsInPlace = (connector: ConnectorDefinition) => + connector.auth_kind === 'oauth' || + connector.auth_kind === 'api_key' || + isMcpPreset(connector); diff --git a/frontend/src/connectors/useConnectorLauncher.tsx b/frontend/src/connectors/useConnectorLauncher.tsx index 696bf86e..f461a12b 100644 --- a/frontend/src/connectors/useConnectorLauncher.tsx +++ b/frontend/src/connectors/useConnectorLauncher.tsx @@ -13,6 +13,7 @@ import { selectToken } from '../preferences/preferenceSlice'; import type { AppDispatch } from '../store'; import ConnectWizard, { type WizardMode } from './ConnectWizard'; import { loadConnectors } from './connectorsSlice'; +import { isMcpPreset } from './launchRules'; import type { ConnectorDefinition } from './types'; export type LaunchOptions = { @@ -29,6 +30,7 @@ type Active = connector: ConnectorDefinition; mode: WizardMode; connectionId?: string; + mcpToolId?: string; } | { kind: 'mcp'; @@ -61,6 +63,19 @@ export default function useConnectorLauncher({ const launch = useCallback( async (connector: ConnectorDefinition, options: LaunchOptions = {}) => { + if (isMcpPreset(connector)) { + setActive({ + kind: 'wizard', + connector, + mode: options.mode === 'reconnect' ? 'reconnect' : 'connect', + connectionId: options.connectionId, + mcpToolId: + typeof options.mcpServer?.id === 'string' + ? options.mcpServer.id + : undefined, + }); + return; + } if (isMcp(connector)) { setActive({ kind: 'mcp', @@ -158,6 +173,7 @@ export default function useConnectorLauncher({ connector={active.connector} mode={active.mode} connectionId={active.connectionId} + mcpToolId={active.mcpToolId} onClose={() => { setActive(null); refresh(); diff --git a/frontend/src/connectors/useMcpOAuth.ts b/frontend/src/connectors/useMcpOAuth.ts new file mode 100644 index 00000000..705ef9ce --- /dev/null +++ b/frontend/src/connectors/useMcpOAuth.ts @@ -0,0 +1,125 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { useSelector } from 'react-redux'; + +import userService from '../api/services/userService'; +import { selectRecentEvents } from '../notifications/notificationsSlice'; +import { selectToken } from '../preferences/preferenceSlice'; + +export type McpOAuthConfig = { + server_url: string; + auth_type: 'oauth'; + oauth_scopes: string[]; + timeout: number; + redirect_uri: string; +}; + +type Handlers = { + /** Signed in: `taskId` is the OAuth task to save with (null when the + * server already had a valid sign-in). */ + onDone: (result: { taskId: string | null }) => void; + onError: (message: string) => void; +}; + +/** + * Sign in to an MCP server over OAuth from a click. The pop-up opens blank + * inside the click (so the browser allows it) and follows the worker's + * `mcp.oauth.*` events: pointed at the provider on `awaiting_redirect`, + * closed on `completed` or `failed`. `blockedUrl` is set when the browser + * blocked the pop-up anyway, so the caller can offer the link. + */ +export default function useMcpOAuth() { + const token = useSelector(selectToken); + const events = useSelector(selectRecentEvents); + const [taskId, setTaskId] = useState(null); + const [pending, setPending] = useState(false); + const [blockedUrl, setBlockedUrl] = useState(null); + const popupRef = useRef(null); + const handlersRef = useRef(null); + const handledRef = useRef>(new Set()); + + const closePopup = () => { + if (popupRef.current && !popupRef.current.closed) popupRef.current.close(); + popupRef.current = null; + }; + + const finish = useCallback(() => { + closePopup(); + setTaskId(null); + setPending(false); + handlersRef.current = null; + }, []); + + useEffect(() => finish, [finish]); + + const start = useCallback( + async (config: McpOAuthConfig, handlers: Handlers) => { + handlersRef.current = handlers; + handledRef.current = new Set(); + setBlockedUrl(null); + setPending(true); + closePopup(); + popupRef.current = window.open( + 'about:blank', + 'mcpOAuth', + 'width=600,height=700', + ); + try { + const response = await userService.testMCPConnection({ config }, token); + const result = await response.json(); + if (result.requires_oauth && result.task_id) { + setTaskId(result.task_id); + return; + } + const done = handlersRef.current; + finish(); + if (result.success) done?.onDone({ taskId: null }); + else done?.onError(result.message || result.error || ''); + } catch { + const done = handlersRef.current; + finish(); + done?.onError(''); + } + }, + [token, finish], + ); + + useEffect(() => { + if (!taskId) return; + // Newest first in the slice; walk oldest first so a redirect buffered + // before its completion still opens the provider. + for (let i = events.length - 1; i >= 0; i--) { + const event = events[i]; + if (event.scope?.id !== taskId || !event.id) continue; + if (handledRef.current.has(event.id)) continue; + handledRef.current.add(event.id); + const payload = (event.payload ?? {}) as Record; + if (event.type === 'mcp.oauth.awaiting_redirect') { + const url = payload.authorization_url as string | undefined; + if (!url) continue; + if (popupRef.current && !popupRef.current.closed) { + popupRef.current.location.href = url; + } else { + popupRef.current = window.open( + url, + 'mcpOAuth', + 'width=600,height=700', + ); + if (!popupRef.current) setBlockedUrl(url); + } + } else if (event.type === 'mcp.oauth.completed') { + const done = handlersRef.current; + const id = taskId; + finish(); + done?.onDone({ taskId: id }); + return; + } else if (event.type === 'mcp.oauth.failed') { + const done = handlersRef.current; + finish(); + done?.onError(String(payload.error ?? '')); + return; + } + } + }, [events, taskId, finish]); + + return { start, cancel: finish, pending, blockedUrl }; +} diff --git a/frontend/src/conversation/ConnectToolCallBar.tsx b/frontend/src/conversation/ConnectToolCallBar.tsx index eb12bd0e..f014c1bb 100644 --- a/frontend/src/conversation/ConnectToolCallBar.tsx +++ b/frontend/src/conversation/ConnectToolCallBar.tsx @@ -12,6 +12,7 @@ import { selectConnectorCatalog, selectConnectorsLoaded, } from '../connectors/connectorsSlice'; +import { reconnectsInPlace } from '../connectors/launchRules'; import useConnectorLauncher from '../connectors/useConnectorLauncher'; import { selectToken } from '../preferences/preferenceSlice'; import type { AppDispatch } from '../store'; @@ -59,13 +60,13 @@ export default function ConnectToolCallBar({ const connect = () => { // No account yet: connect one. The caller's own account that needs // signing in again is reconnected right here (every tool and source on - // it heals). MCP servers and anyone else's account go to the drawer. + // it heals). Custom MCP servers and anyone else's account go to the + // drawer. if (connector && required?.status === 'missing') { launch(connector); return; } - const inPlace = - connector?.auth_kind === 'oauth' || connector?.auth_kind === 'api_key'; + const inPlace = !!connector && reconnectsInPlace(connector); if (connector && required?.connection_id && inPlace) { launch(connector, { mode: 'reconnect', diff --git a/frontend/src/locale/de.json b/frontend/src/locale/de.json index 620e0348..d5523c49 100644 --- a/frontend/src/locale/de.json +++ b/frontend/src/locale/de.json @@ -1240,7 +1240,9 @@ "connectedWithKey": "Verbunden mit dem Schlüssel, der auf {{hint}} endet.", "doneCounts": "{{sources}} werden synchronisiert, {{tools}} bereit.", "doneSources": "{{sources}} werden synchronisiert.", - "doneTools": "{{tools}} bereit." + "doneTools": "{{tools}} bereit.", + "waiting": "Schließe die Anmeldung bei {{name}} im geöffneten Fenster ab.", + "openSignIn": "Anmeldung bei {{name}} öffnen" }, "remove": { "title": "{{name}} entfernen?", diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index ce555adf..75759a9f 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -1246,7 +1246,9 @@ "connectedWithKey": "Connected with the key ending {{hint}}.", "doneCounts": "{{sources}} syncing, {{tools}} ready.", "doneSources": "{{sources}} syncing.", - "doneTools": "{{tools}} ready." + "doneTools": "{{tools}} ready.", + "waiting": "Finish signing in to {{name}} in the window that opened.", + "openSignIn": "Open the {{name}} sign-in" }, "remove": { "title": "Remove {{name}}?", diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index a96888e4..0954f5f1 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -1240,7 +1240,9 @@ "connectedWithKey": "Conectado con la clave que termina en {{hint}}.", "doneCounts": "{{sources}} sincronizando, {{tools}} listas.", "doneSources": "{{sources}} sincronizando.", - "doneTools": "{{tools}} listas." + "doneTools": "{{tools}} listas.", + "waiting": "Termina de iniciar sesión en {{name}} en la ventana que se abrió.", + "openSignIn": "Abrir el inicio de sesión de {{name}}" }, "remove": { "title": "¿Eliminar {{name}}?", diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index a6123e25..95ff6834 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -1228,7 +1228,9 @@ "connectedWithKey": "末尾が {{hint}} のキーで接続しました。", "doneCounts": "{{sources}}を同期中、{{tools}}を利用できます。", "doneSources": "{{sources}}を同期中です。", - "doneTools": "{{tools}}を利用できます。" + "doneTools": "{{tools}}を利用できます。", + "waiting": "開いたウィンドウで {{name}} へのサインインを完了してください。", + "openSignIn": "{{name}} のサインインを開く" }, "remove": { "title": "{{name}} を削除しますか？", diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index 56767fd3..09dc9661 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -1302,7 +1302,9 @@ "connectedWithKey": "Подключено ключом, оканчивающимся на {{hint}}.", "doneCounts": "Синхронизируется: {{sources}}, готово: {{tools}}.", "doneSources": "Синхронизируется: {{sources}}.", - "doneTools": "Готово: {{tools}}." + "doneTools": "Готово: {{tools}}.", + "waiting": "Завершите вход в {{name}} в открывшемся окне.", + "openSignIn": "Открыть вход в {{name}}" }, "remove": { "title": "Удалить {{name}}?", diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index ef07510b..0cc3bfde 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -1228,7 +1228,9 @@ "connectedWithKey": "已使用結尾為 {{hint}} 的金鑰連線。", "doneCounts": "{{sources}}同步中，{{tools}}已就緒。", "doneSources": "{{sources}}同步中。", - "doneTools": "{{tools}}已就緒。" + "doneTools": "{{tools}}已就緒。", + "waiting": "請在開啟的視窗中完成 {{name}} 登入。", + "openSignIn": "開啟 {{name}} 登入" }, "remove": { "title": "要移除 {{name}} 嗎？", diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index 68a9173b..b9191446 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -1228,7 +1228,9 @@ "connectedWithKey": "已使用以 {{hint}} 结尾的密钥连接。", "doneCounts": "{{sources}}正在同步，{{tools}}已就绪。", "doneSources": "{{sources}}正在同步。", - "doneTools": "{{tools}}已就绪。" + "doneTools": "{{tools}}已就绪。", + "waiting": "请在打开的窗口中完成 {{name}} 登录。", + "openSignIn": "打开 {{name}} 登录" }, "remove": { "title": "移除 {{name}}？", diff --git a/frontend/src/modals/MCPServerModal.tsx b/frontend/src/modals/MCPServerModal.tsx index d8a38ba8..d0c150a1 100644 --- a/frontend/src/modals/MCPServerModal.tsx +++ b/frontend/src/modals/MCPServerModal.tsx @@ -94,6 +94,7 @@ export default function MCPServerModal({ const popupOpenedRef = useRef(false); const [oauthCompleted, setOAuthCompleted] = useState(false); const [saveActive, setSaveActive] = useState(false); + const [showAdvanced, setShowAdvanced] = useState(false); const cleanupOAuthListener = useCallback(() => { setOauthTaskId(null); @@ -526,19 +527,6 @@ export default function MCPServerModal({
); - case 'oauth': - return ( - - - handleInputChange('oauth_scopes', e.target.value) - } - placeholder="read, write" - /> - - ); default: return null; } @@ -660,29 +648,59 @@ export default function MCPServerModal({ {renderAuthFields()} - + {formData.auth_type === 'oauth' && ( + + + handleInputChange('oauth_scopes', e.target.value) + } + placeholder="read, write" + /> + + )} + + { + const value = e.target.value; + if (value === '') { + handleInputChange('timeout', ''); + } else { + const numValue = parseInt(value); + if (!isNaN(numValue) && numValue >= 1) { + handleInputChange('timeout', numValue); + } + } + }} + placeholder="30" + min={1} + max={300} + /> + + + ) : null} + {testResult && ( diff --git a/frontend/src/notifications/ConnectionHealthToast.tsx b/frontend/src/notifications/ConnectionHealthToast.tsx index 236509e0..2fa05ff7 100644 --- a/frontend/src/notifications/ConnectionHealthToast.tsx +++ b/frontend/src/notifications/ConnectionHealthToast.tsx @@ -18,6 +18,7 @@ import { selectConnectorCatalog, selectConnectorsLoaded, } from '../connectors/connectorsSlice'; +import { reconnectsInPlace } from '../connectors/launchRules'; import useConnectorLauncher from '../connectors/useConnectorLauncher'; import { formatCount } from '../utils/dateTimeUtils'; import { selectToken } from '../preferences/preferenceSlice'; @@ -106,8 +107,7 @@ export default function ConnectionHealthToast() { const reconnect = (payload: Record) => { const connectionId = String(payload.connection_id ?? ''); const connector = catalog.find((c) => c.key === payload.connector_key); - const inPlace = - connector?.auth_kind === 'oauth' || connector?.auth_kind === 'api_key'; + const inPlace = !!connector && reconnectsInPlace(connector); if (connector && connectionId && inPlace) { launch(connector, { mode: 'reconnect', connectionId }); return true; @@ -137,7 +137,7 @@ export default function ConnectionHealthToast() { - {/* Sign in again right here; MCP servers reconnect from the + {/* Sign in again right here; custom MCP servers reconnect from the connector's drawer. The toast stays until it works. */}
} /> + + + , + ); + }); + // The loader holds its skeleton for 250 ms after the tools arrive. + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 300)); + }); + return setModalState; + }; + + const service = (key: string) => + document.body.querySelector( + `[data-testid="add-tool-service-${key}"]`, + ); + + it('lists services, presets included, before built-in tools', async () => { + await render(); + const text = document.body.textContent ?? ''; + const services = text.indexOf('settings.tools.groupService'); + expect(services).toBeGreaterThan(-1); + expect(services).toBeLessThan(text.indexOf('settings.tools.groupBuiltIn')); + expect(service('mcp:notion')).not.toBeNull(); + // Sync-only and custom connectors are not tools here. + expect(service('google_drive')).toBeNull(); + expect(service('custom_mcp')).toBeNull(); + }); + + it('connects a service that is not connected yet', async () => { + await render(); + await act(async () => service('mcp:notion')!.click()); + expect(launch).toHaveBeenCalledWith( + expect.objectContaining({ key: 'mcp:notion' }), + ); + }); + + it('opens a connected service to manage its tools', async () => { + await render(); + await act(async () => service('telegram')!.click()); + expect(launch).not.toHaveBeenCalled(); + expect(document.body.textContent).toContain('DRAWER'); + }); +}); diff --git a/frontend/src/modals/AddToolModal.tsx b/frontend/src/modals/AddToolModal.tsx index 9e54e6a7..10d20c16 100644 --- a/frontend/src/modals/AddToolModal.tsx +++ b/frontend/src/modals/AddToolModal.tsx @@ -8,7 +8,12 @@ import userService from '../api/services/userService'; import SkeletonLoader from '../components/SkeletonLoader'; import ToolIcon from '../components/ToolIcon'; import { Button } from '../components/ui/button'; -import { Card, CardDescription, CardTitle } from '../components/ui/card'; +import { + Card, + CardDescription, + CardFooter, + CardTitle, +} from '../components/ui/card'; import { Modal, ModalActions } from '../components/ui/modal'; import { SectionHeader } from '../components/ui/section-header'; import { useLoaderState } from '../hooks'; @@ -16,6 +21,10 @@ import { loadConnectors, selectConnectorCatalog, } from '../connectors/connectorsSlice'; +import { ConnectorStateBadge } from '../connectors/ConnectorCard'; +import ConnectorIcon from '../connectors/ConnectorIcon'; +import { connectorDescription, connectorName } from '../connectors/i18n'; +import type { ConnectorDefinition } from '../connectors/types'; import useConnectorLauncher from '../connectors/useConnectorLauncher'; import PairDeviceModal from '../settings/PairDeviceModal'; import type { AppDispatch } from '../store'; @@ -75,6 +84,30 @@ export default function AddToolModal({ }); }; + // Services come from the connector catalog (tool connectors and MCP + // presets): they are added by connecting the service, first in the list. + const services = catalog.filter( + (connector) => + connector.publisher !== 'custom' && + connector.available && + connector.capabilities.some((c) => c === 'read' || c === 'write'), + ); + const builtIn = availableTools.filter( + (tool) => (tool.group ?? 'built_in') === 'built_in', + ); + + const openService = (connector: ConnectorDefinition) => { + setModalState('INACTIVE'); + // Already connected: its tools exist; open its drawer to manage them. + if (connector.connection_count > 0) { + navigate( + `/settings/connectors?connector=${encodeURIComponent(connector.key)}`, + ); + return; + } + launch(connector); + }; + const handleAddTool = (tool: AvailableToolType) => { // ``remote_device`` is created server-side via the pairing redeem // endpoint, not the standard create_tool path. @@ -171,24 +204,60 @@ export default function AddToolModal({
) : ( - (['built_in', 'service'] as const).map((group) => { - const tools = availableTools.filter( - (tool) => (tool.group ?? 'built_in') === group, - ); - if (tools.length === 0) return null; - return ( -
+ <> + {services.length > 0 && ( +
- {tools.map((tool) => ( + {services.map((connector) => ( + + + + ))} +
+
+ )} + {builtIn.length > 0 && ( +
+ +
+ {builtIn.map((tool) => (
- ); - }) + )} + )}
diff --git a/frontend/src/settings/ToolConfig.test.tsx b/frontend/src/settings/ToolConfig.test.tsx index 7950c13d..e6bad295 100644 --- a/frontend/src/settings/ToolConfig.test.tsx +++ b/frontend/src/settings/ToolConfig.test.tsx @@ -19,9 +19,11 @@ vi.mock('../modals/AddActionModal', () => ({ default: () => null })); vi.mock('../modals/ImportSpecModal', () => ({ default: () => null })); const updateTool = vi.fn(); +const createTool = vi.fn(); vi.mock('../api/services/userService', () => ({ default: { updateTool: (...args: unknown[]) => updateTool(...args), + createTool: (...args: unknown[]) => createTool(...args), deleteTool: () => Promise.resolve(), }, })); @@ -393,4 +395,14 @@ describe('ToolConfig', () => { expect(cancel?.dataset.size).toBe('sm'); expect(cancel?.dataset.shape).toBe('pill'); }); + + it('creates a draft OpenAPI tool on its first save, not before', async () => { + createTool.mockResolvedValue({ ok: true }); + updateTool.mockClear(); + await render({ ...apiTool, id: '' } as APIToolType); + expect(createTool).not.toHaveBeenCalled(); + await act(async () => buttonByText('settings.tools.save')!.click()); + expect(updateTool).not.toHaveBeenCalled(); + expect(createTool.mock.calls[0][0]).toMatchObject({ name: 'api_tool' }); + }); }); diff --git a/frontend/src/settings/ToolConfig.tsx b/frontend/src/settings/ToolConfig.tsx index 65e7f9d9..d189585c 100644 --- a/frontend/src/settings/ToolConfig.tsx +++ b/frontend/src/settings/ToolConfig.tsx @@ -245,6 +245,26 @@ export default function ToolConfig({ }); }; + // Saves the tool; a draft without an id (a new OpenAPI tool) is created + // on its first save, so leaving without saving leaves nothing behind. + const persistTool = async (configToSave: Record) => { + const payload = { + name: tool.name, + displayName: tool.displayName, + customName: customName, + description: tool.description, + config: configToSave, + actions: 'actions' in tool ? tool.actions : [], + status: tool.status, + }; + if (tool.id) { + await userService.updateTool({ id: tool.id, ...payload }, token); + return; + } + const response = await userService.createTool(payload, token); + if (!response.ok) throw new Error('create failed'); + }; + const handleSaveChanges = async () => { if (!validateConfig()) return; const configToSave = buildConfigToSave(); @@ -253,19 +273,7 @@ export default function ToolConfig({ setSaveError(''); try { - await userService.updateTool( - { - id: tool.id, - name: tool.name, - displayName: tool.displayName, - customName: customName, - description: tool.description, - config: configToSave, - actions: 'actions' in tool ? tool.actions : [], - status: tool.status, - }, - token, - ); + await persistTool(configToSave); setInitialState({ customName, configValues: { ...configValues }, @@ -282,6 +290,11 @@ export default function ToolConfig({ }; const handleDelete = () => { + // A draft (a new OpenAPI tool not saved yet) has nothing to delete. + if (!tool.id) { + handleGoBack(); + return; + } userService.deleteTool({ id: tool.id }, token).then(() => { handleGoBack(); }); @@ -362,7 +375,8 @@ export default function ToolConfig({ size="sm" shape="pill" onClick={handleSaveChanges} - disabled={!hasUnsavedChanges} + // A draft (no id yet) is saved to create it. + disabled={!hasUnsavedChanges && !!tool.id} loading={saving} > {t('settings.tools.save')} @@ -768,19 +782,7 @@ export default function ToolConfig({ setSaveError(''); try { - await userService.updateTool( - { - id: tool.id, - name: tool.name, - displayName: tool.displayName, - customName: customName, - description: tool.description, - config: configToSave, - actions: 'actions' in tool ? tool.actions : [], - status: tool.status, - }, - token, - ); + await persistTool(configToSave); setShowUnsavedModal(false); handleGoBack(); } catch { diff --git a/frontend/src/settings/Tools.tsx b/frontend/src/settings/Tools.tsx index 9caa56d9..841fb836 100644 --- a/frontend/src/settings/Tools.tsx +++ b/frontend/src/settings/Tools.tsx @@ -28,6 +28,7 @@ import { selectConnectorCatalog, } from '../connectors/connectorsSlice'; import { useLoaderState } from '../hooks'; +import type { AvailableToolType } from '../modals/types'; import AddToolModal from '../modals/AddToolModal'; import ConfirmationModal from '../modals/ConfirmationModal'; import MCPServerModal from '../modals/MCPServerModal'; @@ -283,15 +284,43 @@ export default function Tools() { dispatch(loadConnectors({ token })); }, []); - // The Connectors page creates an OpenAPI tool and sends its id here so - // the spec import opens straight away. - const openToolId = (location.state as { openToolId?: string } | null) - ?.openToolId; + // The Connectors page opens a new OpenAPI tool here as an unsaved draft: + // the spec import opens straight away and the tool is created on save. + const routeState = location.state as { + openToolId?: string; + newApiTool?: boolean; + } | null; + const openToolId = routeState?.openToolId; + const newApiTool = routeState?.newApiTool; React.useEffect(() => { if (!openToolId) return; handleToolAdded(openToolId); navigate(location.pathname, { replace: true, state: null }); }, [openToolId]); + React.useEffect(() => { + if (!newApiTool) return; + navigate(location.pathname, { replace: true, state: null }); + userService + .getAvailableTools(token) + .then((res) => res.json()) + .then((data) => { + const template = (data.data as AvailableToolType[] | undefined)?.find( + (candidate) => candidate.name === 'api_tool', + ); + if (!template) return; + setSelectedTool({ + id: '', + name: template.name, + displayName: template.displayName, + customName: '', + description: template.description, + config: {}, + actions: template.actions, + status: true, + } as unknown as UserToolType); + }) + .catch(() => undefined); + }, [newApiTool]); return (
{selectedTool ? ( diff --git a/frontend/src/upload/Upload.test.tsx b/frontend/src/upload/Upload.test.tsx index d5587db2..37f866ce 100644 --- a/frontend/src/upload/Upload.test.tsx +++ b/frontend/src/upload/Upload.test.tsx @@ -38,6 +38,11 @@ vi.mock('../api/services/userService', () => ({ }, })); +const launch = vi.hoisted(() => vi.fn()); +vi.mock('../connectors/useConnectorLauncher', () => ({ + default: () => ({ launch, modals: null }), +})); + import Upload from './Upload'; Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); @@ -190,6 +195,34 @@ describe('Upload source-type tiles', () => { connectorsState.connections = []; }); + it('hands a connection tile over to the connect wizard', async () => { + launch.mockClear(); + const s3 = { + key: 's3', + icon: 's3', + sync_ingestor: 's3', + auth_kind: 'api_key', + available: true, + missing_settings: [], + }; + connectorsState.catalog = [s3]; + connectorsState.connections = [ + { id: 'k1', connector_key: 's3', status: 'connected' }, + ]; + await render(); + const tile = tiles().find((t) => + t.textContent?.includes('ingestors.s3.label'), + )!; + await act(async () => tile.click()); + // An existing account goes straight to choosing what to sync. + expect(launch).toHaveBeenCalledWith(s3, { + mode: 'sync', + connectionId: 'k1', + }); + connectorsState.catalog = []; + connectorsState.connections = []; + }); + it('picks the saved account once connections finish loading', async () => { connectorsState.catalog = [ { diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index deb33851..aadb7a55 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -30,6 +30,7 @@ import { OptionCard } from '../components/ui/option-card'; import { SectionHeader } from '../components/ui/section-header'; import ConnectorIcon from '../connectors/ConnectorIcon'; import ConnectorSetupNotice from '../connectors/ConnectorSetupNotice'; +import useConnectorLauncher from '../connectors/useConnectorLauncher'; import { formatCount } from '../utils/dateTimeUtils'; import { loadConnectors, @@ -109,6 +110,12 @@ function Upload({ const connections = useSelector(selectConnections); const connectorsLoaded = useSelector(selectConnectorsLoaded); const connectorsEnabled = useSelector(selectConnectorsEnabled); + // A connection tile hands over to the connect wizard, the one flow every + // entry point uses; this modal steps aside and closes with it. + const [handedOver, setHandedOver] = useState(false); + const { launch, modals: connectModals } = useConnectorLauncher({ + onConnected: () => close(), + }); const [files, setfiles] = useState(receivedFile); // Names of the files the last drop turned away (over the size limit or of @@ -1263,7 +1270,24 @@ function Upload({ `modals.uploadDoc.ingestors.${option.value}.label`, )} description={connectionTileState(option.value)} - onClick={() => handleIngestorTypeChange(option.value)} + onClick={() => { + if (connectorsEnabled && connector?.available) { + const account = connections.find( + (c) => + c.connector_key === connector.key && + c.status === 'connected', + ); + setHandedOver(true); + launch( + connector, + account + ? { mode: 'sync', connectionId: account.id } + : {}, + ); + return; + } + handleIngestorTypeChange(option.value); + }} /> ); })} @@ -1278,6 +1302,8 @@ function Upload({ selectedConnector && needsSetup ? ( ) : null; + if (handedOver) return <>{connectModals}; + return ( Date: Mon, 28 Sep 2026 21:52:18 +0100 Subject: [PATCH 025/162] Simplify Admin > Connectors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - One control for members' own MCP servers: the custom MCP server row's switch (it sets both the policy and the instance switch); the separate top-level switch is gone. - Connectors that only sync content show "No tools" instead of a sharing policy, and the policies read "The sharer decides per share", "Always the sharer's account", "Always each person's own account". - A connector that needs server settings can't be switched on until they exist; a tooltip says why. - On phones the table becomes a list ("On · 2 connections · …") and each connector's controls open in a sheet. --- docs/content/Guides/Connectors.mdx | 6 +- docsgpt/api/admin/connectors.py | 1 + frontend/src/admin/Connectors.test.tsx | 147 +++++++--- frontend/src/admin/Connectors.tsx | 258 +++++++++++++----- .../test_policies_and_attribution.py | 2 + 5 files changed, 304 insertions(+), 110 deletions(-) diff --git a/docs/content/Guides/Connectors.mdx b/docs/content/Guides/Connectors.mdx index e9a27957..6cb651e4 100644 --- a/docs/content/Guides/Connectors.mdx +++ b/docs/content/Guides/Connectors.mdx @@ -50,11 +50,11 @@ Tool calls name the service that ran them: "Searched Notion", "Read from Google Admins manage connectors in **Admin > Connectors**. It lists every connector with its status, how many connections use it, and: - **Enabled** turns a connector off for everyone. Members no longer see it (except to manage a connection they already have), its tools stop working, and its sources stop syncing until you turn it back on. Existing connections are kept. -- **Shared tools use** is **Let owners choose** (default), **Always the owner's account** or **Always each member's own account**. -- **Allow custom MCP servers** controls whether members can add their own MCP servers. Presets are enabled one by one. +- **Shared tools use** is **The sharer decides per share** (default), **Always the sharer's account** or **Always each person's own account**. Connectors that only sync content have no tools and show **No tools**. +- The **MCP server** row decides whether members can add their own MCP servers. Presets are switched one by one. - **Setup guide** on an OAuth connector shows the redirect URI to register and which server settings are still missing. -A connector that still needs server settings starts turned off and is hidden from members. It turns on once its settings are present, unless you switched it off; switching it on before then has no effect for members. +A connector that still needs server settings starts turned off and is hidden from members; its switch stays disabled until the settings are present. It turns on once they are, unless you switched it off. On a phone, the page lists the connectors and opens each one's controls in a panel. ### Encryption key diff --git a/docsgpt/api/admin/connectors.py b/docsgpt/api/admin/connectors.py index e727672d..5b17da40 100644 --- a/docsgpt/api/admin/connectors.py +++ b/docsgpt/api/admin/connectors.py @@ -70,6 +70,7 @@ class AdminConnectorsResource(Resource): "category": definition.category, "publisher": definition.publisher, "auth_kind": definition.auth_kind, + "capabilities": list(definition.capabilities), "enabled": service.connector_is_enabled(policies, definition.key), "credential_mode": policy.get("credential_mode", "choose"), "configured": definition.configured, diff --git a/frontend/src/admin/Connectors.test.tsx b/frontend/src/admin/Connectors.test.tsx index 7f5d56c3..95a67a23 100644 --- a/frontend/src/admin/Connectors.test.tsx +++ b/frontend/src/admin/Connectors.test.tsx @@ -30,6 +30,7 @@ const connector = (overrides: Record = {}) => ({ icon: 'google-drive', publisher: 'built_in', auth_kind: 'oauth', + capabilities: ['sync'], enabled: true, credential_mode: 'choose', configured: false, @@ -43,6 +44,29 @@ const connector = (overrides: Record = {}) => ({ ...overrides, }); +const MCP_ROW = connector({ + key: 'custom_mcp', + name: 'MCP server', + icon: 'mcp', + publisher: 'custom', + auth_kind: 'mcp', + capabilities: ['read', 'write'], + configured: true, + required_settings: [], + connection_count: 0, +}); +const NOTION = { + key: 'mcp_notion', + name: 'Notion', + icon: 'notion', + publisher: 'preset', + auth_kind: 'mcp_oauth', + capabilities: ['read', 'write'], + configured: true, + required_settings: [], + connection_count: 0, +}; + const payload = (overrides: Record = {}) => ({ success: true, connectors: [ @@ -53,10 +77,12 @@ const payload = (overrides: Record = {}) => ({ icon: 'notion', publisher: 'preset', auth_kind: 'mcp_oauth', + capabilities: ['read', 'write'], configured: true, required_settings: [], connection_count: 0, }), + MCP_ROW, ], allow_custom_mcp: true, default_encryption_key: false, @@ -106,7 +132,7 @@ describe('Admin Connectors', () => { getAdmin.mockResolvedValue(payload()); await render(); const rows = Array.from(container.querySelectorAll('tbody tr')); - expect(rows).toHaveLength(2); + expect(rows).toHaveLength(3); expect(rows[0].textContent).toContain('Google Drive'); expect(rows[0].textContent).toContain('Needs setup'); expect(rows[0].textContent).toContain('Setup guide'); @@ -130,24 +156,74 @@ describe('Admin Connectors', () => { it('saves a connector toggle as a policy', async () => { getAdmin.mockResolvedValue(payload()); updateAdmin.mockResolvedValue( - payload({ - connectors: [connector({ enabled: false })], - }), + payload({ connectors: [connector({ ...NOTION, enabled: false })] }), ); await render(); - const toggle = container.querySelector( - '[aria-label="Google Drive enabled"]', - )!; - await act(async () => toggle.click()); + const toggle = () => + container.querySelector( + 'table [aria-label="Notion enabled"]', + )!; + await act(async () => toggle().click()); expect(updateAdmin).toHaveBeenCalledWith( - { policies: { google_drive: { enabled: false } } }, + { policies: { mcp_notion: { enabled: false } } }, null, ); + expect(toggle().getAttribute('aria-checked')).toBe('false'); + }); + + it('keeps a connector that needs setup off, and says why', async () => { + getAdmin.mockResolvedValue(payload()); + await render(); + const drive = container.querySelector( + 'table [aria-label="Google Drive enabled"]', + )!; + expect(drive.disabled).toBe(true); + }); + + it('shows no sharing policy for a sync-only connector', async () => { + getAdmin.mockResolvedValue(payload()); + await render(); + const [drive, notion] = Array.from(container.querySelectorAll('tbody tr')); + expect(drive.textContent).toContain('No tools'); expect( - container - .querySelector('[aria-label="Google Drive enabled"]') - ?.getAttribute('aria-checked'), - ).toBe('false'); + notion.querySelector('[aria-label="Notion sharing policy"]'), + ).not.toBeNull(); + }); + + it('turns custom MCP servers off from their own row', async () => { + getAdmin.mockResolvedValue(payload()); + updateAdmin.mockResolvedValue(payload({ allow_custom_mcp: false })); + await render(); + const mcp = () => + container.querySelector( + 'table [aria-label="MCP server enabled"]', + )!; + await act(async () => mcp().click()); + expect(updateAdmin).toHaveBeenCalledWith( + { allow_custom_mcp: false, policies: { custom_mcp: { enabled: false } } }, + null, + ); + expect(mcp().getAttribute('aria-checked')).toBe('false'); + expect(container.querySelector('#allow-custom-mcp')).toBeNull(); + }); + + it('lists connectors on phones and opens their controls in a sheet', async () => { + getAdmin.mockResolvedValue(payload()); + await render(); + const rows = Array.from( + container.querySelectorAll( + '[data-slot="list-row"] button', + ), + ); + expect(rows).toHaveLength(3); + expect(rows[0].textContent).toContain( + 'Needs setup · 3 connections · No tools', + ); + await act(async () => rows[1].click()); + expect( + document.body.querySelector('[data-slot="sheet-content"]'), + ).not.toBeNull(); + expect(document.body.textContent).toContain('Shared tools use'); }); it('reports a failed save in a toast and keeps the page', async () => { @@ -155,14 +231,21 @@ describe('Admin Connectors', () => { updateAdmin.mockResolvedValue({ success: false }); await render(); await act(async () => - container.querySelector('#allow-custom-mcp')!.click(), + container + .querySelector( + 'table [aria-label="Notion enabled"]', + )! + .click(), + ); + expect(updateAdmin).toHaveBeenCalledWith( + { policies: { mcp_notion: { enabled: false } } }, + null, ); - expect(updateAdmin).toHaveBeenCalledWith({ allow_custom_mcp: false }, null); expect(selectActionToast(store.getState())).toMatchObject({ variant: 'destructive', message: 'Could not save the change.', }); - expect(container.querySelectorAll('tbody tr')).toHaveLength(2); + expect(container.querySelectorAll('tbody tr')).toHaveLength(3); }); it('saves one change at a time so a late response cannot win', async () => { @@ -172,34 +255,28 @@ describe('Admin Connectors', () => { () => new Promise((resolve) => pending.push(resolve)), ); await render(); - const toggle = () => + const notion = () => container.querySelector( - '[aria-label="Google Drive enabled"]', + 'table [aria-label="Notion enabled"]', )!; - await act(async () => toggle().click()); - await act(async () => - container.querySelector('#allow-custom-mcp')!.click(), - ); + const mcp = () => + container.querySelector( + 'table [aria-label="MCP server enabled"]', + )!; + await act(async () => notion().click()); + await act(async () => mcp().click()); // The second save waits for the first. expect(updateAdmin).toHaveBeenCalledTimes(1); - await act(async () => - pending[0](payload({ connectors: [connector({ enabled: false })] })), - ); + const off = connector({ ...NOTION, enabled: false }); + await act(async () => pending[0](payload({ connectors: [off, MCP_ROW] }))); expect(updateAdmin).toHaveBeenCalledTimes(2); await act(async () => pending[1]( - payload({ - connectors: [connector({ enabled: false })], - allow_custom_mcp: false, - }), + payload({ connectors: [off, MCP_ROW], allow_custom_mcp: false }), ), ); - expect(toggle().getAttribute('aria-checked')).toBe('false'); - expect( - container - .querySelector('#allow-custom-mcp')! - .getAttribute('aria-checked'), - ).toBe('false'); + expect(notion().getAttribute('aria-checked')).toBe('false'); + expect(mcp().getAttribute('aria-checked')).toBe('false'); }); it('offers a retry when loading fails', async () => { diff --git a/frontend/src/admin/Connectors.tsx b/frontend/src/admin/Connectors.tsx index bffb55a9..e78fbbe1 100644 --- a/frontend/src/admin/Connectors.tsx +++ b/frontend/src/admin/Connectors.tsx @@ -9,6 +9,14 @@ import { Alert, AlertDescription, AlertTitle } from '../components/ui/alert'; import { Badge } from '../components/ui/badge'; import { Button } from '../components/ui/button'; import { Card } from '../components/ui/card'; +import { FormField } from '../components/ui/form-field'; +import { ListRow, ListRows } from '../components/ui/list-row'; +import { + Sheet, + SheetContent, + SheetDescription, + SheetTitle, +} from '../components/ui/sheet'; import { DescriptionItem, DescriptionList, @@ -34,6 +42,11 @@ import { TableHeader, TableRow, } from '../components/ui/table'; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from '../components/ui/tooltip'; import ConnectorIcon from '../connectors/ConnectorIcon'; import { showActionToast } from '../notifications/actionToastSlice'; import { selectToken } from '../preferences/preferenceSlice'; @@ -47,6 +60,7 @@ type AdminConnector = { icon: string; publisher: 'built_in' | 'preset' | 'custom'; auth_kind: string; + capabilities: string[]; enabled: boolean; credential_mode: Policy; configured: boolean; @@ -66,11 +80,14 @@ type AdminConnectorsData = { }; const POLICY_LABELS: Record = { - choose: 'Let owners choose', - member: "Always each member's own account", - owner: "Always the owner's account", + choose: 'The sharer decides per share', + member: "Always each person's own account", + owner: "Always the sharer's account", }; +const hasTools = (connector: AdminConnector) => + connector.capabilities.some((capability) => capability !== 'sync'); + function CodeRow({ value }: { value: string }) { return ( @@ -161,6 +178,7 @@ export default function Connectors() { const [data, setData] = useState(null); const [loading, setLoading] = useState(true); const [guide, setGuide] = useState(null); + const [detailKey, setDetailKey] = useState(null); const load = useCallback(async () => { setLoading(true); @@ -207,6 +225,94 @@ export default function Connectors() { if (!data?.success) return ; + // The sheet (phones) always shows the connector's latest saved state. + const detail = data.connectors.find((c) => c.key === detailKey) ?? null; + + // The custom MCP row is the one switch for members' own MCP servers. + const isCustomMcp = (connector: AdminConnector) => + connector.key === 'custom_mcp'; + const enabledOf = (connector: AdminConnector) => + connector.enabled && (!isCustomMcp(connector) || data.allow_custom_mcp); + const setEnabled = (connector: AdminConnector, on: boolean) => + save( + isCustomMcp(connector) + ? { allow_custom_mcp: on, policies: { custom_mcp: { enabled: on } } } + : { policies: { [connector.key]: { enabled: on } } }, + ); + + const statusBadge = (connector: AdminConnector) => ( + + {connector.configured ? 'Ready' : 'Needs setup'} + + ); + + const summary = (connector: AdminConnector) => + [ + !connector.configured + ? 'Needs setup' + : enabledOf(connector) + ? 'On' + : 'Off', + `${fmtNumber(connector.connection_count)} ${ + connector.connection_count === 1 ? 'connection' : 'connections' + }`, + hasTools(connector) + ? POLICY_LABELS[connector.credential_mode] + : 'No tools', + ].join(' · '); + + const enabledSwitch = (connector: AdminConnector) => { + const control = ( + setEnabled(connector, checked === true)} + /> + ); + // Off until its server settings exist: switching it on would do nothing. + if (connector.configured) return control; + return ( + + + + {control} + + + Add its server settings first + + ); + }; + + const policyControl = (connector: AdminConnector, fullWidth = false) => + hasTools(connector) ? ( + + ) : ( + No tools + ); + return (
@@ -229,22 +335,6 @@ export default function Connectors() { )} - - - - save({ allow_custom_mcp: checked === true }) - } - /> - - -
- - + + {/* Phones: a list; each row opens the connector's controls. */} + + + {data.connectors.map((connector) => ( + + } + title={connector.name} + description={summary(connector)} + trailing={statusBadge(connector)} + > +