Add a built-in GitHub connector for repository sync and read-only tools

One GitHub connection feeds both a Knowledge source and an agent tool.
Users connect with a personal access token, checked against GitHub and
named after the account, or, when an admin registers a GitHub App
(GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET, GITHUB_APP_SLUG), with Sign in
with GitHub. App tokens expire after eight hours and are refreshed before
each sync or tool call; tokens with no expiry are never treated as expired.

The catalog gains an optional second sign-in method (oauth_settings,
exposed as sign_in_methods) without changing any other connector.

Sync lists the repositories the connection can read (the token's own, or
the App installations') and ingests one with the connection's token. The
tool is GitHub's read-only MCP server (api.githubcopilot.com/mcp/readonly):
setup discovers its actions and creates it bound to the connection, and
the executor sends the connection's token only to that server.

docsgpt/api/connector/connections.py

@@ -91,10 +91,22 @@ class ConnectionsList(Resource):
         credentials = body.get("credentials")
         if not isinstance(credentials, dict):
             return _error("credentials must be an object", 400)
+        label = body.get("label") or None
+        if definition.key == "github":
+            # Check the token now, not at the first sync, and name the
+            # connection after the account rather than a hint of the token.
+            from docsgpt.connectors import github
+
+            try:
+                label = label or github.token_account(credentials.get("access_token"))
+            except github.TokenRejected as err:
+                return _error(str(err), 400, code="invalid_credentials")
+            except service.TransientConnectionError as err:
+                return _error(str(err), 502)
         try:
             with db_session() as conn:
                 row, created = service.create_api_key_connection(
-                    conn, user_id, definition, credentials, label=(body.get("label") or None),
+                    conn, user_id, definition, credentials, label=label,
                 )
         except service.EncryptionKeyNotConfigured as err:
             return _error(str(err), 400, code="encryption_key_default")
@@ -211,17 +223,38 @@ class ConnectionSetup(Resource):
         if not user_id:
             return _unauthorized()
         body = _json_body()
+        create_tools = body.get("create_tools", True)
         try:
+            mcp_actions = None
+            with db_readonly() as conn:
+                row = _owned(conn, connection_id, user_id)
+                discover = bool(row) and create_tools and service.needs_mcp_discovery(conn, row)
+            if row is None:
+                return _not_found()
+            if service.normalize_status(row) != service.STATUS_CONNECTED:
+                return _error("Reconnect before setting up", 409, code="reconnect")
+            if discover:
+                # GitHub's tool is its MCP server: read its actions before
+                # the write transaction, not while holding it open.
+                from docsgpt.connectors.mcp import discover_builtin_actions
+
+                try:
+                    mcp_actions = discover_builtin_actions(user_id, row)
+                except service.ConnectionUnavailable:
+                    return _error("Reconnect before setting up", 409, code="reconnect")
+                except Exception as err:
+                    current_app.logger.warning(f"Could not list the MCP server's tools: {err}")
+                    return _error("The service's tools could not be reached. Try again.", 502,
+                                  code="tools_unavailable")
             with db_session() as conn:
                 row = _owned(conn, connection_id, user_id)
                 if row is None:
                     return _not_found()
-                if service.normalize_status(row) != service.STATUS_CONNECTED:
-                    return _error("Reconnect before setting up", 409, code="reconnect")
                 tools = []
-                if body.get("create_tools", True):
+                if create_tools:
                     tools = service.ensure_connection_tools(
                         conn, user_id, row, permissions=body.get("tool_permissions") or None,
+                        mcp_actions=mcp_actions,
                     )
                 tool_payload = [service.serialize_tool(tool) for tool in tools]
             sources = []
@@ -258,7 +291,16 @@ def _start_sync(user_id: str, row: dict, sync: dict):
     frequency = sync.get("frequency") or definition.default_sync_frequency
     if frequency not in _FREQUENCIES:
         return ("Unknown sync frequency", 400)
-    name = (sync.get("name") or "").strip() or definition.name
+    name = (sync.get("name") or "").strip()
+    if definition.sync_ingestor == "github":
+        from docsgpt.parser.remote.github_loader import GitHubLoader
+
+        repo = GitHubLoader.normalize_repo(str(items.get("repo_url") or ""))
+        if not repo:
+            return ("Pick a GitHub repository", 400)
+        items = {**items, "repo_url": repo}
+        name = name or repo
+    name = name or definition.name
     # Validate before claiming the idempotency key: a rejected request must
     # leave the key free for the corrected retry.
     if definition.auth_kind == "oauth":
@@ -307,6 +349,49 @@ def _start_sync(user_id: str, row: dict, sync: dict):
     return {"id": source_id, "task_id": task_id or task.id, "name": name, "sync_frequency": frequency}
 
 
+@connections_ns.route("/connections/<string:connection_id>/repositories")
+class ConnectionRepositories(Resource):
+    @api.doc(
+        description=(
+            "GitHub: the repositories the connection can read, for the sync picker. "
+            "install_url is where a GitHub App sign-in chooses more repositories."
+        )
+    )
+    def get(self, connection_id: str):
+        from docsgpt.connectors import github
+
+        user_id = _user_id()
+        if not user_id:
+            return _unauthorized()
+        with db_readonly() as conn:
+            row = _owned(conn, connection_id, user_id)
+        if row is None or catalog.connector_key_for_row(row) != "github":
+            return _not_found()
+        app_sign_in = (row.get("auth_kind") or "") == "oauth"
+        try:
+            token = service.access_credentials(row).get("access_token")
+            repositories = github.list_repositories(token or "", app=app_sign_in)
+        except service.ConnectionUnavailable:
+            return _error("Reconnect to continue", 409, code="reconnect")
+        except github.TokenRejected as err:
+            service.mark_reconnect_needed(connection_id, str(err))
+            return _error("Reconnect to continue", 409, code="reconnect")
+        except service.TransientConnectionError:
+            return _error("GitHub is not responding. Try again.", 503)
+        except Exception as err:
+            current_app.logger.error(f"Error listing GitHub repositories: {err}", exc_info=True)
+            return _error("Failed to list repositories", 502)
+        install_url = None
+        if app_sign_in:
+            from docsgpt.core.settings import settings
+
+            slug = settings.GITHUB_APP_SLUG
+            install_url = f"https://github.com/apps/{slug}/installations/new" if slug else None
+        return make_response(
+            jsonify({"success": True, "repositories": repositories, "install_url": install_url}), 200,
+        )
+
+
 @connections_ns.route("/connections/<string:connection_id>/reconnect")
 class ConnectionReconnect(Resource):
     @api.doc(

docsgpt/api/connector/routes.py

@@ -106,7 +106,7 @@ def _render_callback_page(
     # The script only carries server-side values: the provider key comes from the
     # supported-connector list rather than the request, and no request text is posted.
     provider_key = next(
-        (key for key in ConnectorCreator.get_supported_connectors() if key == provider_raw.lower()), None,
+        (key for key in ConnectorCreator.get_auth_providers() if key == provider_raw.lower()), None,
     )
     payload = None
     if provider_key and status == "success" and session_token:
@@ -177,12 +177,24 @@ def _render_callback_page(
 
 
 
-def build_authorization(provider: str, user_id: str, connection_id: Optional[str] = None) -> dict:
+def build_authorization(
+    provider: str, user_id: str, connection_id: Optional[str] = None, *, install: bool = False,
+) -> dict:
     """Start an OAuth sign-in for ``provider`` and return its authorization URL.
 
+    Args:
+        provider: The connector, e.g. ``google_drive`` or ``github``.
+        user_id: The caller.
+        connection_id: The caller's connection to sign in again, if any.
+        install: GitHub: send the user to install the GitHub App (where the
+            repositories it can read are chosen) instead of straight to
+            authorization. GitHub returns to the callback with the same
+            state when the app requests authorization during installation.
+
     Raises:
         service.EncryptionKeyNotConfigured: See ``ensure_can_store_credentials``.
         service.ConnectionUnavailable: ``connection_id`` is not the caller's.
+        ValueError: ``install`` for a provider with no installation page.
     """
     service.ensure_can_store_credentials()
     with db_session() as conn:
@@ -191,8 +203,11 @@ def build_authorization(provider: str, user_id: str, connection_id: Optional[str
         json.dumps({"provider": provider, "object_id": str(session_row["id"])}).encode()
     ).decode()
     auth = ConnectorCreator.create_auth(provider)
+    if install and not hasattr(auth, "get_installation_url"):
+        raise ValueError(f"{provider} has no installation page")
+    url = auth.get_installation_url(state=state) if install else auth.get_authorization_url(state=state)
     return {
-        "authorization_url": auth.get_authorization_url(state=state),
+        "authorization_url": url,
         "state": state,
         "callback_origin": _origin_of(settings.CONNECTOR_REDIRECT_BASE_URI),
     }
@@ -207,7 +222,7 @@ class ConnectorAuth(Resource):
             if not provider:
                 return make_response(jsonify({"success": False, "error": "Missing provider"}), 400)
 
-            if not ConnectorCreator.is_supported(provider):
+            if not (ConnectorCreator.is_supported(provider) or ConnectorCreator.has_auth(provider)):
                 return make_response(jsonify({"success": False, "error": f"Unsupported provider: {provider}"}), 400)
 
             decoded_token = request.decoded_token
@@ -215,7 +230,10 @@ class ConnectorAuth(Resource):
                 return make_response(jsonify({"success": False, "error": "Unauthorized"}), 401)
             user_id = decoded_token.get('sub')
             try:
-                started = build_authorization(provider, user_id, request.args.get("connection_id") or None)
+                started = build_authorization(
+                    provider, user_id, request.args.get("connection_id") or None,
+                    install=request.args.get("install") in ("1", "true"),
+                )
             except service.EncryptionKeyNotConfigured as err:
                 return make_response(
                     jsonify({"success": False, "error": str(err), "code": "encryption_key_default"}), 400,
@@ -251,12 +269,24 @@ class ConnectorsCallback(Resource):
             state = request.args.get('state')
             error = request.args.get('error')
 
+            if not state and request.args.get('installation_id'):
+                # The GitHub App was installed from GitHub itself, not from a
+                # DocsGPT sign-in: there is no state to tie the code to a user,
+                # so the code is ignored and the user goes back to DocsGPT.
+                return _render_callback_page(
+                    "success",
+                    "The GitHub App is installed. Return to DocsGPT and refresh the repository list.",
+                    "github",
+                )
+
             state_dict = json.loads(base64.urlsafe_b64decode(state.encode()).decode())
             provider = state_dict.get("provider")
             state_object_id = state_dict.get("object_id")
 
             # Validate provider
-            if not provider or not isinstance(provider, str) or not ConnectorCreator.is_supported(provider):
+            if not provider or not isinstance(provider, str) or not (
+                ConnectorCreator.is_supported(provider) or ConnectorCreator.has_auth(provider)
+            ):
                 return redirect(build_callback_redirect({
                     "status": "error",
                     "message": "Invalid provider"
@@ -295,7 +325,9 @@ class ConnectorsCallback(Resource):
                         user_info = drive_service.about().get(fields="user").execute()
                         user_email = user_info.get('user', {}).get('emailAddress', 'Connected User')
                     else:
-                        user_email = token_info.get('user_info', {}).get('email', 'Connected User')
+                        # GitHub names the account by its login, the others by email.
+                        user_info = token_info.get('user_info') or {}
+                        user_email = user_info.get('email') or user_info.get('login') or 'Connected User'
 
                 except Exception as e:
                     current_app.logger.warning(f"Could not get user info: {e}")