diff --git a/.gitignore b/.gitignore index 1b36e9fc..811ec534 100644 --- a/.gitignore +++ b/.gitignore @@ -183,5 +183,6 @@ application/vectors/ node_modules/ .vscode/settings.json +.vscode/sftp.json /models/ model/ diff --git a/application/core/settings.py b/application/core/settings.py index 5433f832..c9d3cf84 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -72,6 +72,10 @@ class Settings(BaseSettings): MICROSOFT_TENANT_ID: Optional[str] = "common" # Azure AD Tenant ID (or 'common' for multi-tenant) MICROSOFT_AUTHORITY: Optional[str] = None # e.g., "https://login.microsoftonline.com/{tenant_id}" + # Confluence Cloud integration + CONFLUENCE_CLIENT_ID: Optional[str] = None + CONFLUENCE_CLIENT_SECRET: Optional[str] = None + # GitHub source GITHUB_ACCESS_TOKEN: Optional[str] = None # PAT token with read repo access diff --git a/application/parser/connectors/confluence/__init__.py b/application/parser/connectors/confluence/__init__.py new file mode 100644 index 00000000..2b3ba8b7 --- /dev/null +++ b/application/parser/connectors/confluence/__init__.py @@ -0,0 +1,4 @@ +from .auth import ConfluenceAuth +from .loader import ConfluenceLoader + +__all__ = ["ConfluenceAuth", "ConfluenceLoader"] diff --git a/application/parser/connectors/confluence/auth.py b/application/parser/connectors/confluence/auth.py new file mode 100644 index 00000000..0802f4c0 --- /dev/null +++ b/application/parser/connectors/confluence/auth.py @@ -0,0 +1,216 @@ +import datetime +import logging +from typing import Any, Dict, Optional +from urllib.parse import urlencode + +import requests + +from application.core.settings import settings +from application.parser.connectors.base import BaseConnectorAuth + +logger = logging.getLogger(__name__) + + +class ConfluenceAuth(BaseConnectorAuth): + + SCOPES = [ + "read:page:confluence", + "read:space:confluence", + "read:attachment:confluence", + "read:me", + "offline_access", + ] + + AUTH_URL = "https://auth.atlassian.com/authorize" + TOKEN_URL = "https://auth.atlassian.com/oauth/token" + RESOURCES_URL = "https://api.atlassian.com/oauth/token/accessible-resources" + ME_URL = "https://api.atlassian.com/me" + + def __init__(self): + self.client_id = settings.CONFLUENCE_CLIENT_ID + self.client_secret = settings.CONFLUENCE_CLIENT_SECRET + self.redirect_uri = settings.CONNECTOR_REDIRECT_BASE_URI + + if not self.client_id or not self.client_secret: + raise ValueError( + "Confluence OAuth credentials not configured. " + "Please set CONFLUENCE_CLIENT_ID and CONFLUENCE_CLIENT_SECRET in settings." + ) + + def get_authorization_url(self, state: Optional[str] = None) -> str: + params = { + "audience": "api.atlassian.com", + "client_id": self.client_id, + "scope": " ".join(self.SCOPES), + "redirect_uri": self.redirect_uri, + "state": state, + "response_type": "code", + "prompt": "consent", + } + return f"{self.AUTH_URL}?{urlencode(params)}" + + def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]: + if not authorization_code: + raise ValueError("Authorization code is required") + + response = requests.post( + self.TOKEN_URL, + json={ + "grant_type": "authorization_code", + "client_id": self.client_id, + "client_secret": self.client_secret, + "code": authorization_code, + "redirect_uri": self.redirect_uri, + }, + headers={"Content-Type": "application/json"}, + timeout=30, + ) + response.raise_for_status() + token_data = response.json() + + access_token = token_data.get("access_token") + if not access_token: + raise ValueError("OAuth flow did not return an access token") + + refresh_token = token_data.get("refresh_token") + if not refresh_token: + raise ValueError("OAuth flow did not return a refresh token") + + expires_in = token_data.get("expires_in", 3600) + expiry = ( + datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=expires_in) + ).isoformat() + + cloud_id = self._fetch_cloud_id(access_token) + user_info = self._fetch_user_info(access_token) + + return { + "access_token": access_token, + "refresh_token": refresh_token, + "token_uri": self.TOKEN_URL, + "scopes": self.SCOPES, + "expiry": expiry, + "cloud_id": cloud_id, + "user_info": { + "name": user_info.get("display_name", ""), + "email": user_info.get("email", ""), + }, + } + + def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]: + if not refresh_token: + raise ValueError("Refresh token is required") + + response = requests.post( + self.TOKEN_URL, + json={ + "grant_type": "refresh_token", + "client_id": self.client_id, + "client_secret": self.client_secret, + "refresh_token": refresh_token, + }, + headers={"Content-Type": "application/json"}, + timeout=30, + ) + response.raise_for_status() + token_data = response.json() + + access_token = token_data.get("access_token") + new_refresh_token = token_data.get("refresh_token", refresh_token) + + expires_in = token_data.get("expires_in", 3600) + expiry = ( + datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(seconds=expires_in) + ).isoformat() + + cloud_id = self._fetch_cloud_id(access_token) + + return { + "access_token": access_token, + "refresh_token": new_refresh_token, + "token_uri": self.TOKEN_URL, + "scopes": self.SCOPES, + "expiry": expiry, + "cloud_id": cloud_id, + } + + def is_token_expired(self, token_info: Dict[str, Any]) -> bool: + if not token_info: + return True + + expiry = token_info.get("expiry") + if not expiry: + return bool(token_info.get("access_token")) + + try: + expiry_dt = datetime.datetime.fromisoformat(expiry) + now = datetime.datetime.now(datetime.timezone.utc) + return now >= expiry_dt - datetime.timedelta(seconds=60) + except Exception: + return True + + def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]: + from application.core.mongo_db import MongoDB + from application.core.settings import settings as app_settings + + mongo = MongoDB.get_client() + db = mongo[app_settings.MONGO_DB_NAME] + + session = db["connector_sessions"].find_one({"session_token": session_token}) + if not session: + raise ValueError(f"Invalid session token: {session_token}") + + token_info = session.get("token_info") + if not token_info: + raise ValueError("Session missing token information") + + required = ["access_token", "refresh_token", "cloud_id"] + missing = [f for f in required if not token_info.get(f)] + if missing: + raise ValueError(f"Missing required token fields: {missing}") + + return token_info + + def sanitize_token_info( + self, token_info: Dict[str, Any], **extra_fields + ) -> Dict[str, Any]: + return super().sanitize_token_info( + token_info, + cloud_id=token_info.get("cloud_id"), + **extra_fields, + ) + + def _fetch_cloud_id(self, access_token: str) -> str: + response = requests.get( + self.RESOURCES_URL, + headers={ + "Authorization": f"Bearer {access_token}", + "Accept": "application/json", + }, + timeout=30, + ) + response.raise_for_status() + resources = response.json() + + if not resources: + raise ValueError("No accessible Confluence sites found for this account") + + return resources[0]["id"] + + def _fetch_user_info(self, access_token: str) -> Dict[str, Any]: + try: + response = requests.get( + self.ME_URL, + headers={ + "Authorization": f"Bearer {access_token}", + "Accept": "application/json", + }, + timeout=30, + ) + response.raise_for_status() + return response.json() + except Exception as e: + logger.warning("Could not fetch user info: %s", e) + return {} diff --git a/application/parser/connectors/confluence/loader.py b/application/parser/connectors/confluence/loader.py new file mode 100644 index 00000000..3c1ed9b6 --- /dev/null +++ b/application/parser/connectors/confluence/loader.py @@ -0,0 +1,416 @@ +import functools +import logging +import os +from typing import Any, Dict, List, Optional + +import requests + +from application.parser.connectors.base import BaseConnectorLoader +from application.parser.connectors.confluence.auth import ConfluenceAuth +from application.parser.schema.base import Document + +logger = logging.getLogger(__name__) + +API_V2 = "https://api.atlassian.com/ex/confluence/{cloud_id}/wiki/api/v2" +DOWNLOAD_BASE = "https://api.atlassian.com/ex/confluence/{cloud_id}/wiki" + +SUPPORTED_ATTACHMENT_TYPES = { + "application/pdf": ".pdf", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx", + "application/msword": ".doc", + "application/vnd.ms-powerpoint": ".ppt", + "application/vnd.ms-excel": ".xls", + "text/plain": ".txt", + "text/csv": ".csv", + "text/html": ".html", + "text/markdown": ".md", + "application/json": ".json", + "application/epub+zip": ".epub", + "image/jpeg": ".jpg", + "image/png": ".png", +} + + +def _retry_on_auth_failure(func): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) + except requests.exceptions.HTTPError as e: + if e.response is not None and e.response.status_code in (401, 403): + logger.info( + "Auth failure in %s, refreshing token and retrying", func.__name__ + ) + try: + new_token_info = self.auth.refresh_access_token(self.refresh_token) + self.access_token = new_token_info["access_token"] + self.refresh_token = new_token_info.get( + "refresh_token", self.refresh_token + ) + self._persist_refreshed_tokens(new_token_info) + except Exception as refresh_err: + raise ValueError( + f"Authentication failed and could not be refreshed: {refresh_err}" + ) from e + return func(self, *args, **kwargs) + raise + + return wrapper + + +class ConfluenceLoader(BaseConnectorLoader): + + def __init__(self, session_token: str): + self.auth = ConfluenceAuth() + self.session_token = session_token + + token_info = self.auth.get_token_info_from_session(session_token) + self.access_token = token_info["access_token"] + self.refresh_token = token_info["refresh_token"] + self.cloud_id = token_info["cloud_id"] + + self.base_url = API_V2.format(cloud_id=self.cloud_id) + self.download_base = DOWNLOAD_BASE.format(cloud_id=self.cloud_id) + self.next_page_token = None + + def _headers(self) -> Dict[str, str]: + return { + "Authorization": f"Bearer {self.access_token}", + "Accept": "application/json", + } + + def _persist_refreshed_tokens(self, token_info: Dict[str, Any]) -> None: + try: + from application.core.mongo_db import MongoDB + from application.core.settings import settings as app_settings + + sanitized = self.auth.sanitize_token_info(token_info) + mongo = MongoDB.get_client() + db = mongo[app_settings.MONGO_DB_NAME] + db["connector_sessions"].update_one( + {"session_token": self.session_token}, + {"$set": {"token_info": sanitized}}, + ) + except Exception as e: + logger.warning("Failed to persist refreshed tokens: %s", e) + + @_retry_on_auth_failure + def load_data(self, inputs: Dict[str, Any]) -> List[Document]: + folder_id = inputs.get("folder_id") + file_ids = inputs.get("file_ids", []) + limit = inputs.get("limit", 100) + list_only = inputs.get("list_only", False) + page_token = inputs.get("page_token") + search_query = inputs.get("search_query") + self.next_page_token = None + + if file_ids: + return self._load_pages_by_ids(file_ids, list_only, search_query) + + if folder_id: + return self._list_pages_in_space( + folder_id, limit, list_only, page_token, search_query + ) + + return self._list_spaces(limit, page_token, search_query) + + @_retry_on_auth_failure + def download_to_directory(self, local_dir: str, source_config: dict = None) -> dict: + config = source_config or getattr(self, "config", {}) + file_ids = config.get("file_ids", []) + folder_ids = config.get("folder_ids", []) + files_downloaded = 0 + + os.makedirs(local_dir, exist_ok=True) + + if isinstance(file_ids, str): + file_ids = [file_ids] + if isinstance(folder_ids, str): + folder_ids = [folder_ids] + + for page_id in file_ids: + if self._download_page(page_id, local_dir): + files_downloaded += 1 + files_downloaded += self._download_page_attachments(page_id, local_dir) + + for space_id in folder_ids: + files_downloaded += self._download_space(space_id, local_dir) + + return { + "files_downloaded": files_downloaded, + "directory_path": local_dir, + "empty_result": files_downloaded == 0, + "source_type": "confluence", + "config_used": config, + } + + def _list_spaces( + self, limit: int, cursor: Optional[str], search_query: Optional[str] + ) -> List[Document]: + documents: List[Document] = [] + params: Dict[str, Any] = {"limit": min(limit, 250)} + if cursor: + params["cursor"] = cursor + + response = requests.get( + f"{self.base_url}/spaces", + headers=self._headers(), + params=params, + timeout=30, + ) + response.raise_for_status() + data = response.json() + + for space in data.get("results", []): + name = space.get("name", "") + if search_query and search_query.lower() not in name.lower(): + continue + + documents.append( + Document( + text="", + doc_id=space["id"], + extra_info={ + "file_name": name, + "mime_type": "folder", + "size": None, + "created_time": space.get("createdAt"), + "modified_time": None, + "source": "confluence", + "is_folder": True, + "space_key": space.get("key"), + }, + ) + ) + + next_link = data.get("_links", {}).get("next") + self.next_page_token = self._extract_cursor(next_link) + return documents + + def _list_pages_in_space( + self, + space_id: str, + limit: int, + list_only: bool, + cursor: Optional[str], + search_query: Optional[str], + ) -> List[Document]: + documents: List[Document] = [] + params: Dict[str, Any] = {"limit": min(limit, 250)} + if cursor: + params["cursor"] = cursor + + response = requests.get( + f"{self.base_url}/spaces/{space_id}/pages", + headers=self._headers(), + params=params, + timeout=30, + ) + response.raise_for_status() + data = response.json() + + for page in data.get("results", []): + title = page.get("title", "") + if search_query and search_query.lower() not in title.lower(): + continue + + doc = self._page_to_document( + page, load_content=not list_only, space_id=space_id + ) + if doc: + documents.append(doc) + + next_link = data.get("_links", {}).get("next") + self.next_page_token = self._extract_cursor(next_link) + return documents + + def _load_pages_by_ids( + self, page_ids: List[str], list_only: bool, search_query: Optional[str] + ) -> List[Document]: + documents: List[Document] = [] + for page_id in page_ids: + try: + params: Dict[str, str] = {} + if not list_only: + params["body-format"] = "storage" + + response = requests.get( + f"{self.base_url}/pages/{page_id}", + headers=self._headers(), + params=params, + timeout=30, + ) + response.raise_for_status() + page = response.json() + + title = page.get("title", "") + if search_query and search_query.lower() not in title.lower(): + continue + + doc = self._page_to_document(page, load_content=not list_only) + if doc: + documents.append(doc) + except Exception as e: + logger.error("Error loading page %s: %s", page_id, e) + return documents + + def _page_to_document( + self, + page: Dict[str, Any], + load_content: bool = False, + space_id: Optional[str] = None, + ) -> Optional[Document]: + page_id = page.get("id") + title = page.get("title", "Unknown") + version = page.get("version", {}) + modified_time = version.get("createdAt") if isinstance(version, dict) else None + created_time = page.get("createdAt") + resolved_space_id = space_id or page.get("spaceId") + + text = "" + if load_content: + body = page.get("body", {}) + storage = body.get("storage", {}) if isinstance(body, dict) else {} + text = storage.get("value", "") if isinstance(storage, dict) else "" + + return Document( + text=text, + doc_id=str(page_id), + extra_info={ + "file_name": title, + "mime_type": "text/html", + "size": len(text) if text else None, + "created_time": created_time, + "modified_time": modified_time, + "source": "confluence", + "is_folder": False, + "page_id": str(page_id), + "space_id": resolved_space_id, + "cloud_id": self.cloud_id, + }, + ) + + def _download_page(self, page_id: str, local_dir: str) -> bool: + try: + response = requests.get( + f"{self.base_url}/pages/{page_id}", + headers=self._headers(), + params={"body-format": "storage"}, + timeout=30, + ) + response.raise_for_status() + page = response.json() + + title = page.get("title", page_id) + safe_name = "".join(c if c.isalnum() or c in " -_" else "_" for c in title) + body = page.get("body", {}).get("storage", {}).get("value", "") + + file_path = os.path.join(local_dir, f"{safe_name}.html") + with open(file_path, "w", encoding="utf-8") as f: + f.write(body) + + return True + except Exception as e: + logger.error("Error downloading page %s: %s", page_id, e) + return False + + def _download_page_attachments(self, page_id: str, local_dir: str) -> int: + downloaded = 0 + try: + cursor = None + while True: + params: Dict[str, Any] = {"limit": 100} + if cursor: + params["cursor"] = cursor + + response = requests.get( + f"{self.base_url}/pages/{page_id}/attachments", + headers=self._headers(), + params=params, + timeout=30, + ) + response.raise_for_status() + data = response.json() + + for att in data.get("results", []): + media_type = att.get("mediaType", "") + if media_type not in SUPPORTED_ATTACHMENT_TYPES: + continue + + download_link = att.get("_links", {}).get("download") + if not download_link: + continue + + raw_name = att.get("title", att.get("id", "attachment")) + file_name = "".join( + c if c.isalnum() or c in " -_." else "_" + for c in os.path.basename(raw_name) + ) or "attachment" + file_path = os.path.join(local_dir, file_name) + + url = f"{self.download_base}{download_link}" + file_resp = requests.get( + url, headers=self._headers(), timeout=60, stream=True + ) + file_resp.raise_for_status() + + with open(file_path, "wb") as f: + for chunk in file_resp.iter_content(chunk_size=8192): + f.write(chunk) + + downloaded += 1 + + next_link = data.get("_links", {}).get("next") + cursor = self._extract_cursor(next_link) + if not cursor: + break + + except Exception as e: + logger.error("Error downloading attachments for page %s: %s", page_id, e) + return downloaded + + def _download_space(self, space_id: str, local_dir: str) -> int: + downloaded = 0 + cursor = None + while True: + params: Dict[str, Any] = {"limit": 250} + if cursor: + params["cursor"] = cursor + + try: + response = requests.get( + f"{self.base_url}/spaces/{space_id}/pages", + headers=self._headers(), + params=params, + timeout=30, + ) + response.raise_for_status() + data = response.json() + except Exception as e: + logger.error("Error listing pages in space %s: %s", space_id, e) + break + + for page in data.get("results", []): + page_id = page.get("id") + if self._download_page(str(page_id), local_dir): + downloaded += 1 + downloaded += self._download_page_attachments(str(page_id), local_dir) + + next_link = data.get("_links", {}).get("next") + cursor = self._extract_cursor(next_link) + if not cursor: + break + + return downloaded + + @staticmethod + def _extract_cursor(next_link: Optional[str]) -> Optional[str]: + if not next_link: + return None + from urllib.parse import parse_qs, urlparse + + parsed = urlparse(next_link) + cursors = parse_qs(parsed.query).get("cursor") + return cursors[0] if cursors else None diff --git a/application/parser/connectors/connector_creator.py b/application/parser/connectors/connector_creator.py index 609e6407..55371b0e 100644 --- a/application/parser/connectors/connector_creator.py +++ b/application/parser/connectors/connector_creator.py @@ -1,5 +1,7 @@ -from application.parser.connectors.google_drive.loader import GoogleDriveLoader +from application.parser.connectors.confluence.auth import ConfluenceAuth +from application.parser.connectors.confluence.loader import ConfluenceLoader from application.parser.connectors.google_drive.auth import GoogleDriveAuth +from application.parser.connectors.google_drive.loader import GoogleDriveLoader from application.parser.connectors.share_point.auth import SharePointAuth from application.parser.connectors.share_point.loader import SharePointLoader @@ -13,11 +15,13 @@ class ConnectorCreator: """ connectors = { + "confluence": ConfluenceLoader, "google_drive": GoogleDriveLoader, "share_point": SharePointLoader, } auth_providers = { + "confluence": ConfluenceAuth, "google_drive": GoogleDriveAuth, "share_point": SharePointAuth, } diff --git a/docs/content/Guides/Integrations/_meta.js b/docs/content/Guides/Integrations/_meta.js index 6630e25b..3c5354ed 100644 --- a/docs/content/Guides/Integrations/_meta.js +++ b/docs/content/Guides/Integrations/_meta.js @@ -7,6 +7,10 @@ export default { "title": "🔗 SharePoint / OneDrive", "href": "/Guides/Integrations/sharepoint-connector" }, + "confluence-connector": { + "title": "🔗 Confluence", + "href": "/Guides/Integrations/confluence-connector" + }, "mcp-tool-integration": { "title": "🔗 MCP Tools", "href": "/Guides/Integrations/mcp-tool-integration" diff --git a/docs/content/Guides/Integrations/confluence-connector.mdx b/docs/content/Guides/Integrations/confluence-connector.mdx new file mode 100644 index 00000000..e69458ae --- /dev/null +++ b/docs/content/Guides/Integrations/confluence-connector.mdx @@ -0,0 +1,67 @@ +--- +title: Confluence Connector +description: Connect your Confluence Cloud workspace as an external knowledge base to upload and process pages directly. +--- + +import { Callout } from 'nextra/components' +import { Steps } from 'nextra/components' + +# Confluence Connector + +Connect your Confluence Cloud workspace to upload and process pages directly as an external knowledge base. Supports page content and attachments (PDFs, Office files, text files, images, and more). Authentication is handled via Atlassian OAuth 2.0 with automatic token refresh. + +## Setup + + + +### Step 1: Create an OAuth 2.0 App in Atlassian + +1. Go to [developer.atlassian.com/console/myapps](https://developer.atlassian.com/console/myapps/) and click **Create** > **OAuth 2.0 integration** +2. Under **Authorization**, add a callback URL: + - Local: `http://localhost:7091/api/connectors/callback?provider=confluence` + - Production: `https://yourdomain.com/api/connectors/callback?provider=confluence` + +### Step 2: Configure Permissions + +In your app settings, go to **Permissions** and add the **Confluence API**. Enable these scopes: +- `read:page:confluence` +- `read:space:confluence` +- `read:attachment:confluence` + +### Step 3: Get Your Credentials + +Go to **Settings** in your app to find the **Client ID** and **Secret**. Copy both. + +### Step 4: Configure Environment Variables + +Add to your backend `.env` file: + +```env +CONFLUENCE_CLIENT_ID=your-atlassian-client-id +CONFLUENCE_CLIENT_SECRET=your-atlassian-client-secret +``` + +Add to your frontend `.env` file: + +```env +VITE_CONFLUENCE_CLIENT_ID=your-atlassian-client-id +``` + +| Variable | Description | Required | +|----------|-------------|----------| +| `CONFLUENCE_CLIENT_ID` | Client ID from your Atlassian OAuth app | Yes | +| `CONFLUENCE_CLIENT_SECRET` | Client secret from your Atlassian OAuth app | Yes | +| `VITE_CONFLUENCE_CLIENT_ID` | Same Client ID, used by the frontend to show the Confluence option | Yes | + +### Step 5: Restart and Use + +Restart your application, then go to the upload section in DocsGPT and select **Confluence** as the source. You'll be redirected to Atlassian to sign in, then can browse spaces and select pages to process. + + + +## Troubleshooting + +- **Option not appearing** — Verify `VITE_CONFLUENCE_CLIENT_ID` is set in the frontend `.env`, then restart. +- **Authentication failed** — Check that the callback URL matches exactly, including `?provider=confluence`. +- **No accessible sites** — Ensure the authenticating user has access to at least one Confluence Cloud site. +- **Permission denied** — Verify that the Confluence API scopes are enabled in your Atlassian app settings. diff --git a/docs/content/Guides/Integrations/google-drive-connector.mdx b/docs/content/Guides/Integrations/google-drive-connector.mdx index 70d002a2..278dfb83 100644 --- a/docs/content/Guides/Integrations/google-drive-connector.mdx +++ b/docs/content/Guides/Integrations/google-drive-connector.mdx @@ -8,205 +8,66 @@ import { Steps } from 'nextra/components' # Google Drive Connector -The Google Drive Connector allows you to seamlessly connect your Google Drive account as an external knowledge base. This integration enables you to upload and process files directly from your Google Drive without manually downloading and uploading them to DocsGPT. +Connect your Google Drive account to upload and process files directly as an external knowledge base. Supports Google Workspace files (Docs, Sheets, Slides), Office files, PDFs, text files, CSVs, images, and more. Authentication is handled via Google OAuth 2.0 with automatic token refresh. -## Features - -- **Direct File Access**: Browse and select files directly from your Google Drive -- **Comprehensive File Support**: Supports all major document formats including: - - Google Workspace files (Docs, Sheets, Slides) - - Microsoft Office files (.docx, .xlsx, .pptx, .doc, .ppt, .xls) - - PDF documents - - Text files (.txt, .md, .rst, .html, .rtf) - - Data files (.csv, .json) - - Image files (.png, .jpg, .jpeg) - - E-books (.epub) -- **Secure Authentication**: Uses OAuth 2.0 for secure access to your Google Drive -- **Real-time Sync**: Process files directly from Google Drive without local downloads - - -The Google Drive Connector requires proper configuration of Google API credentials. Follow the setup instructions below to enable this feature. - - -## Prerequisites - -Before setting up the Google Drive Connector, you'll need: - -1. A Google Cloud Platform (GCP) project -2. Google Drive API enabled -3. OAuth 2.0 credentials configured -4. DocsGPT instance with proper environment variables - -## Setup Instructions +## Setup ### Step 1: Create a Google Cloud Project -1. Go to the [Google Cloud Console](https://console.cloud.google.com/) -2. Create a new project or select an existing one -3. Note down your Project ID for later use +1. Go to the [Google Cloud Console](https://console.cloud.google.com/) and create a new project (or select an existing one) +2. Navigate to **APIs & Services** > **Library**, search for "Google Drive API", and click **Enable** -### Step 2: Enable Google Drive API +### Step 2: Create OAuth 2.0 Credentials -1. In the Google Cloud Console, navigate to **APIs & Services** > **Library** -2. Search for "Google Drive API" -3. Click on "Google Drive API" and click **Enable** +1. Go to **APIs & Services** > **Credentials** > **Create Credentials** > **OAuth client ID** +2. If prompted, configure the OAuth consent screen (choose **External**, fill in required fields) +3. Select **Web application** as the application type +4. Add your DocsGPT URL to **Authorized JavaScript origins** (e.g. `http://localhost:3000`) +5. Add your callback URL to **Authorized redirect URIs**: + - Local: `http://localhost:7091/api/connectors/callback?provider=google_drive` + - Production: `https://yourdomain.com/api/connectors/callback?provider=google_drive` +6. Click **Create** and copy the **Client ID** and **Client Secret** -### Step 3: Create OAuth 2.0 Credentials +### Step 3: Configure Environment Variables -1. Go to **APIs & Services** > **Credentials** -2. Click **Create Credentials** > **OAuth client ID** -3. If prompted, configure the OAuth consent screen: - - Choose **External** user type (unless you're using Google Workspace) - - Fill in the required fields (App name, User support email, Developer contact) - - Add your domain to **Authorized domains** if deploying publicly -4. For Application type, select **Web application** -5. Add your DocsGPT frontend URL to **Authorized JavaScript origins**: - - For local development: `http://localhost:3000` - - For production: `https://yourdomain.com` -6. Add your DocsGPT callback URL to **Authorized redirect URIs**: - - For local development: `http://localhost:7091/api/connectors/callback?provider=google_drive` - - For production: `https://yourdomain.com/api/connectors/callback?provider=google_drive` -7. Click **Create** and note down the **Client ID** and **Client Secret** - - - -### Step 4: Configure Backend Environment Variables - -Add the following environment variables to your backend configuration: - -**For Docker deployment**, add to your `.env` file in the root directory: +Add to your backend `.env` file: ```env -# Google Drive Connector Configuration -GOOGLE_CLIENT_ID=your_google_client_id_here -GOOGLE_CLIENT_SECRET=your_google_client_secret_here +GOOGLE_CLIENT_ID=your-google-client-id +GOOGLE_CLIENT_SECRET=your-google-client-secret ``` -**For manual deployment**, set these environment variables in your system or application configuration. - -### Step 5: Configure Frontend Environment Variables - -Add the following environment variables to your frontend `.env` file: +Add to your frontend `.env` file: ```env -# Google Drive Frontend Configuration -VITE_GOOGLE_CLIENT_ID=your_google_client_id_here +VITE_GOOGLE_CLIENT_ID=your-google-client-id ``` +| Variable | Description | Required | +|----------|-------------|----------| +| `GOOGLE_CLIENT_ID` | OAuth Client ID from GCP Credentials | Yes | +| `GOOGLE_CLIENT_SECRET` | OAuth Client Secret from GCP Credentials | Yes | +| `VITE_GOOGLE_CLIENT_ID` | Same Client ID, used by the frontend to show the Google Drive option | Yes | + Make sure to use the same Google Client ID in both backend and frontend configurations. -### Step 6: Restart Your Application +### Step 4: Restart and Use -After configuring the environment variables: - -1. **For Docker**: Restart your Docker containers - ```bash - docker-compose down - docker-compose up -d - ``` - -2. **For manual deployment**: Restart both backend and frontend services +Restart your application, then go to the upload section in DocsGPT and select **Google Drive** as the source. You'll be redirected to Google to sign in, then can browse and select files to process. -## Using the Google Drive Connector - -Once configured, you can use the Google Drive Connector to upload files: - - - -### Step 1: Access the Upload Interface - -1. Navigate to the DocsGPT interface -2. Go to the upload/training section -3. You should now see "Google Drive" as an available upload option - -### Step 2: Connect Your Google Account - -1. Select "Google Drive" as your upload method -2. Click "Connect to Google Drive" -3. You'll be redirected to Google's OAuth consent screen -4. Grant the necessary permissions to DocsGPT -5. You'll be redirected back to DocsGPT with a successful connection - -### Step 3: Select Files - -1. Once connected, click "Select Files" -2. The Google Drive picker will open -3. Browse your Google Drive and select the files you want to process -4. Click "Select" to confirm your choices - -### Step 4: Process Files - -1. Review your selected files -2. Click "Train" or "Upload" to process the files -3. DocsGPT will download and process the files from your Google Drive -4. Once processing is complete, the files will be available in your knowledge base - - - -## Supported File Types - -The Google Drive Connector supports the following file types: - -| File Type | Extensions | Description | -|-----------|------------|-------------| -| **Google Workspace** | - | Google Docs, Sheets, Slides (automatically converted) | -| **Microsoft Office** | .docx, .xlsx, .pptx | Modern Office formats | -| **Legacy Office** | .doc, .ppt, .xls | Older Office formats | -| **PDF Documents** | .pdf | Portable Document Format | -| **Text Files** | .txt, .md, .rst, .html, .rtf | Various text formats | -| **Data Files** | .csv, .json | Structured data formats | -| **Images** | .png, .jpg, .jpeg | Image files (with OCR if enabled) | -| **E-books** | .epub | Electronic publication format | - ## Troubleshooting -### Common Issues - -**"Google Drive option not appearing"** -- Verify that `VITE_GOOGLE_CLIENT_ID` is set in frontend environment -- Check that `VITE_GOOGLE_CLIENT_ID` environment variable is present in your frontend configuration -- Check browser console for any JavaScript errors -- Ensure the frontend has been restarted after adding environment variables - -**"Authentication failed"** -- Verify that your OAuth 2.0 credentials are correctly configured -- Check that the redirect URI `http:///api/connectors/callback?provider=google_drive` is correctly added in GCP console -- Ensure the Google Drive API is enabled in your GCP project - -**"Permission denied" errors** -- Verify that the OAuth consent screen is properly configured -- Check that your Google account has access to the files you're trying to select -- Ensure the required scopes are granted during authentication - -**"Files not processing"** -- Check that the backend environment variables are correctly set -- Verify that the OAuth credentials have the necessary permissions -- Check the backend logs for any error messages - -### Environment Variable Checklist - -**Backend (.env in root directory):** -- ✅ `GOOGLE_CLIENT_ID` -- ✅ `GOOGLE_CLIENT_SECRET` - -**Frontend (.env in frontend directory):** -- ✅ `VITE_GOOGLE_CLIENT_ID` - -### Security Considerations - -- Keep your Google Client Secret secure and never expose it in frontend code -- Regularly rotate your OAuth credentials -- Use HTTPS in production to protect authentication tokens -- Ensure proper OAuth consent screen configuration for production use +- **Option not appearing** — Verify `VITE_GOOGLE_CLIENT_ID` is set in the frontend `.env`, then restart. +- **Authentication failed** — Check that the redirect URI matches exactly, including `?provider=google_drive`. Ensure the Google Drive API is enabled. +- **Permission denied** — Verify the OAuth consent screen is configured and the user has access to the target files. +- **Files not processing** — Check backend logs and verify that backend environment variables are correctly set. -For production deployments, make sure to add your actual domain to the OAuth consent screen and authorized origins/redirect URIs. +For production deployments, add your actual domain to the OAuth consent screen and authorized origins/redirect URIs. - - diff --git a/frontend/src/assets/confluence.svg b/frontend/src/assets/confluence.svg new file mode 100644 index 00000000..183807ad --- /dev/null +++ b/frontend/src/assets/confluence.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/src/components/ConnectorAuth.tsx b/frontend/src/components/ConnectorAuth.tsx index eefeb999..d857b8ec 100644 --- a/frontend/src/components/ConnectorAuth.tsx +++ b/frontend/src/components/ConnectorAuth.tsx @@ -1,6 +1,7 @@ import React, { useRef } from 'react'; -import { useSelector } from 'react-redux'; import { useTranslation } from 'react-i18next'; +import { useSelector } from 'react-redux'; + import { useDarkTheme } from '../hooks'; import { selectToken } from '../preferences/preferenceSlice'; @@ -149,7 +150,7 @@ const ConnectorAuth: React.FC = ({ {isConnected ? (
-
+
= ({ {onDisconnect && ( diff --git a/frontend/src/components/FilePicker.tsx b/frontend/src/components/FilePicker.tsx index c096a469..adaef433 100644 --- a/frontend/src/components/FilePicker.tsx +++ b/frontend/src/components/FilePicker.tsx @@ -60,6 +60,10 @@ export const FilePicker: React.FC = ({ displayName: 'SharePoint', rootName: 'My Files', }, + confluence: { + displayName: 'Confluence', + rootName: 'Spaces', + }, } as const; const getProviderConfig = (provider: string) => { @@ -202,7 +206,9 @@ export const FilePicker: React.FC = ({ if (!validateResponse.ok) { removeSessionToken(provider); setIsConnected(false); - setAuthError('Session expired. Please reconnect to Google Drive.'); + setAuthError( + `Session expired. Please reconnect to ${getProviderConfig(provider).displayName}.`, + ); return; } @@ -398,6 +404,7 @@ export const FilePicker: React.FC = ({ { setUserEmail(data.user_email || 'Connected User'); setIsConnected(true); diff --git a/frontend/src/locale/de.json b/frontend/src/locale/de.json index 97b098a4..8bbb2b67 100644 --- a/frontend/src/locale/de.json +++ b/frontend/src/locale/de.json @@ -325,6 +325,14 @@ "s3": { "label": "Amazon S3", "heading": "Inhalt von Amazon S3 hinzufügen" + }, + "share_point": { + "label": "SharePoint", + "heading": "Von SharePoint hochladen" + }, + "confluence": { + "label": "Confluence", + "heading": "Von Confluence hochladen" } }, "connectors": { diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index 1b7c347b..689da277 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -341,6 +341,10 @@ "share_point": { "label": "SharePoint", "heading": "Upload from SharePoint" + }, + "confluence": { + "label": "Confluence", + "heading": "Upload from Confluence" } }, "connectors": { diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index 5a0ed32c..d02308de 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -329,6 +329,10 @@ "share_point": { "label": "SharePoint", "heading": "Subir desde SharePoint" + }, + "confluence": { + "label": "Confluence", + "heading": "Subir desde Confluence" } }, "connectors": { diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index c4fc8b53..c6b304ab 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -329,6 +329,10 @@ "share_point": { "label": "SharePoint", "heading": "SharePointからアップロード" + }, + "confluence": { + "label": "Confluence", + "heading": "Confluenceからアップロード" } }, "connectors": { diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index bec0a093..fbd5f12a 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -329,6 +329,10 @@ "share_point": { "label": "SharePoint", "heading": "Загрузить из SharePoint" + }, + "confluence": { + "label": "Confluence", + "heading": "Загрузить из Confluence" } }, "connectors": { diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index 3f192a90..13fd99df 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -329,6 +329,10 @@ "share_point": { "label": "SharePoint", "heading": "從SharePoint上傳" + }, + "confluence": { + "label": "Confluence", + "heading": "從Confluence上傳" } }, "connectors": { diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index 77c4a874..1bbef6c5 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -329,6 +329,10 @@ "share_point": { "label": "SharePoint", "heading": "从SharePoint上传" + }, + "confluence": { + "label": "Confluence", + "heading": "从Confluence上传" } }, "connectors": { diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index ada545ee..8592817d 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -266,6 +266,23 @@ function Upload({ initialSelectedFolders={selectedFolders} /> ); + case 'confluence_picker': + return ( + { + setSelectedFiles(selectedFileIds); + setSelectedFolders(selectedFolderIds); + }} + provider="confluence" + token={token} + initialSelectedFiles={selectedFiles} + initialSelectedFolders={selectedFolders} + /> + ); default: return null; } @@ -551,6 +568,9 @@ function Upload({ const hasSharePointPicker = schema.some( (field: FormField) => field.type === 'share_point_picker', ); + const hasConfluencePicker = schema.some( + (field: FormField) => field.type === 'confluence_picker', + ); let configData: Record = { ...ingestor.config }; @@ -561,7 +581,8 @@ function Upload({ } else if ( hasRemoteFilePicker || hasGoogleDrivePicker || - hasSharePointPicker + hasSharePointPicker || + hasConfluencePicker ) { const sessionToken = getSessionToken(ingestor.type as string); configData = { @@ -721,6 +742,9 @@ function Upload({ const hasSharePointPicker = schema.some( (field: FormField) => field.type === 'share_point_picker', ); + const hasConfluencePicker = schema.some( + (field: FormField) => field.type === 'confluence_picker', + ); if (hasLocalFilePicker) { if (files.length === 0) { @@ -729,7 +753,8 @@ function Upload({ } else if ( hasRemoteFilePicker || hasGoogleDrivePicker || - hasSharePointPicker + hasSharePointPicker || + hasConfluencePicker ) { if (selectedFiles.length === 0 && selectedFolders.length === 0) { return true; diff --git a/frontend/src/upload/types/ingestor.ts b/frontend/src/upload/types/ingestor.ts index b4440d1a..b30d73a8 100644 --- a/frontend/src/upload/types/ingestor.ts +++ b/frontend/src/upload/types/ingestor.ts @@ -6,8 +6,10 @@ import RedditIcon from '../../assets/reddit.svg'; import DriveIcon from '../../assets/drive.svg'; import S3Icon from '../../assets/s3.svg'; import SharePoint from '../../assets/sharepoint.svg'; +import ConfluenceIcon from '../../assets/confluence.svg'; export type IngestorType = + | 'confluence' | 'crawler' | 'github' | 'reddit' @@ -38,7 +40,8 @@ export type FieldType = | 'local_file_picker' | 'remote_file_picker' | 'google_drive_picker' - | 'share_point_picker'; + | 'share_point_picker' + | 'confluence_picker'; export interface FormField { name: string; @@ -214,6 +217,24 @@ export const IngestorFormSchemas: IngestorSchema[] = [ }, ], }, + { + key: 'confluence', + label: 'Confluence', + icon: ConfluenceIcon, + heading: 'Upload from Confluence', + validate: () => { + const confluenceClientId = import.meta.env.VITE_CONFLUENCE_CLIENT_ID; + return !!confluenceClientId; + }, + fields: [ + { + name: 'files', + label: 'Select Pages from Confluence', + type: 'confluence_picker', + required: true, + }, + ], + }, ]; export const IngestorDefaultConfigs: Record< @@ -261,6 +282,13 @@ export const IngestorDefaultConfigs: Record< recursive: true, }, }, + confluence: { + name: '', + config: { + file_ids: '', + folder_ids: '', + }, + }, }; export interface IngestorOption {