From 6192767451df7606e3b1c21e40c6459b1330989a Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 12 Apr 2026 11:32:24 +0100 Subject: [PATCH] fix: sanitize attachment filenames, drop dateutil dep, add connector docs --- .../parser/connectors/confluence/auth.py | 4 +- .../parser/connectors/confluence/loader.py | 6 +- docs/content/Guides/Integrations/_meta.js | 4 + .../Integrations/confluence-connector.mdx | 67 ++++++ .../Integrations/google-drive-connector.mdx | 203 +++--------------- frontend/src/assets/confluence.svg | 2 +- 6 files changed, 110 insertions(+), 176 deletions(-) create mode 100644 docs/content/Guides/Integrations/confluence-connector.mdx diff --git a/application/parser/connectors/confluence/auth.py b/application/parser/connectors/confluence/auth.py index b46bfdee..0802f4c0 100644 --- a/application/parser/connectors/confluence/auth.py +++ b/application/parser/connectors/confluence/auth.py @@ -145,9 +145,7 @@ class ConfluenceAuth(BaseConnectorAuth): return bool(token_info.get("access_token")) try: - from dateutil import parser - - expiry_dt = parser.parse(expiry) + expiry_dt = datetime.datetime.fromisoformat(expiry) now = datetime.datetime.now(datetime.timezone.utc) return now >= expiry_dt - datetime.timedelta(seconds=60) except Exception: diff --git a/application/parser/connectors/confluence/loader.py b/application/parser/connectors/confluence/loader.py index 6a1b0eea..3c1ed9b6 100644 --- a/application/parser/connectors/confluence/loader.py +++ b/application/parser/connectors/confluence/loader.py @@ -343,7 +343,11 @@ class ConfluenceLoader(BaseConnectorLoader): if not download_link: continue - file_name = att.get("title", att.get("id", "attachment")) + raw_name = att.get("title", att.get("id", "attachment")) + file_name = "".join( + c if c.isalnum() or c in " -_." else "_" + for c in os.path.basename(raw_name) + ) or "attachment" file_path = os.path.join(local_dir, file_name) url = f"{self.download_base}{download_link}" diff --git a/docs/content/Guides/Integrations/_meta.js b/docs/content/Guides/Integrations/_meta.js index 6630e25b..3c5354ed 100644 --- a/docs/content/Guides/Integrations/_meta.js +++ b/docs/content/Guides/Integrations/_meta.js @@ -7,6 +7,10 @@ export default { "title": "🔗 SharePoint / OneDrive", "href": "/Guides/Integrations/sharepoint-connector" }, + "confluence-connector": { + "title": "🔗 Confluence", + "href": "/Guides/Integrations/confluence-connector" + }, "mcp-tool-integration": { "title": "🔗 MCP Tools", "href": "/Guides/Integrations/mcp-tool-integration" diff --git a/docs/content/Guides/Integrations/confluence-connector.mdx b/docs/content/Guides/Integrations/confluence-connector.mdx new file mode 100644 index 00000000..e69458ae --- /dev/null +++ b/docs/content/Guides/Integrations/confluence-connector.mdx @@ -0,0 +1,67 @@ +--- +title: Confluence Connector +description: Connect your Confluence Cloud workspace as an external knowledge base to upload and process pages directly. +--- + +import { Callout } from 'nextra/components' +import { Steps } from 'nextra/components' + +# Confluence Connector + +Connect your Confluence Cloud workspace to upload and process pages directly as an external knowledge base. Supports page content and attachments (PDFs, Office files, text files, images, and more). Authentication is handled via Atlassian OAuth 2.0 with automatic token refresh. + +## Setup + + + +### Step 1: Create an OAuth 2.0 App in Atlassian + +1. Go to [developer.atlassian.com/console/myapps](https://developer.atlassian.com/console/myapps/) and click **Create** > **OAuth 2.0 integration** +2. Under **Authorization**, add a callback URL: + - Local: `http://localhost:7091/api/connectors/callback?provider=confluence` + - Production: `https://yourdomain.com/api/connectors/callback?provider=confluence` + +### Step 2: Configure Permissions + +In your app settings, go to **Permissions** and add the **Confluence API**. Enable these scopes: +- `read:page:confluence` +- `read:space:confluence` +- `read:attachment:confluence` + +### Step 3: Get Your Credentials + +Go to **Settings** in your app to find the **Client ID** and **Secret**. Copy both. + +### Step 4: Configure Environment Variables + +Add to your backend `.env` file: + +```env +CONFLUENCE_CLIENT_ID=your-atlassian-client-id +CONFLUENCE_CLIENT_SECRET=your-atlassian-client-secret +``` + +Add to your frontend `.env` file: + +```env +VITE_CONFLUENCE_CLIENT_ID=your-atlassian-client-id +``` + +| Variable | Description | Required | +|----------|-------------|----------| +| `CONFLUENCE_CLIENT_ID` | Client ID from your Atlassian OAuth app | Yes | +| `CONFLUENCE_CLIENT_SECRET` | Client secret from your Atlassian OAuth app | Yes | +| `VITE_CONFLUENCE_CLIENT_ID` | Same Client ID, used by the frontend to show the Confluence option | Yes | + +### Step 5: Restart and Use + +Restart your application, then go to the upload section in DocsGPT and select **Confluence** as the source. You'll be redirected to Atlassian to sign in, then can browse spaces and select pages to process. + + + +## Troubleshooting + +- **Option not appearing** — Verify `VITE_CONFLUENCE_CLIENT_ID` is set in the frontend `.env`, then restart. +- **Authentication failed** — Check that the callback URL matches exactly, including `?provider=confluence`. +- **No accessible sites** — Ensure the authenticating user has access to at least one Confluence Cloud site. +- **Permission denied** — Verify that the Confluence API scopes are enabled in your Atlassian app settings. diff --git a/docs/content/Guides/Integrations/google-drive-connector.mdx b/docs/content/Guides/Integrations/google-drive-connector.mdx index 70d002a2..278dfb83 100644 --- a/docs/content/Guides/Integrations/google-drive-connector.mdx +++ b/docs/content/Guides/Integrations/google-drive-connector.mdx @@ -8,205 +8,66 @@ import { Steps } from 'nextra/components' # Google Drive Connector -The Google Drive Connector allows you to seamlessly connect your Google Drive account as an external knowledge base. This integration enables you to upload and process files directly from your Google Drive without manually downloading and uploading them to DocsGPT. +Connect your Google Drive account to upload and process files directly as an external knowledge base. Supports Google Workspace files (Docs, Sheets, Slides), Office files, PDFs, text files, CSVs, images, and more. Authentication is handled via Google OAuth 2.0 with automatic token refresh. -## Features - -- **Direct File Access**: Browse and select files directly from your Google Drive -- **Comprehensive File Support**: Supports all major document formats including: - - Google Workspace files (Docs, Sheets, Slides) - - Microsoft Office files (.docx, .xlsx, .pptx, .doc, .ppt, .xls) - - PDF documents - - Text files (.txt, .md, .rst, .html, .rtf) - - Data files (.csv, .json) - - Image files (.png, .jpg, .jpeg) - - E-books (.epub) -- **Secure Authentication**: Uses OAuth 2.0 for secure access to your Google Drive -- **Real-time Sync**: Process files directly from Google Drive without local downloads - - -The Google Drive Connector requires proper configuration of Google API credentials. Follow the setup instructions below to enable this feature. - - -## Prerequisites - -Before setting up the Google Drive Connector, you'll need: - -1. A Google Cloud Platform (GCP) project -2. Google Drive API enabled -3. OAuth 2.0 credentials configured -4. DocsGPT instance with proper environment variables - -## Setup Instructions +## Setup ### Step 1: Create a Google Cloud Project -1. Go to the [Google Cloud Console](https://console.cloud.google.com/) -2. Create a new project or select an existing one -3. Note down your Project ID for later use +1. Go to the [Google Cloud Console](https://console.cloud.google.com/) and create a new project (or select an existing one) +2. Navigate to **APIs & Services** > **Library**, search for "Google Drive API", and click **Enable** -### Step 2: Enable Google Drive API +### Step 2: Create OAuth 2.0 Credentials -1. In the Google Cloud Console, navigate to **APIs & Services** > **Library** -2. Search for "Google Drive API" -3. Click on "Google Drive API" and click **Enable** +1. Go to **APIs & Services** > **Credentials** > **Create Credentials** > **OAuth client ID** +2. If prompted, configure the OAuth consent screen (choose **External**, fill in required fields) +3. Select **Web application** as the application type +4. Add your DocsGPT URL to **Authorized JavaScript origins** (e.g. `http://localhost:3000`) +5. Add your callback URL to **Authorized redirect URIs**: + - Local: `http://localhost:7091/api/connectors/callback?provider=google_drive` + - Production: `https://yourdomain.com/api/connectors/callback?provider=google_drive` +6. Click **Create** and copy the **Client ID** and **Client Secret** -### Step 3: Create OAuth 2.0 Credentials +### Step 3: Configure Environment Variables -1. Go to **APIs & Services** > **Credentials** -2. Click **Create Credentials** > **OAuth client ID** -3. If prompted, configure the OAuth consent screen: - - Choose **External** user type (unless you're using Google Workspace) - - Fill in the required fields (App name, User support email, Developer contact) - - Add your domain to **Authorized domains** if deploying publicly -4. For Application type, select **Web application** -5. Add your DocsGPT frontend URL to **Authorized JavaScript origins**: - - For local development: `http://localhost:3000` - - For production: `https://yourdomain.com` -6. Add your DocsGPT callback URL to **Authorized redirect URIs**: - - For local development: `http://localhost:7091/api/connectors/callback?provider=google_drive` - - For production: `https://yourdomain.com/api/connectors/callback?provider=google_drive` -7. Click **Create** and note down the **Client ID** and **Client Secret** - - - -### Step 4: Configure Backend Environment Variables - -Add the following environment variables to your backend configuration: - -**For Docker deployment**, add to your `.env` file in the root directory: +Add to your backend `.env` file: ```env -# Google Drive Connector Configuration -GOOGLE_CLIENT_ID=your_google_client_id_here -GOOGLE_CLIENT_SECRET=your_google_client_secret_here +GOOGLE_CLIENT_ID=your-google-client-id +GOOGLE_CLIENT_SECRET=your-google-client-secret ``` -**For manual deployment**, set these environment variables in your system or application configuration. - -### Step 5: Configure Frontend Environment Variables - -Add the following environment variables to your frontend `.env` file: +Add to your frontend `.env` file: ```env -# Google Drive Frontend Configuration -VITE_GOOGLE_CLIENT_ID=your_google_client_id_here +VITE_GOOGLE_CLIENT_ID=your-google-client-id ``` +| Variable | Description | Required | +|----------|-------------|----------| +| `GOOGLE_CLIENT_ID` | OAuth Client ID from GCP Credentials | Yes | +| `GOOGLE_CLIENT_SECRET` | OAuth Client Secret from GCP Credentials | Yes | +| `VITE_GOOGLE_CLIENT_ID` | Same Client ID, used by the frontend to show the Google Drive option | Yes | + Make sure to use the same Google Client ID in both backend and frontend configurations. -### Step 6: Restart Your Application +### Step 4: Restart and Use -After configuring the environment variables: - -1. **For Docker**: Restart your Docker containers - ```bash - docker-compose down - docker-compose up -d - ``` - -2. **For manual deployment**: Restart both backend and frontend services +Restart your application, then go to the upload section in DocsGPT and select **Google Drive** as the source. You'll be redirected to Google to sign in, then can browse and select files to process. -## Using the Google Drive Connector - -Once configured, you can use the Google Drive Connector to upload files: - - - -### Step 1: Access the Upload Interface - -1. Navigate to the DocsGPT interface -2. Go to the upload/training section -3. You should now see "Google Drive" as an available upload option - -### Step 2: Connect Your Google Account - -1. Select "Google Drive" as your upload method -2. Click "Connect to Google Drive" -3. You'll be redirected to Google's OAuth consent screen -4. Grant the necessary permissions to DocsGPT -5. You'll be redirected back to DocsGPT with a successful connection - -### Step 3: Select Files - -1. Once connected, click "Select Files" -2. The Google Drive picker will open -3. Browse your Google Drive and select the files you want to process -4. Click "Select" to confirm your choices - -### Step 4: Process Files - -1. Review your selected files -2. Click "Train" or "Upload" to process the files -3. DocsGPT will download and process the files from your Google Drive -4. Once processing is complete, the files will be available in your knowledge base - - - -## Supported File Types - -The Google Drive Connector supports the following file types: - -| File Type | Extensions | Description | -|-----------|------------|-------------| -| **Google Workspace** | - | Google Docs, Sheets, Slides (automatically converted) | -| **Microsoft Office** | .docx, .xlsx, .pptx | Modern Office formats | -| **Legacy Office** | .doc, .ppt, .xls | Older Office formats | -| **PDF Documents** | .pdf | Portable Document Format | -| **Text Files** | .txt, .md, .rst, .html, .rtf | Various text formats | -| **Data Files** | .csv, .json | Structured data formats | -| **Images** | .png, .jpg, .jpeg | Image files (with OCR if enabled) | -| **E-books** | .epub | Electronic publication format | - ## Troubleshooting -### Common Issues - -**"Google Drive option not appearing"** -- Verify that `VITE_GOOGLE_CLIENT_ID` is set in frontend environment -- Check that `VITE_GOOGLE_CLIENT_ID` environment variable is present in your frontend configuration -- Check browser console for any JavaScript errors -- Ensure the frontend has been restarted after adding environment variables - -**"Authentication failed"** -- Verify that your OAuth 2.0 credentials are correctly configured -- Check that the redirect URI `http:///api/connectors/callback?provider=google_drive` is correctly added in GCP console -- Ensure the Google Drive API is enabled in your GCP project - -**"Permission denied" errors** -- Verify that the OAuth consent screen is properly configured -- Check that your Google account has access to the files you're trying to select -- Ensure the required scopes are granted during authentication - -**"Files not processing"** -- Check that the backend environment variables are correctly set -- Verify that the OAuth credentials have the necessary permissions -- Check the backend logs for any error messages - -### Environment Variable Checklist - -**Backend (.env in root directory):** -- ✅ `GOOGLE_CLIENT_ID` -- ✅ `GOOGLE_CLIENT_SECRET` - -**Frontend (.env in frontend directory):** -- ✅ `VITE_GOOGLE_CLIENT_ID` - -### Security Considerations - -- Keep your Google Client Secret secure and never expose it in frontend code -- Regularly rotate your OAuth credentials -- Use HTTPS in production to protect authentication tokens -- Ensure proper OAuth consent screen configuration for production use +- **Option not appearing** — Verify `VITE_GOOGLE_CLIENT_ID` is set in the frontend `.env`, then restart. +- **Authentication failed** — Check that the redirect URI matches exactly, including `?provider=google_drive`. Ensure the Google Drive API is enabled. +- **Permission denied** — Verify the OAuth consent screen is configured and the user has access to the target files. +- **Files not processing** — Check backend logs and verify that backend environment variables are correctly set. -For production deployments, make sure to add your actual domain to the OAuth consent screen and authorized origins/redirect URIs. +For production deployments, add your actual domain to the OAuth consent screen and authorized origins/redirect URIs. - - diff --git a/frontend/src/assets/confluence.svg b/frontend/src/assets/confluence.svg index 0554aee2..183807ad 100644 --- a/frontend/src/assets/confluence.svg +++ b/frontend/src/assets/confluence.svg @@ -1,4 +1,4 @@ - \ No newline at end of file +