diff --git a/HACKTOBERFEST.md b/HACKTOBERFEST.md index 631f73ba..47679960 100644 --- a/HACKTOBERFEST.md +++ b/HACKTOBERFEST.md @@ -7,7 +7,7 @@ All contributors with accepted PRs will receive a cool Holopin! 🤩 (Watch out ### 🏆 Top 50 contributors will recieve a special T-shirt ### 🏆 [LLM Document analysis by LexEU competition](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md): -A separate competition is available for those who sumbit new retrieval / workflow method that will analyze a Document using EU laws. +A separate competition is available for those who submit new retrieval / workflow method that will analyze a Document using EU laws. With 200$, 100$, 50$ prize for 1st, 2nd and 3rd place respectively. You can find more information [here](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md) diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 35b95174..9a22db84 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -40,6 +40,8 @@ if settings.LLM_NAME == "openai": gpt_model = "gpt-3.5-turbo" elif settings.LLM_NAME == "anthropic": gpt_model = "claude-2" +elif settings.LLM_NAME == "groq": + gpt_model = "llama3-8b-8192" if settings.MODEL_NAME: # in case there is particular model name configured gpt_model = settings.MODEL_NAME diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 340d020a..c409e69a 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -363,6 +363,7 @@ class UploadRemote(Resource): ), "name": fields.String(required=True, description="Job name"), "data": fields.String(required=True, description="Data to process"), + "repo_url": fields.String(description="GitHub repository URL"), }, ) ) @@ -377,11 +378,18 @@ class UploadRemote(Resource): return missing_fields try: + if "repo_url" in data: + source_data = data["repo_url"] + loader = "github" + else: + source_data = data["data"] + loader = data["source"] + task = ingest_remote.delay( - source_data=data["data"], + source_data=source_data, job_name=data["name"], user=data["user"], - loader=data["source"], + loader=loader, ) except Exception as err: return make_response(jsonify({"success": False, "error": str(err)}), 400) diff --git a/application/llm/groq.py b/application/llm/groq.py new file mode 100644 index 00000000..b5731a90 --- /dev/null +++ b/application/llm/groq.py @@ -0,0 +1,45 @@ +from application.llm.base import BaseLLM + + + +class GroqLLM(BaseLLM): + + def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): + from openai import OpenAI + + super().__init__(*args, **kwargs) + self.client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1") + self.api_key = api_key + self.user_api_key = user_api_key + + def _raw_gen( + self, + baseself, + model, + messages, + stream=False, + **kwargs + ): + response = self.client.chat.completions.create( + model=model, messages=messages, stream=stream, **kwargs + ) + + return response.choices[0].message.content + + def _raw_gen_stream( + self, + baseself, + model, + messages, + stream=True, + **kwargs + ): + response = self.client.chat.completions.create( + model=model, messages=messages, stream=stream, **kwargs + ) + + for line in response: + # import sys + # print(line.choices[0].delta.content, file=sys.stderr) + if line.choices[0].delta.content is not None: + yield line.choices[0].delta.content diff --git a/application/llm/llm_creator.py b/application/llm/llm_creator.py index 7960778b..6a19de10 100644 --- a/application/llm/llm_creator.py +++ b/application/llm/llm_creator.py @@ -1,3 +1,4 @@ +from application.llm.groq import GroqLLM from application.llm.openai import OpenAILLM, AzureOpenAILLM from application.llm.sagemaker import SagemakerAPILLM from application.llm.huggingface import HuggingFaceLLM @@ -17,6 +18,7 @@ class LLMCreator: "anthropic": AnthropicLLM, "docsgpt": DocsGPTAPILLM, "premai": PremAILLM, + "groq": GroqLLM } @classmethod diff --git a/application/parser/file/bulk.py b/application/parser/file/bulk.py index aec6c8c1..79fc2c45 100644 --- a/application/parser/file/bulk.py +++ b/application/parser/file/bulk.py @@ -10,13 +10,14 @@ from application.parser.file.epub_parser import EpubParser from application.parser.file.html_parser import HTMLParser from application.parser.file.markdown_parser import MarkdownParser from application.parser.file.rst_parser import RstParser -from application.parser.file.tabular_parser import PandasCSVParser +from application.parser.file.tabular_parser import PandasCSVParser,ExcelParser from application.parser.schema.base import Document DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = { ".pdf": PDFParser(), ".docx": DocxParser(), ".csv": PandasCSVParser(), + ".xlsx":ExcelParser(), ".epub": EpubParser(), ".md": MarkdownParser(), ".rst": RstParser(), diff --git a/application/parser/file/tabular_parser.py b/application/parser/file/tabular_parser.py index 81355ae0..b2dbd193 100644 --- a/application/parser/file/tabular_parser.py +++ b/application/parser/file/tabular_parser.py @@ -113,3 +113,68 @@ class PandasCSVParser(BaseParser): return (self._row_joiner).join(text_list) else: return text_list + + +class ExcelParser(BaseParser): + r"""Excel (.xlsx) parser. + + Parses Excel files using Pandas `read_excel` function. + If special parameters are required, use the `pandas_config` dict. + + Args: + concat_rows (bool): whether to concatenate all rows into one document. + If set to False, a Document will be created for each row. + True by default. + + col_joiner (str): Separator to use for joining cols per row. + Set to ", " by default. + + row_joiner (str): Separator to use for joining each row. + Only used when `concat_rows=True`. + Set to "\n" by default. + + pandas_config (dict): Options for the `pandas.read_excel` function call. + Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html + for more information. + Set to empty dict by default, this means pandas will try to figure + out the table structure on its own. + + """ + + def __init__( + self, + *args: Any, + concat_rows: bool = True, + col_joiner: str = ", ", + row_joiner: str = "\n", + pandas_config: dict = {}, + **kwargs: Any + ) -> None: + """Init params.""" + super().__init__(*args, **kwargs) + self._concat_rows = concat_rows + self._col_joiner = col_joiner + self._row_joiner = row_joiner + self._pandas_config = pandas_config + + def _init_parser(self) -> Dict: + """Init parser.""" + return {} + + def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]: + """Parse file.""" + try: + import pandas as pd + except ImportError: + raise ValueError("pandas module is required to read Excel files.") + + df = pd.read_excel(file, **self._pandas_config) + + text_list = df.apply( + lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1 + ).tolist() + + if self._concat_rows: + return (self._row_joiner).join(text_list) + else: + return text_list \ No newline at end of file diff --git a/application/parser/remote/github_loader.py b/application/parser/remote/github_loader.py index e69de29b..49f0ae9c 100644 --- a/application/parser/remote/github_loader.py +++ b/application/parser/remote/github_loader.py @@ -0,0 +1,53 @@ +import base64 +import requests +from typing import List +from application.parser.remote.base import BaseRemote +from langchain_core.documents import Document + +class GitHubLoader(BaseRemote): + def __init__(self): + self.access_token = None + self.headers = { + "Authorization": f"token {self.access_token}" + } if self.access_token else {} + return + + def fetch_file_content(self, repo_url: str, file_path: str) -> str: + url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}" + response = requests.get(url, headers=self.headers) + + if response.status_code == 200: + content = response.json() + if content.get("encoding") == "base64": + try: + decoded_content = base64.b64decode(content["content"]).decode("utf-8") + return f"Filename: {file_path}\n\n{decoded_content}" + except Exception as e: + print(f"Error decoding content for {file_path}: {e}") + raise + else: + return f"Filename: {file_path}\n\n{content['content']}" + else: + response.raise_for_status() + + def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]: + url = f"https://api.github.com/repos/{repo_url}/contents/{path}" + response = requests.get(url, headers={**self.headers, "Accept": "application/vnd.github.v3.raw"}) + contents = response.json() + files = [] + for item in contents: + if item["type"] == "file": + files.append(item["path"]) + elif item["type"] == "dir": + files.extend(self.fetch_repo_files(repo_url, item["path"])) + return files + + def load_data(self, repo_url: str) -> List[Document]: + repo_name = repo_url.split("github.com/")[-1] + files = self.fetch_repo_files(repo_name) + documents = [] + for file_path in files: + content = self.fetch_file_content(repo_name, file_path) + documents.append(Document(page_content=content, metadata={"title": file_path, + "source": f"https://github.com/{repo_name}/blob/main/{file_path}"})) + return documents diff --git a/application/parser/remote/remote_creator.py b/application/parser/remote/remote_creator.py index d2a58f8d..026abd76 100644 --- a/application/parser/remote/remote_creator.py +++ b/application/parser/remote/remote_creator.py @@ -2,6 +2,7 @@ from application.parser.remote.sitemap_loader import SitemapLoader from application.parser.remote.crawler_loader import CrawlerLoader from application.parser.remote.web_loader import WebLoader from application.parser.remote.reddit_loader import RedditPostsLoaderRemote +from application.parser.remote.github_loader import GitHubLoader class RemoteCreator: @@ -10,6 +11,7 @@ class RemoteCreator: "sitemap": SitemapLoader, "crawler": CrawlerLoader, "reddit": RedditPostsLoaderRemote, + "github": GitHubLoader, } @classmethod diff --git a/application/requirements.txt b/application/requirements.txt index d7621cfd..6a57dd12 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -49,6 +49,7 @@ openapi3-parser==1.1.18 orjson==3.10.7 packaging==24.1 pandas==2.2.3 +openpyxl==3.1.5 pathable==0.4.3 pillow==10.4.0 portalocker==2.10.1 diff --git a/application/vectorstore/faiss.py b/application/vectorstore/faiss.py index e6c13bcd..afa55db9 100644 --- a/application/vectorstore/faiss.py +++ b/application/vectorstore/faiss.py @@ -22,7 +22,7 @@ class FaissStore(BaseVectorStore): else: self.docsearch = FAISS.load_local(self.path, embeddings, allow_dangerous_deserialization=True) except Exception: - raise # Just re-raise the exception without assigning to e + raise self.assert_embedding_dimensions(embeddings) diff --git a/application/worker.py b/application/worker.py index 2000523c..f8f38afa 100755 --- a/application/worker.py +++ b/application/worker.py @@ -131,10 +131,10 @@ def ingest_worker( logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": name_job}) file_data = {"name": name_job, "file": filename, "user": user} - download_file(urljoin(settings.API_URL, "/api/download"), file_data, os.path.join(full_path, filename)) if not os.path.exists(full_path): os.makedirs(full_path) + download_file(urljoin(settings.API_URL, "/api/download"), file_data, os.path.join(full_path, filename)) # check if file is .zip and extract it if filename.endswith(".zip"): diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1a6e0ce3..4087e4f5 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,7 +9,6 @@ "version": "0.0.0", "dependencies": { "@reduxjs/toolkit": "^2.2.7", - "@vercel/analytics": "^1.3.1", "chart.js": "^4.4.4", "i18next": "^23.15.1", "i18next-browser-languagedetector": "^8.0.0", @@ -2089,26 +2088,6 @@ "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==" }, - "node_modules/@vercel/analytics": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-1.3.1.tgz", - "integrity": "sha512-xhSlYgAuJ6Q4WQGkzYTLmXwhYl39sWjoMA3nHxfkvG+WdBT25c563a7QhwwKivEOZtPJXifYHR1m2ihoisbWyA==", - "dependencies": { - "server-only": "^0.0.1" - }, - "peerDependencies": { - "next": ">= 13", - "react": "^18 || ^19" - }, - "peerDependenciesMeta": { - "next": { - "optional": true - }, - "react": { - "optional": true - } - } - }, "node_modules/@vitejs/plugin-react": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.3.1.tgz", @@ -8451,11 +8430,6 @@ "semver": "bin/semver.js" } }, - "node_modules/server-only": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/server-only/-/server-only-0.0.1.tgz", - "integrity": "sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==" - }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index 176c4fd9..83d531d6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -20,7 +20,6 @@ }, "dependencies": { "@reduxjs/toolkit": "^2.2.7", - "@vercel/analytics": "^1.3.1", "chart.js": "^4.4.4", "i18next": "^23.15.1", "i18next-browser-languagedetector": "^8.0.0", diff --git a/frontend/src/Hero.tsx b/frontend/src/Hero.tsx index 04da8769..644848dc 100644 --- a/frontend/src/Hero.tsx +++ b/frontend/src/Hero.tsx @@ -37,7 +37,7 @@ export default function Hero({ )}
-
- -

DocsGPT

+
{ + if (isMobile) { + setNavOpen(!navOpen); + } + }}> + + +

DocsGPT

+
{ - dispatch(setConversation([])); - dispatch( - updateConversationId({ - query: { conversationId: null }, - }), - ); + if (isMobile) { + setNavOpen(!navOpen); + } + resetConversation(); }} className={({ isActive }) => - `${ - isActive ? 'bg-gray-3000 dark:bg-transparent' : '' + `${isActive ? 'bg-gray-3000 dark:bg-transparent' : '' } group sticky mx-4 mt-4 flex cursor-pointer gap-2.5 rounded-3xl border border-silver p-3 hover:border-rainy-gray hover:bg-gray-3000 dark:border-purple-taupe dark:text-white dark:hover:bg-transparent` } > @@ -270,6 +280,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { key={conversation.id} conversation={conversation} selectConversation={(id) => handleConversationClick(id)} + onCoversationClick={() => {if (isMobile) { setNavOpen(false) }}} onDeleteConversation={(id) => handleDeleteConversation(id)} onSave={(conversation) => updateConversationName(conversation) @@ -293,21 +304,36 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) { isDocsListOpen={isDocsListOpen} setIsDocsListOpen={setIsDocsListOpen} handleDeleteClick={handleDeleteClick} + handlePostDocumentSelect={(option?: string) => { + if (isMobile) { + setNavOpen(!navOpen) + } + }} /> setUploadModalState('ACTIVE')} - > + onClick={() => { + setUploadModalState('ACTIVE') + if (isMobile) { + setNavOpen(!navOpen); + } + } + }>

{t('sourceDocs')}

{ + if (isMobile) { + setNavOpen(!navOpen); + } + resetConversation(); + }} to="/settings" className={({ isActive }) => - `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${ - isActive ? 'bg-gray-3000 dark:bg-transparent' : '' + `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${isActive ? 'bg-gray-3000 dark:bg-transparent' : '' }` } > @@ -323,10 +349,15 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
{ + if (isMobile) { + setNavOpen(!navOpen); + } + resetConversation(); + }} to="/about" className={({ isActive }) => - `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${ - isActive ? 'bg-gray-3000 dark:bg-[#28292E]' : '' + `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${isActive ? 'bg-gray-3000 dark:bg-[#28292E]' : '' }` } > diff --git a/frontend/src/components/SettingsBar.tsx b/frontend/src/components/SettingsBar.tsx new file mode 100644 index 00000000..2b7c2a33 --- /dev/null +++ b/frontend/src/components/SettingsBar.tsx @@ -0,0 +1,96 @@ +import React, { useCallback, useRef, useState } from 'react'; +import ArrowLeft from '../assets/arrow-left.svg'; +import ArrowRight from '../assets/arrow-right.svg'; +import { useTranslation } from 'react-i18next'; + +type HiddenGradientType = 'left' | 'right' | undefined; + +const useTabs = () => { + const { t } = useTranslation(); + const tabs = [ + t('settings.general.label'), + t('settings.documents.label'), + t('settings.apiKeys.label'), + t('settings.analytics.label'), + t('settings.logs.label'), + ]; + return tabs; +}; + +interface SettingsBarProps { + setActiveTab: React.Dispatch>; + activeTab: string; +} + +const SettingsBar = ({ setActiveTab, activeTab }: SettingsBarProps) => { + const [hiddenGradient, setHiddenGradient] = + useState('left'); + const containerRef = useRef(null); + const tabs = useTabs(); + const scrollTabs = useCallback( + (direction: number) => { + if (containerRef.current) { + const container = containerRef.current; + container.scrollLeft += direction * 100; // Adjust the scroll amount as needed + if (container.scrollLeft === 0) { + setHiddenGradient('left'); + } else if ( + container.scrollLeft + container.offsetWidth === + container.scrollWidth + ) { + setHiddenGradient('right'); + } else { + setHiddenGradient(undefined); + } + } + }, + [containerRef.current], + ); + return ( +
+
+
+ +
+ +
+
+ {tabs.map((tab, index) => ( + + ))} +
+
+ +
+
+ ); +}; + +export default SettingsBar; diff --git a/frontend/src/components/SourceDropdown.tsx b/frontend/src/components/SourceDropdown.tsx index d5146da5..6a348161 100644 --- a/frontend/src/components/SourceDropdown.tsx +++ b/frontend/src/components/SourceDropdown.tsx @@ -11,6 +11,7 @@ type Props = { isDocsListOpen: boolean; setIsDocsListOpen: React.Dispatch>; handleDeleteClick: any; + handlePostDocumentSelect: any; }; function SourceDropdown({ @@ -20,6 +21,7 @@ function SourceDropdown({ setIsDocsListOpen, isDocsListOpen, handleDeleteClick, + handlePostDocumentSelect, // Callback function fired after a document is selected }: Props) { const dispatch = useDispatch(); const { t } = useTranslation(); @@ -85,6 +87,7 @@ function SourceDropdown({ onClick={() => { dispatch(setSelectedDocs(option)); setIsDocsListOpen(false); + handlePostDocumentSelect(option); }} > - + { + handlePostDocumentSelect(null); + }}> {t('none')}
diff --git a/frontend/src/conversation/Conversation.tsx b/frontend/src/conversation/Conversation.tsx index d5908ca3..7bcfbbf6 100644 --- a/frontend/src/conversation/Conversation.tsx +++ b/frontend/src/conversation/Conversation.tsx @@ -54,10 +54,6 @@ export default function Conversation() { } }, []); - useEffect(() => { - fetchStream.current && fetchStream.current.abort(); - }, [conversationId]); - useEffect(() => { if (queries.length) { queries[queries.length - 1].error && setLastQueryReturnedErr(true); diff --git a/frontend/src/conversation/ConversationBubble.tsx b/frontend/src/conversation/ConversationBubble.tsx index 543699ed..3d956e2e 100644 --- a/frontend/src/conversation/ConversationBubble.tsx +++ b/frontend/src/conversation/ConversationBubble.tsx @@ -59,7 +59,12 @@ const ConversationBubble = forwardRef< className={`flex flex-row-reverse self-end flex-wrap ${className}`} > -
+
{message}
diff --git a/frontend/src/conversation/ConversationTile.tsx b/frontend/src/conversation/ConversationTile.tsx index 3a0b85e4..d689f09f 100644 --- a/frontend/src/conversation/ConversationTile.tsx +++ b/frontend/src/conversation/ConversationTile.tsx @@ -22,6 +22,7 @@ interface ConversationProps { interface ConversationTileProps { conversation: ConversationProps; selectConversation: (arg1: string) => void; + onCoversationClick: () => void; //Callback to handle click on conversation tile regardless of selected or not onDeleteConversation: (arg1: string) => void; onSave: ({ name, id }: ConversationProps) => void; } @@ -29,6 +30,7 @@ interface ConversationTileProps { export default function ConversationTile({ conversation, selectConversation, + onCoversationClick, onDeleteConversation, onSave, }: ConversationTileProps) { @@ -90,6 +92,7 @@ export default function ConversationTile({ setIsHovered(false); }} onClick={() => { + onCoversationClick(); conversationId !== conversation.id && selectConversation(conversation.id); }} @@ -158,7 +161,12 @@ export default function ConversationTile({ )} {isOpen && ( -
+