Merge branch 'main' of https://github.com/arc53/DocsGPT

application/api/answer/routes.py

@@ -3,7 +3,6 @@ import datetime
 import json
 import logging
 import os
-import sys
 import traceback
 
 from bson.dbref import DBRef
@@ -263,13 +262,12 @@ def complete_stream(
         data = json.dumps({"type": "end"})
         yield f"data: {data}\n\n"
     except Exception as e:
-        print("\033[91merr", str(e), file=sys.stderr)
+        logger.error(f"Error in stream: {str(e)}")
-        traceback.print_exc()
+        logger.error(traceback.format_exc())
         data = json.dumps(
             {
                 "type": "error",
                 "error": "Please try again later. We apologize for any inconvenience.",
-                "error_exception": str(e),
             }
         )
         yield f"data: {data}\n\n"
@@ -384,7 +382,7 @@ class Stream(Resource):
 
         except ValueError:
             message = "Malformed request body"
-            print("\033[91merr", str(message), file=sys.stderr)
+            current_app.logger.error(f"/stream - error: {message}")
             return Response(
                 error_stream_generate(message),
                 status=400,
@@ -395,13 +393,9 @@ class Stream(Resource):
                 f"/stream - error: {str(e)} - traceback: {traceback.format_exc()}",
                 extra={"error": str(e), "traceback": traceback.format_exc()},
             )
-            message = e.args[0]
             status_code = 400
-            # Custom exceptions with two arguments, index 1 as status code
-            if len(e.args) >= 2:
-                status_code = e.args[1]
             return Response(
-                error_stream_generate(message),
+                error_stream_generate('Unknown error occurred'),
                 status=status_code,
                 mimetype="text/event-stream",
             )

application/api/user/routes.py

@@ -8,7 +8,7 @@ import json
 from bson.binary import Binary, UuidRepresentation
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
-from flask import Blueprint, jsonify, make_response, redirect, request
+from flask import Blueprint, current_app, jsonify, make_response, redirect, request
 from flask_restx import fields, inputs, Namespace, Resource
 from werkzeug.utils import secure_filename
 
@@ -83,7 +83,8 @@ class DeleteConversation(Resource):
         try:
             conversations_collection.delete_one({"_id": ObjectId(conversation_id)})
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error deleting conversation: {err}")
+            return make_response(jsonify({"success": False}), 400)
         return make_response(jsonify({"success": True}), 200)
 
 
@@ -97,7 +98,8 @@ class DeleteAllConversations(Resource):
         try:
             conversations_collection.delete_many({"user": user_id})
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error deleting all conversations: {err}")
+            return make_response(jsonify({"success": False}), 400)
         return make_response(jsonify({"success": True}), 200)
 
 
@@ -114,7 +116,8 @@ class GetConversations(Resource):
                 for conversation in conversations
             ]
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error retrieving conversations: {err}")
+            return make_response(jsonify({"success": False}), 400)
         return make_response(jsonify(list_conversations), 200)
 
 
@@ -138,7 +141,8 @@ class GetSingleConversation(Resource):
             if not conversation:
                 return make_response(jsonify({"status": "not found"}), 404)
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error retrieving conversation: {err}")
+            return make_response(jsonify({"success": False}), 400)
         return make_response(jsonify(conversation["queries"]), 200)
 
 
@@ -170,7 +174,8 @@ class UpdateConversationName(Resource):
                 {"_id": ObjectId(data["id"])}, {"$set": {"name": data["name"]}}
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error updating conversation name: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True}), 200)
 
@@ -221,7 +226,8 @@ class SubmitFeedback(Resource):
             )
 
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error submitting feedback: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True}), 200)
 
@@ -244,7 +250,8 @@ class DeleteByIds(Resource):
             if result:
                 return make_response(jsonify({"success": True}), 200)
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error deleting indexes: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": False}), 400)
 
@@ -277,7 +284,8 @@ class DeleteOldIndexes(Resource):
         except FileNotFoundError:
             pass
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error deleting old indexes: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         sources_collection.delete_one({"_id": ObjectId(source_id)})
         return make_response(jsonify({"success": True}), 200)
@@ -389,8 +397,8 @@ class UploadFile(Resource):
                 )
 
         except Exception as err:
-            print(f"Error: {err}")
+            current_app.logger.error(f"Error uploading file: {err}")
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            return make_response(jsonify({"success": False}), 400)
         return make_response(jsonify({"success": True, "task_id": task.id}), 200)
 
 
@@ -439,7 +447,8 @@ class UploadRemote(Resource):
                 loader=data["source"]
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error uploading remote source: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True, "task_id": task.id}), 200)
 
@@ -471,7 +480,8 @@ class TaskStatus(Resource):
             ):
                 task_meta = str(task_meta)  # Convert to a string representation
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error getting task status: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"status": task.status, "result": task_meta}), 200)
 
@@ -546,7 +556,8 @@ class PaginatedSources(Resource):
             return make_response(jsonify(response), 200)
 
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error retrieving paginated sources: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
 
 @user_ns.route("/api/sources")
@@ -606,7 +617,8 @@ class CombinedJson(Resource):
                 )
 
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error retrieving sources: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify(data), 200)
 
@@ -632,7 +644,8 @@ class CheckDocs(Resource):
             if os.path.exists(vectorstore) or data["docs"] == "default":
                 return {"status": "exists"}, 200
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error checking document: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"status": "not found"}), 404)
 
@@ -670,7 +683,8 @@ class CreatePrompt(Resource):
             )
             new_id = str(resp.inserted_id)
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error creating prompt: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"id": new_id}), 200)
 
@@ -697,7 +711,8 @@ class GetPrompts(Resource):
                     }
                 )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error retrieving prompts: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify(list_prompts), 200)
 
@@ -738,7 +753,8 @@ class GetSinglePrompt(Resource):
 
             prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)})
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error retrieving prompt: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"content": prompt["content"]}), 200)
 
@@ -762,7 +778,8 @@ class DeletePrompt(Resource):
         try:
             prompts_collection.delete_one({"_id": ObjectId(data["id"])})
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error deleting prompt: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True}), 200)
 
@@ -795,7 +812,8 @@ class UpdatePrompt(Resource):
                 {"$set": {"name": data["name"], "content": data["content"]}},
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error updating prompt: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True}), 200)
 
@@ -830,7 +848,8 @@ class GetApiKeys(Resource):
                     }
                 )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error retrieving API keys: {err}")
+            return make_response(jsonify({"success": False}), 400)
         return make_response(jsonify(list_keys), 200)
 
 
@@ -874,7 +893,8 @@ class CreateApiKey(Resource):
             resp = api_key_collection.insert_one(new_api_key)
             new_id = str(resp.inserted_id)
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error creating API key: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"id": new_id, "key": key}), 201)
 
@@ -900,7 +920,8 @@ class DeleteApiKey(Resource):
             if result.deleted_count == 0:
                 return {"success": False, "message": "API Key not found"}, 404
         except Exception as err:
-            return {"success": False, "error": str(err)}, 400
+            current_app.logger.error(f"Error deleting API key: {err}")
+            return {"success": False}, 400
 
         return {"success": True}, 200
 
@@ -1100,7 +1121,8 @@ class ShareConversation(Resource):
                     201,
                 )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error sharing conversation: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
 
 @user_ns.route("/api/shared_conversation/<string:identifier>")
@@ -1155,7 +1177,8 @@ class GetPubliclySharedConversations(Resource):
                 res["api_key"] = shared["api_key"]
             return make_response(jsonify(res), 200)
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error getting shared conversation: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
 
 @user_ns.route("/api/get_message_analytics")
@@ -1196,7 +1219,8 @@ class GetMessageAnalytics(Resource):
                 else None
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error getting API key: {err}")
+            return make_response(jsonify({"success": False}), 400)
         end_date = datetime.datetime.now(datetime.timezone.utc)
 
         if filter_option == "last_hour":
@@ -1289,7 +1313,8 @@ class GetMessageAnalytics(Resource):
                     daily_messages[entry["_id"]["day"]] = entry["total_messages"]
 
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error getting message analytics: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(
             jsonify({"success": True, "messages": daily_messages}), 200
@@ -1331,7 +1356,8 @@ class GetTokenAnalytics(Resource):
                 else None
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error getting API key: {err}")
+            return make_response(jsonify({"success": False}), 400)
         end_date = datetime.datetime.now(datetime.timezone.utc)
 
         if filter_option == "last_hour":
@@ -1440,7 +1466,8 @@ class GetTokenAnalytics(Resource):
                     daily_token_usage[entry["_id"]["day"]] = entry["total_tokens"]
 
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error getting token analytics: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(
             jsonify({"success": True, "token_usage": daily_token_usage}), 200
@@ -1482,7 +1509,8 @@ class GetFeedbackAnalytics(Resource):
                 else None
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error getting API key: {err}")
+            return make_response(jsonify({"success": False}), 400)
         
         end_date = datetime.datetime.now(datetime.timezone.utc)
 
@@ -1583,7 +1611,8 @@ class GetFeedbackAnalytics(Resource):
                 }
 
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error getting feedback analytics: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(
             jsonify({"success": True, "feedback": daily_feedback}), 200
@@ -1625,7 +1654,8 @@ class GetUserLogs(Resource):
                 else None
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error getting API key: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         query = {}
         if api_key:
@@ -1709,7 +1739,8 @@ class ManageSync(Resource):
                 update_data,
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error updating sync frequency: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True}), 200)
 
@@ -1744,7 +1775,8 @@ class TextToSpeech(Resource):
                 200,
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error synthesizing audio: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
 
 @user_ns.route("/api/available_tools")
@@ -1768,7 +1800,8 @@ class AvailableTools(Resource):
                     }
                 )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error getting available tools: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True, "data": tools_metadata}), 200)
 
@@ -1786,7 +1819,8 @@ class GetTools(Resource):
                 tool.pop("_id")
                 user_tools.append(tool)
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error getting user tools: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True, "tools": user_tools}), 200)
 
@@ -1858,7 +1892,8 @@ class CreateTool(Resource):
             resp = user_tools_collection.insert_one(new_tool)
             new_id = str(resp.inserted_id)
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error creating tool: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"id": new_id}), 200)
 
@@ -1909,7 +1944,8 @@ class UpdateTool(Resource):
                 {"$set": update_data},
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error updating tool: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True}), 200)
 
@@ -1941,7 +1977,8 @@ class UpdateToolConfig(Resource):
                 {"$set": {"config": data["config"]}},
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error updating tool config: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True}), 200)
 
@@ -1975,7 +2012,8 @@ class UpdateToolActions(Resource):
                 {"$set": {"actions": data["actions"]}},
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error updating tool actions: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True}), 200)
 
@@ -2007,7 +2045,8 @@ class UpdateToolStatus(Resource):
                 {"$set": {"status": data["status"]}},
             )
         except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+            current_app.logger.error(f"Error updating tool status: {err}")
+            return make_response(jsonify({"success": False}), 400)
 
         return make_response(jsonify({"success": True}), 200)
 
@@ -2033,6 +2072,7 @@ class DeleteTool(Resource):
             if result.deleted_count == 0:
                 return {"success": False, "message": "Tool not found"}, 404
         except Exception as err:
-            return {"success": False, "error": str(err)}, 400
+            current_app.logger.error(f"Error deleting tool: {err}")
+            return {"success": False}, 400
 
         return {"success": True}, 200

application/parser/file/docs_parser.py

@@ -24,26 +24,27 @@ class PDFParser(BaseParser):
             # alternatively you can use local vision capable LLM
             with open(file, "rb") as file_loaded:
                 files = {'file': file_loaded}
-                response = requests.post(doc2md_service, files=files)   
+                response = requests.post(doc2md_service, files=files)
-                data = response.json()["markdown"] 
+                data = response.json()["markdown"]
             return data
 
         try:
-            import PyPDF2
+            from pypdf import PdfReader
         except ImportError:
-            raise ValueError("PyPDF2 is required to read PDF files.")
+            raise ValueError("pypdf is required to read PDF files.")
         text_list = []
         with open(file, "rb") as fp:
             # Create a PDF object
-            pdf = PyPDF2.PdfReader(fp)
+            pdf = PdfReader(fp)
 
             # Get the number of pages in the PDF document
             num_pages = len(pdf.pages)
 
             # Iterate over every page
-            for page in range(num_pages):
+            for page_index in range(num_pages):
                 # Extract the text from the page
-                page_text = pdf.pages[page].extract_text()
+                page = pdf.pages[page_index]
+                page_text = page.extract_text()
                 text_list.append(page_text)
         text = "\n".join(text_list)
 
@@ -66,4 +67,4 @@ class DocxParser(BaseParser):
 
         text = docx2txt.process(file)
 
-        return text
+        return text

application/requirements.txt

@@ -66,7 +66,7 @@ pydantic==2.10.4
 pydantic-core==2.27.2
 pydantic-settings==2.7.1
 pymongo==4.10.1
-pypdf2==3.0.1
+pypdf==5.2.0
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
 python-pptx==1.0.2

deployment/docker-compose-azure.yaml

@@ -1,6 +1,6 @@
 services:
   frontend:
-    build: ./frontend
+    build: ../frontend
     environment:
       - VITE_API_HOST=http://localhost:7091
       - VITE_API_STREAMING=$VITE_API_STREAMING
@@ -10,7 +10,7 @@ services:
       - backend
 
   backend:
-    build: ./application
+    build: ../application
     environment:
       - API_KEY=$OPENAI_API_KEY
       - EMBEDDINGS_KEY=$OPENAI_API_KEY
@@ -25,15 +25,15 @@ services:
     ports:
       - "7091:7091"
     volumes:
-      - ./application/indexes:/app/application/indexes
+      - ../application/indexes:/app/application/indexes
-      - ./application/inputs:/app/application/inputs
+      - ../application/inputs:/app/application/inputs
-      - ./application/vectors:/app/application/vectors
+      - ../application/vectors:/app/application/vectors
     depends_on:
         - redis
         - mongo
 
   worker:
-    build: ./application
+    build: ../application
     command: celery -A application.app.celery worker -l INFO
     environment:
       - API_KEY=$OPENAI_API_KEY

deployment/docker-compose-local.yaml

@@ -1,8 +1,8 @@
 services:
   frontend:
-    build: ./frontend
+    build: ../frontend
     volumes:
-    - ./frontend/src:/app/src
+    - ../frontend/src:/app/src
     environment:
       - VITE_API_HOST=http://localhost:7091
       - VITE_API_STREAMING=$VITE_API_STREAMING

deployment/docker-compose.yaml

@@ -1,8 +1,8 @@
 services:
   frontend:
-    build: ./frontend
+    build: ../frontend
     volumes:
-      - ./frontend/src:/app/src
+      - ../frontend/src:/app/src
     environment:
       - VITE_API_HOST=http://localhost:7091
       - VITE_API_STREAMING=$VITE_API_STREAMING
@@ -12,7 +12,7 @@ services:
       - backend
 
   backend:
-    build: ./application
+    build: ../application
     environment:
       - API_KEY=$API_KEY
       - EMBEDDINGS_KEY=$API_KEY
@@ -26,15 +26,15 @@ services:
     ports:
       - "7091:7091"
     volumes:
-      - ./application/indexes:/app/application/indexes
+      - ../application/indexes:/app/application/indexes
-      - ./application/inputs:/app/application/inputs
+      - ../application/inputs:/app/application/inputs
-      - ./application/vectors:/app/application/vectors
+      - ../application/vectors:/app/application/vectors
     depends_on:
       - redis
       - mongo
 
   worker:
-    build: ./application
+    build: ../application
     command: celery -A application.app.celery worker -l INFO -B
     environment:
       - API_KEY=$API_KEY