- DocsGPT is an open-source AI platform for building intelligent agents and assistants. Features Agent Builder, deep research tools, document analysis (PDF, Office, web content), Multi-model support (choose your provider or run locally), and rich API connectivity for agents with actionable tools and integrations. Deploy anywhere with complete privacy control.
+ DocsGPT is an open-source AI platform for building intelligent agents and assistants. Features Agent Builder, deep research tools, document analysis (PDF, Office, web content, and audio), Multi-model support (choose your provider or run locally), and rich API connectivity for agents with actionable tools and integrations. Deploy anywhere with complete privacy control.
@@ -35,7 +35,8 @@
Key Features:
-
🗂️ Wide Format Support: Reads PDF, DOCX, CSV, XLSX, EPUB, MD, RST, HTML, MDX, JSON, PPTX, and images.
+
🗂️ Wide Format Support: Reads PDF, DOCX, CSV, XLSX, EPUB, MD, RST, HTML, MDX, JSON, PPTX, images, and audio files such as MP3, WAV, M4A, OGG, and WebM.
+
🎙️ Speech Workflows: Record voice input into chat, transcribe audio on the backend, and ingest meeting recordings or voice notes as searchable knowledge.
🌐 Web & Data Integration: Ingests from URLs, sitemaps, Reddit, GitHub and web crawlers.
✅ Reliable Answers: Get accurate, hallucination-free responses with source citations viewable in a clean UI.
🔑 Streamlined API Keys: Generate keys linked to your settings, documents, and models, simplifying chatbot and integration setup.
@@ -158,4 +159,3 @@ The source code license is [MIT](https://opensource.org/license/mit/), as descri
-
diff --git a/application/api/user/agents/folders.py b/application/api/user/agents/folders.py
index 3dd14aa5..c7e5217e 100644
--- a/application/api/user/agents/folders.py
+++ b/application/api/user/agents/folders.py
@@ -5,7 +5,7 @@ Provides virtual folder organization for agents (Google Drive-like structure).
import datetime
from bson.objectid import ObjectId
-from flask import jsonify, make_response, request
+from flask import current_app, jsonify, make_response, request
from flask_restx import Namespace, Resource, fields
from application.api import api
@@ -19,6 +19,11 @@ agents_folders_ns = Namespace(
)
+def _folder_error_response(message: str, err: Exception):
+ current_app.logger.error(f"{message}: {err}", exc_info=True)
+ return make_response(jsonify({"success": False, "message": message}), 400)
+
+
@agents_folders_ns.route("/")
class AgentFolders(Resource):
@api.doc(description="Get all folders for the user")
@@ -40,8 +45,8 @@ class AgentFolders(Resource):
for f in folders
]
return make_response(jsonify({"folders": result}), 200)
- except Exception as e:
- return make_response(jsonify({"success": False, "message": str(e)}), 400)
+ except Exception as err:
+ return _folder_error_response("Failed to fetch folders", err)
@api.doc(description="Create a new folder")
@api.expect(
@@ -82,8 +87,8 @@ class AgentFolders(Resource):
jsonify({"id": str(result.inserted_id), "name": data["name"], "parent_id": parent_id}),
201,
)
- except Exception as e:
- return make_response(jsonify({"success": False, "message": str(e)}), 400)
+ except Exception as err:
+ return _folder_error_response("Failed to create folder", err)
@agents_folders_ns.route("/")
@@ -117,8 +122,8 @@ class AgentFolder(Resource):
}),
200,
)
- except Exception as e:
- return make_response(jsonify({"success": False, "message": str(e)}), 400)
+ except Exception as err:
+ return _folder_error_response("Failed to fetch folder", err)
@api.doc(description="Update a folder")
def put(self, folder_id):
@@ -145,8 +150,8 @@ class AgentFolder(Resource):
if result.matched_count == 0:
return make_response(jsonify({"success": False, "message": "Folder not found"}), 404)
return make_response(jsonify({"success": True}), 200)
- except Exception as e:
- return make_response(jsonify({"success": False, "message": str(e)}), 400)
+ except Exception as err:
+ return _folder_error_response("Failed to update folder", err)
@api.doc(description="Delete a folder")
def delete(self, folder_id):
@@ -165,8 +170,8 @@ class AgentFolder(Resource):
if result.deleted_count == 0:
return make_response(jsonify({"success": False, "message": "Folder not found"}), 404)
return make_response(jsonify({"success": True}), 200)
- except Exception as e:
- return make_response(jsonify({"success": False, "message": str(e)}), 400)
+ except Exception as err:
+ return _folder_error_response("Failed to delete folder", err)
@agents_folders_ns.route("/move_agent")
@@ -211,8 +216,8 @@ class MoveAgentToFolder(Resource):
)
return make_response(jsonify({"success": True}), 200)
- except Exception as e:
- return make_response(jsonify({"success": False, "message": str(e)}), 400)
+ except Exception as err:
+ return _folder_error_response("Failed to move agent", err)
@agents_folders_ns.route("/bulk_move")
@@ -257,5 +262,5 @@ class BulkMoveAgents(Resource):
{"$unset": {"folder_id": ""}},
)
return make_response(jsonify({"success": True}), 200)
- except Exception as e:
- return make_response(jsonify({"success": False, "message": str(e)}), 400)
+ except Exception as err:
+ return _folder_error_response("Failed to move agents", err)
diff --git a/application/api/user/agents/routes.py b/application/api/user/agents/routes.py
index 8f313a7e..02b185c2 100644
--- a/application/api/user/agents/routes.py
+++ b/application/api/user/agents/routes.py
@@ -740,13 +740,7 @@ class UpdateAgent(Resource):
request, existing_agent.get("image", ""), user, storage
)
if error:
- current_app.logger.error(
- f"Image upload error for agent {agent_id}: {error}"
- )
- return make_response(
- jsonify({"success": False, "message": f"Image upload failed: {error}"}),
- 400,
- )
+ return error
update_fields = {}
allowed_fields = [
"name",
@@ -1412,4 +1406,4 @@ class RemoveSharedAgent(Resource):
current_app.logger.error(f"Error removing shared agent: {err}")
return make_response(
jsonify({"success": False, "message": "Server error"}), 500
- )
\ No newline at end of file
+ )
diff --git a/application/api/user/attachments/routes.py b/application/api/user/attachments/routes.py
index 32c4cdca..92158bcb 100644
--- a/application/api/user/attachments/routes.py
+++ b/application/api/user/attachments/routes.py
@@ -1,15 +1,36 @@
"""File attachments and media routes."""
import os
+import tempfile
+from pathlib import Path
from bson.objectid import ObjectId
from flask import current_app, jsonify, make_response, request
from flask_restx import fields, Namespace, Resource
from application.api import api
-from application.api.user.base import agents_collection, storage
-from application.api.user.tasks import store_attachment
+from application.cache import get_redis_instance
from application.core.settings import settings
+from application.stt.constants import (
+ SUPPORTED_AUDIO_EXTENSIONS,
+ SUPPORTED_AUDIO_MIME_TYPES,
+)
+from application.stt.upload_limits import (
+ AudioFileTooLargeError,
+ build_stt_file_size_limit_message,
+ enforce_audio_file_size_limit,
+ is_audio_filename,
+)
+from application.stt.live_session import (
+ apply_live_stt_hypothesis,
+ create_live_stt_session,
+ delete_live_stt_session,
+ finalize_live_stt_session,
+ get_live_stt_transcript_text,
+ load_live_stt_session,
+ save_live_stt_session,
+)
+from application.stt.stt_creator import STTCreator
from application.tts.tts_creator import TTSCreator
from application.utils import safe_filename
@@ -19,6 +40,74 @@ attachments_ns = Namespace(
)
+def _resolve_authenticated_user():
+ decoded_token = getattr(request, "decoded_token", None)
+ api_key = request.form.get("api_key") or request.args.get("api_key")
+
+ if decoded_token:
+ return safe_filename(decoded_token.get("sub"))
+
+ if api_key:
+ from application.api.user.base import agents_collection
+
+ agent = agents_collection.find_one({"key": api_key})
+ if not agent:
+ return make_response(
+ jsonify({"success": False, "message": "Invalid API key"}), 401
+ )
+ return safe_filename(agent.get("user"))
+
+ return None
+
+
+def _get_uploaded_file_size(file) -> int:
+ try:
+ current_position = file.stream.tell()
+ file.stream.seek(0, os.SEEK_END)
+ size_bytes = file.stream.tell()
+ file.stream.seek(current_position)
+ return size_bytes
+ except Exception:
+ return 0
+
+
+def _is_supported_audio_mimetype(mimetype: str) -> bool:
+ if not mimetype:
+ return True
+ normalized = mimetype.split(";")[0].strip().lower()
+ return normalized.startswith("audio/") or normalized in SUPPORTED_AUDIO_MIME_TYPES
+
+
+def _enforce_uploaded_audio_size_limit(file, filename: str) -> None:
+ if not is_audio_filename(filename):
+ return
+ size_bytes = _get_uploaded_file_size(file)
+ if size_bytes:
+ enforce_audio_file_size_limit(size_bytes)
+
+
+def _get_store_attachment_user_error(exc: Exception) -> str:
+ if isinstance(exc, AudioFileTooLargeError):
+ return build_stt_file_size_limit_message()
+ return "Failed to process file"
+
+
+def _require_live_stt_redis():
+ redis_client = get_redis_instance()
+ if redis_client:
+ return redis_client
+ return make_response(
+ jsonify({"success": False, "message": "Live transcription is unavailable"}),
+ 503,
+ )
+
+
+def _parse_bool_form_value(value: str | None) -> bool:
+ if value is None:
+ return False
+ return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
@attachments_ns.route("/store_attachment")
class StoreAttachment(Resource):
@api.expect(
@@ -36,8 +125,9 @@ class StoreAttachment(Resource):
description="Stores one or multiple attachments without vectorization or training. Supports user or API key authentication."
)
def post(self):
- decoded_token = getattr(request, "decoded_token", None)
- api_key = request.form.get("api_key") or request.args.get("api_key")
+ auth_user = _resolve_authenticated_user()
+ if hasattr(auth_user, "status_code"):
+ return auth_user
files = request.files.getlist("file")
if not files:
@@ -51,22 +141,16 @@ class StoreAttachment(Resource):
400,
)
- user = None
- if decoded_token:
- user = safe_filename(decoded_token.get("sub"))
- elif api_key:
- agent = agents_collection.find_one({"key": api_key})
- if not agent:
- return make_response(
- jsonify({"success": False, "message": "Invalid API key"}), 401
- )
- user = safe_filename(agent.get("user"))
- else:
+ user = auth_user
+ if not user:
return make_response(
jsonify({"success": False, "message": "Authentication required"}), 401
)
try:
+ from application.api.user.tasks import store_attachment
+ from application.api.user.base import storage
+
tasks = []
errors = []
original_file_count = len(files)
@@ -75,6 +159,7 @@ class StoreAttachment(Resource):
try:
attachment_id = ObjectId()
original_filename = safe_filename(os.path.basename(file.filename))
+ _enforce_uploaded_audio_size_limit(file, original_filename)
relative_path = f"{settings.UPLOAD_FOLDER}/{user}/attachments/{str(attachment_id)}/{original_filename}"
metadata = storage.save_file(file, relative_path)
@@ -90,15 +175,31 @@ class StoreAttachment(Resource):
"task_id": task.id,
"filename": original_filename,
"attachment_id": str(attachment_id),
+ "upload_index": idx,
})
except Exception as file_err:
current_app.logger.error(f"Error processing file {idx} ({file.filename}): {file_err}", exc_info=True)
errors.append({
+ "upload_index": idx,
"filename": file.filename,
- "error": str(file_err)
+ "error": _get_store_attachment_user_error(file_err),
})
if not tasks:
+ if errors and all(
+ error.get("error") == build_stt_file_size_limit_message()
+ for error in errors
+ ):
+ return make_response(
+ jsonify(
+ {
+ "success": False,
+ "message": build_stt_file_size_limit_message(),
+ "errors": errors,
+ }
+ ),
+ 413,
+ )
return make_response(
jsonify({"status": "error", "message": "No valid files to upload"}),
400,
@@ -135,11 +236,385 @@ class StoreAttachment(Resource):
return make_response(jsonify({"success": False, "error": "Failed to store attachment"}), 400)
+@attachments_ns.route("/stt")
+class SpeechToText(Resource):
+ @api.expect(
+ api.model(
+ "SpeechToTextModel",
+ {
+ "file": fields.Raw(required=True, description="Audio file"),
+ "language": fields.String(
+ required=False, description="Optional transcription language hint"
+ ),
+ },
+ )
+ )
+ @api.doc(description="Transcribe an uploaded audio file")
+ def post(self):
+ auth_user = _resolve_authenticated_user()
+ if hasattr(auth_user, "status_code"):
+ return auth_user
+ if not auth_user:
+ return make_response(
+ jsonify({"success": False, "message": "Authentication required"}),
+ 401,
+ )
+
+ file = request.files.get("file")
+ if not file or file.filename == "":
+ return make_response(
+ jsonify({"success": False, "message": "Missing file"}),
+ 400,
+ )
+
+ filename = safe_filename(os.path.basename(file.filename))
+ suffix = Path(filename).suffix.lower()
+ if suffix not in SUPPORTED_AUDIO_EXTENSIONS:
+ return make_response(
+ jsonify({"success": False, "message": "Unsupported audio format"}),
+ 400,
+ )
+
+ if not _is_supported_audio_mimetype(file.mimetype or ""):
+ return make_response(
+ jsonify({"success": False, "message": "Unsupported audio MIME type"}),
+ 400,
+ )
+
+ try:
+ _enforce_uploaded_audio_size_limit(file, filename)
+ except AudioFileTooLargeError:
+ return make_response(
+ jsonify(
+ {
+ "success": False,
+ "message": build_stt_file_size_limit_message(),
+ }
+ ),
+ 413,
+ )
+
+ temp_path = None
+ try:
+ with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:
+ file.save(temp_file.name)
+ temp_path = Path(temp_file.name)
+
+ stt_instance = STTCreator.create_stt(settings.STT_PROVIDER)
+ transcript = stt_instance.transcribe(
+ temp_path,
+ language=request.form.get("language") or settings.STT_LANGUAGE,
+ timestamps=settings.STT_ENABLE_TIMESTAMPS,
+ diarize=settings.STT_ENABLE_DIARIZATION,
+ )
+ return make_response(jsonify({"success": True, **transcript}), 200)
+ except Exception as err:
+ current_app.logger.error(f"Error transcribing audio: {err}", exc_info=True)
+ return make_response(
+ jsonify({"success": False, "message": "Failed to transcribe audio"}),
+ 400,
+ )
+ finally:
+ if temp_path and temp_path.exists():
+ temp_path.unlink()
+
+
+@attachments_ns.route("/stt/live/start")
+class LiveSpeechToTextStart(Resource):
+ @api.doc(description="Start a live speech-to-text session")
+ def post(self):
+ auth_user = _resolve_authenticated_user()
+ if hasattr(auth_user, "status_code"):
+ return auth_user
+ if not auth_user:
+ return make_response(
+ jsonify({"success": False, "message": "Authentication required"}),
+ 401,
+ )
+
+ redis_client = _require_live_stt_redis()
+ if hasattr(redis_client, "status_code"):
+ return redis_client
+
+ payload = request.get_json(silent=True) or {}
+ session_state = create_live_stt_session(
+ user=auth_user,
+ language=payload.get("language") or settings.STT_LANGUAGE,
+ )
+ save_live_stt_session(redis_client, session_state)
+
+ return make_response(
+ jsonify(
+ {
+ "success": True,
+ "session_id": session_state["session_id"],
+ "language": session_state.get("language"),
+ "committed_text": "",
+ "mutable_text": "",
+ "previous_hypothesis": "",
+ "latest_hypothesis": "",
+ "finalized_text": "",
+ "pending_text": "",
+ "transcript_text": "",
+ }
+ ),
+ 200,
+ )
+
+
+@attachments_ns.route("/stt/live/chunk")
+class LiveSpeechToTextChunk(Resource):
+ @api.expect(
+ api.model(
+ "LiveSpeechToTextChunkModel",
+ {
+ "session_id": fields.String(
+ required=True, description="Live transcription session ID"
+ ),
+ "chunk_index": fields.Integer(
+ required=True, description="Sequential chunk index"
+ ),
+ "is_silence": fields.Boolean(
+ required=False,
+ description="Whether the latest capture window was mostly silence",
+ ),
+ "file": fields.Raw(required=True, description="Audio chunk"),
+ },
+ )
+ )
+ @api.doc(description="Transcribe a chunk for a live speech-to-text session")
+ def post(self):
+ auth_user = _resolve_authenticated_user()
+ if hasattr(auth_user, "status_code"):
+ return auth_user
+ if not auth_user:
+ return make_response(
+ jsonify({"success": False, "message": "Authentication required"}),
+ 401,
+ )
+
+ redis_client = _require_live_stt_redis()
+ if hasattr(redis_client, "status_code"):
+ return redis_client
+
+ session_id = request.form.get("session_id", "").strip()
+ if not session_id:
+ return make_response(
+ jsonify({"success": False, "message": "Missing session_id"}),
+ 400,
+ )
+
+ session_state = load_live_stt_session(redis_client, session_id)
+ if not session_state:
+ return make_response(
+ jsonify(
+ {
+ "success": False,
+ "message": "Live transcription session not found",
+ }
+ ),
+ 404,
+ )
+
+ if safe_filename(str(session_state.get("user", ""))) != auth_user:
+ return make_response(
+ jsonify({"success": False, "message": "Forbidden"}),
+ 403,
+ )
+
+ chunk_index_raw = request.form.get("chunk_index", "").strip()
+ if chunk_index_raw == "":
+ return make_response(
+ jsonify({"success": False, "message": "Missing chunk_index"}),
+ 400,
+ )
+
+ try:
+ chunk_index = int(chunk_index_raw)
+ except ValueError:
+ return make_response(
+ jsonify({"success": False, "message": "Invalid chunk_index"}),
+ 400,
+ )
+ is_silence = _parse_bool_form_value(request.form.get("is_silence"))
+
+ file = request.files.get("file")
+ if not file or file.filename == "":
+ return make_response(
+ jsonify({"success": False, "message": "Missing file"}),
+ 400,
+ )
+
+ filename = safe_filename(os.path.basename(file.filename))
+ suffix = Path(filename).suffix.lower()
+ if suffix not in SUPPORTED_AUDIO_EXTENSIONS:
+ return make_response(
+ jsonify({"success": False, "message": "Unsupported audio format"}),
+ 400,
+ )
+
+ if not _is_supported_audio_mimetype(file.mimetype or ""):
+ return make_response(
+ jsonify({"success": False, "message": "Unsupported audio MIME type"}),
+ 400,
+ )
+
+ try:
+ _enforce_uploaded_audio_size_limit(file, filename)
+ except AudioFileTooLargeError:
+ return make_response(
+ jsonify(
+ {
+ "success": False,
+ "message": build_stt_file_size_limit_message(),
+ }
+ ),
+ 413,
+ )
+
+ temp_path = None
+ try:
+ with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:
+ file.save(temp_file.name)
+ temp_path = Path(temp_file.name)
+
+ session_language = session_state.get("language") or settings.STT_LANGUAGE
+ stt_instance = STTCreator.create_stt(settings.STT_PROVIDER)
+ transcript = stt_instance.transcribe(
+ temp_path,
+ language=session_language,
+ timestamps=False,
+ diarize=False,
+ )
+ if not session_state.get("language") and transcript.get("language"):
+ session_state["language"] = transcript["language"]
+
+ try:
+ apply_live_stt_hypothesis(
+ session_state,
+ str(transcript.get("text", "")),
+ chunk_index,
+ is_silence=is_silence,
+ )
+ except ValueError:
+ current_app.logger.warning(
+ "Invalid live transcription chunk",
+ exc_info=True,
+ )
+ return make_response(
+ jsonify(
+ {
+ "success": False,
+ "message": "Invalid live transcription chunk",
+ }
+ ),
+ 409,
+ )
+ save_live_stt_session(redis_client, session_state)
+
+ return make_response(
+ jsonify(
+ {
+ "success": True,
+ "session_id": session_id,
+ "chunk_index": chunk_index,
+ "chunk_text": transcript.get("text", ""),
+ "is_silence": is_silence,
+ "language": session_state.get("language"),
+ "committed_text": session_state.get("committed_text", ""),
+ "mutable_text": session_state.get("mutable_text", ""),
+ "previous_hypothesis": session_state.get(
+ "previous_hypothesis", ""
+ ),
+ "latest_hypothesis": session_state.get(
+ "latest_hypothesis", ""
+ ),
+ "finalized_text": session_state.get("committed_text", ""),
+ "pending_text": session_state.get("mutable_text", ""),
+ "transcript_text": get_live_stt_transcript_text(session_state),
+ }
+ ),
+ 200,
+ )
+ except Exception as err:
+ current_app.logger.error(
+ f"Error transcribing live audio chunk: {err}", exc_info=True
+ )
+ return make_response(
+ jsonify({"success": False, "message": "Failed to transcribe audio"}),
+ 400,
+ )
+ finally:
+ if temp_path and temp_path.exists():
+ temp_path.unlink()
+
+
+@attachments_ns.route("/stt/live/finish")
+class LiveSpeechToTextFinish(Resource):
+ @api.doc(description="Finish a live speech-to-text session")
+ def post(self):
+ auth_user = _resolve_authenticated_user()
+ if hasattr(auth_user, "status_code"):
+ return auth_user
+ if not auth_user:
+ return make_response(
+ jsonify({"success": False, "message": "Authentication required"}),
+ 401,
+ )
+
+ redis_client = _require_live_stt_redis()
+ if hasattr(redis_client, "status_code"):
+ return redis_client
+
+ payload = request.get_json(silent=True) or {}
+ session_id = str(payload.get("session_id", "")).strip()
+ if not session_id:
+ return make_response(
+ jsonify({"success": False, "message": "Missing session_id"}),
+ 400,
+ )
+
+ session_state = load_live_stt_session(redis_client, session_id)
+ if not session_state:
+ return make_response(
+ jsonify(
+ {
+ "success": False,
+ "message": "Live transcription session not found",
+ }
+ ),
+ 404,
+ )
+
+ if safe_filename(str(session_state.get("user", ""))) != auth_user:
+ return make_response(
+ jsonify({"success": False, "message": "Forbidden"}),
+ 403,
+ )
+
+ final_text = finalize_live_stt_session(session_state)
+ delete_live_stt_session(redis_client, session_id)
+
+ return make_response(
+ jsonify(
+ {
+ "success": True,
+ "session_id": session_id,
+ "language": session_state.get("language"),
+ "text": final_text,
+ }
+ ),
+ 200,
+ )
+
+
@attachments_ns.route("/images/")
class ServeImage(Resource):
@api.doc(description="Serve an image from storage")
def get(self, image_path):
try:
+ from application.api.user.base import storage
+
file_obj = storage.get_file(image_path)
extension = image_path.split(".")[-1].lower()
content_type = f"image/{extension}"
diff --git a/application/api/user/sources/upload.py b/application/api/user/sources/upload.py
index 313f9e7f..7da2c9b7 100644
--- a/application/api/user/sources/upload.py
+++ b/application/api/user/sources/upload.py
@@ -14,7 +14,14 @@ from application.api.user.base import sources_collection
from application.api.user.tasks import ingest, ingest_connector_task, ingest_remote
from application.core.settings import settings
from application.parser.connectors.connector_creator import ConnectorCreator
+from application.parser.file.constants import SUPPORTED_SOURCE_EXTENSIONS
from application.storage.storage_creator import StorageCreator
+from application.stt.upload_limits import (
+ AudioFileTooLargeError,
+ build_stt_file_size_limit_message,
+ enforce_audio_file_size_limit,
+ is_audio_filename,
+)
from application.utils import check_required_fields, safe_filename
@@ -23,6 +30,12 @@ sources_upload_ns = Namespace(
)
+def _enforce_audio_path_size_limit(file_path: str, filename: str) -> None:
+ if not is_audio_filename(filename):
+ return
+ enforce_audio_file_size_limit(os.path.getsize(file_path))
+
+
@sources_upload_ns.route("/upload")
class UploadFile(Resource):
@api.expect(
@@ -78,6 +91,7 @@ class UploadFile(Resource):
with tempfile.TemporaryDirectory() as temp_dir:
temp_file_path = os.path.join(temp_dir, safe_file)
file.save(temp_file_path)
+ _enforce_audio_path_size_limit(temp_file_path, safe_file)
# Only extract actual .zip files, not Office formats (.docx, .xlsx, .pptx)
# which are technically zip archives but should be processed as-is
@@ -102,6 +116,10 @@ class UploadFile(Resource):
os.path.join(root, extracted_file), temp_dir
)
storage_path = f"{base_path}/{rel_path}"
+ _enforce_audio_path_size_limit(
+ os.path.join(root, extracted_file),
+ extracted_file,
+ )
with open(
os.path.join(root, extracted_file), "rb"
@@ -124,29 +142,23 @@ class UploadFile(Resource):
storage.save_file(f, file_path)
task = ingest.delay(
settings.UPLOAD_FOLDER,
- [
- ".rst",
- ".md",
- ".pdf",
- ".txt",
- ".docx",
- ".csv",
- ".epub",
- ".html",
- ".mdx",
- ".json",
- ".xlsx",
- ".pptx",
- ".png",
- ".jpg",
- ".jpeg",
- ],
+ list(SUPPORTED_SOURCE_EXTENSIONS),
job_name,
user,
file_path=base_path,
filename=dir_name,
file_name_map=file_name_map,
)
+ except AudioFileTooLargeError:
+ return make_response(
+ jsonify(
+ {
+ "success": False,
+ "message": build_stt_file_size_limit_message(),
+ }
+ ),
+ 413,
+ )
except Exception as err:
current_app.logger.error(f"Error uploading file: {err}", exc_info=True)
return make_response(jsonify({"success": False}), 400)
diff --git a/application/api/user/utils.py b/application/api/user/utils.py
index 6bae0463..ab2d46a3 100644
--- a/application/api/user/utils.py
+++ b/application/api/user/utils.py
@@ -5,7 +5,14 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
from bson.errors import InvalidId
from bson.objectid import ObjectId
-from flask import jsonify, make_response, request, Response
+from flask import (
+ Response,
+ current_app,
+ has_app_context,
+ jsonify,
+ make_response,
+ request,
+)
from pymongo.collection import Collection
@@ -319,8 +326,10 @@ def safe_db_operation(
try:
result = operation()
return result, None
- except Exception as e:
- return None, error_response(f"{error_message}: {str(e)}")
+ except Exception as err:
+ if has_app_context():
+ current_app.logger.error(f"{error_message}: {err}", exc_info=True)
+ return None, error_response(error_message)
def validate_enum(
diff --git a/application/api/user/workflows/routes.py b/application/api/user/workflows/routes.py
index fc6006bd..66678c81 100644
--- a/application/api/user/workflows/routes.py
+++ b/application/api/user/workflows/routes.py
@@ -30,6 +30,11 @@ from application.api.user.utils import (
workflows_ns = Namespace("workflows", path="/api")
+def _workflow_error_response(message: str, err: Exception):
+ current_app.logger.error(f"{message}: {err}", exc_info=True)
+ return error_response(message)
+
+
def serialize_workflow(w: Dict) -> Dict:
"""Serialize workflow document to API response format."""
return {
@@ -396,11 +401,11 @@ class WorkflowList(Resource):
try:
create_workflow_nodes(workflow_id, nodes_data, 1)
create_workflow_edges(workflow_id, edges_data, 1)
- except Exception as e:
+ except Exception as err:
workflow_nodes_collection.delete_many({"workflow_id": workflow_id})
workflow_edges_collection.delete_many({"workflow_id": workflow_id})
workflows_collection.delete_one({"_id": result.inserted_id})
- return error_response(f"Failed to create workflow structure: {str(e)}")
+ return _workflow_error_response("Failed to create workflow structure", err)
return success_response({"id": workflow_id}, 201)
@@ -470,14 +475,14 @@ class WorkflowDetail(Resource):
try:
create_workflow_nodes(workflow_id, nodes_data, next_graph_version)
create_workflow_edges(workflow_id, edges_data, next_graph_version)
- except Exception as e:
+ except Exception as err:
workflow_nodes_collection.delete_many(
{"workflow_id": workflow_id, "graph_version": next_graph_version}
)
workflow_edges_collection.delete_many(
{"workflow_id": workflow_id, "graph_version": next_graph_version}
)
- return error_response(f"Failed to update workflow structure: {str(e)}")
+ return _workflow_error_response("Failed to update workflow structure", err)
now = datetime.now(timezone.utc)
_, error = safe_db_operation(
@@ -535,7 +540,7 @@ class WorkflowDetail(Resource):
workflow_nodes_collection.delete_many({"workflow_id": workflow_id})
workflow_edges_collection.delete_many({"workflow_id": workflow_id})
workflows_collection.delete_one({"_id": workflow["_id"], "user": user_id})
- except Exception as e:
- return error_response(f"Failed to delete workflow: {str(e)}")
+ except Exception as err:
+ return _workflow_error_response("Failed to delete workflow", err)
return success_response()
diff --git a/application/app.py b/application/app.py
index 489ec840..bb9b3b1b 100644
--- a/application/app.py
+++ b/application/app.py
@@ -19,6 +19,10 @@ from application.api.user.routes import user # noqa: E402
from application.api.connector.routes import connector # noqa: E402
from application.celery_init import celery # noqa: E402
from application.core.settings import settings # noqa: E402
+from application.stt.upload_limits import ( # noqa: E402
+ build_stt_file_size_limit_message,
+ should_reject_stt_request,
+)
if platform.system() == "Windows":
@@ -88,6 +92,23 @@ def generate_token():
return jsonify({"error": "Token generation not allowed in current auth mode"}), 400
+@app.before_request
+def enforce_stt_request_size_limits():
+ if request.method == "OPTIONS":
+ return None
+ if should_reject_stt_request(request.path, request.content_length):
+ return (
+ jsonify(
+ {
+ "success": False,
+ "message": build_stt_file_size_limit_message(),
+ }
+ ),
+ 413,
+ )
+ return None
+
+
@app.before_request
def authenticate_request():
if request.method == "OPTIONS":
diff --git a/application/core/settings.py b/application/core/settings.py
index 50c30a80..41020ebc 100644
--- a/application/core/settings.py
+++ b/application/core/settings.py
@@ -156,6 +156,12 @@ class Settings(BaseSettings):
TTS_PROVIDER: str = "google_tts" # google_tts or elevenlabs
ELEVENLABS_API_KEY: Optional[str] = None
+ STT_PROVIDER: str = "openai" # openai or faster_whisper
+ OPENAI_STT_MODEL: str = "gpt-4o-mini-transcribe"
+ STT_LANGUAGE: Optional[str] = None
+ STT_MAX_FILE_SIZE_MB: int = 25
+ STT_ENABLE_TIMESTAMPS: bool = False
+ STT_ENABLE_DIARIZATION: bool = False
# Tool pre-fetch settings
ENABLE_TOOL_PREFETCH: bool = True
diff --git a/application/parser/file/audio_parser.py b/application/parser/file/audio_parser.py
new file mode 100644
index 00000000..f0fd8daa
--- /dev/null
+++ b/application/parser/file/audio_parser.py
@@ -0,0 +1,48 @@
+from pathlib import Path
+from typing import Dict, Union
+
+from application.core.settings import settings
+from application.parser.file.base_parser import BaseParser
+from application.stt.stt_creator import STTCreator
+from application.stt.upload_limits import enforce_audio_file_size_limit
+
+
+class AudioParser(BaseParser):
+ def __init__(self, parser_config=None):
+ super().__init__(parser_config=parser_config)
+ self._transcript_metadata: Dict[str, Dict] = {}
+
+ def _init_parser(self) -> Dict:
+ return {}
+
+ def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+ _ = errors
+ try:
+ enforce_audio_file_size_limit(file.stat().st_size)
+ except OSError:
+ pass
+ stt = STTCreator.create_stt(settings.STT_PROVIDER)
+ result = stt.transcribe(
+ file,
+ language=settings.STT_LANGUAGE,
+ timestamps=settings.STT_ENABLE_TIMESTAMPS,
+ diarize=settings.STT_ENABLE_DIARIZATION,
+ )
+
+ transcript_metadata = {
+ "transcript_duration_s": result.get("duration_s"),
+ "transcript_language": result.get("language"),
+ "transcript_provider": result.get("provider"),
+ }
+ if result.get("segments"):
+ transcript_metadata["transcript_segments"] = result["segments"]
+
+ self._transcript_metadata[str(file)] = {
+ key: value
+ for key, value in transcript_metadata.items()
+ if value not in (None, [], {})
+ }
+ return result.get("text", "")
+
+ def get_file_metadata(self, file: Path) -> Dict:
+ return self._transcript_metadata.get(str(file), {})
diff --git a/application/parser/file/base_parser.py b/application/parser/file/base_parser.py
index 753a56f9..ff7a095d 100644
--- a/application/parser/file/base_parser.py
+++ b/application/parser/file/base_parser.py
@@ -36,3 +36,8 @@ class BaseParser:
@abstractmethod
def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
"""Parse file."""
+
+ def get_file_metadata(self, file: Path) -> Dict:
+ """Return parser-specific metadata for the most recently parsed file."""
+ _ = file
+ return {}
diff --git a/application/parser/file/bulk.py b/application/parser/file/bulk.py
index bee0dd34..fee27aa0 100644
--- a/application/parser/file/bulk.py
+++ b/application/parser/file/bulk.py
@@ -14,11 +14,17 @@ from application.parser.file.tabular_parser import PandasCSVParser, ExcelParser
from application.parser.file.json_parser import JSONParser
from application.parser.file.pptx_parser import PPTXParser
from application.parser.file.image_parser import ImageParser
+from application.parser.file.audio_parser import AudioParser
from application.parser.schema.base import Document
+from application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
from application.utils import num_tokens_from_string
from application.core.settings import settings
+def _build_audio_parser_mapping() -> Dict[str, BaseParser]:
+ return {extension: AudioParser() for extension in SUPPORTED_AUDIO_EXTENSIONS}
+
+
def get_default_file_extractor(
ocr_enabled: Optional[bool] = None,
) -> Dict[str, BaseParser]:
@@ -70,6 +76,7 @@ def get_default_file_extractor(
".webp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
# Media/subtitles
".vtt": DoclingVTTParser(),
+ **_build_audio_parser_mapping(),
# Specialized XML formats
".xml": DoclingXMLParser(),
# Formats docling doesn't support - use standard parsers
@@ -96,6 +103,7 @@ def get_default_file_extractor(
".png": ImageParser(),
".jpg": ImageParser(),
".jpeg": ImageParser(),
+ **_build_audio_parser_mapping(),
}
@@ -221,11 +229,13 @@ class SimpleDirectoryReader(BaseReader):
for input_file in self.input_files:
suffix_lower = input_file.suffix.lower()
+ parser_metadata = {}
if suffix_lower in self.file_extractor:
parser = self.file_extractor[suffix_lower]
if not parser.parser_config_set:
parser.init_parser()
data = parser.parse_file(input_file, errors=self.errors)
+ parser_metadata = parser.get_file_metadata(input_file)
else:
# do standard read
with open(input_file, "r", errors=self.errors) as f:
@@ -244,6 +254,8 @@ class SimpleDirectoryReader(BaseReader):
'title': input_file.name,
'token_count': file_tokens,
}
+ if parser_metadata:
+ base_metadata.update(parser_metadata)
if hasattr(self, 'input_dir'):
try:
diff --git a/application/parser/file/constants.py b/application/parser/file/constants.py
new file mode 100644
index 00000000..4656727b
--- /dev/null
+++ b/application/parser/file/constants.py
@@ -0,0 +1,27 @@
+"""Shared file-extension constants for parsing and ingestion flows."""
+
+from application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
+
+
+SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS = (
+ ".rst",
+ ".md",
+ ".pdf",
+ ".txt",
+ ".docx",
+ ".csv",
+ ".epub",
+ ".html",
+ ".mdx",
+ ".json",
+ ".xlsx",
+ ".pptx",
+)
+
+SUPPORTED_SOURCE_IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg")
+
+SUPPORTED_SOURCE_EXTENSIONS = (
+ *SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS,
+ *SUPPORTED_SOURCE_IMAGE_EXTENSIONS,
+ *SUPPORTED_AUDIO_EXTENSIONS,
+)
diff --git a/application/stt/__init__.py b/application/stt/__init__.py
new file mode 100644
index 00000000..e2acfcd4
--- /dev/null
+++ b/application/stt/__init__.py
@@ -0,0 +1 @@
+"""Speech-to-text providers."""
diff --git a/application/stt/base.py b/application/stt/base.py
new file mode 100644
index 00000000..16f92f50
--- /dev/null
+++ b/application/stt/base.py
@@ -0,0 +1,15 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+class BaseSTT(ABC):
+ @abstractmethod
+ def transcribe(
+ self,
+ file_path: Path,
+ language: Optional[str] = None,
+ timestamps: bool = False,
+ diarize: bool = False,
+ ) -> Dict[str, Any]:
+ pass
diff --git a/application/stt/constants.py b/application/stt/constants.py
new file mode 100644
index 00000000..3e835799
--- /dev/null
+++ b/application/stt/constants.py
@@ -0,0 +1,15 @@
+SUPPORTED_AUDIO_EXTENSIONS = (".wav", ".mp3", ".m4a", ".ogg", ".webm")
+
+SUPPORTED_AUDIO_MIME_TYPES = {
+ "application/ogg",
+ "audio/aac",
+ "audio/mp3",
+ "audio/mp4",
+ "audio/mpeg",
+ "audio/ogg",
+ "audio/wav",
+ "audio/webm",
+ "audio/x-m4a",
+ "audio/x-wav",
+ "video/webm",
+}
diff --git a/application/stt/faster_whisper_stt.py b/application/stt/faster_whisper_stt.py
new file mode 100644
index 00000000..9c09db6d
--- /dev/null
+++ b/application/stt/faster_whisper_stt.py
@@ -0,0 +1,70 @@
+from pathlib import Path
+from typing import Dict, Optional
+
+from application.stt.base import BaseSTT
+
+
+class FasterWhisperSTT(BaseSTT):
+ def __init__(
+ self,
+ model_size: str = "base",
+ device: str = "auto",
+ compute_type: str = "int8",
+ ):
+ self.model_size = model_size
+ self.device = device
+ self.compute_type = compute_type
+ self._model = None
+
+ def _get_model(self):
+ if self._model is None:
+ try:
+ from faster_whisper import WhisperModel
+ except ImportError as exc:
+ raise ImportError(
+ "faster-whisper is required to use the faster_whisper STT provider."
+ ) from exc
+
+ self._model = WhisperModel(
+ self.model_size,
+ device=self.device,
+ compute_type=self.compute_type,
+ )
+ return self._model
+
+ def transcribe(
+ self,
+ file_path: Path,
+ language: Optional[str] = None,
+ timestamps: bool = False,
+ diarize: bool = False,
+ ) -> Dict[str, object]:
+ _ = diarize
+ model = self._get_model()
+ segments_iter, info = model.transcribe(
+ str(file_path),
+ language=language,
+ word_timestamps=timestamps,
+ )
+
+ segments = []
+ text_parts = []
+ for segment in segments_iter:
+ segment_text = getattr(segment, "text", "").strip()
+ if segment_text:
+ text_parts.append(segment_text)
+ segments.append(
+ {
+ "start": getattr(segment, "start", None),
+ "end": getattr(segment, "end", None),
+ "text": segment_text,
+ }
+ )
+
+ return {
+ "text": " ".join(text_parts).strip(),
+ "language": getattr(info, "language", language),
+ "duration_s": getattr(info, "duration", None),
+ "segments": segments if timestamps else [],
+ "provider": "faster_whisper",
+ }
diff --git a/application/stt/live_session.py b/application/stt/live_session.py
new file mode 100644
index 00000000..e0127976
--- /dev/null
+++ b/application/stt/live_session.py
@@ -0,0 +1,201 @@
+import json
+import re
+import uuid
+from typing import Dict, Optional
+
+
+LIVE_STT_SESSION_PREFIX = "stt_live_session:"
+LIVE_STT_SESSION_TTL_SECONDS = 15 * 60
+LIVE_STT_MUTABLE_TAIL_WORDS = 8
+LIVE_STT_SILENCE_MUTABLE_TAIL_WORDS = 2
+LIVE_STT_MIN_COMMITTED_OVERLAP_WORDS = 2
+
+
+def normalize_transcript_text(text: str) -> str:
+ return " ".join((text or "").split()).strip()
+
+
+def join_transcript_parts(*parts: str) -> str:
+ return " ".join(part for part in map(normalize_transcript_text, parts) if part)
+
+
+def _normalize_word(word: str) -> str:
+ normalized = re.sub(r"[^\w]+", "", word.casefold(), flags=re.UNICODE)
+ return normalized or word.casefold()
+
+
+def _split_words(text: str) -> list[str]:
+ normalized = normalize_transcript_text(text)
+ return normalized.split() if normalized else []
+
+
+def _common_prefix_length(left_words: list[str], right_words: list[str]) -> int:
+ max_index = min(len(left_words), len(right_words))
+ prefix_length = 0
+ for index in range(max_index):
+ if _normalize_word(left_words[index]) != _normalize_word(right_words[index]):
+ break
+ prefix_length += 1
+ return prefix_length
+
+
+def _find_suffix_prefix_overlap(
+ left_words: list[str], right_words: list[str], min_overlap: int
+) -> int:
+ max_overlap = min(len(left_words), len(right_words))
+ if max_overlap < min_overlap:
+ return 0
+
+ left_keys = [_normalize_word(word) for word in left_words]
+ right_keys = [_normalize_word(word) for word in right_words]
+
+ for overlap_size in range(max_overlap, min_overlap - 1, -1):
+ if left_keys[-overlap_size:] == right_keys[:overlap_size]:
+ return overlap_size
+ return 0
+
+
+def strip_committed_prefix(committed_text: str, hypothesis_text: str) -> str:
+ committed_words = _split_words(committed_text)
+ hypothesis_words = _split_words(hypothesis_text)
+ if not committed_words or not hypothesis_words:
+ return normalize_transcript_text(hypothesis_text)
+
+ full_prefix_length = _common_prefix_length(committed_words, hypothesis_words)
+ if full_prefix_length == len(committed_words):
+ return " ".join(hypothesis_words[full_prefix_length:])
+
+ overlap_size = _find_suffix_prefix_overlap(
+ committed_words,
+ hypothesis_words,
+ LIVE_STT_MIN_COMMITTED_OVERLAP_WORDS,
+ )
+ if overlap_size:
+ return " ".join(hypothesis_words[overlap_size:])
+ return " ".join(hypothesis_words)
+
+
+def _calculate_commit_count(
+ previous_hypothesis: str, current_hypothesis: str, is_silence: bool
+) -> int:
+ previous_words = _split_words(previous_hypothesis)
+ current_words = _split_words(current_hypothesis)
+ if not current_words:
+ return 0
+
+ if not previous_words:
+ if is_silence:
+ return max(0, len(current_words) - LIVE_STT_SILENCE_MUTABLE_TAIL_WORDS)
+ return 0
+
+ stable_prefix_length = _common_prefix_length(previous_words, current_words)
+ if not stable_prefix_length:
+ return 0
+
+ mutable_tail_words = (
+ LIVE_STT_SILENCE_MUTABLE_TAIL_WORDS
+ if is_silence
+ else LIVE_STT_MUTABLE_TAIL_WORDS
+ )
+ max_committable_by_tail = max(0, len(current_words) - mutable_tail_words)
+ return min(stable_prefix_length, max_committable_by_tail)
+
+
+def create_live_stt_session(
+ user: str, language: Optional[str] = None
+) -> Dict[str, object]:
+ return {
+ "session_id": str(uuid.uuid4()),
+ "user": user,
+ "language": language,
+ "committed_text": "",
+ "mutable_text": "",
+ "previous_hypothesis": "",
+ "latest_hypothesis": "",
+ "last_chunk_index": -1,
+ }
+
+
+def get_live_stt_session_key(session_id: str) -> str:
+ return f"{LIVE_STT_SESSION_PREFIX}{session_id}"
+
+
+def save_live_stt_session(redis_client, session_state: Dict[str, object]) -> None:
+ redis_client.setex(
+ get_live_stt_session_key(str(session_state["session_id"])),
+ LIVE_STT_SESSION_TTL_SECONDS,
+ json.dumps(session_state),
+ )
+
+
+def load_live_stt_session(redis_client, session_id: str) -> Optional[Dict[str, object]]:
+ raw_session = redis_client.get(get_live_stt_session_key(session_id))
+ if not raw_session:
+ return None
+ if isinstance(raw_session, bytes):
+ raw_session = raw_session.decode("utf-8")
+ return json.loads(raw_session)
+
+
+def delete_live_stt_session(redis_client, session_id: str) -> None:
+ redis_client.delete(get_live_stt_session_key(session_id))
+
+
+def apply_live_stt_hypothesis(
+ session_state: Dict[str, object],
+ hypothesis_text: str,
+ chunk_index: int,
+ is_silence: bool = False,
+) -> Dict[str, object]:
+ last_chunk_index = int(session_state.get("last_chunk_index", -1))
+ if chunk_index < 0:
+ raise ValueError("chunk_index must be non-negative")
+ if chunk_index < last_chunk_index:
+ raise ValueError("chunk_index is older than the last processed chunk")
+ if chunk_index == last_chunk_index:
+ return session_state
+
+ committed_text = normalize_transcript_text(str(session_state.get("committed_text", "")))
+ previous_hypothesis = normalize_transcript_text(
+ str(session_state.get("latest_hypothesis", ""))
+ )
+ current_hypothesis = strip_committed_prefix(committed_text, hypothesis_text)
+
+ if not current_hypothesis and is_silence and previous_hypothesis:
+ committed_text = join_transcript_parts(committed_text, previous_hypothesis)
+ previous_hypothesis = ""
+
+ commit_count = _calculate_commit_count(
+ previous_hypothesis,
+ current_hypothesis,
+ is_silence=is_silence,
+ )
+ current_words = _split_words(current_hypothesis)
+
+ if commit_count:
+ committed_text = join_transcript_parts(
+ committed_text,
+ " ".join(current_words[:commit_count]),
+ )
+ current_hypothesis = " ".join(current_words[commit_count:])
+
+ session_state["committed_text"] = committed_text
+ session_state["mutable_text"] = normalize_transcript_text(current_hypothesis)
+ session_state["previous_hypothesis"] = previous_hypothesis
+ session_state["latest_hypothesis"] = normalize_transcript_text(current_hypothesis)
+ session_state["last_chunk_index"] = chunk_index
+ return session_state
+
+
+def get_live_stt_transcript_text(session_state: Dict[str, object]) -> str:
+ return join_transcript_parts(
+ str(session_state.get("committed_text", "")),
+ str(session_state.get("mutable_text", "")),
+ )
+
+
+def finalize_live_stt_session(session_state: Dict[str, object]) -> str:
+ return join_transcript_parts(
+ str(session_state.get("committed_text", "")),
+ str(session_state.get("latest_hypothesis", "")),
+ )
diff --git a/application/stt/openai_stt.py b/application/stt/openai_stt.py
new file mode 100644
index 00000000..96a46a58
--- /dev/null
+++ b/application/stt/openai_stt.py
@@ -0,0 +1,60 @@
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from openai import OpenAI
+
+from application.core.settings import settings
+from application.stt.base import BaseSTT
+
+
+class OpenAISTT(BaseSTT):
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ base_url: Optional[str] = None,
+ model: Optional[str] = None,
+ ):
+ self.api_key = api_key or settings.OPENAI_API_KEY or settings.API_KEY
+ self.base_url = base_url or settings.OPENAI_BASE_URL or "https://api.openai.com/v1"
+ self.model = model or settings.OPENAI_STT_MODEL
+ self.client = OpenAI(api_key=self.api_key, base_url=self.base_url)
+
+ def transcribe(
+ self,
+ file_path: Path,
+ language: Optional[str] = None,
+ timestamps: bool = False,
+ diarize: bool = False,
+ ) -> Dict[str, Any]:
+ _ = diarize
+ request: Dict[str, Any] = {
+ "file": file_path,
+ "model": self.model,
+ "response_format": "verbose_json",
+ }
+ if language:
+ request["language"] = language
+ if timestamps:
+ request["timestamp_granularities"] = ["segment"]
+
+ with open(file_path, "rb") as audio_file:
+ request["file"] = audio_file
+ response = self.client.audio.transcriptions.create(**request)
+ response_dict = self._to_dict(response)
+ segments = response_dict.get("segments") or []
+
+ return {
+ "text": response_dict.get("text", ""),
+ "language": response_dict.get("language") or language,
+ "duration_s": response_dict.get("duration"),
+ "segments": [self._to_dict(segment) for segment in segments],
+ "provider": "openai",
+ }
+
+ @staticmethod
+ def _to_dict(value: Any) -> Dict[str, Any]:
+ if hasattr(value, "model_dump"):
+ return value.model_dump()
+ if isinstance(value, dict):
+ return value
+ return {}
diff --git a/application/stt/stt_creator.py b/application/stt/stt_creator.py
new file mode 100644
index 00000000..4559443e
--- /dev/null
+++ b/application/stt/stt_creator.py
@@ -0,0 +1,17 @@
+from application.stt.base import BaseSTT
+from application.stt.faster_whisper_stt import FasterWhisperSTT
+from application.stt.openai_stt import OpenAISTT
+
+
+class STTCreator:
+ stt_providers = {
+ "openai": OpenAISTT,
+ "faster_whisper": FasterWhisperSTT,
+ }
+
+ @classmethod
+ def create_stt(cls, stt_type, *args, **kwargs) -> BaseSTT:
+ stt_class = cls.stt_providers.get(stt_type.lower())
+ if not stt_class:
+ raise ValueError(f"No stt class found for type {stt_type}")
+ return stt_class(*args, **kwargs)
diff --git a/application/stt/upload_limits.py b/application/stt/upload_limits.py
new file mode 100644
index 00000000..8a8b938e
--- /dev/null
+++ b/application/stt/upload_limits.py
@@ -0,0 +1,43 @@
+from pathlib import Path
+
+from application.core.settings import settings
+from application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
+from application.utils import safe_filename
+
+
+STT_REQUEST_SIZE_OVERHEAD_BYTES = 1024 * 1024
+STT_SIZE_LIMITED_PATHS = frozenset(("/api/stt", "/api/stt/live/chunk"))
+
+
+class AudioFileTooLargeError(ValueError):
+ pass
+
+
+def get_stt_max_file_size_bytes() -> int:
+ return max(0, settings.STT_MAX_FILE_SIZE_MB) * 1024 * 1024
+
+
+def build_stt_file_size_limit_message() -> str:
+ return f"Audio file exceeds {settings.STT_MAX_FILE_SIZE_MB}MB limit"
+
+
+def is_audio_filename(filename: str | Path | None) -> bool:
+ if not filename:
+ return False
+ safe_name = safe_filename(Path(str(filename)).name)
+ return Path(safe_name).suffix.lower() in SUPPORTED_AUDIO_EXTENSIONS
+
+
+def enforce_audio_file_size_limit(size_bytes: int) -> None:
+ max_size_bytes = get_stt_max_file_size_bytes()
+ if max_size_bytes and size_bytes > max_size_bytes:
+ raise AudioFileTooLargeError(build_stt_file_size_limit_message())
+
+
+def should_reject_stt_request(path: str, content_length: int | None) -> bool:
+ if path not in STT_SIZE_LIMITED_PATHS or content_length is None:
+ return False
+ max_request_size_bytes = (
+ get_stt_max_file_size_bytes() + STT_REQUEST_SIZE_OVERHEAD_BYTES
+ )
+ return content_length > max_request_size_bytes
diff --git a/application/worker.py b/application/worker.py
index 69b22ea8..9821b65b 100755
--- a/application/worker.py
+++ b/application/worker.py
@@ -26,6 +26,7 @@ from application.parser.chunking import Chunker
from application.parser.connectors.connector_creator import ConnectorCreator
from application.parser.embedding_pipeline import embed_and_store_documents
from application.parser.file.bulk import SimpleDirectoryReader, get_default_file_extractor
+from application.parser.file.constants import SUPPORTED_SOURCE_EXTENSIONS
from application.parser.remote.remote_creator import RemoteCreator
from application.parser.schema.base import Document
from application.retriever.retriever_creator import RetrieverCreator
@@ -646,23 +647,7 @@ def reingest_source_worker(self, source_id, user):
reader = SimpleDirectoryReader(
input_dir=temp_dir,
recursive=True,
- required_exts=[
- ".rst",
- ".md",
- ".pdf",
- ".txt",
- ".docx",
- ".csv",
- ".epub",
- ".html",
- ".mdx",
- ".json",
- ".xlsx",
- ".pptx",
- ".png",
- ".jpg",
- ".jpeg",
- ],
+ required_exts=list(SUPPORTED_SOURCE_EXTENSIONS),
exclude_hidden=True,
file_metadata=metadata_from_filename,
)
@@ -1140,17 +1125,25 @@ def attachment_worker(self, file_info, user):
file_extractor = get_default_file_extractor(
ocr_enabled=settings.DOCLING_OCR_ATTACHMENTS_ENABLED
)
- content = storage.process_file(
+ attachment_document = storage.process_file(
relative_path,
lambda local_path, **kwargs: SimpleDirectoryReader(
input_files=[local_path],
exclude_hidden=True,
errors="ignore",
file_extractor=file_extractor,
+ file_metadata=metadata_from_filename,
)
- .load_data()[0]
- .text,
+ .load_data()[0],
)
+ content = attachment_document.text
+ parser_metadata = {
+ key: value
+ for key, value in (attachment_document.extra_info or {}).items()
+ if key.startswith("transcript_")
+ }
+ if parser_metadata:
+ metadata = {**metadata, **parser_metadata}
token_count = num_tokens_from_string(content)
if token_count > 100000:
@@ -1333,23 +1326,7 @@ def ingest_connector(
reader = SimpleDirectoryReader(
input_dir=temp_dir,
recursive=True,
- required_exts=[
- ".rst",
- ".md",
- ".pdf",
- ".txt",
- ".docx",
- ".csv",
- ".epub",
- ".html",
- ".mdx",
- ".json",
- ".xlsx",
- ".pptx",
- ".png",
- ".jpg",
- ".jpeg",
- ],
+ required_exts=list(SUPPORTED_SOURCE_EXTENSIONS),
exclude_hidden=True,
file_metadata=metadata_from_filename,
)
diff --git a/docs/content/Deploying/DocsGPT-Settings.mdx b/docs/content/Deploying/DocsGPT-Settings.mdx
index 3699769c..f5cb0c07 100644
--- a/docs/content/Deploying/DocsGPT-Settings.mdx
+++ b/docs/content/Deploying/DocsGPT-Settings.mdx
@@ -65,6 +65,8 @@ Here are some of the most fundamental settings you'll likely want to configure:
- **`OPENAI_BASE_URL`**: Specifically used when `LLM_PROVIDER` is set to `openai` but you are connecting to a local inference engine (like Ollama, Llama.cpp, etc.) that exposes an OpenAI-compatible API. This setting tells DocsGPT where to find your local LLM server.
+- **`STT_PROVIDER`**: Selects the speech-to-text provider used for microphone transcription in chat and for audio file ingestion through the parser pipeline.
+
## Configuration Examples
Let's look at some concrete examples of how to configure these settings in your `.env` file.
@@ -95,6 +97,49 @@ EMBEDDINGS_NAME=huggingface_sentence-transformers/all-mpnet-base-v2 # You can al
In this case, even though you are using Ollama locally, `LLM_PROVIDER` is set to `openai` because Ollama (and many other local inference engines) are designed to be API-compatible with OpenAI. `OPENAI_BASE_URL` points DocsGPT to the local Ollama server.
+## Speech-to-Text Settings
+
+DocsGPT can transcribe audio in two places:
+
+- Voice input in the chat.
+- Audio file ingestion. Uploaded `.wav`, `.mp3`, `.m4a`, `.ogg`, and `.webm` files are transcribed first and then passed through the normal parser, chunking, embedding, and indexing pipeline.
+
+For an end-to-end walkthrough, see the [Speech and Audio Guide](/Guides/speech-and-audio).
+
+| Setting | Purpose | Typical values |
+| --- | --- | --- |
+| `STT_PROVIDER` | Speech-to-text backend provider. | `openai`, `faster_whisper` |
+| `OPENAI_STT_MODEL` | OpenAI transcription model used when `STT_PROVIDER=openai`. | `gpt-4o-mini-transcribe` |
+| `STT_LANGUAGE` | Optional language hint passed to the provider. Leave unset for auto-detection when supported. | `en`, `es`, unset |
+| `STT_MAX_FILE_SIZE_MB` | Maximum file size accepted by the synchronous `/api/stt` endpoint. | `25` |
+| `STT_ENABLE_TIMESTAMPS` | Include timestamp segments in the normalized transcript response and stored parser metadata. | `true`, `false` |
+| `STT_ENABLE_DIARIZATION` | Reserved provider option for speaker diarization. Some providers may ignore it. | `true`, `false` |
+
+### Example: OpenAI Speech-to-Text
+
+```env
+STT_PROVIDER=openai
+OPENAI_API_KEY=YOUR_OPENAI_API_KEY
+OPENAI_STT_MODEL=gpt-4o-mini-transcribe
+STT_LANGUAGE=
+STT_MAX_FILE_SIZE_MB=25
+STT_ENABLE_TIMESTAMPS=false
+STT_ENABLE_DIARIZATION=false
+```
+
+If you already use `API_KEY` for OpenAI, DocsGPT can reuse that key for transcription. Set `OPENAI_API_KEY` only when you want a dedicated key.
+
+### Example: Local `faster_whisper`
+
+```env
+STT_PROVIDER=faster_whisper
+STT_LANGUAGE=en
+STT_ENABLE_TIMESTAMPS=true
+STT_ENABLE_DIARIZATION=false
+```
+
+`faster_whisper` is an optional backend dependency. Install it in the Python environment used by the DocsGPT API and worker before selecting this provider.
+
## Authentication Settings
DocsGPT includes a JWT (JSON Web Token) based authentication feature for managing sessions or securing local deployments while allowing access.
@@ -180,4 +225,3 @@ These are just the basic settings to get you started. The `settings.py` file con
For a complete list of available settings and their descriptions, refer to the `settings.py` file in `application/core`. Remember to restart your Docker containers after making changes to your `.env` file or `settings.py` for the changes to take effect.
-
diff --git a/docs/content/index.mdx b/docs/content/index.mdx
index 574f19ab..dc5da682 100644
--- a/docs/content/index.mdx
+++ b/docs/content/index.mdx
@@ -59,7 +59,7 @@ export const allGuides = {
# **DocsGPT 🦖**
-DocsGPT is an open-source genAI tool that helps users get reliable answers from any knowledge source, while avoiding hallucinations. It enables quick and reliable information retrieval, with tooling and agentic system capability built in.
+DocsGPT is an open-source genAI tool that helps users get reliable answers from any knowledge source, while avoiding hallucinations. It enables quick and reliable information retrieval, with tooling and agentic system capability built in, including speech-to-text workflows for chat and audio knowledge ingestion.