mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-13 12:14:00 +00:00
build(deps): declare dependencies in pyproject.toml with docling and milvus extras
requirements.txt pinned torch and transformers in core although only docling needs them, and on Linux torch pulls the CUDA 13 stack: 2.7 GB of the 3.0 GB wheel download. Direct dependencies now live in pyproject.toml, uv.lock pins everything, and application/requirements*.txt are exported from the lock by scripts/export_requirements.sh (each file is the core set plus one extra). The docling extra pins torch/torchvision/transformers itself and, on Linux, resolves torch from the CPU-only PyTorch index (no nvidia packages). milvus (pymilvus + milvus-lite, which pulls pyarrow) is the second extra. application/core/optional_deps.py is the one place install hints come from; the milvus store and the docling call sites use it so a missing extra fails with the exact command to run.
This commit is contained in:
Alex committed
1 parent
55a38b3132
commit
196865846c
14 files changed
+10545
-123
No files matched your search
@@ -29,10 +29,18 @@ Use these commands once the dev prerequisites above are satisfied.
|
||||
```bash
|
||||
source .venv/bin/activate # macOS/Linux
|
||||
uv pip install -r application/requirements.txt # or: pip install -r application/requirements.txt
|
||||
# Optional docling parser engine (OCR / read_document structured output):
|
||||
# uv pip install -r application/requirements-docling.txt
|
||||
# Optional extras (not installed by default; each file = core + the extra):
|
||||
# uv pip install -r application/requirements-docling.txt # docling parser engine (OCR backend, structured output)
|
||||
# uv pip install -r application/requirements-milvus.txt # VECTOR_STORE=milvus
|
||||
# With uv alone: `uv sync --extra docling` (pyproject.toml + uv.lock are the source of truth).
|
||||
```
|
||||
|
||||
Dependencies are declared in `pyproject.toml` and locked in `uv.lock`; the
|
||||
`application/requirements*.txt` files are exported from the lock. To add or
|
||||
bump a package: edit `pyproject.toml`, run `uv lock`, then
|
||||
`bash scripts/export_requirements.sh` (CI fails if the exports are stale).
|
||||
Never edit the requirements files by hand.
|
||||
|
||||
Run the API. For local dev, prefer the ASGI entrypoint under uvicorn — it
|
||||
serves the **whole** app, matches production, and hot-reloads:
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Optional dependency extras and the one place their install hints come from.
|
||||
|
||||
Heavy or niche packages are not installed by default. Each extra maps to a
|
||||
pyproject extra and to an exported ``application/requirements-<extra>.txt``,
|
||||
so a missing module can always be explained with the exact command to run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
import sys
|
||||
from types import ModuleType
|
||||
from typing import Dict, Tuple
|
||||
|
||||
#: Extra name -> top-level modules it provides. Keep in sync with
|
||||
#: ``[project.optional-dependencies]`` in pyproject.toml.
|
||||
EXTRAS: Dict[str, Tuple[str, ...]] = {
|
||||
"docling": ("docling", "rapidocr"),
|
||||
"milvus": ("pymilvus",),
|
||||
}
|
||||
|
||||
_MODULE_TO_EXTRA: Dict[str, str] = {
|
||||
module: extra for extra, modules in EXTRAS.items() for module in modules
|
||||
}
|
||||
|
||||
|
||||
def install_hint(extra: str) -> str:
|
||||
"""Install command for ``extra``, for error messages and logs."""
|
||||
return (
|
||||
f"pip install -r application/requirements-{extra}.txt "
|
||||
f"(or: uv sync --extra {extra}; Docker: --build-arg EXTRAS={extra})"
|
||||
)
|
||||
|
||||
|
||||
def extra_for(module: str) -> str | None:
|
||||
"""Extra that provides top-level module ``module``, if any."""
|
||||
return _MODULE_TO_EXTRA.get(module.split(".")[0])
|
||||
|
||||
|
||||
def is_available(module: str) -> bool:
|
||||
"""Whether ``module`` can be imported, without importing it."""
|
||||
root = module.split(".")[0]
|
||||
if root in sys.modules:
|
||||
return sys.modules[root] is not None
|
||||
try:
|
||||
return importlib.util.find_spec(root) is not None
|
||||
except (ImportError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def missing_message(module: str, purpose: str | None = None) -> str:
|
||||
"""Human-readable explanation that ``module`` is absent and how to add it."""
|
||||
extra = extra_for(module)
|
||||
what = f"{module} is not installed"
|
||||
if purpose:
|
||||
what += f" ({purpose})"
|
||||
if extra:
|
||||
return f"{what}. It is part of the optional '{extra}' extra: {install_hint(extra)}"
|
||||
return f"{what}. Install it with: pip install {module}"
|
||||
|
||||
|
||||
def require(module: str, purpose: str | None = None) -> ModuleType:
|
||||
"""Import ``module`` or raise ``ImportError`` naming the extra to install.
|
||||
|
||||
Args:
|
||||
module: Importable module path, e.g. ``"pymilvus"``.
|
||||
purpose: Short note on what needed it, included in the error.
|
||||
|
||||
Returns:
|
||||
The imported module.
|
||||
|
||||
Raises:
|
||||
ImportError: With the install hint when the module is absent.
|
||||
"""
|
||||
try:
|
||||
return importlib.import_module(module)
|
||||
except ImportError as exc:
|
||||
raise ImportError(missing_message(module, purpose)) from exc
|
||||
@@ -19,6 +19,7 @@ from application.parser.schema.base import Document
|
||||
from application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
|
||||
from application.utils import num_tokens_from_string
|
||||
from application.core.settings import settings
|
||||
from application.core.optional_deps import install_hint
|
||||
|
||||
|
||||
def _build_audio_parser_mapping() -> Dict[str, BaseParser]:
|
||||
@@ -199,8 +200,9 @@ def _docling_file_extractor(
|
||||
logging.log(
|
||||
missing_log_level,
|
||||
"docling is not installed. Using standard parsers%s. For layout-model "
|
||||
"parsing, install with: pip install -r application/requirements-docling.txt",
|
||||
"parsing, install the docling extra: %s",
|
||||
" with native OCR" if ocr_enabled else "",
|
||||
install_hint("docling"),
|
||||
)
|
||||
return _legacy_file_extractor(pdf_text_fast_path, ocr_enabled=ocr_enabled)
|
||||
|
||||
|
||||
@@ -26,6 +26,8 @@ from application.parser.file.ocr_parser import VALID_OCR_ENGINES as _VALID_OCR_E
|
||||
from application.parser.file.ocr_parser import collapse_cjk_spaces
|
||||
from application.utils import truncate_to_line_boundary
|
||||
|
||||
from application.core.optional_deps import install_hint
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -591,10 +593,7 @@ class DoclingParser(BaseParser):
|
||||
logger.info(f" ocr_engine={self.ocr_engine or settings.OCR_ENGINE}")
|
||||
|
||||
if importlib.util.find_spec("docling.document_converter") is None:
|
||||
raise ImportError(
|
||||
"docling is required for DoclingParser. "
|
||||
"Install it with: pip install -r application/requirements-docling.txt"
|
||||
)
|
||||
raise ImportError(f"docling is required for DoclingParser. {install_hint('docling')}")
|
||||
|
||||
# Create converter with hybrid OCR (smart: text direct, bitmaps OCR'd)
|
||||
self._converter = self._create_converter()
|
||||
|
||||
@@ -39,6 +39,8 @@ from application.parser.file.base_parser import (
|
||||
module_available,
|
||||
)
|
||||
|
||||
from application.core.optional_deps import install_hint
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Every engine ``OCR_ENGINE`` accepts. ``auto``, ``ocrmac`` and ``rapidocr``
|
||||
@@ -129,8 +131,8 @@ def resolve_ocr_backend(requested: Optional[str] = None) -> str:
|
||||
docling_installed = module_available("docling")
|
||||
if backend == "docling" and not docling_installed:
|
||||
logger.warning(
|
||||
"OCR_BACKEND=docling but docling is not installed (pip install -r "
|
||||
"application/requirements-docling.txt); using the native OCR backend"
|
||||
"OCR_BACKEND=docling but docling is not installed (%s); using the native OCR backend",
|
||||
install_hint("docling"),
|
||||
)
|
||||
return "native"
|
||||
if backend == "auto":
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,977 @@
|
||||
# GENERATED by scripts/export_requirements.sh from uv.lock -- do not edit.
|
||||
#
|
||||
# Core runtime plus the milvus extra (VECTOR_STORE=milvus): pymilvus and the
|
||||
# embedded milvus-lite server, which pulls pyarrow.
|
||||
# Docker: --build-arg EXTRAS=milvus
|
||||
|
||||
a2wsgi==1.10.10
|
||||
# via docsgpt
|
||||
aiofile==3.12.3
|
||||
# via py-key-value-aio
|
||||
aiofiles==25.1.0
|
||||
# via daytona
|
||||
aiohappyeyeballs==2.7.1
|
||||
# via aiohttp
|
||||
aiohttp==3.14.3
|
||||
# via
|
||||
# aiohttp-retry
|
||||
# daytona
|
||||
# daytona-analytics-api-client-async
|
||||
# daytona-api-client-async
|
||||
# daytona-toolbox-api-client-async
|
||||
# python-socketio
|
||||
aiohttp-retry==2.9.1
|
||||
# via
|
||||
# daytona-analytics-api-client-async
|
||||
# daytona-api-client-async
|
||||
# daytona-toolbox-api-client-async
|
||||
aiosignal==1.4.0
|
||||
# via aiohttp
|
||||
alembic==1.19.2
|
||||
# via docsgpt
|
||||
amqp==5.3.1
|
||||
# via kombu
|
||||
aniso8601==10.0.1
|
||||
# via flask-restx
|
||||
annotated-doc==0.0.5
|
||||
# via typer
|
||||
annotated-types==0.8.0
|
||||
# via pydantic
|
||||
anthropic==0.121.0
|
||||
# via docsgpt
|
||||
anyio==4.15.1
|
||||
# via
|
||||
# anthropic
|
||||
# google-genai
|
||||
# httpx
|
||||
# httpx-ws
|
||||
# mcp
|
||||
# openai
|
||||
# py-key-value-aio
|
||||
# sse-starlette
|
||||
# starlette
|
||||
# watchfiles
|
||||
asgiref==3.12.1
|
||||
# via opentelemetry-instrumentation-asgi
|
||||
attrs==26.1.0
|
||||
# via
|
||||
# aiohttp
|
||||
# cyclopts
|
||||
# jsonschema
|
||||
# jsonschema-path
|
||||
# referencing
|
||||
authlib==1.8.0
|
||||
# via fastmcp-slim
|
||||
beartype==0.22.9
|
||||
# via py-key-value-aio
|
||||
beautifulsoup4==4.15.0
|
||||
# via
|
||||
# docsgpt
|
||||
# markdownify
|
||||
bidict==0.24.1
|
||||
# via python-socketio
|
||||
billiard==4.2.4
|
||||
# via celery
|
||||
blinker==1.9.0
|
||||
# via flask
|
||||
boto3==1.43.67
|
||||
# via docsgpt
|
||||
botocore==1.43.89
|
||||
# via
|
||||
# boto3
|
||||
# s3transfer
|
||||
cachetools==7.1.8
|
||||
# via
|
||||
# py-key-value-aio
|
||||
# pymilvus
|
||||
caio==0.12.2
|
||||
# via aiofile
|
||||
cel-python==0.5.0
|
||||
# via docsgpt
|
||||
celery==5.6.3
|
||||
# via
|
||||
# celery-redbeat
|
||||
# docsgpt
|
||||
celery-redbeat==2.4.2
|
||||
# via docsgpt
|
||||
certifi==2026.7.22
|
||||
# via
|
||||
# httpcore
|
||||
# httpx
|
||||
# requests
|
||||
cffi==2.1.1 ; platform_python_implementation != 'PyPy'
|
||||
# via cryptography
|
||||
chardet==7.6.0
|
||||
# via prance
|
||||
charset-normalizer==3.5.1
|
||||
# via requests
|
||||
click==8.1.8
|
||||
# via
|
||||
# celery
|
||||
# click-didyoumean
|
||||
# click-plugins
|
||||
# click-repl
|
||||
# ddgs
|
||||
# flask
|
||||
# gtts
|
||||
# uvicorn
|
||||
click-didyoumean==0.3.1
|
||||
# via celery
|
||||
click-plugins==1.1.1.2
|
||||
# via celery
|
||||
click-repl==0.3.0
|
||||
# via celery
|
||||
colorama==0.4.6 ; sys_platform == 'win32'
|
||||
# via
|
||||
# click
|
||||
# loguru
|
||||
# tqdm
|
||||
# typer
|
||||
croniter==6.2.4
|
||||
# via docsgpt
|
||||
cryptography==50.0.0
|
||||
# via
|
||||
# authlib
|
||||
# docsgpt
|
||||
# google-auth
|
||||
# joserfc
|
||||
# msal
|
||||
# pyjwt
|
||||
# secretstorage
|
||||
cyclopts==4.24.0
|
||||
# via fastmcp-slim
|
||||
dataclasses-json==0.6.7
|
||||
# via docsgpt
|
||||
daytona==0.205.1
|
||||
# via docsgpt
|
||||
daytona-analytics-api-client==0.205.1
|
||||
# via daytona
|
||||
daytona-analytics-api-client-async==0.205.1
|
||||
# via daytona
|
||||
daytona-api-client==0.205.1
|
||||
# via daytona
|
||||
daytona-api-client-async==0.205.1
|
||||
# via daytona
|
||||
daytona-toolbox-api-client==0.205.1
|
||||
# via daytona
|
||||
daytona-toolbox-api-client-async==0.205.1
|
||||
# via daytona
|
||||
ddgs==9.16.0
|
||||
# via docsgpt
|
||||
decorator==5.3.1
|
||||
# via retry
|
||||
defusedxml==0.7.1
|
||||
# via
|
||||
# docsgpt
|
||||
# praw
|
||||
deprecated==1.3.1
|
||||
# via daytona
|
||||
distro==1.9.0
|
||||
# via
|
||||
# anthropic
|
||||
# google-genai
|
||||
# openai
|
||||
dnspython==2.8.0
|
||||
# via email-validator
|
||||
docstring-parser==0.18.0
|
||||
# via
|
||||
# anthropic
|
||||
# cyclopts
|
||||
docx2txt==0.9
|
||||
# via docsgpt
|
||||
ecdsa==0.19.2
|
||||
# via python-jose
|
||||
elevenlabs==2.62.0
|
||||
# via docsgpt
|
||||
email-validator==2.3.0
|
||||
# via pydantic
|
||||
et-xmlfile==2.0.0
|
||||
# via openpyxl
|
||||
exceptiongroup==1.3.1
|
||||
# via fastmcp-slim
|
||||
faiss-cpu==1.15.0
|
||||
# via
|
||||
# docsgpt
|
||||
# milvus-lite
|
||||
fast-ebook==0.2.0
|
||||
# via docsgpt
|
||||
fastembed==0.8.0
|
||||
# via docsgpt
|
||||
fastmcp==3.4.6
|
||||
# via docsgpt
|
||||
fastmcp-slim==3.4.6
|
||||
# via fastmcp
|
||||
filelock==3.32.5
|
||||
# via
|
||||
# huggingface-hub
|
||||
# tldextract
|
||||
firecrawl-anydoc==0.2.3
|
||||
# via docsgpt
|
||||
flask==3.1.3
|
||||
# via
|
||||
# docsgpt
|
||||
# flask-restx
|
||||
flask-restx==1.3.2
|
||||
# via docsgpt
|
||||
flatbuffers==25.12.19
|
||||
# via onnxruntime
|
||||
frozenlist==1.8.0
|
||||
# via
|
||||
# aiohttp
|
||||
# aiosignal
|
||||
fsspec==2026.7.0
|
||||
# via huggingface-hub
|
||||
google-api-core==2.36.0
|
||||
# via google-api-python-client
|
||||
google-api-python-client==2.198.0
|
||||
# via docsgpt
|
||||
google-auth==2.57.1
|
||||
# via
|
||||
# google-api-core
|
||||
# google-api-python-client
|
||||
# google-auth-httplib2
|
||||
# google-auth-oauthlib
|
||||
# google-genai
|
||||
google-auth-httplib2==0.4.2
|
||||
# via google-api-python-client
|
||||
google-auth-oauthlib==1.4.0
|
||||
# via docsgpt
|
||||
google-genai==2.17.0
|
||||
# via docsgpt
|
||||
google-re2==1.1.20251105
|
||||
# via cel-python
|
||||
googleapis-common-protos==1.75.3
|
||||
# via
|
||||
# google-api-core
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
greenlet==3.5.5 ; platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'
|
||||
# via sqlalchemy
|
||||
griffelib==2.3.0
|
||||
# via fastmcp-slim
|
||||
grpcio==1.83.1
|
||||
# via
|
||||
# milvus-lite
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# pymilvus
|
||||
# qdrant-client
|
||||
gtts==2.5.4
|
||||
# via docsgpt
|
||||
gunicorn==26.0.0
|
||||
# via
|
||||
# docsgpt
|
||||
# uvicorn-worker
|
||||
h11==0.16.0
|
||||
# via
|
||||
# httpcore
|
||||
# uvicorn
|
||||
# wsproto
|
||||
h2==4.4.1
|
||||
# via httpx
|
||||
hf-xet==1.6.0 ; platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'
|
||||
# via huggingface-hub
|
||||
hpack==4.2.0
|
||||
# via h2
|
||||
httpcore==1.0.9
|
||||
# via
|
||||
# httpx
|
||||
# httpx-ws
|
||||
httplib2==0.32.0
|
||||
# via
|
||||
# google-api-python-client
|
||||
# google-auth-httplib2
|
||||
httptools==0.8.0
|
||||
# via uvicorn
|
||||
httpx==0.28.1
|
||||
# via
|
||||
# anthropic
|
||||
# daytona
|
||||
# elevenlabs
|
||||
# fastmcp-slim
|
||||
# google-genai
|
||||
# httpx-ws
|
||||
# huggingface-hub
|
||||
# mcp
|
||||
# openai
|
||||
# qdrant-client
|
||||
httpx-sse==0.4.3
|
||||
# via mcp
|
||||
httpx-ws==0.9.0
|
||||
# via daytona
|
||||
huggingface-hub==1.16.1
|
||||
# via
|
||||
# fastembed
|
||||
# tokenizers
|
||||
hyperframe==6.1.0
|
||||
# via h2
|
||||
idna==3.19
|
||||
# via
|
||||
# anyio
|
||||
# email-validator
|
||||
# httpx
|
||||
# requests
|
||||
# tldextract
|
||||
# yarl
|
||||
importlib-resources==7.1.0
|
||||
# via flask-restx
|
||||
itsdangerous==2.2.0
|
||||
# via flask
|
||||
jaraco-classes==3.4.0
|
||||
# via keyring
|
||||
jaraco-context==6.1.2
|
||||
# via keyring
|
||||
jaraco-functools==4.6.0
|
||||
# via keyring
|
||||
jeepney==0.9.0 ; sys_platform == 'linux'
|
||||
# via
|
||||
# keyring
|
||||
# secretstorage
|
||||
jinja2==3.1.6
|
||||
# via
|
||||
# docsgpt
|
||||
# flask
|
||||
jiter==0.16.0
|
||||
# via
|
||||
# anthropic
|
||||
# openai
|
||||
jmespath==1.1.0
|
||||
# via
|
||||
# boto3
|
||||
# botocore
|
||||
# cel-python
|
||||
joserfc==1.7.5
|
||||
# via
|
||||
# authlib
|
||||
# fastmcp-slim
|
||||
jsonref==1.1.0
|
||||
# via fastmcp-slim
|
||||
jsonschema==4.26.0
|
||||
# via
|
||||
# flask-restx
|
||||
# mcp
|
||||
# openapi-schema-validator
|
||||
# openapi-spec-validator
|
||||
jsonschema-path==0.5.0
|
||||
# via
|
||||
# fastmcp-slim
|
||||
# openapi-spec-validator
|
||||
jsonschema-specifications==2025.9.1
|
||||
# via
|
||||
# jsonschema
|
||||
# openapi-schema-validator
|
||||
keyring==25.7.0
|
||||
# via py-key-value-aio
|
||||
kombu==5.6.2
|
||||
# via
|
||||
# celery
|
||||
# docsgpt
|
||||
lark==1.3.1
|
||||
# via cel-python
|
||||
lazy-object-proxy==1.12.0
|
||||
# via openapi-spec-validator
|
||||
loguru==0.7.3
|
||||
# via fastembed
|
||||
lxml==6.1.3
|
||||
# via
|
||||
# ddgs
|
||||
# python-pptx
|
||||
mako==1.4.1
|
||||
# via alembic
|
||||
markdown-it-py==4.2.0
|
||||
# via rich
|
||||
markdownify==1.2.3
|
||||
# via docsgpt
|
||||
markupsafe==3.0.3
|
||||
# via
|
||||
# flask
|
||||
# jinja2
|
||||
# mako
|
||||
# werkzeug
|
||||
marshmallow==3.26.2
|
||||
# via dataclasses-json
|
||||
mcp==1.29.1
|
||||
# via fastmcp-slim
|
||||
mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
milvus-lite==3.2.0 ; sys_platform != 'win32'
|
||||
# via docsgpt
|
||||
mmh3==5.3.0
|
||||
# via fastembed
|
||||
more-itertools==11.1.0
|
||||
# via
|
||||
# jaraco-classes
|
||||
# jaraco-functools
|
||||
msal==1.37.0
|
||||
# via docsgpt
|
||||
multidict==6.7.1
|
||||
# via
|
||||
# aiohttp
|
||||
# yarl
|
||||
mypy-extensions==1.1.0
|
||||
# via typing-inspect
|
||||
networkx==3.6.1
|
||||
# via docsgpt
|
||||
numpy==2.5.1
|
||||
# via
|
||||
# docsgpt
|
||||
# faiss-cpu
|
||||
# fastembed
|
||||
# milvus-lite
|
||||
# onnxruntime
|
||||
# pandas
|
||||
# qdrant-client
|
||||
oauthlib==3.3.1
|
||||
# via requests-oauthlib
|
||||
obstore==0.11.1
|
||||
# via daytona
|
||||
onnxruntime==1.28.0
|
||||
# via
|
||||
# docsgpt
|
||||
# fastembed
|
||||
openai==2.53.0
|
||||
# via docsgpt
|
||||
openapi-pydantic==0.5.1
|
||||
# via fastmcp-slim
|
||||
openapi-schema-validator==0.9.0
|
||||
# via openapi-spec-validator
|
||||
openapi-spec-validator==0.9.0
|
||||
# via openapi3-parser
|
||||
openapi3-parser==1.1.22
|
||||
# via docsgpt
|
||||
openpyxl==3.1.5
|
||||
# via docsgpt
|
||||
opentelemetry-api==1.44.0
|
||||
# via
|
||||
# daytona
|
||||
# fastmcp-slim
|
||||
# google-api-core
|
||||
# opentelemetry-distro
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
# opentelemetry-instrumentation
|
||||
# opentelemetry-instrumentation-aiohttp-client
|
||||
# opentelemetry-instrumentation-asgi
|
||||
# opentelemetry-instrumentation-celery
|
||||
# opentelemetry-instrumentation-dbapi
|
||||
# opentelemetry-instrumentation-flask
|
||||
# opentelemetry-instrumentation-logging
|
||||
# opentelemetry-instrumentation-psycopg
|
||||
# opentelemetry-instrumentation-redis
|
||||
# opentelemetry-instrumentation-requests
|
||||
# opentelemetry-instrumentation-sqlalchemy
|
||||
# opentelemetry-instrumentation-starlette
|
||||
# opentelemetry-instrumentation-wsgi
|
||||
# opentelemetry-sdk
|
||||
# opentelemetry-semantic-conventions
|
||||
opentelemetry-distro==0.65b0
|
||||
# via docsgpt
|
||||
opentelemetry-exporter-otlp==1.44.0
|
||||
# via docsgpt
|
||||
opentelemetry-exporter-otlp-proto-common==1.44.0
|
||||
# via
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
opentelemetry-exporter-otlp-proto-grpc==1.44.0
|
||||
# via opentelemetry-exporter-otlp
|
||||
opentelemetry-exporter-otlp-proto-http==1.44.0
|
||||
# via
|
||||
# daytona
|
||||
# opentelemetry-exporter-otlp
|
||||
opentelemetry-instrumentation==0.65b0
|
||||
# via
|
||||
# opentelemetry-distro
|
||||
# opentelemetry-instrumentation-aiohttp-client
|
||||
# opentelemetry-instrumentation-asgi
|
||||
# opentelemetry-instrumentation-celery
|
||||
# opentelemetry-instrumentation-dbapi
|
||||
# opentelemetry-instrumentation-flask
|
||||
# opentelemetry-instrumentation-logging
|
||||
# opentelemetry-instrumentation-psycopg
|
||||
# opentelemetry-instrumentation-redis
|
||||
# opentelemetry-instrumentation-requests
|
||||
# opentelemetry-instrumentation-sqlalchemy
|
||||
# opentelemetry-instrumentation-starlette
|
||||
# opentelemetry-instrumentation-wsgi
|
||||
opentelemetry-instrumentation-aiohttp-client==0.65b0
|
||||
# via daytona
|
||||
opentelemetry-instrumentation-asgi==0.65b0
|
||||
# via opentelemetry-instrumentation-starlette
|
||||
opentelemetry-instrumentation-celery==0.65b0
|
||||
# via docsgpt
|
||||
opentelemetry-instrumentation-dbapi==0.65b0
|
||||
# via opentelemetry-instrumentation-psycopg
|
||||
opentelemetry-instrumentation-flask==0.65b0
|
||||
# via docsgpt
|
||||
opentelemetry-instrumentation-logging==0.65b0
|
||||
# via docsgpt
|
||||
opentelemetry-instrumentation-psycopg==0.65b0
|
||||
# via docsgpt
|
||||
opentelemetry-instrumentation-redis==0.65b0
|
||||
# via docsgpt
|
||||
opentelemetry-instrumentation-requests==0.65b0
|
||||
# via docsgpt
|
||||
opentelemetry-instrumentation-sqlalchemy==0.65b0
|
||||
# via docsgpt
|
||||
opentelemetry-instrumentation-starlette==0.65b0
|
||||
# via docsgpt
|
||||
opentelemetry-instrumentation-wsgi==0.65b0
|
||||
# via opentelemetry-instrumentation-flask
|
||||
opentelemetry-proto==1.44.0
|
||||
# via
|
||||
# opentelemetry-exporter-otlp-proto-common
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
opentelemetry-sdk==1.44.0
|
||||
# via
|
||||
# daytona
|
||||
# opentelemetry-distro
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
opentelemetry-semantic-conventions==0.65b0
|
||||
# via
|
||||
# opentelemetry-instrumentation
|
||||
# opentelemetry-instrumentation-aiohttp-client
|
||||
# opentelemetry-instrumentation-asgi
|
||||
# opentelemetry-instrumentation-celery
|
||||
# opentelemetry-instrumentation-dbapi
|
||||
# opentelemetry-instrumentation-flask
|
||||
# opentelemetry-instrumentation-logging
|
||||
# opentelemetry-instrumentation-redis
|
||||
# opentelemetry-instrumentation-requests
|
||||
# opentelemetry-instrumentation-sqlalchemy
|
||||
# opentelemetry-instrumentation-starlette
|
||||
# opentelemetry-instrumentation-wsgi
|
||||
# opentelemetry-sdk
|
||||
opentelemetry-util-http==0.65b0
|
||||
# via
|
||||
# opentelemetry-instrumentation-aiohttp-client
|
||||
# opentelemetry-instrumentation-asgi
|
||||
# opentelemetry-instrumentation-flask
|
||||
# opentelemetry-instrumentation-requests
|
||||
# opentelemetry-instrumentation-starlette
|
||||
# opentelemetry-instrumentation-wsgi
|
||||
orjson==3.12.0
|
||||
# via pymilvus
|
||||
packaging==26.3
|
||||
# via
|
||||
# faiss-cpu
|
||||
# fastmcp-slim
|
||||
# gunicorn
|
||||
# huggingface-hub
|
||||
# kombu
|
||||
# marshmallow
|
||||
# onnxruntime
|
||||
# opentelemetry-instrumentation
|
||||
# opentelemetry-instrumentation-flask
|
||||
# opentelemetry-instrumentation-sqlalchemy
|
||||
# prance
|
||||
pandas==3.0.5
|
||||
# via
|
||||
# docsgpt
|
||||
# pymilvus
|
||||
pathable==0.6.0
|
||||
# via jsonschema-path
|
||||
pdf2image==1.17.0
|
||||
# via docsgpt
|
||||
pendulum==3.2.0
|
||||
# via cel-python
|
||||
pgvector==0.5.0
|
||||
# via docsgpt
|
||||
pillow==12.3.0
|
||||
# via
|
||||
# docsgpt
|
||||
# fastembed
|
||||
# pdf2image
|
||||
# python-pptx
|
||||
platformdirs==4.11.7
|
||||
# via fastmcp-slim
|
||||
portalocker==3.2.0
|
||||
# via qdrant-client
|
||||
prance==26.7.19.0
|
||||
# via openapi3-parser
|
||||
praw==8.0.2
|
||||
# via docsgpt
|
||||
prawcore==4.0.0
|
||||
# via praw
|
||||
primp==2.0.0
|
||||
# via ddgs
|
||||
prompt-toolkit==3.0.53
|
||||
# via click-repl
|
||||
propcache==0.5.2
|
||||
# via
|
||||
# aiohttp
|
||||
# yarl
|
||||
proto-plus==1.28.4
|
||||
# via google-api-core
|
||||
protobuf==7.36.1
|
||||
# via
|
||||
# google-api-core
|
||||
# googleapis-common-protos
|
||||
# onnxruntime
|
||||
# opentelemetry-proto
|
||||
# proto-plus
|
||||
# pymilvus
|
||||
# qdrant-client
|
||||
psycopg==3.3.5
|
||||
# via docsgpt
|
||||
psycopg-binary==3.3.5 ; implementation_name != 'pypy'
|
||||
# via psycopg
|
||||
psycopg-pool==3.3.1
|
||||
# via psycopg
|
||||
py==1.11.0
|
||||
# via retry
|
||||
py-key-value-aio==0.4.5
|
||||
# via fastmcp-slim
|
||||
py-rust-stemmers==0.1.8
|
||||
# via fastembed
|
||||
pyarrow==25.0.1 ; sys_platform != 'win32'
|
||||
# via milvus-lite
|
||||
pyasn1==0.6.4
|
||||
# via
|
||||
# pyasn1-modules
|
||||
# python-jose
|
||||
# rsa
|
||||
pyasn1-modules==0.4.2
|
||||
# via google-auth
|
||||
pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy'
|
||||
# via cffi
|
||||
pydantic==2.13.5
|
||||
# via
|
||||
# anthropic
|
||||
# daytona
|
||||
# daytona-analytics-api-client
|
||||
# daytona-analytics-api-client-async
|
||||
# daytona-api-client
|
||||
# daytona-api-client-async
|
||||
# daytona-toolbox-api-client
|
||||
# daytona-toolbox-api-client-async
|
||||
# docsgpt
|
||||
# elevenlabs
|
||||
# fastmcp-slim
|
||||
# google-genai
|
||||
# mcp
|
||||
# openai
|
||||
# openapi-pydantic
|
||||
# openapi-schema-validator
|
||||
# openapi-spec-validator
|
||||
# pydantic-settings
|
||||
# qdrant-client
|
||||
pydantic-core==2.46.5
|
||||
# via
|
||||
# elevenlabs
|
||||
# pydantic
|
||||
pydantic-settings==2.15.0
|
||||
# via
|
||||
# docsgpt
|
||||
# fastmcp-slim
|
||||
# mcp
|
||||
# openapi-schema-validator
|
||||
# openapi-spec-validator
|
||||
pygments==2.21.0
|
||||
# via
|
||||
# rich
|
||||
# rich-rst
|
||||
pyjwt==2.13.0
|
||||
# via
|
||||
# mcp
|
||||
# msal
|
||||
pymilvus==3.0.1
|
||||
# via docsgpt
|
||||
pyparsing==3.3.2
|
||||
# via httplib2
|
||||
pypdf==6.15.0
|
||||
# via docsgpt
|
||||
pypdfium2==5.12.1
|
||||
# via docsgpt
|
||||
pyperclip==1.11.0
|
||||
# via fastmcp-slim
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# botocore
|
||||
# celery
|
||||
# celery-redbeat
|
||||
# croniter
|
||||
# daytona-analytics-api-client
|
||||
# daytona-analytics-api-client-async
|
||||
# daytona-api-client
|
||||
# daytona-api-client-async
|
||||
# daytona-toolbox-api-client
|
||||
# daytona-toolbox-api-client-async
|
||||
# docsgpt
|
||||
# pandas
|
||||
# pendulum
|
||||
python-dotenv==1.2.3
|
||||
# via
|
||||
# daytona
|
||||
# docsgpt
|
||||
# fastmcp-slim
|
||||
# pydantic-settings
|
||||
# pymilvus
|
||||
# uvicorn
|
||||
python-engineio==4.14.0
|
||||
# via python-socketio
|
||||
python-jose==3.5.0
|
||||
# via docsgpt
|
||||
python-multipart==0.0.32
|
||||
# via
|
||||
# daytona
|
||||
# fastmcp-slim
|
||||
# mcp
|
||||
python-pptx==1.0.2
|
||||
# via docsgpt
|
||||
python-socketio==5.16.4
|
||||
# via daytona
|
||||
pywin32==312 ; sys_platform == 'win32'
|
||||
# via
|
||||
# mcp
|
||||
# portalocker
|
||||
pywin32-ctypes==0.2.3 ; sys_platform == 'win32'
|
||||
# via keyring
|
||||
pyyaml==6.0.3
|
||||
# via
|
||||
# cel-python
|
||||
# docsgpt
|
||||
# fastmcp-slim
|
||||
# huggingface-hub
|
||||
# jsonschema-path
|
||||
# uvicorn
|
||||
qdrant-client==1.19.0
|
||||
# via docsgpt
|
||||
redis==7.4.0
|
||||
# via
|
||||
# celery-redbeat
|
||||
# docsgpt
|
||||
referencing==0.37.0
|
||||
# via
|
||||
# flask-restx
|
||||
# jsonschema
|
||||
# jsonschema-path
|
||||
# jsonschema-specifications
|
||||
# openapi-schema-validator
|
||||
regex==2026.9.3
|
||||
# via tiktoken
|
||||
requests==2.34.2
|
||||
# via
|
||||
# docsgpt
|
||||
# elevenlabs
|
||||
# fastembed
|
||||
# google-api-core
|
||||
# google-auth
|
||||
# google-genai
|
||||
# gtts
|
||||
# msal
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
# prance
|
||||
# prawcore
|
||||
# pymilvus
|
||||
# python-socketio
|
||||
# requests-file
|
||||
# requests-oauthlib
|
||||
# tiktoken
|
||||
# tldextract
|
||||
requests-file==3.0.1
|
||||
# via tldextract
|
||||
requests-oauthlib==2.0.0
|
||||
# via google-auth-oauthlib
|
||||
retry==0.9.2
|
||||
# via docsgpt
|
||||
rfc3339-validator==0.1.4
|
||||
# via openapi-schema-validator
|
||||
rich==15.0.0
|
||||
# via
|
||||
# cyclopts
|
||||
# fastmcp-slim
|
||||
# rich-rst
|
||||
# typer
|
||||
rich-rst==2.1.0
|
||||
# via cyclopts
|
||||
rpds-py==2026.6.3
|
||||
# via
|
||||
# jsonschema
|
||||
# referencing
|
||||
rsa==4.9.1
|
||||
# via python-jose
|
||||
ruamel-yaml==0.19.1
|
||||
# via prance
|
||||
s3transfer==0.19.2
|
||||
# via boto3
|
||||
secretstorage==3.5.0 ; sys_platform == 'linux'
|
||||
# via keyring
|
||||
shellingham==1.5.4
|
||||
# via typer
|
||||
simple-websocket==1.1.0
|
||||
# via python-engineio
|
||||
six==1.17.0
|
||||
# via
|
||||
# ecdsa
|
||||
# markdownify
|
||||
# python-dateutil
|
||||
# rfc3339-validator
|
||||
sniffio==1.3.1
|
||||
# via
|
||||
# anthropic
|
||||
# google-genai
|
||||
# openai
|
||||
soupsieve==2.9.2
|
||||
# via beautifulsoup4
|
||||
sqlalchemy==2.0.52
|
||||
# via
|
||||
# alembic
|
||||
# docsgpt
|
||||
sse-starlette==3.4.11
|
||||
# via mcp
|
||||
starlette==1.6.0
|
||||
# via
|
||||
# docsgpt
|
||||
# fastmcp-slim
|
||||
# mcp
|
||||
# sse-starlette
|
||||
tenacity==9.1.4
|
||||
# via
|
||||
# celery-redbeat
|
||||
# google-genai
|
||||
tiktoken==0.13.0
|
||||
# via docsgpt
|
||||
tldextract==5.3.2
|
||||
# via docsgpt
|
||||
tokenizers==0.22.2
|
||||
# via
|
||||
# docsgpt
|
||||
# fastembed
|
||||
toml==0.10.2
|
||||
# via daytona
|
||||
tqdm==4.67.3
|
||||
# via
|
||||
# docsgpt
|
||||
# fastembed
|
||||
# huggingface-hub
|
||||
# openai
|
||||
typer==0.26.8
|
||||
# via huggingface-hub
|
||||
typing-extensions==4.16.0
|
||||
# via
|
||||
# aiohttp
|
||||
# aiosignal
|
||||
# alembic
|
||||
# anthropic
|
||||
# anyio
|
||||
# beautifulsoup4
|
||||
# daytona
|
||||
# daytona-analytics-api-client
|
||||
# daytona-analytics-api-client-async
|
||||
# daytona-api-client
|
||||
# daytona-api-client-async
|
||||
# daytona-toolbox-api-client
|
||||
# daytona-toolbox-api-client-async
|
||||
# elevenlabs
|
||||
# exceptiongroup
|
||||
# fastmcp-slim
|
||||
# google-genai
|
||||
# grpcio
|
||||
# huggingface-hub
|
||||
# mcp
|
||||
# obstore
|
||||
# openai
|
||||
# opentelemetry-api
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
# opentelemetry-sdk
|
||||
# opentelemetry-semantic-conventions
|
||||
# psycopg
|
||||
# psycopg-pool
|
||||
# py-key-value-aio
|
||||
# pydantic
|
||||
# pydantic-core
|
||||
# python-pptx
|
||||
# referencing
|
||||
# sqlalchemy
|
||||
# starlette
|
||||
# typing-inspect
|
||||
# typing-inspection
|
||||
typing-inspect==0.9.0
|
||||
# via dataclasses-json
|
||||
typing-inspection==0.4.4
|
||||
# via
|
||||
# mcp
|
||||
# pydantic
|
||||
# pydantic-settings
|
||||
tzdata==2026.3
|
||||
# via
|
||||
# kombu
|
||||
# pandas
|
||||
# pendulum
|
||||
# psycopg
|
||||
# tzlocal
|
||||
tzlocal==5.4.4
|
||||
# via celery
|
||||
uncalled-for==0.4.0
|
||||
# via fastmcp-slim
|
||||
update-checker==1.0.0
|
||||
# via praw
|
||||
uritemplate==4.2.0
|
||||
# via google-api-python-client
|
||||
urllib3==2.7.0
|
||||
# via
|
||||
# botocore
|
||||
# daytona
|
||||
# daytona-analytics-api-client
|
||||
# daytona-api-client
|
||||
# daytona-toolbox-api-client
|
||||
# qdrant-client
|
||||
# requests
|
||||
uvicorn==0.52.4
|
||||
# via
|
||||
# docsgpt
|
||||
# fastmcp-slim
|
||||
# mcp
|
||||
# uvicorn-worker
|
||||
uvicorn-worker==0.4.0
|
||||
# via docsgpt
|
||||
uvloop==0.22.1 ; platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'
|
||||
# via uvicorn
|
||||
vine==5.1.0
|
||||
# via
|
||||
# amqp
|
||||
# celery
|
||||
# kombu
|
||||
watchfiles==1.2.0
|
||||
# via
|
||||
# fastmcp-slim
|
||||
# uvicorn
|
||||
wcwidth==0.8.3
|
||||
# via prompt-toolkit
|
||||
websocket-client==1.9.0
|
||||
# via
|
||||
# docsgpt
|
||||
# praw
|
||||
# python-socketio
|
||||
websockets==16.1.1
|
||||
# via
|
||||
# elevenlabs
|
||||
# fastmcp-slim
|
||||
# google-genai
|
||||
# uvicorn
|
||||
werkzeug==3.1.8
|
||||
# via
|
||||
# docsgpt
|
||||
# flask
|
||||
# flask-restx
|
||||
win32-setctime==1.2.0 ; sys_platform == 'win32'
|
||||
# via loguru
|
||||
wrapt==2.4.0
|
||||
# via
|
||||
# deprecated
|
||||
# opentelemetry-instrumentation
|
||||
# opentelemetry-instrumentation-aiohttp-client
|
||||
# opentelemetry-instrumentation-dbapi
|
||||
# opentelemetry-instrumentation-redis
|
||||
# opentelemetry-instrumentation-sqlalchemy
|
||||
wsproto==1.3.2
|
||||
# via
|
||||
# daytona
|
||||
# httpx-ws
|
||||
# simple-websocket
|
||||
xlsxwriter==3.2.9
|
||||
# via python-pptx
|
||||
yarl==1.24.5
|
||||
# via aiohttp
|
||||
+930
-98
File diff suppressed because it is too large.
Load diff
@@ -4,6 +4,7 @@ import uuid
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from application.core.optional_deps import require
|
||||
from application.core.settings import settings
|
||||
from application.vectorstore.base import BaseVectorStore
|
||||
from application.vectorstore.document_class import Document
|
||||
@@ -41,7 +42,8 @@ class MilvusStore(BaseVectorStore):
|
||||
def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
|
||||
super().__init__()
|
||||
with _without_milvus_uri_env():
|
||||
from pymilvus import DataType, MilvusClient
|
||||
pymilvus = require("pymilvus", "VECTOR_STORE=milvus")
|
||||
DataType, MilvusClient = pymilvus.DataType, pymilvus.MilvusClient
|
||||
|
||||
self._DataType = DataType
|
||||
self._source_id = str(source_id).replace("application/indexes/", "").rstrip("/")
|
||||
|
||||
@@ -94,13 +94,25 @@ To run the DocsGPT backend locally, you'll need to set up a Python environment a
|
||||
pip install -r application/requirements.txt
|
||||
```
|
||||
|
||||
Optionally add the docling parser engine (OCR, `structured` output for
|
||||
`read_document`; the default anydoc engine does not need it):
|
||||
Dependencies are declared in `pyproject.toml` and locked in `uv.lock`; the
|
||||
`requirements*.txt` files are exported from that lock, so with
|
||||
[uv](https://docs.astral.sh/uv/) installed `uv sync` sets up the same
|
||||
environment (plus the test tools) in one step.
|
||||
|
||||
Optional extras are not installed by default. Add them when you need the
|
||||
feature (each file is the core set plus the extra):
|
||||
|
||||
```bash
|
||||
pip install -r application/requirements-docling.txt
|
||||
pip install -r application/requirements-docling.txt # docling parser engine: OCR backend, read_document structured output
|
||||
pip install -r application/requirements-milvus.txt # VECTOR_STORE=milvus
|
||||
# or with uv: uv sync --extra docling --extra milvus
|
||||
```
|
||||
|
||||
A feature whose extra is missing fails with the exact command to run.
|
||||
After changing `pyproject.toml`, run `uv lock` and
|
||||
`bash scripts/export_requirements.sh` so the exported files stay in sync
|
||||
(CI checks this).
|
||||
|
||||
5. **Run the Backend:**
|
||||
|
||||
For local development, run the ASGI composition under uvicorn. It serves the **whole** application, hot-reloads on source changes, and matches the production runtime:
|
||||
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
[project]
|
||||
name = "docsgpt"
|
||||
# Bump together with application/version.py and frontend/package.json.
|
||||
version = "0.19.0"
|
||||
description = "DocsGPT backend: chat with your documents, agents, and tools."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
license = { file = "LICENSE" }
|
||||
|
||||
# Direct dependencies only. Transitive pins live in uv.lock; the pip-facing
|
||||
# files under application/ (requirements*.txt) are exported from that lock by
|
||||
# scripts/export_requirements.sh and must not be edited by hand.
|
||||
dependencies = [
|
||||
"a2wsgi==1.10.10",
|
||||
"alembic>=1.13,<2",
|
||||
"anthropic==0.121.0",
|
||||
"beautifulsoup4==4.15.0",
|
||||
"boto3==1.43.67",
|
||||
"cel-python==0.5.0",
|
||||
"celery==5.6.3",
|
||||
"celery-redbeat==2.4.2",
|
||||
"croniter==6.2.4",
|
||||
"cryptography==50.0.0",
|
||||
"dataclasses-json==0.6.7",
|
||||
"daytona==0.205.1",
|
||||
"ddgs>=8.0.0",
|
||||
"defusedxml==0.7.1",
|
||||
"docx2txt==0.9",
|
||||
"elevenlabs==2.62.0",
|
||||
"faiss-cpu==1.15.0",
|
||||
"fast-ebook",
|
||||
# Default document converter (DOC_PARSER_ENGINE=anydoc): a Rust extension
|
||||
# with no model downloads. The docling engine is the `docling` extra.
|
||||
"firecrawl-anydoc==0.2.3",
|
||||
"fastembed==0.8.0",
|
||||
"fastmcp==3.4.6",
|
||||
"Flask==3.1.3",
|
||||
"flask-restx==1.3.2",
|
||||
"google-api-python-client==2.198.0",
|
||||
"google-auth-oauthlib==1.4.0",
|
||||
"google-genai==2.17.0",
|
||||
"gTTS==2.5.4",
|
||||
"gunicorn==26.0.0",
|
||||
"jinja2==3.1.6",
|
||||
"kombu==5.6.2",
|
||||
"markdownify==1.2.3",
|
||||
"msal==1.37.0",
|
||||
"networkx==3.6.1",
|
||||
"numpy==2.5.1",
|
||||
# fastembed's runtime: local embeddings execute on it.
|
||||
"onnxruntime==1.28.0",
|
||||
"openai==2.53.0",
|
||||
"openapi3-parser==1.1.22",
|
||||
# pandas reads .xlsx through openpyxl but does not depend on it.
|
||||
"openpyxl==3.1.5",
|
||||
"opentelemetry-distro>=0.50b0,<1",
|
||||
"opentelemetry-exporter-otlp>=1.29.0,<2",
|
||||
"opentelemetry-instrumentation-celery>=0.50b0,<1",
|
||||
"opentelemetry-instrumentation-flask>=0.50b0,<1",
|
||||
"opentelemetry-instrumentation-logging>=0.50b0,<1",
|
||||
"opentelemetry-instrumentation-psycopg>=0.50b0,<1",
|
||||
"opentelemetry-instrumentation-redis>=0.50b0,<1",
|
||||
"opentelemetry-instrumentation-requests>=0.50b0,<1",
|
||||
"opentelemetry-instrumentation-sqlalchemy>=0.50b0,<1",
|
||||
"opentelemetry-instrumentation-starlette>=0.50b0,<1",
|
||||
"pandas==3.0.5",
|
||||
"pdf2image>=1.17.0",
|
||||
"pgvector>=0.5,<1",
|
||||
"pillow==12.3.0",
|
||||
"praw==8.0.2",
|
||||
"psycopg[binary,pool]>=3.1,<4",
|
||||
"pydantic",
|
||||
"pydantic-settings",
|
||||
"pypdf==6.15.0",
|
||||
"pypdfium2==5.12.1",
|
||||
"python-dateutil==2.9.0.post0",
|
||||
"python-dotenv",
|
||||
"python-jose==3.5.0",
|
||||
"python-pptx==1.0.2",
|
||||
"PyYAML",
|
||||
"qdrant-client==1.19.0",
|
||||
"redis==7.4.0",
|
||||
"requests==2.34.2",
|
||||
"retry==0.9.2",
|
||||
"sqlalchemy>=2.0,<3",
|
||||
"starlette>=1.0,<2",
|
||||
"tiktoken==0.13.0",
|
||||
"tldextract==5.3.2",
|
||||
"tokenizers==0.22.2",
|
||||
"tqdm==4.67.3",
|
||||
"uvicorn[standard]>=0.30,<1",
|
||||
"uvicorn-worker>=0.4,<1",
|
||||
"websocket-client==1.9.0",
|
||||
"werkzeug>=3.1.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
# Docling parser engine: DOC_PARSER_ENGINE=docling, the docling OCR backend
|
||||
# (layout-model hybrid OCR, ocrmac/rapidocr engines), .adoc/.vtt/.xml
|
||||
# attachment parsing, and read_document's `structured` output. Pulls torch and
|
||||
# transformers; on Linux torch resolves from the CPU-only PyTorch index (see
|
||||
# [tool.uv.sources]) so the extra does not drag the CUDA stack in.
|
||||
docling = [
|
||||
"docling==2.119.0",
|
||||
"rapidocr==3.9.2",
|
||||
# docling's model stack. Declared here (not left transitive) so the pins
|
||||
# hold and the CPU index source below applies. transformers is capped by
|
||||
# docling-core at <5.9: 5.9+ breaks the PDF layout model on Apple Silicon.
|
||||
"torch==2.11.0",
|
||||
"torchvision==0.26.0",
|
||||
"transformers==5.8.1",
|
||||
]
|
||||
# VECTOR_STORE=milvus. milvus-lite (the embedded server) pulls pyarrow.
|
||||
milvus = [
|
||||
"pymilvus==3.0.1",
|
||||
"milvus-lite==3.2.0; sys_platform != 'win32'",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
# Mirrors tests/requirements.txt for `uv sync`; pip users install that file.
|
||||
dev = [
|
||||
"pytest>=8.0.0",
|
||||
"pytest-asyncio>=0.23",
|
||||
"pytest-cov>=4.1.0",
|
||||
"pytest-xdist>=3.5",
|
||||
"coverage>=7.4.0",
|
||||
"pytest-postgresql>=6.0.0",
|
||||
"jupyter-client>=8.0",
|
||||
"python-docx>=1.1",
|
||||
"reportlab>=4.0,<5",
|
||||
"ruff",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
# The repo is run in place (`application.*` imported from the checkout), not
|
||||
# installed as a distribution.
|
||||
package = false
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cpu"
|
||||
url = "https://download.pytorch.org/whl/cpu"
|
||||
explicit = true
|
||||
|
||||
[tool.uv.sources]
|
||||
# PyPI's Linux torch wheels depend on the full CUDA 13 stack (~2.7 GB of
|
||||
# wheels). The docling extra runs its models on CPU, so take torch from the
|
||||
# CPU index there. macOS and Windows PyPI wheels are CPU-only already.
|
||||
torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }]
|
||||
torchvision = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }]
|
||||
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerate the pip-facing requirements files from uv.lock.
|
||||
#
|
||||
# pyproject.toml declares the direct dependencies and the optional extras;
|
||||
# uv.lock pins everything. pip users, the Dockerfile and CI install from the
|
||||
# exported files, so run this after any change to pyproject.toml or uv.lock:
|
||||
#
|
||||
# uv lock # or: uv lock --upgrade-package <name>
|
||||
# bash scripts/export_requirements.sh
|
||||
#
|
||||
# Each exported file is a complete environment (core plus the named extra),
|
||||
# so `pip install -r application/requirements-docling.txt` on its own works,
|
||||
# and installing it on top of requirements.txt only adds the extra's packages.
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
UV=(uv)
|
||||
if ! uv --version 2>/dev/null | grep -qE '^uv 0\.([89]|[1-9][0-9])\.'; then
|
||||
# `uv export` needs a current uv; run one through uvx without touching the
|
||||
# machine's install.
|
||||
UV=(uv tool run --from 'uv>=0.8' uv)
|
||||
fi
|
||||
|
||||
export_file() {
|
||||
local out="$1"; shift
|
||||
local header="$1"; shift
|
||||
local index_url="${INDEX_URL:-}"
|
||||
{
|
||||
echo "# GENERATED by scripts/export_requirements.sh from uv.lock -- do not edit."
|
||||
echo "#"
|
||||
# shellcheck disable=SC2001
|
||||
echo "$header" | sed 's/^/# /'
|
||||
echo
|
||||
# uv export records the wheel's origin in uv.lock but writes no index
|
||||
# directive; pip needs one to find the +cpu torch build.
|
||||
if [ -n "$index_url" ]; then echo "--extra-index-url $index_url"; echo; fi
|
||||
"${UV[@]}" export --frozen --no-hashes --no-dev --no-emit-project --no-header --quiet "$@"
|
||||
} > "$out"
|
||||
echo "wrote $out ($(grep -cE '^[A-Za-z0-9]' "$out") packages)"
|
||||
}
|
||||
|
||||
export_file application/requirements.txt \
|
||||
"Core runtime. Optional extras live in requirements-<extra>.txt:
|
||||
docling DOC_PARSER_ENGINE=docling, docling OCR backend, read_document structured output
|
||||
milvus VECTOR_STORE=milvus"
|
||||
|
||||
INDEX_URL=https://download.pytorch.org/whl/cpu export_file application/requirements-docling.txt \
|
||||
"Core runtime plus the docling extra: DOC_PARSER_ENGINE=docling, the docling
|
||||
OCR backend (layout-model hybrid OCR, ocrmac/rapidocr engines), .adoc/.vtt/.xml
|
||||
attachment parsing, and read_document's 'structured' output. The default
|
||||
anydoc engine needs none of this, and OCR itself does not either:
|
||||
OCR_ENABLED=true with the tesseract binary (or a DeepSeek-OCR endpoint) runs
|
||||
through application/parser/file/ocr_parser.py.
|
||||
On Linux torch comes from the CPU-only PyTorch index (no CUDA stack); a GPU
|
||||
deployment can reinstall torch from PyPI on top.
|
||||
Docker: --build-arg EXTRAS=docling" \
|
||||
--extra docling
|
||||
|
||||
export_file application/requirements-milvus.txt \
|
||||
"Core runtime plus the milvus extra (VECTOR_STORE=milvus): pymilvus and the
|
||||
embedded milvus-lite server, which pulls pyarrow.
|
||||
Docker: --build-arg EXTRAS=milvus" \
|
||||
--extra milvus
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Optional-extra bookkeeping: hints name the extra, require() explains absence."""
|
||||
|
||||
import sys
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from application.core import optional_deps
|
||||
|
||||
|
||||
class TestExtras:
|
||||
def test_every_extra_module_maps_back(self):
|
||||
for extra, modules in optional_deps.EXTRAS.items():
|
||||
for module in modules:
|
||||
assert optional_deps.extra_for(module) == extra
|
||||
|
||||
def test_submodule_resolves_to_its_extra(self):
|
||||
assert optional_deps.extra_for("docling.document_converter") == "docling"
|
||||
|
||||
def test_unknown_module_has_no_extra(self):
|
||||
assert optional_deps.extra_for("flask") is None
|
||||
|
||||
def test_install_hint_names_every_install_route(self):
|
||||
hint = optional_deps.install_hint("milvus")
|
||||
assert "requirements-milvus.txt" in hint
|
||||
assert "--extra milvus" in hint
|
||||
assert "EXTRAS=milvus" in hint
|
||||
|
||||
|
||||
class TestMissingMessage:
|
||||
def test_extra_module_points_at_the_extra(self):
|
||||
message = optional_deps.missing_message("pymilvus", "VECTOR_STORE=milvus")
|
||||
assert "pymilvus is not installed (VECTOR_STORE=milvus)" in message
|
||||
assert "'milvus' extra" in message
|
||||
assert optional_deps.install_hint("milvus") in message
|
||||
|
||||
def test_plain_module_gets_a_pip_line(self):
|
||||
assert optional_deps.missing_message("boto3") == "boto3 is not installed. Install it with: pip install boto3"
|
||||
|
||||
|
||||
class TestRequire:
|
||||
def test_returns_the_module_when_present(self):
|
||||
assert optional_deps.require("json").dumps({}) == "{}"
|
||||
|
||||
def test_absent_module_raises_with_hint(self):
|
||||
with patch.dict(sys.modules, {"pymilvus": None}):
|
||||
with pytest.raises(ImportError) as excinfo:
|
||||
optional_deps.require("pymilvus", "VECTOR_STORE=milvus")
|
||||
assert "'milvus' extra" in str(excinfo.value)
|
||||
|
||||
|
||||
class TestIsAvailable:
|
||||
def test_stubbed_module_counts_as_present(self):
|
||||
with patch.dict(sys.modules, {"docling": types.ModuleType("docling")}):
|
||||
assert optional_deps.is_available("docling.document_converter")
|
||||
|
||||
def test_blocked_module_counts_as_absent(self):
|
||||
with patch.dict(sys.modules, {"docling": None}):
|
||||
assert not optional_deps.is_available("docling")
|
||||
Reference in new issue
Block a user