mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 00:16:20 +00:00
build(deps): declare dependencies in pyproject.toml with docling and milvus extras
requirements.txt pinned torch and transformers in core although only docling needs them, and on Linux torch pulls the CUDA 13 stack: 2.7 GB of the 3.0 GB wheel download. Direct dependencies now live in pyproject.toml, uv.lock pins everything, and application/requirements*.txt are exported from the lock by scripts/export_requirements.sh (each file is the core set plus one extra). The docling extra pins torch/torchvision/transformers itself and, on Linux, resolves torch from the CPU-only PyTorch index (no nvidia packages). milvus (pymilvus + milvus-lite, which pulls pyarrow) is the second extra. application/core/optional_deps.py is the one place install hints come from; the milvus store and the docling call sites use it so a missing extra fails with the exact command to run.
This commit is contained in:
1 parent
55a38b3132
commit
196865846c
14 files changed
+10545
-123
No files matched your search
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerate the pip-facing requirements files from uv.lock.
|
||||
#
|
||||
# pyproject.toml declares the direct dependencies and the optional extras;
|
||||
# uv.lock pins everything. pip users, the Dockerfile and CI install from the
|
||||
# exported files, so run this after any change to pyproject.toml or uv.lock:
|
||||
#
|
||||
# uv lock # or: uv lock --upgrade-package <name>
|
||||
# bash scripts/export_requirements.sh
|
||||
#
|
||||
# Each exported file is a complete environment (core plus the named extra),
|
||||
# so `pip install -r application/requirements-docling.txt` on its own works,
|
||||
# and installing it on top of requirements.txt only adds the extra's packages.
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
UV=(uv)
|
||||
if ! uv --version 2>/dev/null | grep -qE '^uv 0\.([89]|[1-9][0-9])\.'; then
|
||||
# `uv export` needs a current uv; run one through uvx without touching the
|
||||
# machine's install.
|
||||
UV=(uv tool run --from 'uv>=0.8' uv)
|
||||
fi
|
||||
|
||||
export_file() {
|
||||
local out="$1"; shift
|
||||
local header="$1"; shift
|
||||
local index_url="${INDEX_URL:-}"
|
||||
{
|
||||
echo "# GENERATED by scripts/export_requirements.sh from uv.lock -- do not edit."
|
||||
echo "#"
|
||||
# shellcheck disable=SC2001
|
||||
echo "$header" | sed 's/^/# /'
|
||||
echo
|
||||
# uv export records the wheel's origin in uv.lock but writes no index
|
||||
# directive; pip needs one to find the +cpu torch build.
|
||||
if [ -n "$index_url" ]; then echo "--extra-index-url $index_url"; echo; fi
|
||||
"${UV[@]}" export --frozen --no-hashes --no-dev --no-emit-project --no-header --quiet "$@"
|
||||
} > "$out"
|
||||
echo "wrote $out ($(grep -cE '^[A-Za-z0-9]' "$out") packages)"
|
||||
}
|
||||
|
||||
export_file application/requirements.txt \
|
||||
"Core runtime. Optional extras live in requirements-<extra>.txt:
|
||||
docling DOC_PARSER_ENGINE=docling, docling OCR backend, read_document structured output
|
||||
milvus VECTOR_STORE=milvus"
|
||||
|
||||
INDEX_URL=https://download.pytorch.org/whl/cpu export_file application/requirements-docling.txt \
|
||||
"Core runtime plus the docling extra: DOC_PARSER_ENGINE=docling, the docling
|
||||
OCR backend (layout-model hybrid OCR, ocrmac/rapidocr engines), .adoc/.vtt/.xml
|
||||
attachment parsing, and read_document's 'structured' output. The default
|
||||
anydoc engine needs none of this, and OCR itself does not either:
|
||||
OCR_ENABLED=true with the tesseract binary (or a DeepSeek-OCR endpoint) runs
|
||||
through application/parser/file/ocr_parser.py.
|
||||
On Linux torch comes from the CPU-only PyTorch index (no CUDA stack); a GPU
|
||||
deployment can reinstall torch from PyPI on top.
|
||||
Docker: --build-arg EXTRAS=docling" \
|
||||
--extra docling
|
||||
|
||||
export_file application/requirements-milvus.txt \
|
||||
"Core runtime plus the milvus extra (VECTOR_STORE=milvus): pymilvus and the
|
||||
embedded milvus-lite server, which pulls pyarrow.
|
||||
Docker: --build-arg EXTRAS=milvus" \
|
||||
--extra milvus
|
||||
Reference in new issue
Block a user