mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-13 12:14:00 +00:00
Fixes batch 2
This commit is contained in:
1 parent
c17caf24ec
commit
4e14a79923
16 files changed
+278
-31
No files matched your search
@@ -46,7 +46,7 @@ RUN pip install --no-cache-dir --upgrade pip && \
|
||||
# backend, read_document's structured output) — OFF by default: it pulls the
|
||||
# layout/OCR model stack and adds gigabytes to the image. anydoc (in
|
||||
# requirements.txt) is the default parser and needs none of it, and OCR runs
|
||||
# natively on tesseract (below) without it.
|
||||
# natively on tesseract (below, also opt-in) without it.
|
||||
ARG INSTALL_DOCLING=false
|
||||
RUN if [ "$INSTALL_DOCLING" = "true" ]; then \
|
||||
pip install --no-cache-dir -r requirements-docling.txt; \
|
||||
@@ -67,12 +67,14 @@ RUN apt-get update && \
|
||||
ln -s /usr/bin/python3.12 /usr/bin/python && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# OCR (OCR_ENABLED=true) runs on the system tesseract binary by default,
|
||||
# with or without the docling extra, so ship it in every image (~35 MB).
|
||||
# Opt out with --build-arg INSTALL_TESSERACT=false (e.g. a DeepSeek-OCR-only
|
||||
# deployment). Extra language packs are a deployment concern
|
||||
# (apt: tesseract-ocr-<lang>, then list them in OCR_LANGS).
|
||||
ARG INSTALL_TESSERACT=true
|
||||
# Optional tesseract OCR engine (OCR_ENABLED=true with OCR_ENGINE=tesseract,
|
||||
# the default engine) — OFF by default like every other OCR dependency; OCR
|
||||
# itself is off unless configured. Opt in with --build-arg
|
||||
# INSTALL_TESSERACT=true (setup.sh writes it to .env when OCR is enabled);
|
||||
# ~35 MB of system packages. Extra language packs are a deployment concern
|
||||
# (apt: tesseract-ocr-<lang>, then list them in OCR_LANGS). A DeepSeek-OCR
|
||||
# endpoint (OCR_ENGINE=deepseek) needs none of this.
|
||||
ARG INSTALL_TESSERACT=false
|
||||
RUN if [ "$INSTALL_TESSERACT" = "true" ]; then \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng && \
|
||||
|
||||
@@ -165,8 +165,9 @@ class Settings(BaseSettings):
|
||||
# menu):
|
||||
# tesseract — recommended: best classic-engine accuracy (perfect EN word
|
||||
# recall, 0.000 bilingual CER, 100% table cells), ~35 MB, CPU-only.
|
||||
# Needs the system binary + language packs (the Docker image ships
|
||||
# them unless built with INSTALL_TESSERACT=false). Both backends.
|
||||
# Needs the system binary + language packs: an optional install like
|
||||
# every OCR dependency (build with INSTALL_TESSERACT=true, or apt/brew
|
||||
# install tesseract-ocr for a local run). Both backends.
|
||||
# deepseek — DeepSeek-OCR against an Ollama/vLLM endpoint
|
||||
# (OCR_DEEPSEEK_*). Best table/CJK quality; the worker stays light
|
||||
# (no layout models) but each page costs seconds on the model server.
|
||||
|
||||
@@ -186,6 +186,13 @@ def _pick_parser(suffix: str, *, ocr_enabled: bool, engine: str):
|
||||
legacy = _legacy_parser_for(suffix)
|
||||
if legacy is not None:
|
||||
return legacy
|
||||
from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
|
||||
|
||||
if suffix in ANYDOC_GAINED_SUFFIXES:
|
||||
# anydoc-only office formats have no legacy parser. Falling through
|
||||
# would run the configured engine the caller explicitly opted out
|
||||
# of; images/audio/.txt still fall through (no legacy alternative).
|
||||
return None
|
||||
if engine in ("docling", "anydoc"):
|
||||
extractor = get_default_file_extractor(ocr_enabled=ocr_enabled, engine=engine)
|
||||
else:
|
||||
@@ -529,6 +536,16 @@ def _shape(
|
||||
}
|
||||
|
||||
parser = _pick_parser(suffix, ocr_enabled=ocr_enabled, engine=engine)
|
||||
# Under OCR_BACKEND=native with the docling engine the PDF parser is the
|
||||
# native OCR parser wrapping a DoclingParser as its text_parser. A PDF
|
||||
# with a text layer on every page is handed to that parser untouched, so
|
||||
# when tables are wanted run it directly: the single docling conversion
|
||||
# below then yields both the markdown and the tables. Scans and mixed
|
||||
# documents keep the native parser (and the exclusion just below).
|
||||
if include_tables and output != "chunks" and _is_native_ocr_parser(parser):
|
||||
delegate = parser.text_layer_delegate(path)
|
||||
if _is_docling_parser(delegate):
|
||||
parser = delegate
|
||||
# Tables come from a Docling conversion, so they are only collected when
|
||||
# a Docling parser is actually in play: under anydoc/fast a table pass
|
||||
# would be a second, full docling conversion that costs more than the
|
||||
@@ -569,6 +586,11 @@ def _shape(
|
||||
from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
|
||||
|
||||
if suffix in ANYDOC_GAINED_SUFFIXES:
|
||||
if engine == "fast":
|
||||
return {
|
||||
"error": f"The fast engine has no parser for {suffix} files; "
|
||||
"use engine='anydoc' or 'auto'."
|
||||
}
|
||||
return {
|
||||
"error": f"No parser is available for {suffix} files: firecrawl-anydoc "
|
||||
"is not installed (pip install firecrawl-anydoc)"
|
||||
|
||||
@@ -103,8 +103,8 @@ def _resolve_ocr_engine(requested: Optional[str]) -> str:
|
||||
if engine == "tesseract" and shutil.which("tesseract") is None:
|
||||
logger.warning(
|
||||
"OCR_ENGINE=tesseract but no tesseract binary is on PATH (install "
|
||||
"tesseract-ocr plus language packs; the Docker image ships them unless "
|
||||
"built with INSTALL_TESSERACT=false); using docling auto-selection"
|
||||
"tesseract-ocr plus language packs: Docker builds need "
|
||||
"INSTALL_TESSERACT=true); using docling auto-selection"
|
||||
)
|
||||
return "auto"
|
||||
if engine == "ocrmac" and (
|
||||
|
||||
@@ -315,9 +315,9 @@ class TesseractEngine:
|
||||
if not self.available():
|
||||
raise OcrUnavailableError(
|
||||
"OCR_ENGINE=tesseract but no tesseract binary is on PATH. Install "
|
||||
"tesseract-ocr plus language packs (the Docker image ships them "
|
||||
"unless built with INSTALL_TESSERACT=false), or set "
|
||||
"OCR_ENGINE=deepseek / install the docling extra."
|
||||
"tesseract-ocr plus language packs (Docker: build with "
|
||||
"INSTALL_TESSERACT=true; local: apt/brew install tesseract-ocr), "
|
||||
"or set OCR_ENGINE=deepseek / install the docling extra."
|
||||
)
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
@@ -699,6 +699,32 @@ class NativeOcrPdfParser(BaseParser):
|
||||
self.last_engine = engine.name
|
||||
return texts
|
||||
|
||||
def text_layer_delegate(self, file: Path) -> Optional[BaseParser]:
|
||||
"""The ``text_parser`` that ``parse_file`` would hand ``file`` to, or None.
|
||||
|
||||
None when there is no text parser, the file cannot be opened, or any
|
||||
page falls below ``min_text_chars`` (those are OCR'd here). Callers
|
||||
that want a by-product of the delegate's conversion (docling tables)
|
||||
can run it directly instead of converting the document twice.
|
||||
"""
|
||||
if self.text_parser is None:
|
||||
return None
|
||||
try:
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
pdf = pdfium.PdfDocument(str(Path(file)))
|
||||
except Exception: # noqa: BLE001 - a probe; parse_file reports the real error
|
||||
return None
|
||||
try:
|
||||
counts = self._text_layer_counts(pdf)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
finally:
|
||||
pdf.close()
|
||||
if counts and all(count >= self.min_text_chars for count in counts):
|
||||
return self.text_parser
|
||||
return None
|
||||
|
||||
def _delegate_text(self, path: Path, errors: str) -> Union[str, List[str]]:
|
||||
text_parser = self.text_parser
|
||||
assert text_parser is not None
|
||||
|
||||
@@ -29,7 +29,7 @@ import zlib
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterator, List, Optional, Union
|
||||
|
||||
_OBJ = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL)
|
||||
_OBJ_HEADER = re.compile(rb"\d+\s+\d+\s+obj")
|
||||
_TYPE0 = re.compile(rb"/Subtype\s*/Type0")
|
||||
# The CJK expectation is keyed on CIDSystemInfo /Ordering ALONE — the
|
||||
# authoritative "this PDF maps text through a CJK character collection"
|
||||
@@ -99,6 +99,28 @@ def _stream_bodies(raw: bytes) -> Iterator[bytes]:
|
||||
yield body
|
||||
|
||||
|
||||
def _object_bodies(raw: bytes) -> Iterator[bytes]:
|
||||
"""The bytes between each ``N G obj`` header and the next ``endobj``.
|
||||
|
||||
A forward walk for the same reason as ``_stream_bodies``: the regex
|
||||
``\\d+\\s+\\d+\\s+obj(.*?)endobj`` rescans to end-of-file for every header
|
||||
that has no ``endobj`` after it, so a malformed (or crafted) PDF with many
|
||||
such headers costs headers x size — 2000 orphan headers in 1 MB measured
|
||||
13.5 s, on the default upload path. Advancing past each ``endobj`` keeps
|
||||
the walk linear and yields the same bodies.
|
||||
"""
|
||||
pos = 0
|
||||
while True:
|
||||
header = _OBJ_HEADER.search(raw, pos)
|
||||
if header is None:
|
||||
return
|
||||
end = raw.find(b"endobj", header.end())
|
||||
if end < 0:
|
||||
return
|
||||
yield raw[header.end():end]
|
||||
pos = end + 6
|
||||
|
||||
|
||||
def _decompressed_streams(raw: bytes) -> Iterator[bytes]:
|
||||
"""Each Flate stream in ``raw`` that decompresses, one at a time, capped."""
|
||||
for body in _stream_bodies(raw):
|
||||
@@ -145,8 +167,7 @@ def check_pdf_fonts(data: bytes) -> Dict[str, Union[int, bool]]:
|
||||
if b"/ToUnicode" not in unit:
|
||||
bare += 1
|
||||
|
||||
for match in _OBJ.finditer(data):
|
||||
body = match.group(1)
|
||||
for body in _object_bodies(data):
|
||||
if _TYPE0.search(body):
|
||||
count_font(body)
|
||||
# Each decompressed stream is scanned for every signal in one pass and
|
||||
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
context: ../application
|
||||
args:
|
||||
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
|
||||
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true}
|
||||
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
|
||||
env_file:
|
||||
- ../.env
|
||||
environment:
|
||||
@@ -39,7 +39,7 @@ services:
|
||||
context: ../application
|
||||
args:
|
||||
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
|
||||
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true}
|
||||
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
|
||||
# `parsing` queue carries read_document/parse_document; required for its await to resolve.
|
||||
command: celery -A application.app.celery worker -l INFO -Q docsgpt,parsing
|
||||
env_file:
|
||||
|
||||
@@ -21,8 +21,10 @@ services:
|
||||
# Bake the optional docling engine (layout-model OCR backend, structured
|
||||
# output) into the image: set INSTALL_DOCLING=true in ../.env or the shell.
|
||||
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
|
||||
# The tesseract binary behind OCR_ENABLED=true; on by default (~35 MB).
|
||||
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true}
|
||||
# Bake the tesseract binary behind OCR_ENABLED=true (~35 MB): set
|
||||
# INSTALL_TESSERACT=true in ../.env or the shell (setup.sh does this
|
||||
# when OCR is enabled). Off by default, like docling.
|
||||
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
|
||||
env_file:
|
||||
- ../.env
|
||||
environment:
|
||||
@@ -49,7 +51,7 @@ services:
|
||||
context: ../application
|
||||
args:
|
||||
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
|
||||
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true}
|
||||
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
|
||||
# Consumes the default queue AND the dedicated `parsing` queue (read_document /
|
||||
# parse_document). Without `parsing` here the read_document await never resolves.
|
||||
# For heavy/OCR parsing run a separate worker with `-Q parsing`.
|
||||
|
||||
@@ -216,7 +216,7 @@ for the engines and flows.
|
||||
| `MARKUP_MAX_BYTES` | `8000000` | anydoc engine: HTML/XHTML larger than this are head-truncated before the `markdownify` parser runs (its tree costs ~50x the input). `0` disables the gate. |
|
||||
| `PDF_TRUST_CHECK` | `true` | Trust-check anydoc's PDF output for the two silent-loss classes (Type0 fonts without ToUnicode, CJK-declaring PDFs with CJK-less text). Flagged files re-parse on Docling when installed, else carry `parse_warnings` metadata. |
|
||||
| `ANYDOC_TABLEIZE` | `false` | Rewrite dot-leader / whitespace-aligned table runs in anydoc's PDF markdown into GFM tables. |
|
||||
| `OCR_ENGINE` | `tesseract` | OCR engine when OCR is on: `tesseract` (recommended, both backends), `deepseek` (VLM endpoint, both backends), or the Docling-only `auto`, `ocrmac`, `rapidocr`. Unavailable engines degrade with a warning. See the [OCR guide](/Guides/ocr). |
|
||||
| `OCR_ENGINE` | `tesseract` | OCR engine when OCR is on: `tesseract` (recommended, both backends; optional system package — build with `INSTALL_TESSERACT=true` or `apt`/`brew` install it), `deepseek` (VLM endpoint, both backends), or the Docling-only `auto`, `ocrmac`, `rapidocr`. Unavailable engines degrade with a warning. See the [OCR guide](/Guides/ocr). |
|
||||
| `OCR_LANGS` | `eng` | Tesseract language packs, `+`-separated (e.g. `eng+chi_sim+deu`). |
|
||||
| `OCR_DEEPSEEK_URL` | `http://localhost:11434/v1/chat/completions` | OpenAI-compatible endpoint for `OCR_ENGINE=deepseek` (Ollama or vLLM). |
|
||||
| `OCR_DEEPSEEK_MODEL` | `deepseek-ocr:3b` | Model name at that endpoint. |
|
||||
|
||||
@@ -43,6 +43,34 @@ and — when it is the OCR backend — scanned PDFs and images. Without Docling
|
||||
those formats use the standard parsers; OCR still works through the native
|
||||
backend, and with OCR off images are only read when `PARSE_IMAGE_REMOTE=true`.
|
||||
|
||||
## Installing tesseract
|
||||
|
||||
Nothing OCR-related ships in the base install: the default engine,
|
||||
`tesseract`, is a ~35 MB system package that you opt into like any other OCR
|
||||
dependency. Locally:
|
||||
|
||||
```bash
|
||||
# Debian/Ubuntu # macOS
|
||||
apt-get install tesseract-ocr tesseract-ocr-eng brew install tesseract
|
||||
```
|
||||
|
||||
Docker images build without it by default; opt in with the build argument:
|
||||
|
||||
```bash
|
||||
docker build --build-arg INSTALL_TESSERACT=true ./application
|
||||
```
|
||||
|
||||
`deployment/docker-compose.yaml` forwards the same switch, so setting
|
||||
`INSTALL_TESSERACT=true` in `.env` (or the shell) bakes tesseract plus the
|
||||
English pack into locally built backend and worker images; `setup.sh` writes
|
||||
it when you answer yes to the OCR question after choosing to build images
|
||||
locally. Pre-built Docker Hub images (`docker-compose-hub.yaml`) do not
|
||||
include it, so `setup.sh` leaves OCR at its default (off) for them; to OCR
|
||||
there, point `OCR_ENGINE=deepseek` at a DeepSeek-OCR endpoint or install
|
||||
`tesseract-ocr` in a derived image. With `OCR_ENABLED=true` and no binary on
|
||||
`PATH`, scanned pages fail with an install hint (text-layer documents are
|
||||
unaffected).
|
||||
|
||||
## Installing the docling engine
|
||||
|
||||
docling is not part of the base install, and OCR does not need it (see
|
||||
@@ -112,7 +140,7 @@ Docling is not the only way to OCR. The `native` backend renders the pages
|
||||
that need it with pypdfium2 and Pillow — both already core dependencies —
|
||||
and feeds them straight into tesseract or a DeepSeek-OCR endpoint. No ML
|
||||
models load in the worker, and nothing beyond the ~35 MB tesseract binary
|
||||
(shipped in the Docker image) is installed.
|
||||
(see [Installing tesseract](#installing-tesseract)) is needed.
|
||||
|
||||
| `OCR_BACKEND` | What runs | When to pick it |
|
||||
|---|---|---|
|
||||
@@ -135,7 +163,7 @@ scans (all engines driven through docling so layout handling is identical):
|
||||
|
||||
| `OCR_ENGINE` | Backends | Role | Notes |
|
||||
|---|---|---|---|
|
||||
| `tesseract` | native, docling | **recommended default** | Best classic-engine accuracy in the bench: perfect EN word recall on all docs, 0.000 CER on the bilingual page, 100% table cells, robust to mild degradation. ~35 MB of *system* packages, CPU-only. Needs the `tesseract` binary + language packs — shipped in the Docker image (opt out with `--build-arg INSTALL_TESSERACT=false`); set languages via `OCR_LANGS` (e.g. `eng+chi_sim`). |
|
||||
| `tesseract` | native, docling | **recommended default** | Best classic-engine accuracy in the bench: perfect EN word recall on all docs, 0.000 CER on the bilingual page, 100% table cells, robust to mild degradation. ~35 MB of *system* packages, CPU-only. Needs the `tesseract` binary + language packs — an optional install like every OCR dependency (see [Installing tesseract](#installing-tesseract)); set languages via `OCR_LANGS` (e.g. `eng+chi_sim`). |
|
||||
| `deepseek` | native, docling | best quality, heavy on the *server* | DeepSeek-OCR against an OpenAI-compatible endpoint. Only engine that reconstructs totals rows as table rows; near-perfect CJK; barely affected by degradation. The ingestion worker stays light (no layout models); the model runs in Ollama or vLLM. Costs: a GPU/Apple-Silicon endpoint, ~seconds per page, and occasional silent drops of page-level elements (titles). |
|
||||
| `auto` | docling | convenience | docling picks: `ocrmac` on macOS (excellent, ~1 s/page), `rapidocr` on Linux — see below before relying on it server-side. Also docling's automatic fallback whenever the selected engine is not installed. The native backend runs tesseract for it. |
|
||||
| `ocrmac` | docling | macOS only | Best raw accuracy and fastest of all classic engines; irrelevant for Linux deploys. |
|
||||
@@ -233,8 +261,8 @@ With `OCR_BACKEND=docling`, OCR behavior is different for PDFs vs images:
|
||||
- Image parser defaults to full-page OCR (the whole image is visual content).
|
||||
|
||||
The engine and its languages come from `OCR_ENGINE` and `OCR_LANGS` (see the
|
||||
table above). The Docker image ships only the English tesseract pack; for
|
||||
other languages install their packs in the image (e.g.
|
||||
table above). `INSTALL_TESSERACT=true` installs only the English tesseract
|
||||
pack; for other languages install their packs in the image (e.g.
|
||||
`apt-get install tesseract-ocr-chi-sim`) and list them in `OCR_LANGS`
|
||||
(`eng+chi_sim`).
|
||||
|
||||
|
||||
@@ -362,10 +362,22 @@ configure_doc_processing() {
|
||||
echo -e "${GREEN}PDF-as-image parsing enabled.${NC}"
|
||||
fi
|
||||
|
||||
# OCR needs the tesseract binary, an optional system package that only
|
||||
# locally built images can include (INSTALL_TESSERACT build arg). The
|
||||
# pre-built Docker Hub images ship without it, so there OCR stays off
|
||||
# (its default) rather than being switched on to fail on every scan.
|
||||
if [[ "$COMPOSE_FILE" != "$COMPOSE_FILE_LOCAL" ]]; then
|
||||
echo -e "${YELLOW}OCR for scanned PDFs and images stays off: the pre-built Docker Hub images do not include tesseract. To use OCR, rerun setup and choose option 5 (build images locally), or point OCR_ENGINE=deepseek at a DeepSeek-OCR endpoint in .env.${NC}"
|
||||
return
|
||||
fi
|
||||
|
||||
read -p "$(echo -e "${DEFAULT_FG}Enable OCR for scanned PDFs and images? (y/N): ${NC}")" ocr_enabled
|
||||
if [[ "$ocr_enabled" =~ ^[yY]$ ]]; then
|
||||
echo "OCR_ENABLED=true" >> "$ENV_FILE"
|
||||
echo -e "${GREEN}OCR enabled (tesseract, shipped in the Docker image; set OCR_ENGINE=deepseek for a DeepSeek-OCR endpoint).${NC}"
|
||||
# Bakes tesseract into the locally built images (docker compose
|
||||
# --env-file .env build).
|
||||
echo "INSTALL_TESSERACT=true" >> "$ENV_FILE"
|
||||
echo -e "${GREEN}OCR enabled. tesseract will be built into the images (INSTALL_TESSERACT=true); set OCR_ENGINE=deepseek in .env for a DeepSeek-OCR endpoint instead.${NC}"
|
||||
read -p "$(echo -e "${DEFAULT_FG}Also install the Docling layout engine for OCR (better tables/reading order, several GB heavier)? (y/N): ${NC}")" docling_ocr
|
||||
if [[ "$docling_ocr" =~ ^[yY]$ ]]; then
|
||||
# Locally built images include docling via this build arg; it becomes
|
||||
|
||||
@@ -13,6 +13,7 @@ Three behaviours matter and are asserted separately:
|
||||
import importlib.machinery
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -319,9 +320,7 @@ def test_init_parser_imports_for_real_not_just_find_spec(monkeypatch):
|
||||
|
||||
# --- PDF trust check + tableize wiring (PR 3) -----------------------------------
|
||||
|
||||
from pathlib import Path as _Path
|
||||
|
||||
FIXTURES = _Path(__file__).parent / "fixtures"
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
CID_PDF = FIXTURES / "nda_en_zh_cid_font.pdf"
|
||||
|
||||
|
||||
|
||||
@@ -158,8 +158,12 @@ class TestOcrEngineSelection:
|
||||
|
||||
def test_build_tesseract_reads_ocr_langs(self, settings, monkeypatch):
|
||||
pytest.importorskip("docling")
|
||||
import application.parser.file.ocr_parser as op
|
||||
from application.parser.file.docling_parser import _build_ocr_options
|
||||
|
||||
# Pack inventory unknown: the resolved list is passed through untouched
|
||||
# (a host with tesseract but no chi_sim pack would otherwise drop it).
|
||||
monkeypatch.setattr(op, "tesseract_languages", lambda: None)
|
||||
monkeypatch.setattr(settings, "OCR_LANGS", "eng+chi_sim")
|
||||
options = _build_ocr_options("tesseract", None, True)
|
||||
|
||||
|
||||
@@ -964,3 +964,38 @@ def test_delegate_parse_lets_setup_errors_through():
|
||||
|
||||
with pytest.raises(ImportError, match="docling is required"):
|
||||
delegate_parse(_NeedsLib(), Path("x.pdf"), "ignore")
|
||||
|
||||
|
||||
class TestTextLayerDelegate:
|
||||
"""``text_layer_delegate`` mirrors ``parse_file``'s delegation decision."""
|
||||
|
||||
def test_every_page_with_text_returns_the_text_parser(self, monkeypatch):
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
class _Pdf:
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(pdfium, "PdfDocument", lambda path: _Pdf())
|
||||
text_parser = object()
|
||||
parser = op.NativeOcrPdfParser(text_parser=text_parser, min_text_chars=20)
|
||||
monkeypatch.setattr(parser, "_text_layer_counts", lambda pdf: [50, 60])
|
||||
assert parser.text_layer_delegate(Path("x.pdf")) is text_parser
|
||||
|
||||
def test_a_scanned_page_keeps_the_native_path(self, monkeypatch):
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
class _Pdf:
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(pdfium, "PdfDocument", lambda path: _Pdf())
|
||||
parser = op.NativeOcrPdfParser(text_parser=object(), min_text_chars=20)
|
||||
monkeypatch.setattr(parser, "_text_layer_counts", lambda pdf: [50, 3])
|
||||
assert parser.text_layer_delegate(Path("x.pdf")) is None
|
||||
|
||||
def test_no_text_parser_or_unreadable_file_is_none(self, tmp_path):
|
||||
assert op.NativeOcrPdfParser().text_layer_delegate(Path("x.pdf")) is None
|
||||
broken = tmp_path / "broken.pdf"
|
||||
broken.write_bytes(b"not a pdf")
|
||||
assert op.NativeOcrPdfParser(text_parser=object()).text_layer_delegate(broken) is None
|
||||
@@ -182,6 +182,36 @@ class TestStreamWalk:
|
||||
assert result["type0_no_tounicode"] == 1
|
||||
|
||||
|
||||
class TestObjectWalk:
|
||||
def test_bodies_match_the_regex_reference(self):
|
||||
"""The forward walk yields exactly what ``\\d+\\s+\\d+\\s+obj(.*?)endobj`` did."""
|
||||
import re
|
||||
|
||||
from application.parser.file.pdf_trust import _object_bodies
|
||||
|
||||
reference = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL)
|
||||
data = (
|
||||
TYPE0_BARE
|
||||
+ b"7 0 obj no terminator here " # orphan header: body runs to the next endobj
|
||||
+ SIMPLE_FONT
|
||||
+ b"junk 12 0 obj<</A/B>>endobj"
|
||||
+ _stream(b"x")
|
||||
+ b"99 0 obj trailing orphan"
|
||||
)
|
||||
assert list(_object_bodies(data)) == [m.group(1) for m in reference.finditer(data)]
|
||||
|
||||
def test_orphan_headers_are_linear(self):
|
||||
"""Headers with no ``endobj`` after them: the old regex rescanned to EOF for
|
||||
each one (2000 in 1 MB took 13.5 s), pinning an ingest worker per upload."""
|
||||
import time
|
||||
|
||||
data = TYPE0_BARE + b"1 0 obj\n" * 20_000 + b"\x00" * 500_000
|
||||
started = time.perf_counter()
|
||||
result = check_pdf_fonts(data)
|
||||
assert time.perf_counter() - started < 1.0
|
||||
assert result["type0_no_tounicode"] == 1
|
||||
|
||||
|
||||
# --- the CJK cross-check is confirmed against pdfium's text layer --------------
|
||||
|
||||
|
||||
|
||||
@@ -724,6 +724,31 @@ def test_fast_engine_covers_the_whole_legacy_map():
|
||||
assert type(dr._legacy_parser_for(suffix)).__name__ == name
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", [".odt", ".doc", ".rtf", ".xls", ".ods", ".odp"])
|
||||
def test_fast_engine_is_terminal_for_anydoc_only_suffixes(monkeypatch, suffix):
|
||||
"""A suffix only anydoc reads has no legacy parser; `fast` must report that
|
||||
rather than silently running the configured engine the caller opted out of."""
|
||||
|
||||
def _boom(*args, **kwargs):
|
||||
raise AssertionError("fast fell through to the configured engine map")
|
||||
|
||||
monkeypatch.setattr(dr, "get_default_file_extractor", _boom)
|
||||
monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)
|
||||
|
||||
assert dr._pick_parser(suffix, ocr_enabled=False, engine="fast") is None
|
||||
out = parse_document_bytes(b"\xd0\xcf\x11\xe0 bytes", f"doc{suffix}", output="text", engine="fast")
|
||||
assert "fast engine" in out["error"] and suffix in out["error"]
|
||||
|
||||
|
||||
def test_fast_engine_still_falls_through_for_images_and_audio(monkeypatch):
|
||||
"""Formats with no legacy parser and no anydoc alternative keep the configured map."""
|
||||
seen = []
|
||||
monkeypatch.setattr(dr, "get_default_file_extractor", lambda **kwargs: seen.append(kwargs) or {})
|
||||
for suffix in (".png", ".jpg", ".mp3", ".txt"):
|
||||
dr._pick_parser(suffix, ocr_enabled=True, engine="fast")
|
||||
assert len(seen) == 4
|
||||
|
||||
|
||||
def test_tables_are_not_collected_when_the_pdf_parser_is_not_docling(monkeypatch):
|
||||
"""Under OCR_BACKEND=native the docling engine hands PDFs to the native OCR
|
||||
parser; a vanilla DocumentConverter table pass would OCR the scan again."""
|
||||
@@ -746,6 +771,46 @@ def test_tables_are_not_collected_when_the_pdf_parser_is_not_docling(monkeypatch
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_tables_survive_native_ocr_delegation_for_text_only_pdfs(monkeypatch):
|
||||
"""OCR_BACKEND=native under the docling engine wraps a DoclingParser as the
|
||||
native parser's text_parser. A PDF with a text layer on every page is handed
|
||||
to that parser, so tables must ride that single conversion instead of being
|
||||
dropped by the scan exclusion."""
|
||||
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from application.parser.file.ocr_parser import NativeOcrPdfParser
|
||||
|
||||
class _Docling(DoclingParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._parser_config = {"ready": True}
|
||||
|
||||
docling = _Docling()
|
||||
native = NativeOcrPdfParser(text_parser=docling)
|
||||
native._parser_config = {}
|
||||
monkeypatch.setattr(native, "text_layer_delegate", lambda path: docling)
|
||||
monkeypatch.setattr(
|
||||
native, "parse_file", lambda path, errors="ignore": pytest.fail("native OCR ran on a text-only PDF")
|
||||
)
|
||||
|
||||
calls = []
|
||||
|
||||
def _fake_structured(path, *, ocr_enabled, include_tables, parser=None):
|
||||
calls.append(parser)
|
||||
return {"markdown": "# text pdf", "tables": [{"markdown": "| t |"}], "structured": {}, "page_count": 1}
|
||||
|
||||
monkeypatch.setattr(dr, "_pick_parser", lambda *a, **k: native)
|
||||
monkeypatch.setattr(dr, "_effective_engine", lambda engine: "docling")
|
||||
monkeypatch.setattr(dr, "_docling_structured", _fake_structured)
|
||||
monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)
|
||||
|
||||
out = parse_document_bytes(b"%PDF-1.4", "text.pdf", engine="docling", include_tables=True)
|
||||
|
||||
assert out["content"] == "# text pdf"
|
||||
assert out["tables"] == [{"markdown": "| t |"}]
|
||||
assert calls == [docling]
|
||||
|
||||
|
||||
def test_binary_office_suffix_without_anydoc_is_an_error_not_text(monkeypatch):
|
||||
monkeypatch.setattr(dr, "_pick_parser", lambda *a, **k: None)
|
||||
monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)
|
||||
|
||||
Reference in new issue
Block a user