Fixes batch 2

This commit is contained in:
Pavel committed 2026-09-04 13:38:29 +04:00
1 parent c17caf24ec
commit 4e14a79923
16 files changed
+278 -31

No files matched your search

+9 -7
View File
@@ -46,7 +46,7 @@ RUN pip install --no-cache-dir --upgrade pip && \
# backend, read_document's structured output) — OFF by default: it pulls the
# layout/OCR model stack and adds gigabytes to the image. anydoc (in
# requirements.txt) is the default parser and needs none of it, and OCR runs
# natively on tesseract (below) without it.
# natively on tesseract (below, also opt-in) without it.
ARG INSTALL_DOCLING=false
RUN if [ "$INSTALL_DOCLING" = "true" ]; then \
pip install --no-cache-dir -r requirements-docling.txt; \
@@ -67,12 +67,14 @@ RUN apt-get update && \
ln -s /usr/bin/python3.12 /usr/bin/python && \
rm -rf /var/lib/apt/lists/*
# OCR (OCR_ENABLED=true) runs on the system tesseract binary by default,
# with or without the docling extra, so ship it in every image (~35 MB).
# Opt out with --build-arg INSTALL_TESSERACT=false (e.g. a DeepSeek-OCR-only
# deployment). Extra language packs are a deployment concern
# (apt: tesseract-ocr-<lang>, then list them in OCR_LANGS).
ARG INSTALL_TESSERACT=true
# Optional tesseract OCR engine (OCR_ENABLED=true with OCR_ENGINE=tesseract,
# the default engine) — OFF by default like every other OCR dependency; OCR
# itself is off unless configured. Opt in with --build-arg
# INSTALL_TESSERACT=true (setup.sh writes it to .env when OCR is enabled);
# ~35 MB of system packages. Extra language packs are a deployment concern
# (apt: tesseract-ocr-<lang>, then list them in OCR_LANGS). A DeepSeek-OCR
# endpoint (OCR_ENGINE=deepseek) needs none of this.
ARG INSTALL_TESSERACT=false
RUN if [ "$INSTALL_TESSERACT" = "true" ]; then \
apt-get update && \
apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng && \
+3 -2
View File
@@ -165,8 +165,9 @@ class Settings(BaseSettings):
# menu):
# tesseract — recommended: best classic-engine accuracy (perfect EN word
# recall, 0.000 bilingual CER, 100% table cells), ~35 MB, CPU-only.
# Needs the system binary + language packs (the Docker image ships
# them unless built with INSTALL_TESSERACT=false). Both backends.
# Needs the system binary + language packs: an optional install like
# every OCR dependency (build with INSTALL_TESSERACT=true, or apt/brew
# install tesseract-ocr for a local run). Both backends.
# deepseek — DeepSeek-OCR against an Ollama/vLLM endpoint
# (OCR_DEEPSEEK_*). Best table/CJK quality; the worker stays light
# (no layout models) but each page costs seconds on the model server.
+22
View File
@@ -186,6 +186,13 @@ def _pick_parser(suffix: str, *, ocr_enabled: bool, engine: str):
legacy = _legacy_parser_for(suffix)
if legacy is not None:
return legacy
from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
if suffix in ANYDOC_GAINED_SUFFIXES:
# anydoc-only office formats have no legacy parser. Falling through
# would run the configured engine the caller explicitly opted out
# of; images/audio/.txt still fall through (no legacy alternative).
return None
if engine in ("docling", "anydoc"):
extractor = get_default_file_extractor(ocr_enabled=ocr_enabled, engine=engine)
else:
@@ -529,6 +536,16 @@ def _shape(
}
parser = _pick_parser(suffix, ocr_enabled=ocr_enabled, engine=engine)
# Under OCR_BACKEND=native with the docling engine the PDF parser is the
# native OCR parser wrapping a DoclingParser as its text_parser. A PDF
# with a text layer on every page is handed to that parser untouched, so
# when tables are wanted run it directly: the single docling conversion
# below then yields both the markdown and the tables. Scans and mixed
# documents keep the native parser (and the exclusion just below).
if include_tables and output != "chunks" and _is_native_ocr_parser(parser):
delegate = parser.text_layer_delegate(path)
if _is_docling_parser(delegate):
parser = delegate
# Tables come from a Docling conversion, so they are only collected when
# a Docling parser is actually in play: under anydoc/fast a table pass
# would be a second, full docling conversion that costs more than the
@@ -569,6 +586,11 @@ def _shape(
from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
if suffix in ANYDOC_GAINED_SUFFIXES:
if engine == "fast":
return {
"error": f"The fast engine has no parser for {suffix} files; "
"use engine='anydoc' or 'auto'."
}
return {
"error": f"No parser is available for {suffix} files: firecrawl-anydoc "
"is not installed (pip install firecrawl-anydoc)"
+2 -2
View File
@@ -103,8 +103,8 @@ def _resolve_ocr_engine(requested: Optional[str]) -> str:
if engine == "tesseract" and shutil.which("tesseract") is None:
logger.warning(
"OCR_ENGINE=tesseract but no tesseract binary is on PATH (install "
"tesseract-ocr plus language packs; the Docker image ships them unless "
"built with INSTALL_TESSERACT=false); using docling auto-selection"
"tesseract-ocr plus language packs: Docker builds need "
"INSTALL_TESSERACT=true); using docling auto-selection"
)
return "auto"
if engine == "ocrmac" and (
+29 -3
View File
@@ -315,9 +315,9 @@ class TesseractEngine:
if not self.available():
raise OcrUnavailableError(
"OCR_ENGINE=tesseract but no tesseract binary is on PATH. Install "
"tesseract-ocr plus language packs (the Docker image ships them "
"unless built with INSTALL_TESSERACT=false), or set "
"OCR_ENGINE=deepseek / install the docling extra."
"tesseract-ocr plus language packs (Docker: build with "
"INSTALL_TESSERACT=true; local: apt/brew install tesseract-ocr), "
"or set OCR_ENGINE=deepseek / install the docling extra."
)
try:
completed = subprocess.run(
@@ -699,6 +699,32 @@ class NativeOcrPdfParser(BaseParser):
self.last_engine = engine.name
return texts
def text_layer_delegate(self, file: Path) -> Optional[BaseParser]:
"""The ``text_parser`` that ``parse_file`` would hand ``file`` to, or None.
None when there is no text parser, the file cannot be opened, or any
page falls below ``min_text_chars`` (those are OCR'd here). Callers
that want a by-product of the delegate's conversion (docling tables)
can run it directly instead of converting the document twice.
"""
if self.text_parser is None:
return None
try:
import pypdfium2 as pdfium
pdf = pdfium.PdfDocument(str(Path(file)))
except Exception: # noqa: BLE001 - a probe; parse_file reports the real error
return None
try:
counts = self._text_layer_counts(pdf)
except Exception: # noqa: BLE001
return None
finally:
pdf.close()
if counts and all(count >= self.min_text_chars for count in counts):
return self.text_parser
return None
def _delegate_text(self, path: Path, errors: str) -> Union[str, List[str]]:
text_parser = self.text_parser
assert text_parser is not None
+24 -3
View File
@@ -29,7 +29,7 @@ import zlib
from pathlib import Path
from typing import Dict, Iterator, List, Optional, Union
_OBJ = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL)
_OBJ_HEADER = re.compile(rb"\d+\s+\d+\s+obj")
_TYPE0 = re.compile(rb"/Subtype\s*/Type0")
# The CJK expectation is keyed on CIDSystemInfo /Ordering ALONE — the
# authoritative "this PDF maps text through a CJK character collection"
@@ -99,6 +99,28 @@ def _stream_bodies(raw: bytes) -> Iterator[bytes]:
yield body
def _object_bodies(raw: bytes) -> Iterator[bytes]:
"""The bytes between each ``N G obj`` header and the next ``endobj``.
A forward walk for the same reason as ``_stream_bodies``: the regex
``\\d+\\s+\\d+\\s+obj(.*?)endobj`` rescans to end-of-file for every header
that has no ``endobj`` after it, so a malformed (or crafted) PDF with many
such headers costs headers x size — 2000 orphan headers in 1 MB measured
13.5 s, on the default upload path. Advancing past each ``endobj`` keeps
the walk linear and yields the same bodies.
"""
pos = 0
while True:
header = _OBJ_HEADER.search(raw, pos)
if header is None:
return
end = raw.find(b"endobj", header.end())
if end < 0:
return
yield raw[header.end():end]
pos = end + 6
def _decompressed_streams(raw: bytes) -> Iterator[bytes]:
"""Each Flate stream in ``raw`` that decompresses, one at a time, capped."""
for body in _stream_bodies(raw):
@@ -145,8 +167,7 @@ def check_pdf_fonts(data: bytes) -> Dict[str, Union[int, bool]]:
if b"/ToUnicode" not in unit:
bare += 1
for match in _OBJ.finditer(data):
body = match.group(1)
for body in _object_bodies(data):
if _TYPE0.search(body):
count_font(body)
# Each decompressed stream is scanned for every signal in one pass and
+2 -2
View File
@@ -14,7 +14,7 @@ services:
context: ../application
args:
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true}
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
env_file:
- ../.env
environment:
@@ -39,7 +39,7 @@ services:
context: ../application
args:
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true}
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
# `parsing` queue carries read_document/parse_document; required for its await to resolve.
command: celery -A application.app.celery worker -l INFO -Q docsgpt,parsing
env_file:
+5 -3
View File
@@ -21,8 +21,10 @@ services:
# Bake the optional docling engine (layout-model OCR backend, structured
# output) into the image: set INSTALL_DOCLING=true in ../.env or the shell.
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
# The tesseract binary behind OCR_ENABLED=true; on by default (~35 MB).
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true}
# Bake the tesseract binary behind OCR_ENABLED=true (~35 MB): set
# INSTALL_TESSERACT=true in ../.env or the shell (setup.sh does this
# when OCR is enabled). Off by default, like docling.
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
env_file:
- ../.env
environment:
@@ -49,7 +51,7 @@ services:
context: ../application
args:
INSTALL_DOCLING: ${INSTALL_DOCLING:-false}
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true}
INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false}
# Consumes the default queue AND the dedicated `parsing` queue (read_document /
# parse_document). Without `parsing` here the read_document await never resolves.
# For heavy/OCR parsing run a separate worker with `-Q parsing`.
+1 -1
View File
@@ -216,7 +216,7 @@ for the engines and flows.
| `MARKUP_MAX_BYTES` | `8000000` | anydoc engine: HTML/XHTML larger than this are head-truncated before the `markdownify` parser runs (its tree costs ~50x the input). `0` disables the gate. |
| `PDF_TRUST_CHECK` | `true` | Trust-check anydoc's PDF output for the two silent-loss classes (Type0 fonts without ToUnicode, CJK-declaring PDFs with CJK-less text). Flagged files re-parse on Docling when installed, else carry `parse_warnings` metadata. |
| `ANYDOC_TABLEIZE` | `false` | Rewrite dot-leader / whitespace-aligned table runs in anydoc's PDF markdown into GFM tables. |
| `OCR_ENGINE` | `tesseract` | OCR engine when OCR is on: `tesseract` (recommended, both backends), `deepseek` (VLM endpoint, both backends), or the Docling-only `auto`, `ocrmac`, `rapidocr`. Unavailable engines degrade with a warning. See the [OCR guide](/Guides/ocr). |
| `OCR_ENGINE` | `tesseract` | OCR engine when OCR is on: `tesseract` (recommended, both backends; optional system package — build with `INSTALL_TESSERACT=true` or `apt`/`brew` install it), `deepseek` (VLM endpoint, both backends), or the Docling-only `auto`, `ocrmac`, `rapidocr`. Unavailable engines degrade with a warning. See the [OCR guide](/Guides/ocr). |
| `OCR_LANGS` | `eng` | Tesseract language packs, `+`-separated (e.g. `eng+chi_sim+deu`). |
| `OCR_DEEPSEEK_URL` | `http://localhost:11434/v1/chat/completions` | OpenAI-compatible endpoint for `OCR_ENGINE=deepseek` (Ollama or vLLM). |
| `OCR_DEEPSEEK_MODEL` | `deepseek-ocr:3b` | Model name at that endpoint. |
+32 -4
View File
@@ -43,6 +43,34 @@ and — when it is the OCR backend — scanned PDFs and images. Without Docling
those formats use the standard parsers; OCR still works through the native
backend, and with OCR off images are only read when `PARSE_IMAGE_REMOTE=true`.
## Installing tesseract
Nothing OCR-related ships in the base install: the default engine,
`tesseract`, is a ~35 MB system package that you opt into like any other OCR
dependency. Locally:
```bash
# Debian/Ubuntu # macOS
apt-get install tesseract-ocr tesseract-ocr-eng brew install tesseract
```
Docker images build without it by default; opt in with the build argument:
```bash
docker build --build-arg INSTALL_TESSERACT=true ./application
```
`deployment/docker-compose.yaml` forwards the same switch, so setting
`INSTALL_TESSERACT=true` in `.env` (or the shell) bakes tesseract plus the
English pack into locally built backend and worker images; `setup.sh` writes
it when you answer yes to the OCR question after choosing to build images
locally. Pre-built Docker Hub images (`docker-compose-hub.yaml`) do not
include it, so `setup.sh` leaves OCR at its default (off) for them; to OCR
there, point `OCR_ENGINE=deepseek` at a DeepSeek-OCR endpoint or install
`tesseract-ocr` in a derived image. With `OCR_ENABLED=true` and no binary on
`PATH`, scanned pages fail with an install hint (text-layer documents are
unaffected).
## Installing the docling engine
docling is not part of the base install, and OCR does not need it (see
@@ -112,7 +140,7 @@ Docling is not the only way to OCR. The `native` backend renders the pages
that need it with pypdfium2 and Pillow — both already core dependencies —
and feeds them straight into tesseract or a DeepSeek-OCR endpoint. No ML
models load in the worker, and nothing beyond the ~35 MB tesseract binary
(shipped in the Docker image) is installed.
(see [Installing tesseract](#installing-tesseract)) is needed.
| `OCR_BACKEND` | What runs | When to pick it |
|---|---|---|
@@ -135,7 +163,7 @@ scans (all engines driven through docling so layout handling is identical):
| `OCR_ENGINE` | Backends | Role | Notes |
|---|---|---|---|
| `tesseract` | native, docling | **recommended default** | Best classic-engine accuracy in the bench: perfect EN word recall on all docs, 0.000 CER on the bilingual page, 100% table cells, robust to mild degradation. ~35 MB of *system* packages, CPU-only. Needs the `tesseract` binary + language packs — shipped in the Docker image (opt out with `--build-arg INSTALL_TESSERACT=false`); set languages via `OCR_LANGS` (e.g. `eng+chi_sim`). |
| `tesseract` | native, docling | **recommended default** | Best classic-engine accuracy in the bench: perfect EN word recall on all docs, 0.000 CER on the bilingual page, 100% table cells, robust to mild degradation. ~35 MB of *system* packages, CPU-only. Needs the `tesseract` binary + language packs — an optional install like every OCR dependency (see [Installing tesseract](#installing-tesseract)); set languages via `OCR_LANGS` (e.g. `eng+chi_sim`). |
| `deepseek` | native, docling | best quality, heavy on the *server* | DeepSeek-OCR against an OpenAI-compatible endpoint. Only engine that reconstructs totals rows as table rows; near-perfect CJK; barely affected by degradation. The ingestion worker stays light (no layout models); the model runs in Ollama or vLLM. Costs: a GPU/Apple-Silicon endpoint, ~seconds per page, and occasional silent drops of page-level elements (titles). |
| `auto` | docling | convenience | docling picks: `ocrmac` on macOS (excellent, ~1 s/page), `rapidocr` on Linux — see below before relying on it server-side. Also docling's automatic fallback whenever the selected engine is not installed. The native backend runs tesseract for it. |
| `ocrmac` | docling | macOS only | Best raw accuracy and fastest of all classic engines; irrelevant for Linux deploys. |
@@ -233,8 +261,8 @@ With `OCR_BACKEND=docling`, OCR behavior is different for PDFs vs images:
- Image parser defaults to full-page OCR (the whole image is visual content).
The engine and its languages come from `OCR_ENGINE` and `OCR_LANGS` (see the
table above). The Docker image ships only the English tesseract pack; for
other languages install their packs in the image (e.g.
table above). `INSTALL_TESSERACT=true` installs only the English tesseract
pack; for other languages install their packs in the image (e.g.
`apt-get install tesseract-ocr-chi-sim`) and list them in `OCR_LANGS`
(`eng+chi_sim`).
+13 -1
View File
@@ -362,10 +362,22 @@ configure_doc_processing() {
echo -e "${GREEN}PDF-as-image parsing enabled.${NC}"
fi
# OCR needs the tesseract binary, an optional system package that only
# locally built images can include (INSTALL_TESSERACT build arg). The
# pre-built Docker Hub images ship without it, so there OCR stays off
# (its default) rather than being switched on to fail on every scan.
if [[ "$COMPOSE_FILE" != "$COMPOSE_FILE_LOCAL" ]]; then
echo -e "${YELLOW}OCR for scanned PDFs and images stays off: the pre-built Docker Hub images do not include tesseract. To use OCR, rerun setup and choose option 5 (build images locally), or point OCR_ENGINE=deepseek at a DeepSeek-OCR endpoint in .env.${NC}"
return
fi
read -p "$(echo -e "${DEFAULT_FG}Enable OCR for scanned PDFs and images? (y/N): ${NC}")" ocr_enabled
if [[ "$ocr_enabled" =~ ^[yY]$ ]]; then
echo "OCR_ENABLED=true" >> "$ENV_FILE"
echo -e "${GREEN}OCR enabled (tesseract, shipped in the Docker image; set OCR_ENGINE=deepseek for a DeepSeek-OCR endpoint).${NC}"
# Bakes tesseract into the locally built images (docker compose
# --env-file .env build).
echo "INSTALL_TESSERACT=true" >> "$ENV_FILE"
echo -e "${GREEN}OCR enabled. tesseract will be built into the images (INSTALL_TESSERACT=true); set OCR_ENGINE=deepseek in .env for a DeepSeek-OCR endpoint instead.${NC}"
read -p "$(echo -e "${DEFAULT_FG}Also install the Docling layout engine for OCR (better tables/reading order, several GB heavier)? (y/N): ${NC}")" docling_ocr
if [[ "$docling_ocr" =~ ^[yY]$ ]]; then
# Locally built images include docling via this build arg; it becomes
+2 -3
View File
@@ -13,6 +13,7 @@ Three behaviours matter and are asserted separately:
import importlib.machinery
import sys
import types
from pathlib import Path
import pytest
@@ -319,9 +320,7 @@ def test_init_parser_imports_for_real_not_just_find_spec(monkeypatch):
# --- PDF trust check + tableize wiring (PR 3) -----------------------------------
from pathlib import Path as _Path
FIXTURES = _Path(__file__).parent / "fixtures"
FIXTURES = Path(__file__).parent / "fixtures"
CID_PDF = FIXTURES / "nda_en_zh_cid_font.pdf"
+4
View File
@@ -158,8 +158,12 @@ class TestOcrEngineSelection:
def test_build_tesseract_reads_ocr_langs(self, settings, monkeypatch):
pytest.importorskip("docling")
import application.parser.file.ocr_parser as op
from application.parser.file.docling_parser import _build_ocr_options
# Pack inventory unknown: the resolved list is passed through untouched
# (a host with tesseract but no chi_sim pack would otherwise drop it).
monkeypatch.setattr(op, "tesseract_languages", lambda: None)
monkeypatch.setattr(settings, "OCR_LANGS", "eng+chi_sim")
options = _build_ocr_options("tesseract", None, True)
+35
View File
@@ -964,3 +964,38 @@ def test_delegate_parse_lets_setup_errors_through():
with pytest.raises(ImportError, match="docling is required"):
delegate_parse(_NeedsLib(), Path("x.pdf"), "ignore")
class TestTextLayerDelegate:
"""``text_layer_delegate`` mirrors ``parse_file``'s delegation decision."""
def test_every_page_with_text_returns_the_text_parser(self, monkeypatch):
import pypdfium2 as pdfium
class _Pdf:
def close(self):
pass
monkeypatch.setattr(pdfium, "PdfDocument", lambda path: _Pdf())
text_parser = object()
parser = op.NativeOcrPdfParser(text_parser=text_parser, min_text_chars=20)
monkeypatch.setattr(parser, "_text_layer_counts", lambda pdf: [50, 60])
assert parser.text_layer_delegate(Path("x.pdf")) is text_parser
def test_a_scanned_page_keeps_the_native_path(self, monkeypatch):
import pypdfium2 as pdfium
class _Pdf:
def close(self):
pass
monkeypatch.setattr(pdfium, "PdfDocument", lambda path: _Pdf())
parser = op.NativeOcrPdfParser(text_parser=object(), min_text_chars=20)
monkeypatch.setattr(parser, "_text_layer_counts", lambda pdf: [50, 3])
assert parser.text_layer_delegate(Path("x.pdf")) is None
def test_no_text_parser_or_unreadable_file_is_none(self, tmp_path):
assert op.NativeOcrPdfParser().text_layer_delegate(Path("x.pdf")) is None
broken = tmp_path / "broken.pdf"
broken.write_bytes(b"not a pdf")
assert op.NativeOcrPdfParser(text_parser=object()).text_layer_delegate(broken) is None
+30
View File
@@ -182,6 +182,36 @@ class TestStreamWalk:
assert result["type0_no_tounicode"] == 1
class TestObjectWalk:
def test_bodies_match_the_regex_reference(self):
"""The forward walk yields exactly what ``\\d+\\s+\\d+\\s+obj(.*?)endobj`` did."""
import re
from application.parser.file.pdf_trust import _object_bodies
reference = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL)
data = (
TYPE0_BARE
+ b"7 0 obj no terminator here " # orphan header: body runs to the next endobj
+ SIMPLE_FONT
+ b"junk 12 0 obj<</A/B>>endobj"
+ _stream(b"x")
+ b"99 0 obj trailing orphan"
)
assert list(_object_bodies(data)) == [m.group(1) for m in reference.finditer(data)]
def test_orphan_headers_are_linear(self):
"""Headers with no ``endobj`` after them: the old regex rescanned to EOF for
each one (2000 in 1 MB took 13.5 s), pinning an ingest worker per upload."""
import time
data = TYPE0_BARE + b"1 0 obj\n" * 20_000 + b"\x00" * 500_000
started = time.perf_counter()
result = check_pdf_fonts(data)
assert time.perf_counter() - started < 1.0
assert result["type0_no_tounicode"] == 1
# --- the CJK cross-check is confirmed against pdfium's text layer --------------
+65
View File
@@ -724,6 +724,31 @@ def test_fast_engine_covers_the_whole_legacy_map():
assert type(dr._legacy_parser_for(suffix)).__name__ == name
@pytest.mark.parametrize("suffix", [".odt", ".doc", ".rtf", ".xls", ".ods", ".odp"])
def test_fast_engine_is_terminal_for_anydoc_only_suffixes(monkeypatch, suffix):
"""A suffix only anydoc reads has no legacy parser; `fast` must report that
rather than silently running the configured engine the caller opted out of."""
def _boom(*args, **kwargs):
raise AssertionError("fast fell through to the configured engine map")
monkeypatch.setattr(dr, "get_default_file_extractor", _boom)
monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)
assert dr._pick_parser(suffix, ocr_enabled=False, engine="fast") is None
out = parse_document_bytes(b"\xd0\xcf\x11\xe0 bytes", f"doc{suffix}", output="text", engine="fast")
assert "fast engine" in out["error"] and suffix in out["error"]
def test_fast_engine_still_falls_through_for_images_and_audio(monkeypatch):
"""Formats with no legacy parser and no anydoc alternative keep the configured map."""
seen = []
monkeypatch.setattr(dr, "get_default_file_extractor", lambda **kwargs: seen.append(kwargs) or {})
for suffix in (".png", ".jpg", ".mp3", ".txt"):
dr._pick_parser(suffix, ocr_enabled=True, engine="fast")
assert len(seen) == 4
def test_tables_are_not_collected_when_the_pdf_parser_is_not_docling(monkeypatch):
"""Under OCR_BACKEND=native the docling engine hands PDFs to the native OCR
parser; a vanilla DocumentConverter table pass would OCR the scan again."""
@@ -746,6 +771,46 @@ def test_tables_are_not_collected_when_the_pdf_parser_is_not_docling(monkeypatch
assert calls == []
def test_tables_survive_native_ocr_delegation_for_text_only_pdfs(monkeypatch):
"""OCR_BACKEND=native under the docling engine wraps a DoclingParser as the
native parser's text_parser. A PDF with a text layer on every page is handed
to that parser, so tables must ride that single conversion instead of being
dropped by the scan exclusion."""
from application.parser.file.docling_parser import DoclingParser
from application.parser.file.ocr_parser import NativeOcrPdfParser
class _Docling(DoclingParser):
def __init__(self):
super().__init__()
self._parser_config = {"ready": True}
docling = _Docling()
native = NativeOcrPdfParser(text_parser=docling)
native._parser_config = {}
monkeypatch.setattr(native, "text_layer_delegate", lambda path: docling)
monkeypatch.setattr(
native, "parse_file", lambda path, errors="ignore": pytest.fail("native OCR ran on a text-only PDF")
)
calls = []
def _fake_structured(path, *, ocr_enabled, include_tables, parser=None):
calls.append(parser)
return {"markdown": "# text pdf", "tables": [{"markdown": "| t |"}], "structured": {}, "page_count": 1}
monkeypatch.setattr(dr, "_pick_parser", lambda *a, **k: native)
monkeypatch.setattr(dr, "_effective_engine", lambda engine: "docling")
monkeypatch.setattr(dr, "_docling_structured", _fake_structured)
monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)
out = parse_document_bytes(b"%PDF-1.4", "text.pdf", engine="docling", include_tables=True)
assert out["content"] == "# text pdf"
assert out["tables"] == [{"markdown": "| t |"}]
assert calls == [docling]
def test_binary_office_suffix_without_anydoc_is_an_error_not_text(monkeypatch):
monkeypatch.setattr(dr, "_pick_parser", lambda *a, **k: None)
monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)