diff --git a/application/Dockerfile b/application/Dockerfile index 89bf4825..ffc60ad8 100644 --- a/application/Dockerfile +++ b/application/Dockerfile @@ -46,7 +46,7 @@ RUN pip install --no-cache-dir --upgrade pip && \ # backend, read_document's structured output) — OFF by default: it pulls the # layout/OCR model stack and adds gigabytes to the image. anydoc (in # requirements.txt) is the default parser and needs none of it, and OCR runs -# natively on tesseract (below) without it. +# natively on tesseract (below, also opt-in) without it. ARG INSTALL_DOCLING=false RUN if [ "$INSTALL_DOCLING" = "true" ]; then \ pip install --no-cache-dir -r requirements-docling.txt; \ @@ -67,12 +67,14 @@ RUN apt-get update && \ ln -s /usr/bin/python3.12 /usr/bin/python && \ rm -rf /var/lib/apt/lists/* -# OCR (OCR_ENABLED=true) runs on the system tesseract binary by default, -# with or without the docling extra, so ship it in every image (~35 MB). -# Opt out with --build-arg INSTALL_TESSERACT=false (e.g. a DeepSeek-OCR-only -# deployment). Extra language packs are a deployment concern -# (apt: tesseract-ocr-, then list them in OCR_LANGS). -ARG INSTALL_TESSERACT=true +# Optional tesseract OCR engine (OCR_ENABLED=true with OCR_ENGINE=tesseract, +# the default engine) — OFF by default like every other OCR dependency; OCR +# itself is off unless configured. Opt in with --build-arg +# INSTALL_TESSERACT=true (setup.sh writes it to .env when OCR is enabled); +# ~35 MB of system packages. Extra language packs are a deployment concern +# (apt: tesseract-ocr-, then list them in OCR_LANGS). A DeepSeek-OCR +# endpoint (OCR_ENGINE=deepseek) needs none of this. +ARG INSTALL_TESSERACT=false RUN if [ "$INSTALL_TESSERACT" = "true" ]; then \ apt-get update && \ apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng && \ diff --git a/application/core/settings.py b/application/core/settings.py index 8fdcee32..2babc6fd 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -165,8 +165,9 @@ class Settings(BaseSettings): # menu): # tesseract — recommended: best classic-engine accuracy (perfect EN word # recall, 0.000 bilingual CER, 100% table cells), ~35 MB, CPU-only. - # Needs the system binary + language packs (the Docker image ships - # them unless built with INSTALL_TESSERACT=false). Both backends. + # Needs the system binary + language packs: an optional install like + # every OCR dependency (build with INSTALL_TESSERACT=true, or apt/brew + # install tesseract-ocr for a local run). Both backends. # deepseek — DeepSeek-OCR against an Ollama/vLLM endpoint # (OCR_DEEPSEEK_*). Best table/CJK quality; the worker stays light # (no layout models) but each page costs seconds on the model server. diff --git a/application/parser/document_reader.py b/application/parser/document_reader.py index 7ae4554c..4a8a2e4b 100644 --- a/application/parser/document_reader.py +++ b/application/parser/document_reader.py @@ -186,6 +186,13 @@ def _pick_parser(suffix: str, *, ocr_enabled: bool, engine: str): legacy = _legacy_parser_for(suffix) if legacy is not None: return legacy + from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES + + if suffix in ANYDOC_GAINED_SUFFIXES: + # anydoc-only office formats have no legacy parser. Falling through + # would run the configured engine the caller explicitly opted out + # of; images/audio/.txt still fall through (no legacy alternative). + return None if engine in ("docling", "anydoc"): extractor = get_default_file_extractor(ocr_enabled=ocr_enabled, engine=engine) else: @@ -529,6 +536,16 @@ def _shape( } parser = _pick_parser(suffix, ocr_enabled=ocr_enabled, engine=engine) + # Under OCR_BACKEND=native with the docling engine the PDF parser is the + # native OCR parser wrapping a DoclingParser as its text_parser. A PDF + # with a text layer on every page is handed to that parser untouched, so + # when tables are wanted run it directly: the single docling conversion + # below then yields both the markdown and the tables. Scans and mixed + # documents keep the native parser (and the exclusion just below). + if include_tables and output != "chunks" and _is_native_ocr_parser(parser): + delegate = parser.text_layer_delegate(path) + if _is_docling_parser(delegate): + parser = delegate # Tables come from a Docling conversion, so they are only collected when # a Docling parser is actually in play: under anydoc/fast a table pass # would be a second, full docling conversion that costs more than the @@ -569,6 +586,11 @@ def _shape( from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES if suffix in ANYDOC_GAINED_SUFFIXES: + if engine == "fast": + return { + "error": f"The fast engine has no parser for {suffix} files; " + "use engine='anydoc' or 'auto'." + } return { "error": f"No parser is available for {suffix} files: firecrawl-anydoc " "is not installed (pip install firecrawl-anydoc)" diff --git a/application/parser/file/docling_parser.py b/application/parser/file/docling_parser.py index e1a41cba..77ee25b5 100644 --- a/application/parser/file/docling_parser.py +++ b/application/parser/file/docling_parser.py @@ -103,8 +103,8 @@ def _resolve_ocr_engine(requested: Optional[str]) -> str: if engine == "tesseract" and shutil.which("tesseract") is None: logger.warning( "OCR_ENGINE=tesseract but no tesseract binary is on PATH (install " - "tesseract-ocr plus language packs; the Docker image ships them unless " - "built with INSTALL_TESSERACT=false); using docling auto-selection" + "tesseract-ocr plus language packs: Docker builds need " + "INSTALL_TESSERACT=true); using docling auto-selection" ) return "auto" if engine == "ocrmac" and ( diff --git a/application/parser/file/ocr_parser.py b/application/parser/file/ocr_parser.py index 61c15f19..991f1894 100644 --- a/application/parser/file/ocr_parser.py +++ b/application/parser/file/ocr_parser.py @@ -315,9 +315,9 @@ class TesseractEngine: if not self.available(): raise OcrUnavailableError( "OCR_ENGINE=tesseract but no tesseract binary is on PATH. Install " - "tesseract-ocr plus language packs (the Docker image ships them " - "unless built with INSTALL_TESSERACT=false), or set " - "OCR_ENGINE=deepseek / install the docling extra." + "tesseract-ocr plus language packs (Docker: build with " + "INSTALL_TESSERACT=true; local: apt/brew install tesseract-ocr), " + "or set OCR_ENGINE=deepseek / install the docling extra." ) try: completed = subprocess.run( @@ -699,6 +699,32 @@ class NativeOcrPdfParser(BaseParser): self.last_engine = engine.name return texts + def text_layer_delegate(self, file: Path) -> Optional[BaseParser]: + """The ``text_parser`` that ``parse_file`` would hand ``file`` to, or None. + + None when there is no text parser, the file cannot be opened, or any + page falls below ``min_text_chars`` (those are OCR'd here). Callers + that want a by-product of the delegate's conversion (docling tables) + can run it directly instead of converting the document twice. + """ + if self.text_parser is None: + return None + try: + import pypdfium2 as pdfium + + pdf = pdfium.PdfDocument(str(Path(file))) + except Exception: # noqa: BLE001 - a probe; parse_file reports the real error + return None + try: + counts = self._text_layer_counts(pdf) + except Exception: # noqa: BLE001 + return None + finally: + pdf.close() + if counts and all(count >= self.min_text_chars for count in counts): + return self.text_parser + return None + def _delegate_text(self, path: Path, errors: str) -> Union[str, List[str]]: text_parser = self.text_parser assert text_parser is not None diff --git a/application/parser/file/pdf_trust.py b/application/parser/file/pdf_trust.py index 17ad3003..a14f3db5 100644 --- a/application/parser/file/pdf_trust.py +++ b/application/parser/file/pdf_trust.py @@ -29,7 +29,7 @@ import zlib from pathlib import Path from typing import Dict, Iterator, List, Optional, Union -_OBJ = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL) +_OBJ_HEADER = re.compile(rb"\d+\s+\d+\s+obj") _TYPE0 = re.compile(rb"/Subtype\s*/Type0") # The CJK expectation is keyed on CIDSystemInfo /Ordering ALONE — the # authoritative "this PDF maps text through a CJK character collection" @@ -99,6 +99,28 @@ def _stream_bodies(raw: bytes) -> Iterator[bytes]: yield body +def _object_bodies(raw: bytes) -> Iterator[bytes]: + """The bytes between each ``N G obj`` header and the next ``endobj``. + + A forward walk for the same reason as ``_stream_bodies``: the regex + ``\\d+\\s+\\d+\\s+obj(.*?)endobj`` rescans to end-of-file for every header + that has no ``endobj`` after it, so a malformed (or crafted) PDF with many + such headers costs headers x size — 2000 orphan headers in 1 MB measured + 13.5 s, on the default upload path. Advancing past each ``endobj`` keeps + the walk linear and yields the same bodies. + """ + pos = 0 + while True: + header = _OBJ_HEADER.search(raw, pos) + if header is None: + return + end = raw.find(b"endobj", header.end()) + if end < 0: + return + yield raw[header.end():end] + pos = end + 6 + + def _decompressed_streams(raw: bytes) -> Iterator[bytes]: """Each Flate stream in ``raw`` that decompresses, one at a time, capped.""" for body in _stream_bodies(raw): @@ -145,8 +167,7 @@ def check_pdf_fonts(data: bytes) -> Dict[str, Union[int, bool]]: if b"/ToUnicode" not in unit: bare += 1 - for match in _OBJ.finditer(data): - body = match.group(1) + for body in _object_bodies(data): if _TYPE0.search(body): count_font(body) # Each decompressed stream is scanned for every signal in one pass and diff --git a/deployment/docker-compose-azure.yaml b/deployment/docker-compose-azure.yaml index d991009a..99cd0988 100644 --- a/deployment/docker-compose-azure.yaml +++ b/deployment/docker-compose-azure.yaml @@ -14,7 +14,7 @@ services: context: ../application args: INSTALL_DOCLING: ${INSTALL_DOCLING:-false} - INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true} + INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false} env_file: - ../.env environment: @@ -39,7 +39,7 @@ services: context: ../application args: INSTALL_DOCLING: ${INSTALL_DOCLING:-false} - INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true} + INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false} # `parsing` queue carries read_document/parse_document; required for its await to resolve. command: celery -A application.app.celery worker -l INFO -Q docsgpt,parsing env_file: diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index 4a974fb3..4835c4f9 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -21,8 +21,10 @@ services: # Bake the optional docling engine (layout-model OCR backend, structured # output) into the image: set INSTALL_DOCLING=true in ../.env or the shell. INSTALL_DOCLING: ${INSTALL_DOCLING:-false} - # The tesseract binary behind OCR_ENABLED=true; on by default (~35 MB). - INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true} + # Bake the tesseract binary behind OCR_ENABLED=true (~35 MB): set + # INSTALL_TESSERACT=true in ../.env or the shell (setup.sh does this + # when OCR is enabled). Off by default, like docling. + INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false} env_file: - ../.env environment: @@ -49,7 +51,7 @@ services: context: ../application args: INSTALL_DOCLING: ${INSTALL_DOCLING:-false} - INSTALL_TESSERACT: ${INSTALL_TESSERACT:-true} + INSTALL_TESSERACT: ${INSTALL_TESSERACT:-false} # Consumes the default queue AND the dedicated `parsing` queue (read_document / # parse_document). Without `parsing` here the read_document await never resolves. # For heavy/OCR parsing run a separate worker with `-Q parsing`. diff --git a/docs/content/Deploying/DocsGPT-Settings.mdx b/docs/content/Deploying/DocsGPT-Settings.mdx index d51d128d..cceb9ce8 100644 --- a/docs/content/Deploying/DocsGPT-Settings.mdx +++ b/docs/content/Deploying/DocsGPT-Settings.mdx @@ -216,7 +216,7 @@ for the engines and flows. | `MARKUP_MAX_BYTES` | `8000000` | anydoc engine: HTML/XHTML larger than this are head-truncated before the `markdownify` parser runs (its tree costs ~50x the input). `0` disables the gate. | | `PDF_TRUST_CHECK` | `true` | Trust-check anydoc's PDF output for the two silent-loss classes (Type0 fonts without ToUnicode, CJK-declaring PDFs with CJK-less text). Flagged files re-parse on Docling when installed, else carry `parse_warnings` metadata. | | `ANYDOC_TABLEIZE` | `false` | Rewrite dot-leader / whitespace-aligned table runs in anydoc's PDF markdown into GFM tables. | -| `OCR_ENGINE` | `tesseract` | OCR engine when OCR is on: `tesseract` (recommended, both backends), `deepseek` (VLM endpoint, both backends), or the Docling-only `auto`, `ocrmac`, `rapidocr`. Unavailable engines degrade with a warning. See the [OCR guide](/Guides/ocr). | +| `OCR_ENGINE` | `tesseract` | OCR engine when OCR is on: `tesseract` (recommended, both backends; optional system package — build with `INSTALL_TESSERACT=true` or `apt`/`brew` install it), `deepseek` (VLM endpoint, both backends), or the Docling-only `auto`, `ocrmac`, `rapidocr`. Unavailable engines degrade with a warning. See the [OCR guide](/Guides/ocr). | | `OCR_LANGS` | `eng` | Tesseract language packs, `+`-separated (e.g. `eng+chi_sim+deu`). | | `OCR_DEEPSEEK_URL` | `http://localhost:11434/v1/chat/completions` | OpenAI-compatible endpoint for `OCR_ENGINE=deepseek` (Ollama or vLLM). | | `OCR_DEEPSEEK_MODEL` | `deepseek-ocr:3b` | Model name at that endpoint. | diff --git a/docs/content/Guides/ocr.mdx b/docs/content/Guides/ocr.mdx index 9c0a6b44..4850d8ea 100644 --- a/docs/content/Guides/ocr.mdx +++ b/docs/content/Guides/ocr.mdx @@ -43,6 +43,34 @@ and — when it is the OCR backend — scanned PDFs and images. Without Docling those formats use the standard parsers; OCR still works through the native backend, and with OCR off images are only read when `PARSE_IMAGE_REMOTE=true`. +## Installing tesseract + +Nothing OCR-related ships in the base install: the default engine, +`tesseract`, is a ~35 MB system package that you opt into like any other OCR +dependency. Locally: + +```bash +# Debian/Ubuntu # macOS +apt-get install tesseract-ocr tesseract-ocr-eng brew install tesseract +``` + +Docker images build without it by default; opt in with the build argument: + +```bash +docker build --build-arg INSTALL_TESSERACT=true ./application +``` + +`deployment/docker-compose.yaml` forwards the same switch, so setting +`INSTALL_TESSERACT=true` in `.env` (or the shell) bakes tesseract plus the +English pack into locally built backend and worker images; `setup.sh` writes +it when you answer yes to the OCR question after choosing to build images +locally. Pre-built Docker Hub images (`docker-compose-hub.yaml`) do not +include it, so `setup.sh` leaves OCR at its default (off) for them; to OCR +there, point `OCR_ENGINE=deepseek` at a DeepSeek-OCR endpoint or install +`tesseract-ocr` in a derived image. With `OCR_ENABLED=true` and no binary on +`PATH`, scanned pages fail with an install hint (text-layer documents are +unaffected). + ## Installing the docling engine docling is not part of the base install, and OCR does not need it (see @@ -112,7 +140,7 @@ Docling is not the only way to OCR. The `native` backend renders the pages that need it with pypdfium2 and Pillow — both already core dependencies — and feeds them straight into tesseract or a DeepSeek-OCR endpoint. No ML models load in the worker, and nothing beyond the ~35 MB tesseract binary -(shipped in the Docker image) is installed. +(see [Installing tesseract](#installing-tesseract)) is needed. | `OCR_BACKEND` | What runs | When to pick it | |---|---|---| @@ -135,7 +163,7 @@ scans (all engines driven through docling so layout handling is identical): | `OCR_ENGINE` | Backends | Role | Notes | |---|---|---|---| -| `tesseract` | native, docling | **recommended default** | Best classic-engine accuracy in the bench: perfect EN word recall on all docs, 0.000 CER on the bilingual page, 100% table cells, robust to mild degradation. ~35 MB of *system* packages, CPU-only. Needs the `tesseract` binary + language packs — shipped in the Docker image (opt out with `--build-arg INSTALL_TESSERACT=false`); set languages via `OCR_LANGS` (e.g. `eng+chi_sim`). | +| `tesseract` | native, docling | **recommended default** | Best classic-engine accuracy in the bench: perfect EN word recall on all docs, 0.000 CER on the bilingual page, 100% table cells, robust to mild degradation. ~35 MB of *system* packages, CPU-only. Needs the `tesseract` binary + language packs — an optional install like every OCR dependency (see [Installing tesseract](#installing-tesseract)); set languages via `OCR_LANGS` (e.g. `eng+chi_sim`). | | `deepseek` | native, docling | best quality, heavy on the *server* | DeepSeek-OCR against an OpenAI-compatible endpoint. Only engine that reconstructs totals rows as table rows; near-perfect CJK; barely affected by degradation. The ingestion worker stays light (no layout models); the model runs in Ollama or vLLM. Costs: a GPU/Apple-Silicon endpoint, ~seconds per page, and occasional silent drops of page-level elements (titles). | | `auto` | docling | convenience | docling picks: `ocrmac` on macOS (excellent, ~1 s/page), `rapidocr` on Linux — see below before relying on it server-side. Also docling's automatic fallback whenever the selected engine is not installed. The native backend runs tesseract for it. | | `ocrmac` | docling | macOS only | Best raw accuracy and fastest of all classic engines; irrelevant for Linux deploys. | @@ -233,8 +261,8 @@ With `OCR_BACKEND=docling`, OCR behavior is different for PDFs vs images: - Image parser defaults to full-page OCR (the whole image is visual content). The engine and its languages come from `OCR_ENGINE` and `OCR_LANGS` (see the -table above). The Docker image ships only the English tesseract pack; for -other languages install their packs in the image (e.g. +table above). `INSTALL_TESSERACT=true` installs only the English tesseract +pack; for other languages install their packs in the image (e.g. `apt-get install tesseract-ocr-chi-sim`) and list them in `OCR_LANGS` (`eng+chi_sim`). diff --git a/setup.sh b/setup.sh index 66c4b6a6..29089f86 100755 --- a/setup.sh +++ b/setup.sh @@ -362,10 +362,22 @@ configure_doc_processing() { echo -e "${GREEN}PDF-as-image parsing enabled.${NC}" fi + # OCR needs the tesseract binary, an optional system package that only + # locally built images can include (INSTALL_TESSERACT build arg). The + # pre-built Docker Hub images ship without it, so there OCR stays off + # (its default) rather than being switched on to fail on every scan. + if [[ "$COMPOSE_FILE" != "$COMPOSE_FILE_LOCAL" ]]; then + echo -e "${YELLOW}OCR for scanned PDFs and images stays off: the pre-built Docker Hub images do not include tesseract. To use OCR, rerun setup and choose option 5 (build images locally), or point OCR_ENGINE=deepseek at a DeepSeek-OCR endpoint in .env.${NC}" + return + fi + read -p "$(echo -e "${DEFAULT_FG}Enable OCR for scanned PDFs and images? (y/N): ${NC}")" ocr_enabled if [[ "$ocr_enabled" =~ ^[yY]$ ]]; then echo "OCR_ENABLED=true" >> "$ENV_FILE" - echo -e "${GREEN}OCR enabled (tesseract, shipped in the Docker image; set OCR_ENGINE=deepseek for a DeepSeek-OCR endpoint).${NC}" + # Bakes tesseract into the locally built images (docker compose + # --env-file .env build). + echo "INSTALL_TESSERACT=true" >> "$ENV_FILE" + echo -e "${GREEN}OCR enabled. tesseract will be built into the images (INSTALL_TESSERACT=true); set OCR_ENGINE=deepseek in .env for a DeepSeek-OCR endpoint instead.${NC}" read -p "$(echo -e "${DEFAULT_FG}Also install the Docling layout engine for OCR (better tables/reading order, several GB heavier)? (y/N): ${NC}")" docling_ocr if [[ "$docling_ocr" =~ ^[yY]$ ]]; then # Locally built images include docling via this build arg; it becomes diff --git a/tests/parser/file/test_anydoc_parser.py b/tests/parser/file/test_anydoc_parser.py index 6f86fa07..399cd8f3 100644 --- a/tests/parser/file/test_anydoc_parser.py +++ b/tests/parser/file/test_anydoc_parser.py @@ -13,6 +13,7 @@ Three behaviours matter and are asserted separately: import importlib.machinery import sys import types +from pathlib import Path import pytest @@ -319,9 +320,7 @@ def test_init_parser_imports_for_real_not_just_find_spec(monkeypatch): # --- PDF trust check + tableize wiring (PR 3) ----------------------------------- -from pathlib import Path as _Path - -FIXTURES = _Path(__file__).parent / "fixtures" +FIXTURES = Path(__file__).parent / "fixtures" CID_PDF = FIXTURES / "nda_en_zh_cid_font.pdf" diff --git a/tests/parser/file/test_docling_parser.py b/tests/parser/file/test_docling_parser.py index 147d7d4a..682f5435 100644 --- a/tests/parser/file/test_docling_parser.py +++ b/tests/parser/file/test_docling_parser.py @@ -158,8 +158,12 @@ class TestOcrEngineSelection: def test_build_tesseract_reads_ocr_langs(self, settings, monkeypatch): pytest.importorskip("docling") + import application.parser.file.ocr_parser as op from application.parser.file.docling_parser import _build_ocr_options + # Pack inventory unknown: the resolved list is passed through untouched + # (a host with tesseract but no chi_sim pack would otherwise drop it). + monkeypatch.setattr(op, "tesseract_languages", lambda: None) monkeypatch.setattr(settings, "OCR_LANGS", "eng+chi_sim") options = _build_ocr_options("tesseract", None, True) diff --git a/tests/parser/file/test_ocr_parser.py b/tests/parser/file/test_ocr_parser.py index e2e3f6e2..2764c3a4 100644 --- a/tests/parser/file/test_ocr_parser.py +++ b/tests/parser/file/test_ocr_parser.py @@ -964,3 +964,38 @@ def test_delegate_parse_lets_setup_errors_through(): with pytest.raises(ImportError, match="docling is required"): delegate_parse(_NeedsLib(), Path("x.pdf"), "ignore") + + +class TestTextLayerDelegate: + """``text_layer_delegate`` mirrors ``parse_file``'s delegation decision.""" + + def test_every_page_with_text_returns_the_text_parser(self, monkeypatch): + import pypdfium2 as pdfium + + class _Pdf: + def close(self): + pass + + monkeypatch.setattr(pdfium, "PdfDocument", lambda path: _Pdf()) + text_parser = object() + parser = op.NativeOcrPdfParser(text_parser=text_parser, min_text_chars=20) + monkeypatch.setattr(parser, "_text_layer_counts", lambda pdf: [50, 60]) + assert parser.text_layer_delegate(Path("x.pdf")) is text_parser + + def test_a_scanned_page_keeps_the_native_path(self, monkeypatch): + import pypdfium2 as pdfium + + class _Pdf: + def close(self): + pass + + monkeypatch.setattr(pdfium, "PdfDocument", lambda path: _Pdf()) + parser = op.NativeOcrPdfParser(text_parser=object(), min_text_chars=20) + monkeypatch.setattr(parser, "_text_layer_counts", lambda pdf: [50, 3]) + assert parser.text_layer_delegate(Path("x.pdf")) is None + + def test_no_text_parser_or_unreadable_file_is_none(self, tmp_path): + assert op.NativeOcrPdfParser().text_layer_delegate(Path("x.pdf")) is None + broken = tmp_path / "broken.pdf" + broken.write_bytes(b"not a pdf") + assert op.NativeOcrPdfParser(text_parser=object()).text_layer_delegate(broken) is None diff --git a/tests/parser/file/test_pdf_trust.py b/tests/parser/file/test_pdf_trust.py index 9ca76d18..c495817d 100644 --- a/tests/parser/file/test_pdf_trust.py +++ b/tests/parser/file/test_pdf_trust.py @@ -182,6 +182,36 @@ class TestStreamWalk: assert result["type0_no_tounicode"] == 1 +class TestObjectWalk: + def test_bodies_match_the_regex_reference(self): + """The forward walk yields exactly what ``\\d+\\s+\\d+\\s+obj(.*?)endobj`` did.""" + import re + + from application.parser.file.pdf_trust import _object_bodies + + reference = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL) + data = ( + TYPE0_BARE + + b"7 0 obj no terminator here " # orphan header: body runs to the next endobj + + SIMPLE_FONT + + b"junk 12 0 obj<>endobj" + + _stream(b"x") + + b"99 0 obj trailing orphan" + ) + assert list(_object_bodies(data)) == [m.group(1) for m in reference.finditer(data)] + + def test_orphan_headers_are_linear(self): + """Headers with no ``endobj`` after them: the old regex rescanned to EOF for + each one (2000 in 1 MB took 13.5 s), pinning an ingest worker per upload.""" + import time + + data = TYPE0_BARE + b"1 0 obj\n" * 20_000 + b"\x00" * 500_000 + started = time.perf_counter() + result = check_pdf_fonts(data) + assert time.perf_counter() - started < 1.0 + assert result["type0_no_tounicode"] == 1 + + # --- the CJK cross-check is confirmed against pdfium's text layer -------------- diff --git a/tests/parser/test_document_reader.py b/tests/parser/test_document_reader.py index f110431b..b2627e8a 100644 --- a/tests/parser/test_document_reader.py +++ b/tests/parser/test_document_reader.py @@ -724,6 +724,31 @@ def test_fast_engine_covers_the_whole_legacy_map(): assert type(dr._legacy_parser_for(suffix)).__name__ == name +@pytest.mark.parametrize("suffix", [".odt", ".doc", ".rtf", ".xls", ".ods", ".odp"]) +def test_fast_engine_is_terminal_for_anydoc_only_suffixes(monkeypatch, suffix): + """A suffix only anydoc reads has no legacy parser; `fast` must report that + rather than silently running the configured engine the caller opted out of.""" + + def _boom(*args, **kwargs): + raise AssertionError("fast fell through to the configured engine map") + + monkeypatch.setattr(dr, "get_default_file_extractor", _boom) + monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None) + + assert dr._pick_parser(suffix, ocr_enabled=False, engine="fast") is None + out = parse_document_bytes(b"\xd0\xcf\x11\xe0 bytes", f"doc{suffix}", output="text", engine="fast") + assert "fast engine" in out["error"] and suffix in out["error"] + + +def test_fast_engine_still_falls_through_for_images_and_audio(monkeypatch): + """Formats with no legacy parser and no anydoc alternative keep the configured map.""" + seen = [] + monkeypatch.setattr(dr, "get_default_file_extractor", lambda **kwargs: seen.append(kwargs) or {}) + for suffix in (".png", ".jpg", ".mp3", ".txt"): + dr._pick_parser(suffix, ocr_enabled=True, engine="fast") + assert len(seen) == 4 + + def test_tables_are_not_collected_when_the_pdf_parser_is_not_docling(monkeypatch): """Under OCR_BACKEND=native the docling engine hands PDFs to the native OCR parser; a vanilla DocumentConverter table pass would OCR the scan again.""" @@ -746,6 +771,46 @@ def test_tables_are_not_collected_when_the_pdf_parser_is_not_docling(monkeypatch assert calls == [] +def test_tables_survive_native_ocr_delegation_for_text_only_pdfs(monkeypatch): + """OCR_BACKEND=native under the docling engine wraps a DoclingParser as the + native parser's text_parser. A PDF with a text layer on every page is handed + to that parser, so tables must ride that single conversion instead of being + dropped by the scan exclusion.""" + + from application.parser.file.docling_parser import DoclingParser + from application.parser.file.ocr_parser import NativeOcrPdfParser + + class _Docling(DoclingParser): + def __init__(self): + super().__init__() + self._parser_config = {"ready": True} + + docling = _Docling() + native = NativeOcrPdfParser(text_parser=docling) + native._parser_config = {} + monkeypatch.setattr(native, "text_layer_delegate", lambda path: docling) + monkeypatch.setattr( + native, "parse_file", lambda path, errors="ignore": pytest.fail("native OCR ran on a text-only PDF") + ) + + calls = [] + + def _fake_structured(path, *, ocr_enabled, include_tables, parser=None): + calls.append(parser) + return {"markdown": "# text pdf", "tables": [{"markdown": "| t |"}], "structured": {}, "page_count": 1} + + monkeypatch.setattr(dr, "_pick_parser", lambda *a, **k: native) + monkeypatch.setattr(dr, "_effective_engine", lambda engine: "docling") + monkeypatch.setattr(dr, "_docling_structured", _fake_structured) + monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None) + + out = parse_document_bytes(b"%PDF-1.4", "text.pdf", engine="docling", include_tables=True) + + assert out["content"] == "# text pdf" + assert out["tables"] == [{"markdown": "| t |"}] + assert calls == [docling] + + def test_binary_office_suffix_without_anydoc_is_an_error_not_text(monkeypatch): monkeypatch.setattr(dr, "_pick_parser", lambda *a, **k: None) monkeypatch.setattr(dr, "_zip_bomb_reason", lambda *a, **k: None)