diff --git a/application/core/settings.py b/application/core/settings.py index 9ddc7328..8fdcee32 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -183,9 +183,10 @@ class Settings(BaseSettings): OCR_LANGS: str = "eng" OCR_DEEPSEEK_URL: str = "http://localhost:11434/v1/chat/completions" OCR_DEEPSEEK_MODEL: str = "deepseek-ocr:3b" - # Native backend only: seconds allowed per page request to the DeepSeek - # endpoint (pages are sent one at a time). A 3B model on a laptop needs - # minutes; a vLLM GPU deployment, seconds. + # Seconds allowed per page request to the DeepSeek endpoint, on both + # backends (native sends pages one at a time; docling's VLM pipeline + # keeps its own concurrency). A 3B model on a laptop needs minutes; a + # vLLM GPU deployment, seconds. OCR_DEEPSEEK_TIMEOUT: float = 300.0 # Native backend only: resolution at which pages without a text layer are # rendered before OCR. 200 suits tesseract; clamped to 72-600. diff --git a/application/parser/file/anydoc_parser.py b/application/parser/file/anydoc_parser.py index cebb220c..68d3e974 100644 --- a/application/parser/file/anydoc_parser.py +++ b/application/parser/file/anydoc_parser.py @@ -68,6 +68,12 @@ ANYDOC_SUFFIXES: Tuple[str, ...] = ( # else in ``ANYDOC_SUFFIXES`` is anydoc-only ("gained") and is mapped to a # fallback-less ``AnydocParser`` in every parser map. _CORE_SUFFIXES = frozenset({".pdf", ".docx", ".pptx", ".xlsx", ".csv"}) + +# Spreadsheets anydoc may refuse on its fixed, non-configurable limits +# (2M XML nodes per part, 128 MiB decompressed per part — a ~265k-row sheet +# trips them) that the plain tabular parsers read fine in a few hundred MB. +# For these a ResourceLimitError is a reason to delegate, not to fail. +_TABULAR_SUFFIXES = frozenset({".csv", ".xlsx", ".xlsm", ".xlsb", ".xls", ".ods"}) ANYDOC_GAINED_SUFFIXES: Tuple[str, ...] = tuple( suffix for suffix in ANYDOC_SUFFIXES if suffix not in _CORE_SUFFIXES ) @@ -171,9 +177,14 @@ class AnydocParser(BaseParser): content = anydoc.to_markdown(str(path)) except anydoc.ResourceLimitError as exc: # anydoc refused because the file is too expensive (declared - # decompression size, nesting depth, node count). A heavier - # engine would spend exactly what was just refused, so this is - # terminal rather than a reason to delegate. + # decompression size, nesting depth, node count). For office + # documents a heavier engine would spend exactly what was just + # refused, so this is terminal. Spreadsheets are the exception: + # anydoc's cell/node limits sit below what ordinary business + # exports reach, and the tabular fallback (docling's gate hands + # oversized sheets to pandas/openpyxl) is lighter, not heavier. + if path.suffix.lower() in _TABULAR_SUFFIXES and self.fallback_parser is not None: + return self._delegate(path, errors, f"{type(exc).__name__}: {exc}") self.last_engine = None raise DocumentParseError( f"Failed to parse {path.name}: {type(exc).__name__}: {exc}" @@ -304,10 +315,13 @@ class AnydocParser(BaseParser): Runs only when the fallback parser both has OCR on and exposes ``ocr_pages`` (the native OCR PDF parser, or the docling PDF parser). With OCR off the fallback is the plain pypdf parser and mixed - documents keep today's behaviour: text pages only. An OCR failure is - logged and the anydoc text kept — a partial document beats a - rejected upload, and the loud-failure path stays reserved for fully - scanned files. + documents keep today's behaviour: text pages only. Pages are OCR'd + one at a time so a failure on one page costs that page, not the OCR + of every other; the anydoc text is always kept — a partial document + beats a rejected upload, and the loud-failure path stays reserved + for fully scanned files. anydoc's PDF markdown carries no page + separators, so the OCR'd pages are appended after the text pages + rather than spliced into position. """ fallback = self.fallback_parser if not (getattr(fallback, "ocr_enabled", False) and hasattr(fallback, "ocr_pages")): @@ -326,7 +340,6 @@ class AnydocParser(BaseParser): try: if not fallback.parser_config_set: fallback.init_parser() - texts = fallback.ocr_pages(path, indices) except Exception: # noqa: BLE001 - keep the text pages rather than fail the file logger.warning( "OCR of the scanned pages of %s failed; indexing the text pages only", @@ -334,7 +347,20 @@ class AnydocParser(BaseParser): exc_info=True, ) return content - extra = [texts[index] for index in indices if texts.get(index, "").strip()] + extra: List[str] = [] + for index in indices: + try: + text = fallback.ocr_pages(path, [index]).get(index, "") + except Exception: # noqa: BLE001 - one page's failure must not drop the others + logger.warning( + "OCR of page %d of %s failed; skipping that page", + index + 1, + path.name, + exc_info=True, + ) + continue + if text and text.strip(): + extra.append(text) if not extra: return content self._last_ocr_pages = (path, len(extra)) diff --git a/application/parser/file/docling_parser.py b/application/parser/file/docling_parser.py index 47d96836..365e24a4 100644 --- a/application/parser/file/docling_parser.py +++ b/application/parser/file/docling_parser.py @@ -543,6 +543,10 @@ class DoclingParser(BaseParser): vlm_options = vlm_model_specs.DEEPSEEKOCR_OLLAMA.model_copy(deep=True) vlm_options.url = settings.OCR_DEEPSEEK_URL vlm_options.params["model"] = settings.OCR_DEEPSEEK_MODEL + # docling's spec default is 90 s per request, which a laptop-hosted + # 3B model overruns; honour the same per-page budget the native + # backend gives the endpoint. + vlm_options.timeout = float(settings.OCR_DEEPSEEK_TIMEOUT) pipeline_options = VlmPipelineOptions( vlm_options=vlm_options, enable_remote_services=True ) @@ -622,7 +626,12 @@ class DoclingParser(BaseParser): tesseract's CJK models space every glyph ("互相 保密 协议"); the native backend collapses that in ``TesseractEngine`` and the docling path needs the same treatment or Chinese scans index worse here than there. - Other engines (and OCR off) return the export untouched. + Other engines (and OCR off) return the export untouched. Only the PDF + and image parsers construct with OCR on — the office/markup parsers + never OCR, so their born-digital text must not be rewritten. On a + hybrid-OCR PDF the collapse still covers the whole export, native + text included; deliberate spaces inside CJK runs are rare enough + there that indexing scans correctly wins. """ if self.ocr_enabled and self._active_ocr_engine == "tesseract": return collapse_cjk_spaces(content) @@ -866,21 +875,21 @@ class DoclingDocxParser(DoclingParser): """Docling-based DOCX parser.""" def __init__(self): - super().__init__(export_format="markdown") + super().__init__(ocr_enabled=False, export_format="markdown") class DoclingPPTXParser(DoclingParser): """Docling-based PPTX parser.""" def __init__(self): - super().__init__(export_format="markdown") + super().__init__(ocr_enabled=False, export_format="markdown") class DoclingXLSXParser(DoclingParser): """Docling-based XLSX parser with table structure.""" def __init__(self): - super().__init__(table_structure=True, export_format="markdown") + super().__init__(ocr_enabled=False, table_structure=True, export_format="markdown") def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]: """Parse an XLSX file, delegating oversized files to ``ExcelParser``.""" @@ -899,7 +908,7 @@ class DoclingHTMLParser(DoclingParser): """Docling-based HTML parser.""" def __init__(self): - super().__init__(export_format="markdown") + super().__init__(ocr_enabled=False, export_format="markdown") def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]: """Parse HTML, truncating oversized element-dense markup first.""" @@ -933,7 +942,7 @@ class DoclingCSVParser(DoclingParser): """Docling-based CSV parser.""" def __init__(self): - super().__init__(table_structure=True, export_format="markdown") + super().__init__(ocr_enabled=False, table_structure=True, export_format="markdown") def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]: """Parse a CSV file, delegating oversized files to the plain ``CSVParser``.""" @@ -952,21 +961,21 @@ class DoclingMarkdownParser(DoclingParser): """Docling-based Markdown parser.""" def __init__(self): - super().__init__(export_format="markdown") + super().__init__(ocr_enabled=False, export_format="markdown") class DoclingAsciiDocParser(DoclingParser): """Docling-based AsciiDoc parser.""" def __init__(self): - super().__init__(export_format="markdown") + super().__init__(ocr_enabled=False, export_format="markdown") class DoclingVTTParser(DoclingParser): """Docling-based WebVTT (video text tracks) parser.""" def __init__(self): - super().__init__(export_format="markdown") + super().__init__(ocr_enabled=False, export_format="markdown") def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]: """Parse WebVTT, truncating oversized cue-dense tracks first.""" @@ -977,4 +986,4 @@ class DoclingXMLParser(DoclingParser): """Docling-based XML parser (USPTO, JATS).""" def __init__(self): - super().__init__(export_format="markdown") + super().__init__(ocr_enabled=False, export_format="markdown") diff --git a/application/parser/file/ocr_parser.py b/application/parser/file/ocr_parser.py index 2d4938b9..331d9b27 100644 --- a/application/parser/file/ocr_parser.py +++ b/application/parser/file/ocr_parser.py @@ -56,6 +56,12 @@ DEFAULT_TEXT_LAYER_MIN_CHARS = 32 _DEFAULT_MIN_CHARS_PER_PAGE = 20 _DEFAULT_RENDER_DPI = 200 _MIN_RENDER_DPI, _MAX_RENDER_DPI = 72, 600 +# Pixel budget for one rendered page. pypdfium2 allocates width*height*4 +# bytes up front with no cap of its own, so a 14400x14400 pt page (the PDF +# maximum; a 191-byte file) at 200 dpi would ask for 40000x40000 px = 6 GiB. +# 40 MP is ~10x a Letter page at 200 dpi and still OCRs an A0 poster at a +# usable ~125 dpi; anything larger renders at the scale that fits. +_MAX_RENDER_PIXELS = 40_000_000 _TESSERACT_TIMEOUT_SECONDS = 300 # docling's DeepSeek-OCR prompt minus its ``<|grounding|>`` prefix: grounding # makes the model wrap every element in ref/det tags with bounding boxes, @@ -368,9 +374,32 @@ def build_native_ocr_engine(engine: Optional[str] = None, languages: Optional[Li # --------------------------------------------------------------------------- +def render_scale(page_width_pt: float, page_height_pt: float, dpi: int) -> float: + """Render scale for a page of the given size at ``dpi``, capped by ``_MAX_RENDER_PIXELS``. + + The cap is what stands between an oversized (or hostile) MediaBox and a + multi-gigabyte bitmap allocation in the ingest worker. + """ + scale = dpi / 72.0 + area = max(1.0, float(page_width_pt)) * max(1.0, float(page_height_pt)) + max_scale = (_MAX_RENDER_PIXELS / area) ** 0.5 + return min(scale, max_scale) + + def _render_page(page, dpi: int): - """Render a pypdfium2 page to a PIL image at ``dpi``.""" - return page.render(scale=dpi / 72.0).to_pil() + """Render a pypdfium2 page to a PIL image at ``dpi``, within the pixel budget.""" + width_pt, height_pt = page.get_size() + scale = render_scale(width_pt, height_pt, dpi) + if scale < dpi / 72.0: + logger.warning( + "Page is %.0fx%.0f pt; rendering at %.0f dpi instead of %d to stay within %d MP", + width_pt, + height_pt, + scale * 72.0, + dpi, + _MAX_RENDER_PIXELS // 1_000_000, + ) + return page.render(scale=scale).to_pil() def text_layer_counts(path: Path) -> List[int]: @@ -399,20 +428,54 @@ def text_layer_counts(path: Path) -> List[int]: return counts +def _page_has_image(page) -> bool: + """Whether the page draws at least one image XObject (something OCR could read).""" + import pypdfium2.raw as pdfium_c + + try: + return next(iter(page.get_objects(filter=(pdfium_c.FPDF_PAGEOBJ_IMAGE,))), None) is not None + except Exception: # noqa: BLE001 - treat an unreadable page as image-bearing + return True + + def scanned_page_indices(path: Path, min_chars: int = DEFAULT_TEXT_LAYER_MIN_CHARS) -> List[int]: - """0-based indices of the pages whose text layer is too thin to be real. + """0-based indices of the pages that look scanned: no text layer, or a thin one over an image. The probe behind mixed-document handling: a text-layer converter (anydoc) reads the text pages and silently skips these, so whoever owns OCR has to - fill them in. Returns [] when the file cannot be probed — the caller then - keeps what it has rather than failing a successful parse. + fill them in. A page with no text at all always qualifies. A page with a + thin text layer (under ``min_chars``: a Bates stamp on a scan, a page + number under a full-page figure) qualifies only when it also draws an + image — a cover page reading just "Annual Report 2025" has nothing to OCR, + and OCR-ing it would duplicate text the converter already read. Returns [] + when the file cannot be probed — the caller then keeps what it has rather + than failing a successful parse. """ + import pypdfium2 as pdfium + + floor = max(1, int(min_chars)) + indices: List[int] = [] try: - counts = text_layer_counts(Path(path)) + pdf = pdfium.PdfDocument(str(path)) + try: + for index in range(len(pdf)): + page = pdf[index] + try: + textpage = page.get_textpage() + try: + count = max(0, textpage.count_chars()) + finally: + textpage.close() + if count == 0 or (count < floor and _page_has_image(page)): + indices.append(index) + finally: + page.close() + finally: + pdf.close() except Exception: # noqa: BLE001 - a probe must never fail the parse logger.warning("Could not probe %s for scanned pages", Path(path).name, exc_info=True) return [] - return [index for index, count in enumerate(counts) if count < max(1, int(min_chars))] + return indices def _check_near_empty(name: str, engine: str, content: str, pages: int, ocr_pages: int) -> None: diff --git a/application/parser/file/pdf_trust.py b/application/parser/file/pdf_trust.py index 0b5686e5..17ad3003 100644 --- a/application/parser/file/pdf_trust.py +++ b/application/parser/file/pdf_trust.py @@ -27,10 +27,9 @@ silent-drop cases with zero false positives on the other 14 PDFs, and cost import re import zlib from pathlib import Path -from typing import Dict, Iterator, List, Union +from typing import Dict, Iterator, List, Optional, Union _OBJ = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL) -_STREAM = re.compile(rb"stream\r?\n(.*?)\r?\nendstream", re.DOTALL) _TYPE0 = re.compile(rb"/Subtype\s*/Type0") # The CJK expectation is keyed on CIDSystemInfo /Ordering ALONE — the # authoritative "this PDF maps text through a CJK character collection" @@ -49,6 +48,7 @@ _WINDOW_BEFORE, _WINDOW_AFTER = 200, 800 # Extracted text with fewer CJK characters than this, from a PDF that # declares CJK fonts, is treated as a silent drop. _MIN_CJK_CHARS = 10 +_CJK_PROBLEM_PREFIX = "PDF declares CJK fonts" _CJK_RANGES = ( ("一", "鿿"), # CJK Unified Ideographs @@ -64,13 +64,46 @@ _CJK_RANGES = ( _STREAM_INFLATE_CAP = 4_000_000 +def _stream_bodies(raw: bytes) -> Iterator[bytes]: + """The raw bytes between each ``stream`` / ``endstream`` keyword pair. + + A ``find`` walk rather than a regex: ``stream\\r?\\n(.*?)\\r?\\nendstream`` + misses every stream whose data runs straight into ``endstream`` with no + EOL (common in the wild), and when it misses one its lazy group scans on + to the *next* stream's terminator — quadratic on large files (a 10 MB + file measured 32 s, matching a fifth of its streams) and a merged, + undecodable body for the rest. + """ + pos = 0 + while True: + pos = raw.find(b"stream", pos) + if pos < 0: + return + if raw[max(0, pos - 3):pos] == b"end": + pos += 6 + continue + start = pos + 6 + if raw[start:start + 2] == b"\r\n": + start += 2 + elif raw[start:start + 1] in (b"\n", b"\r"): + start += 1 + end = raw.find(b"endstream", start) + if end < 0: + return + body = raw[start:end] + if body.endswith(b"\r\n"): + body = body[:-2] + elif body.endswith((b"\n", b"\r")): + body = body[:-1] + pos = end + 9 + yield body + + def _decompressed_streams(raw: bytes) -> Iterator[bytes]: """Each Flate stream in ``raw`` that decompresses, one at a time, capped.""" - for match in _STREAM.finditer(raw): + for body in _stream_bodies(raw): try: - inflated = zlib.decompressobj().decompress( - match.group(1), _STREAM_INFLATE_CAP - ) + inflated = zlib.decompressobj().decompress(body, _STREAM_INFLATE_CAP) except zlib.error: continue yield inflated @@ -162,16 +195,65 @@ def verify_extraction(data: bytes, markdown: str) -> List[str]: cjk = _cjk_chars(markdown, _MIN_CJK_CHARS) if cjk < _MIN_CJK_CHARS: problems.append( - f"PDF declares CJK fonts but the extracted text contains only " + f"{_CJK_PROBLEM_PREFIX} but the extracted text contains only " f"{cjk} CJK character(s)" ) return problems +def _text_layer_cjk_chars(file: Path, up_to: int) -> Optional[int]: + """CJK characters in the text layer as pypdfium2 reads it, stopping at ``up_to``. + + pdfium ships Adobe's predefined CMaps, so it is an independent reference + for whether the document *has* CJK text to lose. None when pypdfium2 is + unavailable or the file cannot be read that way. + """ + try: + import pypdfium2 as pdfium + except ImportError: + return None + try: + pdf = pdfium.PdfDocument(str(file)) + except Exception: # noqa: BLE001 - a reference probe, not a parse + return None + count = 0 + try: + for index in range(len(pdf)): + page = pdf[index] + try: + textpage = page.get_textpage() + try: + text = textpage.get_text_bounded() + finally: + textpage.close() + finally: + page.close() + count += _cjk_chars(text, up_to - count) + if count >= up_to: + break + except Exception: # noqa: BLE001 + return None + finally: + pdf.close() + return count + + def verify_pdf_file(file: Path, markdown: str) -> List[str]: - """``verify_extraction`` for a file on disk; unreadable files trust the output.""" + """``verify_extraction`` for a file on disk; unreadable files trust the output. + + The CJK cross-check is confirmed against pdfium's own text layer: a + stray ``/Ordering (Japan1)`` in an otherwise Latin document (a 48-page + Quartz export in the corpus carries one among 258 Identity fonts) is not + a dropped Chinese column, and must not send the file to a full docling + re-parse. + """ try: data = Path(file).read_bytes() except OSError: return [] - return verify_extraction(data, markdown) + problems = verify_extraction(data, markdown) + if any(problem.startswith(_CJK_PROBLEM_PREFIX) for problem in problems): + reference = _text_layer_cjk_chars(Path(file), _MIN_CJK_CHARS) + if reference is not None and reference < _MIN_CJK_CHARS: + problems = [p for p in problems if not p.startswith(_CJK_PROBLEM_PREFIX)] + return problems diff --git a/docs/content/Deploying/DocsGPT-Settings.mdx b/docs/content/Deploying/DocsGPT-Settings.mdx index 35298ba7..c6f3b03c 100644 --- a/docs/content/Deploying/DocsGPT-Settings.mdx +++ b/docs/content/Deploying/DocsGPT-Settings.mdx @@ -220,7 +220,7 @@ for the engines and flows. | `OCR_LANGS` | `eng` | Tesseract language packs, `+`-separated (e.g. `eng+chi_sim+deu`). | | `OCR_DEEPSEEK_URL` | `http://localhost:11434/v1/chat/completions` | OpenAI-compatible endpoint for `OCR_ENGINE=deepseek` (Ollama or vLLM). | | `OCR_DEEPSEEK_MODEL` | `deepseek-ocr:3b` | Model name at that endpoint. | -| `OCR_DEEPSEEK_TIMEOUT` | `300` | Native backend: seconds allowed per page request to the DeepSeek endpoint (pages are sent one at a time). | +| `OCR_DEEPSEEK_TIMEOUT` | `300` | Seconds allowed per page request to the DeepSeek endpoint, on both backends (the native backend sends pages one at a time). | | `OCR_RENDER_DPI` | `200` | Native backend: resolution at which pages without a text layer are rendered before OCR (clamped to 72-600). | | `OCR_MIN_CHARS_PER_PAGE` | `20` | Chars-per-page floor below which an OCR'd parse is treated as an OCR dropout and fails loudly instead of indexing an empty document; `0` disables. Alias: `DOCLING_OCR_MIN_CHARS_PER_PAGE`. | diff --git a/docs/content/Guides/ocr.mdx b/docs/content/Guides/ocr.mdx index decc29f8..90140242 100644 --- a/docs/content/Guides/ocr.mdx +++ b/docs/content/Guides/ocr.mdx @@ -142,14 +142,15 @@ For `deepseek`, point the worker at an OpenAI-compatible endpoint: OCR_ENGINE=deepseek OCR_DEEPSEEK_URL=http://localhost:11434/v1/chat/completions # Ollama default OCR_DEEPSEEK_MODEL=deepseek-ocr:3b -OCR_DEEPSEEK_TIMEOUT=300 # native backend: seconds per page +OCR_DEEPSEEK_TIMEOUT=300 # seconds per page request, both backends ``` Ollama works out of the box (`ollama pull deepseek-ocr:3b`); for real throughput serve `deepseek-ai/DeepSeek-OCR` with vLLM on a GPU and set the URL accordingly. The native backend sends pages one at a time, so a slow laptop-hosted model only needs a generous `OCR_DEEPSEEK_TIMEOUT`; docling's -VLM pipeline uses its own 90 s timeout and four concurrent requests. +VLM pipeline honours the same timeout per request but keeps its own four +concurrent requests. A selected engine that is not available degrades rather than failing parses: under docling, a missing tesseract binary or non-macOS `ocrmac` falls back to @@ -177,7 +178,7 @@ difference matters more than for tesseract: - **docling** uses its VLM pipeline, which *replaces* the whole converter: every page of every PDF that reaches docling goes to the model, text layer or not (a 39-page text PDF measured 721 s against 20 s on the other paths), - with four concurrent requests and a fixed 90 s timeout. In testing it also + with four concurrent requests and `OCR_DEEPSEEK_TIMEOUT` per request. In testing it also dropped page-level elements — a title, an intro paragraph — that the same model kept through the native path, and it logs a burst of pydantic serialization warnings per conversion. Its one advantage was table diff --git a/tests/parser/file/test_anydoc_parser.py b/tests/parser/file/test_anydoc_parser.py index fb6292a8..feba529a 100644 --- a/tests/parser/file/test_anydoc_parser.py +++ b/tests/parser/file/test_anydoc_parser.py @@ -534,3 +534,81 @@ def test_non_scan_refusals_do_not_trigger_the_guard(tmp_path, monkeypatch): out = AnydocParser(fallback_parser=_RecordingFallback(result="tiny")).parse_file(path) assert out == "tiny" + + +# --- spreadsheets: a resource limit delegates, other formats stay terminal ---- + + +def test_resource_limit_on_a_spreadsheet_delegates_to_the_tabular_fallback(tmp_path, monkeypatch): + """anydoc refuses a 265k-row sheet on fixed limits that openpyxl/pandas + read fine, so for tabular suffixes the limit is a reason to fall back.""" + fake = _fake_anydoc(monkeypatch, None) + + def _refuse(path): + raise fake.ResourceLimitError("resource limit exceeded (max_xml_nodes): 2000000") + + fake.to_markdown = _refuse + fallback = _RecordingFallback() + parser = AnydocParser(fallback_parser=fallback) + path = tmp_path / "big.xlsx" + path.write_bytes(b"PK") + + assert parser.parse_file(path) == FALLBACK_TEXT + assert fallback.calls == [path] + assert parser.last_engine == "_RecordingFallback" + + +def test_resource_limit_on_a_spreadsheet_without_fallback_is_terminal(tmp_path, monkeypatch): + fake = _fake_anydoc(monkeypatch, None) + + def _refuse(path): + raise fake.ResourceLimitError("resource limit exceeded (max_entry_bytes)") + + fake.to_markdown = _refuse + path = tmp_path / "big.xlsx" + path.write_bytes(b"PK") + + with pytest.raises(DocumentParseError, match="max_entry_bytes"): + AnydocParser().parse_file(path) + + +# --- mixed documents: one page's OCR failure costs that page only ------------ + + +class _PerPageOcrFallback(BaseParser): + ocr_enabled = True + + def __init__(self, failing_index): + super().__init__() + self.failing_index = failing_index + self.requests = [] + + def _init_parser(self): + return {} + + def parse_file(self, file, errors="ignore"): + return "unused" + + def ocr_pages(self, file, indices): + self.requests.append(list(indices)) + if self.failing_index in indices: + raise DocumentParseError("tesseract failed on this page") + return {index: f"OCR TEXT PAGE {index + 1}" for index in indices} + + +def test_one_failing_page_does_not_drop_the_other_pages_ocr(tmp_path, monkeypatch): + pytest.importorskip("pypdfium2") + pytest.importorskip("pypdf") + _fake_anydoc(monkeypatch, lambda path: "Text pages read by anydoc.") + fallback = _PerPageOcrFallback(failing_index=0) + parser = AnydocParser(fallback_parser=fallback) + path = _scanned_pdf(tmp_path / "mixed.pdf", pages=3) # every page probes as scanned + + text = parser.parse_file(path) + + assert fallback.requests == [[0], [1], [2]] + assert "OCR TEXT PAGE 1" not in text + assert "OCR TEXT PAGE 2" in text and "OCR TEXT PAGE 3" in text + assert text.startswith("Text pages read by anydoc.") + # (The font-less blank-page fixture also earns a trust-check warning.) + assert parser.get_file_metadata(path)["ocr_pages"] == 2 diff --git a/tests/parser/file/test_docling_parser.py b/tests/parser/file/test_docling_parser.py index 0d47471f..3bc6f6df 100644 --- a/tests/parser/file/test_docling_parser.py +++ b/tests/parser/file/test_docling_parser.py @@ -568,7 +568,51 @@ class TestDoclingParserGaps: parser = DoclingCSVParser() assert parser.export_format == "markdown" - assert parser.ocr_enabled is True + assert parser.ocr_enabled is False + + +@pytest.mark.unit +class TestNonOcrParsersLeaveTextAlone: + """Office/markup docling parsers never OCR, so the tesseract CJK + post-processing must not touch their born-digital exports.""" + + @pytest.mark.parametrize( + "name", + [ + "DoclingDocxParser", + "DoclingPPTXParser", + "DoclingXLSXParser", + "DoclingHTMLParser", + "DoclingCSVParser", + "DoclingMarkdownParser", + "DoclingAsciiDocParser", + "DoclingVTTParser", + "DoclingXMLParser", + ], + ) + def test_ocr_is_off_by_construction(self, name): + import application.parser.file.docling_parser as dp + + assert getattr(dp, name)().ocr_enabled is False + + def test_cjk_spaces_survive_in_a_docx_export(self, monkeypatch): + from application.core.settings import settings + from application.parser.file.docling_parser import DoclingDocxParser + + monkeypatch.setattr(settings, "OCR_ENGINE", "tesseract") + parser = DoclingDocxParser() + # Simulate what _create_converter records for an OCR-on parser; the + # docx parser is OCR-off so the collapse must stay inert regardless. + parser._active_ocr_engine = "tesseract" + text = "\u5f20 \u4e09 \u548c \u674e \u56db\n\uff21\uff22\uff23 \uff24\uff25\uff26" + assert parser._postprocess_ocr_text(text) == text + + def test_pdf_parser_with_tesseract_still_collapses(self): + from application.parser.file.docling_parser import DoclingPDFParser + + parser = DoclingPDFParser(ocr_enabled=True) + parser._active_ocr_engine = "tesseract" + assert parser._postprocess_ocr_text("\u4e92\u76f8 \u4fdd\u5bc6") == "\u4e92\u76f8\u4fdd\u5bc6" # ===================================================================== @@ -1331,11 +1375,11 @@ class TestOCRDropoutGuard: assert create.call_count == 1 def test_guard_does_not_apply_to_non_ocr_formats(self): - """DOCX parsers inherit ocr_enabled=True but never OCR anything.""" + """DOCX parsers construct with OCR off: they never OCR anything.""" from application.parser.file.docling_parser import DoclingDocxParser parser = DoclingDocxParser() - assert parser.ocr_enabled is True + assert parser.ocr_enabled is False converter = MagicMock() converter.convert.return_value = _mock_conversion("Hi", pages=1) parser._converter = converter diff --git a/tests/parser/file/test_ocr_parser.py b/tests/parser/file/test_ocr_parser.py index d8b11be9..3f608b35 100644 --- a/tests/parser/file/test_ocr_parser.py +++ b/tests/parser/file/test_ocr_parser.py @@ -10,6 +10,7 @@ import sys from pathlib import Path from unittest.mock import MagicMock, patch +import logging import pytest pytest.importorskip("pypdfium2") @@ -748,7 +749,7 @@ class TestAnydocMixedDocuments: content = parser.parse_file(mixed_pdf) assert "Digital text page" in content - assert "text pages only" in caplog.text + assert "skipping that page" in caplog.text assert parser.get_file_metadata(mixed_pdf) == {} def test_all_text_document_never_probes_ocr(self, tmp_path): @@ -761,3 +762,88 @@ class TestAnydocMixedDocuments: parser = AnydocParser(fallback_parser=fallback) parser.parse_file(pdf) fallback.ocr_pages.assert_not_called() + + +# --------------------------------------------------------------------------- +# Render budget: an oversized MediaBox must not become a multi-GB bitmap +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestRenderBudget: + def test_letter_page_is_not_clamped(self): + assert op.render_scale(612, 792, 200) == pytest.approx(200 / 72) + + def test_pdf_maximum_page_is_clamped_to_the_budget(self): + scale = op.render_scale(14400, 14400, 200) + assert scale < 200 / 72 + assert (14400 * scale) * (14400 * scale) <= op._MAX_RENDER_PIXELS * 1.001 + + def test_huge_page_renders_within_budget(self, tmp_path, caplog): + """A 14400x14400 pt blank page (the PDF maximum, a few hundred bytes) + would be 40000x40000 px = 6 GiB at 200 dpi without the cap.""" + pytest.importorskip("pypdf") + import pypdfium2 as pdfium + from pypdf import PdfWriter + + path = tmp_path / "huge.pdf" + writer = PdfWriter() + writer.add_blank_page(width=14400, height=14400) + with open(path, "wb") as fh: + writer.write(fh) + + pdf = pdfium.PdfDocument(str(path)) + try: + page = pdf[0] + try: + with caplog.at_level(logging.WARNING, logger="application.parser.file.ocr_parser"): + image = op._render_page(page, 200) + finally: + page.close() + finally: + pdf.close() + assert image.width * image.height <= op._MAX_RENDER_PIXELS * 1.001 + assert "stay within" in caplog.text + + +# --------------------------------------------------------------------------- +# Scanned-page probe: thin text layers count only over an image +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +class TestScannedPageProbeImageGate: + def _cover_and_figure_pdf(self, path: Path) -> Path: + pytest.importorskip("reportlab") + from reportlab.lib.pagesizes import letter + from reportlab.lib.utils import ImageReader + from reportlab.pdfgen import canvas + + c = canvas.Canvas(str(path), pagesize=letter) + # Page 1: a short title and nothing else — not a scan, nothing to OCR. + c.setFont("Helvetica", 36) + c.drawString(72, 500, "Annual Report 2025") + c.showPage() + # Page 2: a page number under a full-page raster figure — a scan with a stamp. + img = Image.new("RGB", (300, 400), "white") + ImageDraw.Draw(img).rectangle((50, 50, 250, 350), outline="black", width=3) + c.drawImage(ImageReader(img), 72, 150, width=450, height=600) + c.setFont("Helvetica", 10) + c.drawString(300, 60, "17") + c.showPage() + # Page 3: ordinary text. + c.setFont("Helvetica", 12) + c.drawString(72, 700, "A full paragraph of ordinary body text that anydoc reads directly.") + c.showPage() + c.save() + return path + + def test_short_text_page_without_image_is_not_scanned(self, tmp_path): + path = self._cover_and_figure_pdf(tmp_path / "cover.pdf") + counts = op.text_layer_counts(path) + assert 0 < counts[0] < op.DEFAULT_TEXT_LAYER_MIN_CHARS + assert 0 < counts[1] < op.DEFAULT_TEXT_LAYER_MIN_CHARS + assert op.scanned_page_indices(path) == [1] + + def test_pages_with_no_text_at_all_still_count(self, tmp_path): + assert op.scanned_page_indices(_image_pdf(tmp_path / "scan.pdf", pages=2)) == [0, 1] diff --git a/tests/parser/file/test_pdf_trust.py b/tests/parser/file/test_pdf_trust.py index ad9ea421..9ca76d18 100644 --- a/tests/parser/file/test_pdf_trust.py +++ b/tests/parser/file/test_pdf_trust.py @@ -146,3 +146,79 @@ def test_flate_bomb_streams_are_capped(): def test_verify_pdf_file_unreadable_trusts_output(tmp_path): assert verify_pdf_file(tmp_path / "missing.pdf", "text") == [] + + +# --- stream scanning is a keyword walk, not a regex ----------------------------- + + +class TestStreamWalk: + def test_stream_without_eol_before_endstream_is_seen(self): + body = zlib.compress(b"<>") + data = b"2 0 obj<>stream\n" + body + b"endstream endobj\n" + assert check_pdf_fonts(data)["type0"] == 1 + + def test_undecodable_stream_does_not_swallow_the_next_one(self): + """A non-Flate (image) stream before a font stream must not merge the two.""" + image = b"3 0 obj<>stream\n\xff\xd8 not zlib at all\nendstream endobj\n" + font = _stream(b"<>") + result = check_pdf_fonts(image + font) + assert result["type0"] == 1 + assert result["type0_no_tounicode"] == 0 + + def test_crlf_streams_decompress(self): + body = zlib.compress(b"/Ordering (GB1)") + data = b"2 0 obj<>stream\r\n" + body + b"\r\nendstream endobj\n" + assert check_pdf_fonts(data)["expects_cjk"] is True + + def test_large_file_with_many_streams_is_linear(self): + """1000 undecodable streams: the old regex went quadratic here.""" + import time + + chunk = b"9 0 obj<>stream\n" + b"\x00" * 30 + b"endstream endobj\n" + data = chunk * 1000 + TYPE0_BARE + started = time.perf_counter() + result = check_pdf_fonts(data) + assert time.perf_counter() - started < 1.0 + assert result["type0_no_tounicode"] == 1 + + +# --- the CJK cross-check is confirmed against pdfium's text layer -------------- + + +def _latin_pdf(path: Path) -> Path: + import pytest + + pytest.importorskip("reportlab") + from reportlab.pdfgen import canvas + + c = canvas.Canvas(str(path)) + c.drawString(72, 700, "Plain English content, nothing else on the page.") + c.showPage() + c.save() + return path + + +def test_stray_cjk_ordering_without_cjk_text_is_not_flagged(tmp_path): + """A Quartz-style export declares Japan1 on one unused font: pdfium sees no + CJK either, so anydoc dropped nothing and the file must not be rerouted.""" + import pytest + + pytest.importorskip("pypdfium2") + path = _latin_pdf(tmp_path / "latin.pdf") + with open(path, "ab") as fh: + fh.write(b"\n% trailing junk\n" + _obj(b"<>")) + markdown = "Plain English content, nothing else on the page." + + # The byte-level check alone still fires... + assert any(p.startswith("PDF declares CJK fonts") for p in verify_extraction(path.read_bytes(), markdown)) + # ...and the file-level check withdraws it. + assert [p for p in verify_pdf_file(path, markdown) if p.startswith("PDF declares CJK fonts")] == [] + + +def test_real_cjk_drop_survives_the_cross_check(): + import pytest + + pytest.importorskip("pypdfium2") + fixture = FIXTURES / "nda_en_zh_cid_font.pdf" + problems = verify_pdf_file(fixture, "English only, the Chinese column is gone.") + assert any(p.startswith("PDF declares CJK fonts") for p in problems) diff --git a/tests/parser/file/test_pdfium_parser.py b/tests/parser/file/test_pdfium_parser.py index d3dc9f40..624c38c7 100644 --- a/tests/parser/file/test_pdfium_parser.py +++ b/tests/parser/file/test_pdfium_parser.py @@ -165,6 +165,9 @@ def docling_engine(monkeypatch): from application.core.settings import settings monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "docling") + # A developer .env with OCR on (and OCR_BACKEND=native) would otherwise + # swap the native OCR parser in for the docling one these tests expect. + monkeypatch.setattr(settings, "OCR_ENABLED", False) def test_extractor_defaults_to_docling_for_pdf(docling_engine):