diff --git a/application/agents/tools/read_document.py b/application/agents/tools/read_document.py index 52ce7183..fae6ea1d 100644 --- a/application/agents/tools/read_document.py +++ b/application/agents/tools/read_document.py @@ -102,7 +102,8 @@ class ReadDocumentTool(Tool): "enum": ["markdown", "text", "structured", "chunks"], "description": "Shape of the parsed result (default: markdown). Note: " "`structured` always uses the Docling engine regardless of `engine` " - "(the `fast` engine is markdown/text only).", + "(requires the optional docling install; the other engines are " + "markdown/text only).", }, "ocr": { "type": "string", @@ -115,9 +116,11 @@ class ReadDocumentTool(Tool): }, "engine": { "type": "string", - "enum": ["auto", "docling", "fast"], - "description": "Parser engine (default: auto). `fast` is a lighter " - "markdown/text-only engine; it is ignored when `output='structured'`, " + "enum": ["auto", "docling", "anydoc", "fast"], + "description": "Parser engine (default: auto = the server's configured " + "engine). `anydoc` is the fast Markdown converter, `docling` the " + "layout-model engine (optional install, needed for tables/OCR), " + "`fast` a plain-text legacy path. Ignored when `output='structured'`, " "which always uses Docling.", }, "max_chars": { diff --git a/application/core/settings.py b/application/core/settings.py index ea226b84..927ffa44 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -106,6 +106,15 @@ class Settings(BaseSettings): UPLOAD_MAX_ARCHIVE_DEPTH: int = Field(default=3, ge=0) PARSE_PDF_AS_IMAGE: bool = False PARSE_IMAGE_REMOTE: bool = False + # Document parser for source ingestion, chat attachments and the + # read_document tool. "anydoc" (default): firecrawl-anydoc, a Rust + # converter with no ML models — milliseconds per file, ~100 MB peak RSS. + # "docling": the layout/table-model pipeline (optional install; needed + # for OCR and for read_document's structured output). Files anydoc + # cannot convert (scanned PDFs, malformed input) fall back to docling + # when it is installed, otherwise to the legacy parsers. Rollback to the + # previous behaviour is this one variable. + DOC_PARSER_ENGINE: str = "anydoc" DOCLING_OCR_ENABLED: bool = False # Enable OCR for docling parsers (PDF, images) DOCLING_OCR_ATTACHMENTS_ENABLED: bool = False # Enable OCR for docling when parsing attachments # Pages docling's threaded pipeline buffers in flight; the library @@ -114,6 +123,12 @@ class Settings(BaseSettings): DOCLING_COMPILE_TORCH_MODELS: bool = False DOCLING_TABULAR_MAX_BYTES: int = 2_000_000 DOCLING_MARKUP_MAX_BYTES: int = 8_000_000 + # HTML/XHTML larger than this (bytes) are head-truncated before the + # markdownify parser runs (the anydoc engine's HTML path). The tree that + # path builds costs ~50x the input — 30 MB of HTML measured at 1.6 GB RSS — + # and the upload cap is 100 MB, so the gate is what keeps one upload from + # taking the ingest worker down. 0 disables it. + MARKUP_MAX_BYTES: int = 8_000_000 # Chars-per-page floor below which an OCR'd PDF/image parse is treated as a docling # pipeline dropout (long-running workers were observed returning zero characters for # every scanned page after a long scanned PDF, with no error) rather than as content. diff --git a/application/parser/document_reader.py b/application/parser/document_reader.py index 57bb2a98..05d7e7b0 100644 --- a/application/parser/document_reader.py +++ b/application/parser/document_reader.py @@ -1,10 +1,11 @@ """In-process document parsing for the ``read_document`` tool, run on the Celery parsing worker. ``parse_document_bytes`` turns untrusted document bytes into a bounded, shaped -result (markdown/text/structured/chunks) using the BACKEND parsers (Docling by -default). It applies the same untrusted-content safeguards as uploads — an -extension whitelist, a byte cap, ``safe_filename`` staging into a temp file, and -temp cleanup — so a hostile filename or document is treated as inert data. +result (markdown/text/structured/chunks) using the BACKEND parsers (the engine +selected by ``DOC_PARSER_ENGINE``; ``structured`` output always needs Docling). +It applies the same untrusted-content safeguards as uploads — an extension +whitelist, a byte cap, ``safe_filename`` staging into a temp file, and temp +cleanup — so a hostile filename or document is treated as inert data. """ from __future__ import annotations @@ -40,7 +41,7 @@ _MAX_PAGE_SELECTOR_TOKENS = 20_000 _VALID_OUTPUTS = ("markdown", "text", "structured", "chunks") _VALID_OCR = ("auto", "on", "off") -_VALID_ENGINES = ("auto", "docling", "fast") +_VALID_ENGINES = ("auto", "docling", "anydoc", "fast") def truncate_text_head_tail(text: str, max_bytes: Optional[int] = None) -> str: @@ -102,7 +103,13 @@ def _max_input_bytes() -> int: return int(getattr(settings, "SANDBOX_MAX_INPUT_BYTES", 25 * 1024 * 1024)) -_ZIP_CONTAINER_EXTENSIONS = frozenset({".docx", ".xlsx", ".pptx", ".epub"}) +# Every zip-packaged format a parser map can route: OOXML and its macro/ +# binary variants, OpenDocument, EPUB. Chat attachments have no extension +# whitelist, so the anydoc engine can receive any of these. +_ZIP_CONTAINER_EXTENSIONS = frozenset({ + ".docx", ".docm", ".xlsx", ".xlsm", ".xlsb", ".pptx", ".pptm", ".ppsx", ".ppsm", + ".odt", ".ods", ".odp", ".epub", +}) def _zip_bomb_reason(source: Union[bytes, str, Path], suffix: str) -> Optional[str]: @@ -165,13 +172,24 @@ def _resolve_ocr_enabled(ocr: str) -> bool: return bool(getattr(settings, "DOCLING_OCR_ENABLED", False)) +def _effective_engine(engine: str) -> str: + """Resolve ``auto`` to the server's ``DOC_PARSER_ENGINE``; other values pass through.""" + if engine != "auto": + return engine + configured = getattr(settings, "DOC_PARSER_ENGINE", None) or "anydoc" + return str(configured).strip().lower() + + def _pick_parser(suffix: str, *, ocr_enabled: bool, engine: str): """Select the parser for ``suffix`` honoring the requested engine; None when unsupported.""" if engine == "fast": legacy = _legacy_parser_for(suffix) if legacy is not None: return legacy - extractor = get_default_file_extractor(ocr_enabled=ocr_enabled) + if engine in ("docling", "anydoc"): + extractor = get_default_file_extractor(ocr_enabled=ocr_enabled, engine=engine) + else: + extractor = get_default_file_extractor(ocr_enabled=ocr_enabled) return extractor.get(suffix) @@ -491,7 +509,12 @@ def _shape( } parser = _pick_parser(suffix, ocr_enabled=ocr_enabled, engine=engine) - wants_tables = include_tables and engine != "fast" and output != "chunks" + # Tables come from a Docling conversion, so they are only collected when + # Docling is the engine in play: under anydoc/fast a table pass would be + # a second, full docling conversion that costs more than the whole read. + wants_tables = ( + include_tables and output != "chunks" and _effective_engine(engine) == "docling" + ) # A Docling-backed parser already converts the whole document to produce its text. # When tables are also requested, reuse that single conversion for both the markdown diff --git a/application/parser/file/anydoc_parser.py b/application/parser/file/anydoc_parser.py new file mode 100644 index 00000000..1593c8d6 --- /dev/null +++ b/application/parser/file/anydoc_parser.py @@ -0,0 +1,164 @@ +"""anydoc parser. + +Converts documents to GitHub-Flavored Markdown with `firecrawl-anydoc +`_: a single Rust extension with no ML +models and no Python dependencies. Measured against docling on a 20-document +corpus it was 20-160x faster per file (1.2 s vs 193 s on a 150-page annual +report), imported in 6 ms instead of 3.9 s, and peaked at 107 MB RSS instead +of 2.3 GB, with equivalent output on clean office files, CSV and text-layer +PDFs. + +anydoc never OCRs. It *detects* a scanned or image-only PDF and raises +``UnsupportedError`` ("OCR is required"), which is the routing point: a +parser given a ``fallback_parser`` (docling when installed, otherwise the +legacy parser for the suffix) delegates there; without one the file fails +loudly as ``DocumentParseError`` rather than being stored empty. +""" +import logging +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union + +from application.parser.file.base_parser import ( + BaseParser, + DocumentParseError, + delegate_parse, + module_available, +) + +logger = logging.getLogger(__name__) + +# Suffixes anydoc 0.2.3 converts, verified with ``anydoc.format_from_extension``. +# ``.epub`` is deliberately absent: it stays on ``EpubParser``. +ANYDOC_SUFFIXES: Tuple[str, ...] = ( + # Word processing + ".pdf", + ".docx", + ".docm", + ".doc", + ".odt", + ".rtf", + # Presentations + ".pptx", + ".pptm", + ".ppsx", + ".ppsm", + ".ppt", + ".pps", + ".pot", + ".odp", + # Spreadsheets / tabular + ".xlsx", + ".xlsm", + ".xlsb", + ".xls", + ".ods", + ".csv", +) + + +def anydoc_available() -> bool: + """Whether the ``anydoc`` extension can be imported.""" + return module_available("anydoc") + + +class AnydocParser(BaseParser): + """Markdown conversion via ``anydoc.to_markdown`` with a typed fallback. + + Attributes: + fallback_parser: Parser used when anydoc cannot convert the file + (scanned PDF, unsupported or malformed input). ``None`` means + such files raise ``DocumentParseError``. + last_engine: Name of the engine that produced the most recent parse: + ``"anydoc"``, or the fallback parser's ``last_engine`` / class + name when it was delegated to. Mirrors ``PdfiumTextParser`` so + the attachment worker records what actually ran. + """ + + def __init__( + self, + fallback_parser: Optional[BaseParser] = None, + parser_config: Optional[Dict] = None, + ) -> None: + super().__init__(parser_config) + self.fallback_parser = fallback_parser + self.last_engine: Optional[str] = None + + def _init_parser(self) -> Dict: + # Import for real rather than trusting ``find_spec``: a wheel whose + # native extension fails to load (glibc/arch mismatch) has a spec but + # no importable module, and that must fail here — where the ingest + # task treats it as a setup error — not as a bare ImportError from + # ``parse_file`` mid-batch, which ``load_data`` does not catch. + try: + import anydoc # noqa: F401 + except ImportError as exc: + raise ImportError( + "firecrawl-anydoc is required for AnydocParser. " + "Install it with: pip install firecrawl-anydoc" + ) from exc + fallback = self.fallback_parser + return {"fallback_parser": type(fallback).__name__ if fallback else None} + + def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]: + """Convert ``file`` to Markdown. + + Args: + file: Path to the document. + errors: Decoding error policy; anydoc decodes internally, so this + is only forwarded to the fallback parser. + + Returns: + The document as Markdown. + + Raises: + DocumentParseError: When neither anydoc nor the fallback could + produce text for the file, or anydoc hit a resource limit. + """ + import anydoc + + path = Path(file) + try: + content = anydoc.to_markdown(str(path)) + except anydoc.ResourceLimitError as exc: + # anydoc refused because the file is too expensive (declared + # decompression size, nesting depth, node count). A heavier + # engine would spend exactly what was just refused, so this is + # terminal rather than a reason to delegate. + self.last_engine = None + raise DocumentParseError( + f"Failed to parse {path.name}: {type(exc).__name__}: {exc}" + ) from exc + except anydoc.ConvertError as exc: + # Typed, per-document failures another engine may still handle: + # UnsupportedError (scanned PDF / unknown format), MalformedError, + # EncryptedError, MissingPartError. + return self._delegate(path, errors, f"{type(exc).__name__}: {exc}") + except OSError as exc: + raise DocumentParseError( + f"Failed to parse {path.name}: the file could not be read." + ) from exc + except Exception as exc: + logger.error(f"anydoc failed on {path.name}: {exc}", exc_info=True) + raise DocumentParseError( + f"Failed to parse {path.name} with anydoc: {exc}" + ) from exc + + if not content or not content.strip(): + return self._delegate(path, errors, "anydoc produced no text") + + self.last_engine = "anydoc" + return content + + def _delegate(self, path: Path, errors: str, reason: str) -> Union[str, List[str]]: + """Hand ``path`` to the fallback parser, or fail loudly without one.""" + fallback = self.fallback_parser + if fallback is None: + self.last_engine = None + raise DocumentParseError(f"Failed to parse {path.name}: {reason}") + logger.warning( + f"anydoc could not convert {path.name} ({reason}); " + f"falling back to {type(fallback).__name__}" + ) + result = delegate_parse(fallback, path, errors) + self.last_engine = getattr(fallback, "last_engine", None) or type(fallback).__name__ + return result diff --git a/application/parser/file/base_parser.py b/application/parser/file/base_parser.py index ad55eb7f..d0ae5ae4 100644 --- a/application/parser/file/base_parser.py +++ b/application/parser/file/base_parser.py @@ -1,9 +1,14 @@ """Base parser and config class.""" +import importlib.util +import logging +import sys from abc import abstractmethod from pathlib import Path from typing import Dict, List, Optional, Union +logger = logging.getLogger(__name__) + class DocumentParseError(Exception): """A file could not be converted to text. @@ -57,3 +62,67 @@ class BaseParser: """Return parser-specific metadata for the most recently parsed file.""" _ = file return {} + + +def module_available(name: str) -> bool: + """Whether top-level module ``name`` can be imported, without importing it. + + Cheap enough to call while building a parser map. Honors ``sys.modules`` + first so tests that stub a module in (or block one out with ``None``) + are seen the same way the import system would see them; a stub with no + ``__spec__`` counts as present rather than raising. + + Args: + name: Top-level module name, e.g. ``"docling"``. + + Returns: + True when an import of ``name`` would succeed. + """ + if name in sys.modules: + return sys.modules[name] is not None + try: + return importlib.util.find_spec(name) is not None + except (ImportError, ValueError): + return False + + +def delegate_parse( + parser: "BaseParser", file: Path, errors: str +) -> Union[str, List[str]]: + """Run a fallback parser, normalizing its failures to ``DocumentParseError``. + + Used wherever one parser hands a file to another (docling to the plain + tabular parsers for oversized sheets, anydoc to docling/legacy parsers + for files it cannot convert). Wrapping matters twice over: the Celery + tasks that own these paths list ``DocumentParseError`` in + ``dont_autoretry_for``, so a deterministic content error fails once + instead of retrying four times; and ``SimpleDirectoryReader.load_data`` + catches only ``DocumentParseError``, so one poison file is skipped into + ``failed_files`` instead of aborting an entire multi-file ingest. + + Args: + parser: The parser to delegate to; initialized here if needed. + file: Path to the file being parsed. + errors: Decoding error policy, forwarded to the parser. + + Returns: + The parsed text, or list of row strings. + + Raises: + DocumentParseError: If the fallback parser fails for any reason. + """ + try: + if not parser.parser_config_set: + parser.init_parser() + return parser.parse_file(file, errors) + except DocumentParseError: + raise + except Exception as e: + logger.error( + f"Fallback parse of {Path(file).name} with " + f"{type(parser).__name__} failed: {e}", + exc_info=True, + ) + raise DocumentParseError( + f"Failed to parse {Path(file).name}: the file could not be read." + ) from e diff --git a/application/parser/file/bulk.py b/application/parser/file/bulk.py index d8e5ae8d..78da38aa 100644 --- a/application/parser/file/bulk.py +++ b/application/parser/file/bulk.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Callable, Dict, List, Optional, Tuple, Union from application.parser.file.base import BaseReader -from application.parser.file.base_parser import BaseParser, DocumentParseError +from application.parser.file.base_parser import BaseParser, DocumentParseError, module_available from application.parser.file.docs_parser import DocxParser, PDFParser from application.parser.file.epub_parser import EpubParser from application.parser.file.html_parser import HTMLParser @@ -49,28 +49,66 @@ def _wrap_pdf_fast_path(pdf_parser: BaseParser) -> BaseParser: ) -def get_default_file_extractor( - ocr_enabled: Optional[bool] = None, - pdf_text_fast_path: bool = False, -) -> Dict[str, BaseParser]: - """Get the default file extractor. - - Uses docling parsers by default for advanced document processing. - Falls back to standard parsers if docling is not installed. +def _legacy_file_extractor(pdf_text_fast_path: bool = False) -> Dict[str, BaseParser]: + """Parser map that needs neither docling nor anydoc. Args: - ocr_enabled: Enable OCR in the docling parsers. Defaults to - ``settings.DOCLING_OCR_ENABLED``. - pdf_text_fast_path: Read PDFs via their embedded text layer - (pypdfium2) instead of docling, falling back to docling for scans. - Off by default: it trades docling's structural markdown for speed, - which suits attachments (read into a prompt) but not source - ingestion (chunked and embedded for retrieval). + pdf_text_fast_path: Put the pypdfium2 text-layer parser in front of + the legacy PDF parser. + + Returns: + Dict[str, BaseParser]: Parser keyed by lower-case file suffix. + """ + legacy_pdf: BaseParser = PDFParser() + if pdf_text_fast_path: + legacy_pdf = _wrap_pdf_fast_path(legacy_pdf) + return { + ".pdf": legacy_pdf, + ".docx": DocxParser(), + ".csv": PandasCSVParser(), + ".xlsx": ExcelParser(), + ".epub": EpubParser(), + ".md": MarkdownParser(), + ".rst": RstParser(), + ".html": HTMLParser(), + ".xhtml": HTMLParser(), + ".mdx": MarkdownParser(), + ".json": JSONParser(), + ".pptx": PPTXParser(), + ".png": ImageParser(), + ".jpg": ImageParser(), + ".jpeg": ImageParser(), + **_build_audio_parser_mapping(), + } + + +def _docling_file_extractor( + ocr_enabled: bool, + pdf_text_fast_path: bool = False, + *, + missing_log_level: int = logging.WARNING, +) -> Dict[str, BaseParser]: + """docling parser map, or the legacy map when docling is not installed. + + The availability check has to happen here, not at parse time: + ``DoclingParser`` only imports docling inside ``_init_parser``, and the + ``ImportError`` that raises is not a ``DocumentParseError``, so a map + that handed out docling parsers on a docling-less install would abort + the whole ingest on its first file instead of degrading. + + Args: + ocr_enabled: Enable OCR in the docling PDF/image parsers. + pdf_text_fast_path: Put the pypdfium2 text-layer parser in front of + the PDF parser (attachments only; see ``get_default_file_extractor``). + missing_log_level: Level at which a missing docling install is logged; + informational when docling is only the *fallback* engine. Returns: Dict[str, BaseParser]: Parser keyed by lower-case file suffix. """ try: + if not module_available("docling"): + raise ImportError("docling is not installed") from application.parser.file.docling_parser import ( DoclingPDFParser, DoclingDocxParser, @@ -83,71 +121,140 @@ def get_default_file_extractor( DoclingVTTParser, DoclingXMLParser, ) - if ocr_enabled is None: - ocr_enabled = settings.DOCLING_OCR_ENABLED - pdf_parser: BaseParser = DoclingPDFParser(ocr_enabled=ocr_enabled) - if pdf_text_fast_path: - pdf_parser = _wrap_pdf_fast_path(pdf_parser) - return { - # Documents - ".pdf": pdf_parser, - ".docx": DoclingDocxParser(), - ".pptx": DoclingPPTXParser(), - ".xlsx": DoclingXLSXParser(), - # Web formats - ".html": DoclingHTMLParser(), - ".xhtml": DoclingHTMLParser(), - # Data formats - ".csv": DoclingCSVParser(), - ".json": JSONParser(), # Keep JSON parser (specialized handling) - # Text/markup formats - ".md": MarkdownParser(), # Keep markdown parser (specialized handling) - ".mdx": MarkdownParser(), - ".rst": RstParser(), - ".adoc": DoclingAsciiDocParser(), - ".asciidoc": DoclingAsciiDocParser(), - # Images (with OCR) - only use Docling when OCR is enabled - ".png": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), - ".jpg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), - ".jpeg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), - ".tiff": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), - ".tif": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), - ".bmp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), - ".webp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), - # Media/subtitles - ".vtt": DoclingVTTParser(), - **_build_audio_parser_mapping(), - # Specialized XML formats - ".xml": DoclingXMLParser(), - # Formats docling doesn't support - use standard parsers - ".epub": EpubParser(), - } except ImportError: - logging.warning( + logging.log( + missing_log_level, "docling is not installed. Using standard parsers. " - "For advanced document parsing, install with: pip install docling" + "For OCR and layout-model parsing, install with: pip install docling", ) - # Fallback to standard parsers - legacy_pdf: BaseParser = PDFParser() - if pdf_text_fast_path: - legacy_pdf = _wrap_pdf_fast_path(legacy_pdf) - return { - ".pdf": legacy_pdf, - ".docx": DocxParser(), - ".csv": PandasCSVParser(), - ".xlsx": ExcelParser(), - ".epub": EpubParser(), - ".md": MarkdownParser(), - ".rst": RstParser(), - ".html": HTMLParser(), - ".mdx": MarkdownParser(), - ".json": JSONParser(), - ".pptx": PPTXParser(), - ".png": ImageParser(), - ".jpg": ImageParser(), - ".jpeg": ImageParser(), - **_build_audio_parser_mapping(), - } + return _legacy_file_extractor(pdf_text_fast_path) + + pdf_parser: BaseParser = DoclingPDFParser(ocr_enabled=ocr_enabled) + if pdf_text_fast_path: + pdf_parser = _wrap_pdf_fast_path(pdf_parser) + return { + # Documents + ".pdf": pdf_parser, + ".docx": DoclingDocxParser(), + ".pptx": DoclingPPTXParser(), + ".xlsx": DoclingXLSXParser(), + # Web formats + ".html": DoclingHTMLParser(), + ".xhtml": DoclingHTMLParser(), + # Data formats + ".csv": DoclingCSVParser(), + ".json": JSONParser(), # Keep JSON parser (specialized handling) + # Text/markup formats + ".md": MarkdownParser(), # Keep markdown parser (specialized handling) + ".mdx": MarkdownParser(), + ".rst": RstParser(), + ".adoc": DoclingAsciiDocParser(), + ".asciidoc": DoclingAsciiDocParser(), + # Images (with OCR) - only use Docling when OCR is enabled + ".png": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), + ".jpg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), + ".jpeg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), + ".tiff": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), + ".tif": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), + ".bmp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), + ".webp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(), + # Media/subtitles + ".vtt": DoclingVTTParser(), + **_build_audio_parser_mapping(), + # Specialized XML formats + ".xml": DoclingXMLParser(), + # Formats docling doesn't support - use standard parsers + ".epub": EpubParser(), + } + + +def _anydoc_file_extractor(ocr_enabled: bool, pdf_text_fast_path: bool = False) -> Dict[str, BaseParser]: + """anydoc parser map, layered over the docling (or legacy) map. + + anydoc takes every format it converts; each of those parsers gets the + base map's parser for the same suffix as ``fallback_parser``, so a + scanned PDF still reaches docling+OCR when docling is installed and the + legacy parser otherwise. docling, when installed, also keeps serving + what anydoc cannot: image OCR and ``.adoc``/``.vtt``/``.xml``. HTML goes + to the markdownify-based ``HTMLMarkdownParser``. + + ``pdf_text_fast_path`` is deliberately ignored here: anydoc already reads + the text layer in milliseconds *and* keeps headings/tables, and does its + own scanned-PDF detection, so putting pypdfium2 in front of it would + only lose structure. + + Args: + ocr_enabled: Enable OCR in the docling fallback/image parsers. + pdf_text_fast_path: Honored only when anydoc is not installed and the + map degrades to the base engine. + + Returns: + Dict[str, BaseParser]: Parser keyed by lower-case file suffix. + """ + from application.parser.file.anydoc_parser import ( + ANYDOC_SUFFIXES, + AnydocParser, + anydoc_available, + ) + from application.parser.file.html_parser import HTMLMarkdownParser + + if not anydoc_available(): + logging.warning( + "firecrawl-anydoc is not installed; DOC_PARSER_ENGINE=anydoc is using " + "the %s parsers instead. Install with: pip install firecrawl-anydoc", + "docling" if module_available("docling") else "standard", + ) + return _docling_file_extractor(ocr_enabled, pdf_text_fast_path) + + base = _docling_file_extractor( + ocr_enabled, pdf_text_fast_path=False, missing_log_level=logging.INFO + ) + extractor = dict(base) + for suffix in ANYDOC_SUFFIXES: + extractor[suffix] = AnydocParser(fallback_parser=base.get(suffix)) + extractor[".html"] = HTMLMarkdownParser() + extractor[".xhtml"] = HTMLMarkdownParser() + return extractor + + +def get_default_file_extractor( + ocr_enabled: Optional[bool] = None, + pdf_text_fast_path: bool = False, + engine: Optional[str] = None, +) -> Dict[str, BaseParser]: + """Get the default file extractor. + + The engine is ``settings.DOC_PARSER_ENGINE`` unless overridden: + + * ``anydoc`` (default): firecrawl-anydoc for office/PDF/tabular formats, + markdownify for HTML, docling (if installed) for image OCR and as the + fallback for files anydoc rejects; legacy parsers otherwise. + * ``docling``: the docling map, with the legacy parsers when docling is + not installed. + + Args: + ocr_enabled: Enable OCR in the docling parsers. Defaults to + ``settings.DOCLING_OCR_ENABLED``. + pdf_text_fast_path: Read PDFs via their embedded text layer + (pypdfium2) instead of docling, falling back to docling for scans. + Off by default: it trades docling's structural markdown for speed, + which suits attachments (read into a prompt) but not source + ingestion (chunked and embedded for retrieval). A no-op under the + anydoc engine, which is already that fast and keeps structure. + engine: ``"anydoc"`` or ``"docling"``; ``None`` reads the setting. + + Returns: + Dict[str, BaseParser]: Parser keyed by lower-case file suffix. + """ + if ocr_enabled is None: + ocr_enabled = settings.DOCLING_OCR_ENABLED + selected = (engine or getattr(settings, "DOC_PARSER_ENGINE", None) or "anydoc") + selected = str(selected).strip().lower() + if selected == "docling": + return _docling_file_extractor(ocr_enabled, pdf_text_fast_path) + if selected != "anydoc": + logging.warning("Unknown DOC_PARSER_ENGINE %r; using anydoc", selected) + return _anydoc_file_extractor(ocr_enabled, pdf_text_fast_path) # For backwards compatibility diff --git a/application/parser/file/docling_parser.py b/application/parser/file/docling_parser.py index 9d6ee0d9..74222353 100644 --- a/application/parser/file/docling_parser.py +++ b/application/parser/file/docling_parser.py @@ -15,7 +15,11 @@ import zipfile from pathlib import Path from typing import Dict, List, Optional, Tuple, Union -from application.parser.file.base_parser import BaseParser, DocumentParseError +from application.parser.file.base_parser import ( + BaseParser, + DocumentParseError, + delegate_parse as _delegate, +) from application.utils import truncate_to_line_boundary logger = logging.getLogger(__name__) @@ -118,46 +122,6 @@ def _exceeds_tabular_gate(file: Path) -> bool: return _tabular_content_size(file) > max_bytes -def _delegate( - parser: BaseParser, file: Path, errors: str -) -> Union[str, List[str]]: - """Run a lightweight fallback parser, normalizing its failures. - - Anything the fallback raises is converted to ``DocumentParseError``, - which matters twice over. The Celery tasks that own these paths list - ``DocumentParseError`` in ``dont_autoretry_for``, so a deterministic - content error now fails once instead of retrying four times (each retry - re-downloading the file from S3); and ``SimpleDirectoryReader.load_data`` - catches only ``DocumentParseError``, so one poison spreadsheet is skipped - into ``failed_files`` instead of aborting an entire multi-file ingest. - - This restores the symmetry with ``DoclingParser.parse_file``, whose own - failures are already wrapped — the delegation branch bypassed that. - - Args: - parser: The lightweight parser to delegate to. - file: Path to the file being parsed. - errors: Decoding error policy, forwarded to the parser. - - Returns: - The parsed text, or list of row strings. - - Raises: - DocumentParseError: If the fallback parser fails for any reason. - """ - try: - return parser.parse_file(file, errors) - except DocumentParseError: - raise - except Exception as e: - logger.error( - f"Lightweight parse of {file.name} failed: {e}", exc_info=True - ) - raise DocumentParseError( - f"Failed to parse {file.name}: the file could not be read." - ) from e - - def _capped_markup_copy(file: Path) -> Optional[str]: """Head-truncate an oversized markup file to a temp copy for docling. diff --git a/application/parser/file/html_parser.py b/application/parser/file/html_parser.py index dbf1c6bf..37cd1a0a 100644 --- a/application/parser/file/html_parser.py +++ b/application/parser/file/html_parser.py @@ -1,12 +1,136 @@ -"""HTML parser. - -Contains parser for html files. +"""HTML parsers. +``HTMLParser`` returns the visible text of a page (the ``fast`` engine and +the legacy no-docling map rely on that plain-text contract). +``HTMLMarkdownParser`` returns Markdown instead and is what the anydoc +engine maps ``.html``/``.xhtml`` to: anydoc has no HTML support, and +markdownify — already a dependency of the web crawler — keeps links, GFM +tables, headings and byte-exact code blocks at a few ms per page. """ +import logging +import re from pathlib import Path -from typing import Dict, Union +from typing import Dict, Optional, Tuple, Union +from application.core.settings import settings from application.parser.file.base_parser import BaseParser +from application.utils import truncate_to_line_boundary + +logger = logging.getLogger(__name__) + +# The crawler's conventions (``crawler_markdown.py`` / ``read_webpage.py``), +# so file and web ingestion produce the same Markdown shape. +MARKDOWNIFY_OPTIONS = {"heading_style": "ATX", "newline_style": "BACKSLASH"} + +# Elements whose text is never document content. ``title`` is reported via +# ``get_file_metadata`` instead of leaking in as a stray first line. +_DROP_TAGS = ("title", "script", "style", "noscript", "template") + + +def html_to_markdown(html: Union[str, bytes]) -> str: + """Convert an HTML/XHTML document to Markdown. + + Args: + html: The markup. Pass the raw bytes when you have them: BeautifulSoup + then decodes by the document's own BOM / ```` + instead of the process locale, so a windows-1250 page keeps its + diacritics. + + Returns: + Markdown with runs of blank lines collapsed to one. + """ + from bs4 import BeautifulSoup + + return soup_to_markdown(BeautifulSoup(html, "html.parser")) + + +def soup_to_markdown(soup) -> str: + """Convert a parsed ``BeautifulSoup`` tree to Markdown. + + Destructive: the non-content elements in ``_DROP_TAGS`` (including + ````) are removed from ``soup``; read the title first. + + Args: + soup: The parsed document. + + Returns: + Markdown with runs of blank lines collapsed to one. + """ + from markdownify import MarkdownConverter + + for tag in soup.find_all(_DROP_TAGS): + tag.decompose() + markdown = MarkdownConverter(**MARKDOWNIFY_OPTIONS).convert_soup(soup) + return re.sub(r"\n{3,}", "\n\n", markdown).strip() + + +def _soup_title(soup) -> Optional[str]: + """The document's ``<title>`` text, or None.""" + if soup.title and soup.title.string: + return str(soup.title.string) + return None + + +def _trim_torn_utf8_tail(data: bytes) -> bytes: + """Drop a trailing partial multi-byte UTF-8 sequence left by a byte cut. + + A cut that lands inside a multi-byte character matters more than one + lost glyph: BeautifulSoup decodes strictly and, when the declared UTF-8 + fails on the torn tail, retries as windows-1252 — and if every other + non-ASCII byte happens to be cp1252-valid (German umlauts are), the + *whole* document comes back as mojibake. Only a UTF-8 tail is + recognised; a single-byte page loses at most the final byte. + + Args: + data: A head window whose last bytes may be a torn sequence. + + Returns: + ``data`` minus any trailing incomplete UTF-8 sequence. + """ + end = len(data) + trailing = 0 + while end > 0 and trailing < 3 and 0x80 <= data[end - 1] < 0xC0: + end -= 1 + trailing += 1 + if end == 0: + return data + lead = data[end - 1] + if lead < 0xC0: + return data + need = 2 if lead < 0xE0 else 3 if lead < 0xF0 else 4 + if trailing + 1 < need: + return data[: end - 1] + return data + + +def read_markup_head(file: Path, max_bytes: int) -> bytes: + """Read a markup file, head-truncated to ``max_bytes`` on a line boundary. + + HTML has no lightweight fallback parser to bound it by other means (the + way oversized CSV/XLSX go to the plain tabular parsers), and the soup + + markdownify tree costs ~50x the input — 30 MB measured at 1.6 GB RSS, + against a 100 MB upload cap. Parsing only the head keeps one upload from + taking the ingest worker down; the cut lands on a line boundary when one + is reasonably close, and the lenient HTML parser copes with the torn tail. + + Args: + file: Path to the markup file. + max_bytes: Size gate; ``<= 0`` reads the whole file. + + Returns: + The file's bytes, or its first ``max_bytes`` when it is larger. + """ + with open(file, "rb") as fh: + if max_bytes <= 0: + return fh.read() + head = fh.read(max_bytes + 1) + if len(head) <= max_bytes: + return head + logger.warning( + f"Markup {Path(file).name} exceeds MARKUP_MAX_BYTES ({max_bytes}); " + f"parsing the first {max_bytes} bytes to bound memory" + ) + return _trim_torn_utf8_tail(truncate_to_line_boundary(head[:max_bytes])) class HTMLParser(BaseParser): @@ -40,3 +164,51 @@ class HTMLParser(BaseParser): if soup.title and soup.title.string: return {"title": str(soup.title.string)} return {} + + +class HTMLMarkdownParser(HTMLParser): + """HTML/XHTML to Markdown, for chunking and retrieval. + + Differs from ``HTMLParser`` in three ways: the body comes back as + Markdown; the file is read as bytes so the document's own charset + declaration wins over the process locale; and input is head-truncated at + ``MARKUP_MAX_BYTES`` (the docling HTML path has the same gate under + ``DOCLING_MARKUP_MAX_BYTES``). + """ + + def __init__(self, parser_config: Optional[Dict] = None) -> None: + super().__init__(parser_config) + # (path, title) of the most recent ``parse_file``, so the metadata + # call that follows it does not build the soup a second time. + self._last_title: Optional[Tuple[Path, Optional[str]]] = None + + @staticmethod + def _read(file: Path) -> bytes: + return read_markup_head(Path(file), int(settings.MARKUP_MAX_BYTES)) + + def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]: + """Convert an HTML file to Markdown. + + ``errors`` is accepted for the ``BaseParser`` contract but unused: + decoding is done by BeautifulSoup from the raw bytes. + """ + from bs4 import BeautifulSoup + + _ = errors + soup = BeautifulSoup(self._read(file), "html.parser") + self._last_title = (Path(file), _soup_title(soup)) + return soup_to_markdown(soup) + + def get_file_metadata(self, file: Path) -> Dict: + """Return the document title, decoded the same way as the body.""" + from bs4 import BeautifulSoup + + last = self._last_title + if last is not None and last[0] == Path(file): + title = last[1] + else: + try: + title = _soup_title(BeautifulSoup(self._read(file), "html.parser")) + except OSError: + return {} + return {"title": title} if title else {} diff --git a/application/parser/remote/crawler_markdown.py b/application/parser/remote/crawler_markdown.py index b51aad75..dec25e06 100644 --- a/application/parser/remote/crawler_markdown.py +++ b/application/parser/remote/crawler_markdown.py @@ -110,8 +110,9 @@ class CrawlerLoader(BaseRemote): language = language_tag.get('lang', 'en') if language_tag else "en" markdownified = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH") - # Reduce sequences of more than two newlines to exactly three - markdownified = re.sub(r'\n{3,}', '\n\n\n', markdownified) + # Collapse runs of blank lines to a single one — the same shape + # ``html_to_markdown`` gives uploaded HTML files. + markdownified = re.sub(r'\n{3,}', '\n\n', markdownified) return title, language, markdownified def _extract_links(self, html_content, current_url): diff --git a/application/requirements.txt b/application/requirements.txt index d914ef83..98c6b536 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -13,6 +13,9 @@ dataclasses-json==0.6.7 daytona==0.205.1 defusedxml==0.7.1 docling==2.119.0 +# Default document converter (DOC_PARSER_ENGINE=anydoc); docling above is +# the optional heavy engine for OCR/structured output. +firecrawl-anydoc==0.2.3 rapidocr==3.9.2 onnxruntime==1.28.0 docx2txt==0.9 diff --git a/application/worker.py b/application/worker.py index 1268ca39..44ea4340 100755 --- a/application/worker.py +++ b/application/worker.py @@ -1739,10 +1739,11 @@ def attachment_worker(self, file_info, user): scope={"kind": "attachment", "id": str(attachment_id)}, ) - # Attachments only: PDFs are read via their text layer where one - # exists (see ``ATTACHMENT_PDF_TEXT_FAST_PATH``). Source ingestion + # Attachments only: under the docling engine PDFs are read via their + # text layer where one exists (``ATTACHMENT_PDF_TEXT_FAST_PATH``; a + # no-op under anydoc, which is already that fast). Source ingestion # calls SimpleDirectoryReader without a ``file_extractor`` and so keeps - # the docling default, which is what retrieval quality depends on. + # the ``DOC_PARSER_ENGINE`` default map, which retrieval quality depends on. file_extractor = get_default_file_extractor( ocr_enabled=settings.DOCLING_OCR_ATTACHMENTS_ENABLED, pdf_text_fast_path=settings.ATTACHMENT_PDF_TEXT_FAST_PATH, diff --git a/docs/content/Deploying/DocsGPT-Settings.mdx b/docs/content/Deploying/DocsGPT-Settings.mdx index b71c93d6..f344edac 100644 --- a/docs/content/Deploying/DocsGPT-Settings.mdx +++ b/docs/content/Deploying/DocsGPT-Settings.mdx @@ -198,6 +198,22 @@ These byte/count limits and the non-negative nesting depth can be overridden in The request limit intentionally applies only to public upload routes; internal worker index transfers use their own trusted service boundary. +## Document Parsing + +Uploaded sources and chat attachments are converted to Markdown before +chunking or prompting. See the [Document Parsing and OCR](/Guides/ocr) guide +for the engines and flows. + +| Setting | Default | Description | +| --- | --- | --- | +| `DOC_PARSER_ENGINE` | `anydoc` | Parser engine: `anydoc` (fast Rust converter, no ML models) or `docling` (layout/table models, OCR, structured output; heavy optional install). Files anydoc cannot read fall back to Docling when installed. | +| `DOCLING_OCR_ENABLED` | `false` | OCR for source ingestion (Docling only). | +| `DOCLING_OCR_ATTACHMENTS_ENABLED` | `false` | OCR for chat attachments (Docling only). | +| `ATTACHMENT_PDF_TEXT_FAST_PATH` | `true` | Under the `docling` engine, read PDF attachments through their text layer (pypdfium2) and only hand scans to Docling. Not needed under `anydoc`. | +| `DOCLING_TABULAR_MAX_BYTES` | `2000000` | Docling engine: CSV/XLSX larger than this (by content) use the lightweight tabular parsers. | +| `DOCLING_MARKUP_MAX_BYTES` | `8000000` | Docling engine: HTML/VTT larger than this are head-truncated before parsing. | +| `MARKUP_MAX_BYTES` | `8000000` | anydoc engine: HTML/XHTML larger than this are head-truncated before the `markdownify` parser runs (its tree costs ~50x the input). `0` disables the gate. | + ## Speech-to-Text Settings DocsGPT can transcribe audio in two places: diff --git a/docs/content/Guides/ocr.mdx b/docs/content/Guides/ocr.mdx index 9ac09af3..3109ac52 100644 --- a/docs/content/Guides/ocr.mdx +++ b/docs/content/Guides/ocr.mdx @@ -1,13 +1,49 @@ --- -title: OCR for Sources and Attachments -description: How OCR works in DocsGPT, how to configure it, and what changes for source ingestion vs chat attachments. +title: Document Parsing and OCR +description: Which engine parses uploaded documents, how OCR works in DocsGPT, how to configure it, and what changes for source ingestion vs chat attachments. --- import { Callout } from 'nextra/components' -# Docling OCR for Sources and Attachments +# Document Parsing and OCR for Sources and Attachments -DocsGPT uses Docling as the default parser layer for many document formats. OCR is optional and controlled by two settings: +## Parser engine + +DocsGPT converts uploaded documents to Markdown before chunking, embedding or +handing them to a model. The engine is selected with one setting: + +```env +DOC_PARSER_ENGINE=anydoc +``` + +- `anydoc` (default): [firecrawl-anydoc](https://github.com/firecrawl/anydoc), + a Rust converter with no ML models. It reads PDF, DOCX, PPTX, XLSX and CSV + in milliseconds with ~100 MB peak memory; HTML/XHTML is converted with + `markdownify`, head-truncated at `MARKUP_MAX_BYTES`. Source uploads and the + `read_document` tool accept the same file types as before (the + `SUPPORTED_SOURCE_EXTENSIONS` whitelist is unchanged); chat attachments, + which are not whitelisted by extension, can additionally be parsed from + anydoc's other formats (DOC, PPT/PPS/POT, XLS, ODT/ODS/ODP, RTF and the + macro-enabled Office variants). It never performs OCR: + a scanned or image-only PDF is *detected* and handed to the fallback + parser — Docling when it is installed, the legacy text parsers otherwise — + and if nothing can read the file the upload fails with a clear error + instead of storing an empty document. +- `docling`: the previous default. Docling's layout and table models produce + structured Markdown, support OCR, and back the `read_document` tool's + `structured` output, at the cost of a large dependency tree (torch, + transformers, ONNX models) and seconds to minutes per PDF. Switching back is + this one variable; nothing else changes. + +With `DOC_PARSER_ENGINE=anydoc`, Docling still handles what anydoc cannot when +it is installed: image OCR (`DOCLING_OCR_ENABLED`), `.adoc`, `.vtt` and `.xml` +sources, and the fallback for files anydoc rejects. Without Docling those +formats use the standard parsers, and images are only read when +`PARSE_IMAGE_REMOTE=true`. + +## Docling OCR + +OCR is optional, Docling-only, and controlled by two settings: ```env DOCLING_OCR_ENABLED=false @@ -24,7 +60,7 @@ DOCLING_OCR_ATTACHMENTS_ENABLED=false 1. Files are uploaded through `/api/upload`. 2. Ingestion runs asynchronously in Celery (`ingest_worker`). 3. `SimpleDirectoryReader` parses files with `get_default_file_extractor`. -4. For PDFs and image formats, Docling parsers are used. OCR in this path is controlled by `DOCLING_OCR_ENABLED`. +4. Documents are parsed by the `DOC_PARSER_ENGINE` engine; images (and, under `anydoc`, scanned PDFs) reach Docling when it is installed. OCR in this path is controlled by `DOCLING_OCR_ENABLED`. 5. Parsed text is chunked, embedded, and stored in the vector store. 6. Retrieval during chat uses this indexed text and returns source citations. @@ -90,7 +126,7 @@ After changing these settings, restart the API and Celery worker. ## Legacy Fallback Notes -- If Docling is unavailable, DocsGPT falls back to legacy parsers. +- If Docling is unavailable, DocsGPT falls back to legacy parsers for the formats anydoc does not cover, and anydoc's own refusals become upload errors. - With OCR disabled, text-based PDFs can still parse, but scanned/image-heavy content may produce little text. - For image parsing without Docling OCR, the legacy image parser only extracts text when `PARSE_IMAGE_REMOTE=true`. diff --git a/tests/parser/file/test_anydoc_parser.py b/tests/parser/file/test_anydoc_parser.py new file mode 100644 index 00000000..b7751d69 --- /dev/null +++ b/tests/parser/file/test_anydoc_parser.py @@ -0,0 +1,313 @@ +"""Tests for the anydoc parser (the default ``DOC_PARSER_ENGINE``). + +Three behaviours matter and are asserted separately: + +* a file anydoc converts comes back as Markdown, with ``last_engine`` set; +* a file it refuses (a scanned PDF, an unknown format) goes to the fallback + parser when one is configured, and every failure surfaces as + ``DocumentParseError`` — never as an empty document or a raw traceback; +* the suffix list the parser map is built from matches what the installed + anydoc actually accepts. +""" + +import importlib.machinery +import sys +import types + +import pytest + +from application.parser.file.base_parser import BaseParser, DocumentParseError + +anydoc = pytest.importorskip("anydoc") + +from application.parser.file.anydoc_parser import ( # noqa: E402 — after importorskip + ANYDOC_SUFFIXES, + AnydocParser, + anydoc_available, +) + + +class _RecordingFallback(BaseParser): + """Stands in for docling / a legacy parser so delegation is observable.""" + + def __init__(self, result="fallback text"): + super().__init__(parser_config={}) + self.calls = [] + self._result = result + + def _init_parser(self): + return {} + + def parse_file(self, file, errors="ignore"): + self.calls.append(file) + return self._result + + +class _ExplodingFallback(BaseParser): + def _init_parser(self): + return {} + + def parse_file(self, file, errors="ignore"): + raise RuntimeError("fallback blew up") + + +def _scanned_pdf(path, pages=2): + """A PDF with pages but no text layer: what a scan looks like to a parser.""" + from pypdf import PdfWriter + + writer = PdfWriter() + for _ in range(pages): + writer.add_blank_page(width=612, height=792) + with open(path, "wb") as fh: + writer.write(fh) + return path + + +def _fake_anydoc(monkeypatch, to_markdown): + """Install a stub ``anydoc`` module whose ``to_markdown`` is ``to_markdown``.""" + fake = types.ModuleType("anydoc") + fake.__spec__ = importlib.machinery.ModuleSpec("anydoc", None) + + class ConvertError(Exception): + pass + + class UnsupportedError(ConvertError): + pass + + class ResourceLimitError(ConvertError): + pass + + fake.ConvertError = ConvertError + fake.UnsupportedError = UnsupportedError + fake.ResourceLimitError = ResourceLimitError + fake.to_markdown = to_markdown + monkeypatch.setitem(sys.modules, "anydoc", fake) + return fake + + +# --- the suffix list is what anydoc really supports --------------------------- + + +def test_every_listed_suffix_is_an_anydoc_format(): + for suffix in ANYDOC_SUFFIXES: + assert anydoc.format_from_extension(suffix) is not None, suffix + + +def test_epub_and_html_are_not_routed_to_anydoc(): + assert ".epub" not in ANYDOC_SUFFIXES + assert ".html" not in ANYDOC_SUFFIXES + + +# --- successful conversion ----------------------------------------------------- + + +def test_csv_converts_to_markdown_table(tmp_path): + path = tmp_path / "items.csv" + path.write_text("name,qty\nwidget,3\ngadget,5\n") + parser = AnydocParser() + parser.init_parser() + + out = parser.parse_file(path) + + assert isinstance(out, str) + assert "widget" in out and "gadget" in out + assert "|" in out # rendered as a GFM table, not flat text + assert parser.last_engine == "anydoc" + + +def test_xlsx_converts_with_cell_text(tmp_path): + openpyxl = pytest.importorskip("openpyxl") + path = tmp_path / "book.xlsx" + wb = openpyxl.Workbook() + ws = wb.active + ws.append(["city", "population"]) + ws.append(["Ljubljana", 295000]) + wb.save(path) + + out = AnydocParser().parse_file(path) + + assert "Ljubljana" in out + assert "295000" in out + + +def test_accepts_path_and_str(tmp_path): + path = tmp_path / "a.csv" + path.write_text("x,y\n1,2\n") + assert AnydocParser().parse_file(str(path)) == AnydocParser().parse_file(path) + + +# --- refusal routes to the fallback -------------------------------------------- + + +def test_scanned_pdf_delegates_to_fallback(tmp_path): + path = _scanned_pdf(tmp_path / "scan.pdf") + fallback = _RecordingFallback() + parser = AnydocParser(fallback_parser=fallback) + + out = parser.parse_file(path) + + assert out == "fallback text" + assert fallback.calls == [path] + assert parser.last_engine == "_RecordingFallback" + + +def test_fallback_last_engine_is_reported_when_it_has_one(tmp_path): + path = _scanned_pdf(tmp_path / "scan.pdf") + fallback = _RecordingFallback() + fallback.last_engine = "pypdfium2" + + parser = AnydocParser(fallback_parser=fallback) + parser.parse_file(path) + + assert parser.last_engine == "pypdfium2" + + +def test_scanned_pdf_without_fallback_is_document_parse_error(tmp_path): + path = _scanned_pdf(tmp_path / "scan.pdf") + parser = AnydocParser() + + with pytest.raises(DocumentParseError, match="scan.pdf"): + parser.parse_file(path) + assert parser.last_engine is None + + +def test_unknown_format_without_fallback_is_document_parse_error(tmp_path): + path = tmp_path / "blob.xyz" + path.write_bytes(b"hello") + + with pytest.raises(DocumentParseError, match="blob.xyz"): + AnydocParser().parse_file(path) + + +def test_fallback_failure_is_document_parse_error(tmp_path): + """A fallback that raises must not leak a bare exception (load_data skips only DocumentParseError).""" + path = _scanned_pdf(tmp_path / "scan.pdf") + + with pytest.raises(DocumentParseError): + AnydocParser(fallback_parser=_ExplodingFallback()).parse_file(path) + + +def test_fallback_document_parse_error_passes_through(tmp_path): + class _Refusing(BaseParser): + def _init_parser(self): + return {} + + def parse_file(self, file, errors="ignore"): + raise DocumentParseError("scan needs OCR") + + path = _scanned_pdf(tmp_path / "scan.pdf") + with pytest.raises(DocumentParseError, match="scan needs OCR"): + AnydocParser(fallback_parser=_Refusing()).parse_file(path) + + +def test_missing_file_is_document_parse_error(tmp_path): + with pytest.raises(DocumentParseError, match="could not be read"): + AnydocParser().parse_file(tmp_path / "missing.docx") + + +def test_empty_output_delegates(tmp_path, monkeypatch): + """Whitespace-only output is treated as a failed conversion, not stored.""" + _fake_anydoc(monkeypatch, lambda path: " \n\n ") + fallback = _RecordingFallback("real text") + path = tmp_path / "x.docx" + path.write_bytes(b"irrelevant") + + out = AnydocParser(fallback_parser=fallback).parse_file(path) + + assert out == "real text" + assert fallback.calls == [path] + + +def test_typed_convert_error_delegates(tmp_path, monkeypatch): + fake = _fake_anydoc(monkeypatch, None) + + def _refuse(path): + raise fake.UnsupportedError("PDF has no extractable text; OCR is required") + + fake.to_markdown = _refuse + fallback = _RecordingFallback() + path = tmp_path / "x.pdf" + path.write_bytes(b"%PDF-1.4") + + assert AnydocParser(fallback_parser=fallback).parse_file(path) == "fallback text" + + +def test_typed_convert_error_message_reaches_the_user(tmp_path, monkeypatch): + fake = _fake_anydoc(monkeypatch, None) + + def _refuse(path): + raise fake.UnsupportedError("PDF has no extractable text; OCR is required") + + fake.to_markdown = _refuse + path = tmp_path / "x.pdf" + path.write_bytes(b"%PDF-1.4") + + with pytest.raises(DocumentParseError, match="OCR is required"): + AnydocParser().parse_file(path) + + +def test_resource_limit_is_terminal_even_with_fallback(tmp_path, monkeypatch): + """anydoc refusing a file as too expensive must not hand it to a heavier + engine that would then spend exactly what was refused.""" + fake = _fake_anydoc(monkeypatch, None) + + def _refuse(path): + raise fake.ResourceLimitError( + "resource limit exceeded (max_entry_bytes): word/document.xml declares 400400167 bytes" + ) + + fake.to_markdown = _refuse + fallback = _RecordingFallback() + parser = AnydocParser(fallback_parser=fallback) + path = tmp_path / "bomb.docx" + path.write_bytes(b"PK") + + with pytest.raises(DocumentParseError, match="max_entry_bytes"): + parser.parse_file(path) + assert fallback.calls == [] + assert parser.last_engine is None + + +def test_unexpected_exception_is_document_parse_error(tmp_path, monkeypatch): + def _crash(path): + raise RuntimeError("panic") + + _fake_anydoc(monkeypatch, _crash) + path = tmp_path / "x.docx" + path.write_bytes(b"irrelevant") + + with pytest.raises(DocumentParseError, match="x.docx"): + AnydocParser().parse_file(path) + + +# --- init / availability -------------------------------------------------------- + + +def test_init_parser_records_fallback(): + parser = AnydocParser(fallback_parser=_RecordingFallback()) + parser.init_parser() + assert parser.parser_config == {"fallback_parser": "_RecordingFallback"} + + bare = AnydocParser() + bare.init_parser() + assert bare.parser_config == {"fallback_parser": None} + + +def test_init_parser_without_anydoc_raises_import_error(monkeypatch): + monkeypatch.setitem(sys.modules, "anydoc", None) + assert not anydoc_available() + with pytest.raises(ImportError, match="firecrawl-anydoc"): + AnydocParser().init_parser() + + +def test_init_parser_imports_for_real_not_just_find_spec(monkeypatch): + """A wheel whose native extension fails to load has a spec but no importable + module; that must surface at init, not as a bare ImportError from parse_file + mid-ingest (which load_data does not catch).""" + import application.parser.file.anydoc_parser as mod + + monkeypatch.setattr(mod, "anydoc_available", lambda: True) + monkeypatch.setitem(sys.modules, "anydoc", None) + + with pytest.raises(ImportError, match="firecrawl-anydoc"): + AnydocParser().init_parser() diff --git a/tests/parser/file/test_bulk.py b/tests/parser/file/test_bulk.py index 5a81d2d4..40794e18 100644 --- a/tests/parser/file/test_bulk.py +++ b/tests/parser/file/test_bulk.py @@ -508,3 +508,123 @@ class TestGetDefaultFileExtractor: mock_fn.return_value = {".pdf": MagicMock(), ".md": MagicMock()} result = mock_fn() assert ".pdf" in result + + +# ===================================================================== +# DOC_PARSER_ENGINE switch +# ===================================================================== + + +@pytest.mark.unit +class TestParserEngineSwitch: + """``get_default_file_extractor`` honours ``DOC_PARSER_ENGINE`` / ``engine``.""" + + @pytest.fixture + def settings(self): + from application.core.settings import settings + + return settings + + def test_default_engine_is_anydoc(self, settings, monkeypatch): + pytest.importorskip("anydoc") + from application.parser.file.anydoc_parser import ANYDOC_SUFFIXES, AnydocParser + from application.parser.file.bulk import get_default_file_extractor + from application.parser.file.html_parser import HTMLMarkdownParser + + monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "anydoc") + extractor = get_default_file_extractor() + + for suffix in ANYDOC_SUFFIXES: + assert isinstance(extractor[suffix], AnydocParser), suffix + assert isinstance(extractor[".html"], HTMLMarkdownParser) + assert isinstance(extractor[".xhtml"], HTMLMarkdownParser) + # Specialised parsers are untouched by the engine choice. + assert type(extractor[".md"]).__name__ == "MarkdownParser" + assert type(extractor[".json"]).__name__ == "JSONParser" + assert type(extractor[".epub"]).__name__ == "EpubParser" + + def test_anydoc_falls_back_to_docling_when_installed(self): + pytest.importorskip("anydoc") + pytest.importorskip("docling") + from application.parser.file.bulk import get_default_file_extractor + + extractor = get_default_file_extractor(engine="anydoc") + + assert type(extractor[".pdf"].fallback_parser).__name__ == "DoclingPDFParser" + assert type(extractor[".csv"].fallback_parser).__name__ == "DoclingCSVParser" + assert extractor[".doc"].fallback_parser is None # nothing else reads .doc + # docling keeps the formats anydoc cannot read. + assert type(extractor[".vtt"]).__name__ == "DoclingVTTParser" + + def test_anydoc_fallback_honours_ocr_flag(self): + pytest.importorskip("anydoc") + pytest.importorskip("docling") + from application.parser.file.bulk import get_default_file_extractor + + extractor = get_default_file_extractor(engine="anydoc", ocr_enabled=True) + + assert extractor[".pdf"].fallback_parser.ocr_enabled is True + assert type(extractor[".png"]).__name__ == "DoclingImageParser" + + def test_anydoc_falls_back_to_legacy_without_docling(self, monkeypatch): + pytest.importorskip("anydoc") + import sys + + from application.parser.file.bulk import get_default_file_extractor + + monkeypatch.setitem(sys.modules, "docling", None) + extractor = get_default_file_extractor(engine="anydoc") + + assert type(extractor[".pdf"].fallback_parser).__name__ == "PDFParser" + assert type(extractor[".xlsx"].fallback_parser).__name__ == "ExcelParser" + assert type(extractor[".png"]).__name__ == "ImageParser" + + def test_docling_engine_keeps_docling_map(self): + pytest.importorskip("docling") + from application.parser.file.bulk import get_default_file_extractor + + extractor = get_default_file_extractor(engine="docling") + + assert type(extractor[".pdf"]).__name__ == "DoclingPDFParser" + assert type(extractor[".html"]).__name__ == "DoclingHTMLParser" + + def test_setting_selects_docling(self, settings, monkeypatch): + pytest.importorskip("docling") + from application.parser.file.bulk import get_default_file_extractor + + monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "docling") + assert type(get_default_file_extractor()[".pdf"]).__name__ == "DoclingPDFParser" + + def test_unknown_engine_uses_anydoc(self): + pytest.importorskip("anydoc") + from application.parser.file.anydoc_parser import AnydocParser + from application.parser.file.bulk import get_default_file_extractor + + assert isinstance(get_default_file_extractor(engine="ghost")[".pdf"], AnydocParser) + + def test_missing_anydoc_degrades_to_base_engine(self, monkeypatch): + import sys + + from application.parser.file.bulk import get_default_file_extractor + + monkeypatch.setitem(sys.modules, "anydoc", None) + extractor = get_default_file_extractor(engine="anydoc") + + assert type(extractor[".pdf"]).__name__ in ("DoclingPDFParser", "PDFParser") + assert type(extractor[".html"]).__name__ in ("DoclingHTMLParser", "HTMLParser") + + def test_missing_docling_map_is_usable(self, monkeypatch): + """Regression: with docling absent the map handed out docling parsers whose + ``init_parser`` raised ``ImportError`` — not a ``DocumentParseError`` — so + the first file aborted the whole ingest instead of degrading.""" + import sys + + from application.parser.file.bulk import get_default_file_extractor + + monkeypatch.setitem(sys.modules, "docling", None) + extractor = get_default_file_extractor(engine="docling") + + assert type(extractor[".pdf"]).__name__ == "PDFParser" + assert ".xhtml" in extractor + for suffix in (".pdf", ".docx", ".csv", ".xlsx", ".html", ".xhtml", ".pptx"): + extractor[suffix].init_parser() diff --git a/tests/parser/file/test_html_parser.py b/tests/parser/file/test_html_parser.py index 9726ddcb..a1b9a4fc 100644 --- a/tests/parser/file/test_html_parser.py +++ b/tests/parser/file/test_html_parser.py @@ -54,3 +54,180 @@ def test_html_parser_metadata_without_title(tmp_path): def test_html_parser_metadata_unreadable_file(tmp_path): assert HTMLParser().get_file_metadata(tmp_path / "missing.html") == {} + + +# --- HTMLMarkdownParser: the anydoc engine's HTML path --------------------------- + +from application.parser.file.html_parser import HTMLMarkdownParser, html_to_markdown # noqa: E402 + +RICH_HTML = """<html><head><title>Doc Title + + +

Heading

+

See the ref for details.

+
h1h2
12
+
typing.__name__, typing.__spec__.name
+""" + + +@pytest.fixture +def rich_html_file(tmp_path): + path = tmp_path / "rich.html" + path.write_text(RICH_HTML) + return path + + +def test_markdown_parser_keeps_structure(rich_html_file): + out = HTMLMarkdownParser().parse_file(rich_html_file) + + assert isinstance(out, str) + assert "# Heading" in out + assert "[the ref](../ref.html#anchor)" in out # links kept, hrefs verbatim + assert "| h1 | h2 |" in out and "| 1 | 2 |" in out # GFM table + assert "typing.__name__, typing.__spec__.name" in out # code byte-exact + + +def test_markdown_parser_drops_non_content(rich_html_file): + out = HTMLMarkdownParser().parse_file(rich_html_file) + + assert "var secret" not in out + assert "color: red" not in out + assert "Doc Title" not in out # title is metadata, not body text + + +def test_markdown_parser_reports_title_metadata(rich_html_file): + assert HTMLMarkdownParser().get_file_metadata(rich_html_file) == {"title": "Doc Title"} + + +def test_markdown_parser_differs_from_plain_text_contract(rich_html_file): + """``HTMLParser`` stays plain text (the ``fast`` engine's contract); only the new class emits Markdown.""" + plain = HTMLParser().parse_file(rich_html_file) + assert "# Heading" not in plain + assert "[the ref]" not in plain + + +def test_markdown_parser_handles_xhtml(tmp_path): + path = tmp_path / "page.xhtml" + path.write_text( + '\n' + '\n' + 'X' + "

Section

Body & entities.

" + ) + out = HTMLMarkdownParser().parse_file(path) + assert "## Section" in out + assert "Body & entities." in out + + +def test_markdown_parser_init(): + parser = HTMLMarkdownParser() + assert not parser.parser_config_set + parser.init_parser() + assert parser.parser_config_set + + +def test_html_to_markdown_collapses_blank_runs(): + out = html_to_markdown("

a





b

") + assert "\n\n\n" not in out + assert out.startswith("a") and out.endswith("b") + + +# --- HTMLMarkdownParser: input bounding and decoding ------------------------------ + + +def test_markdown_parser_head_truncates_oversized_markup(tmp_path, monkeypatch): + """Markup past MARKUP_MAX_BYTES is not parsed: the soup+markdownify tree costs + ~50x the input, and the upload cap is 100 MB.""" + from application.core.settings import settings + + monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", 600) + path = tmp_path / "big.html" + body = "".join(f"

para {i}

\n" for i in range(300)) + path.write_text("Big\n" + body + "") + assert path.stat().st_size > 600 + + out = HTMLMarkdownParser().parse_file(path) + + assert "para 0" in out + assert "para 299" not in out + assert HTMLMarkdownParser().get_file_metadata(path) == {"title": "Big"} + + +def test_markdown_parser_gate_disabled_reads_everything(tmp_path, monkeypatch): + from application.core.settings import settings + + monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", 0) + path = tmp_path / "big.html" + body = "".join(f"

para {i}

\n" for i in range(300)) + path.write_text("\n" + body + "") + + out = HTMLMarkdownParser().parse_file(path) + + assert "para 299" in out + + +def test_markdown_parser_honours_declared_charset(tmp_path): + """The file is handed to BeautifulSoup as bytes so ```` wins + over the process locale (docling did the same; the text parser does not).""" + path = tmp_path / "cp1250.html" + path.write_bytes( + 'Plzeň' + "

Jiří z Plzně

".encode("cp1250") + ) + + assert "Jiří z Plzně" in HTMLMarkdownParser().parse_file(path) + assert HTMLMarkdownParser().get_file_metadata(path) == {"title": "Plzeň"} + + +def test_markdown_parser_metadata_unreadable_file(tmp_path): + assert HTMLMarkdownParser().get_file_metadata(tmp_path / "missing.html") == {} + + +def test_markdown_parser_cut_inside_utf8_char_does_not_mojibake_the_page(tmp_path, monkeypatch): + """A byte cut mid-character makes strict UTF-8 fail; BeautifulSoup would then + retry as windows-1252, which *succeeds* on umlauts and garbles everything.""" + from application.core.settings import settings + + path = tmp_path / "minified.html" + body = "

Größe Übermaß schön für

" * 400 # no newlines: the cut is arbitrary + path.write_bytes( + ('Größe' + body + "") + .encode("utf-8") + ) + for cap in (2001, 2002, 2003): # one of these lands inside a 2-byte sequence + monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", cap) + out = HTMLMarkdownParser().parse_file(path) + assert "Größe Übermaß schön für" in out, cap + assert "Ã" not in out, cap + + +def test_trim_torn_utf8_tail(): + from application.parser.file.html_parser import _trim_torn_utf8_tail as trim + + e_acute, snowman, emoji = "é".encode(), "☃".encode(), "😀".encode() + assert trim(b"abc") == b"abc" + assert trim(b"abc" + e_acute) == b"abc" + e_acute # complete 2-byte + assert trim(b"abc" + e_acute[:1]) == b"abc" # torn 2-byte + assert trim(b"abc" + snowman[:2]) == b"abc" # torn 3-byte + assert trim(b"abc" + emoji[:3]) == b"abc" # torn 4-byte + assert trim(b"abc" + emoji) == b"abc" + emoji + assert trim(b"") == b"" + assert trim(b"\xe8") == b"" # lone lead byte (also a cp1250 letter: at most one glyph lost) + + +def test_markdown_parser_metadata_reuses_last_parse(tmp_path, rich_html_file, monkeypatch): + """The metadata call after parse_file must not build the soup again.""" + import application.parser.file.html_parser as mod + + parser = HTMLMarkdownParser() + parser.parse_file(rich_html_file) + calls = [] + monkeypatch.setattr(mod, "read_markup_head", lambda *a, **k: calls.append(a) or b"") + assert parser.get_file_metadata(rich_html_file) == {"title": "Doc Title"} + assert calls == [] + # A different file is read afresh. + other = tmp_path / "other.html" + other.write_text("Other") + parser.get_file_metadata(other) + assert len(calls) == 1 diff --git a/tests/parser/file/test_pdfium_parser.py b/tests/parser/file/test_pdfium_parser.py index 62e92491..22b44848 100644 --- a/tests/parser/file/test_pdfium_parser.py +++ b/tests/parser/file/test_pdfium_parser.py @@ -153,9 +153,17 @@ def test_threshold_sends_sparse_text_to_fallback(tmp_path, parser_cls): # --- extractor wiring: the guard that source ingestion keeps docling --------- +# (under the docling engine; the anydoc engine ignores the fast path, see below) -def test_extractor_defaults_to_docling_for_pdf(): +@pytest.fixture +def docling_engine(monkeypatch): + from application.core.settings import settings + + monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "docling") + + +def test_extractor_defaults_to_docling_for_pdf(docling_engine): """Sources must be unaffected: no fast path unless explicitly requested.""" from application.parser.file.bulk import get_default_file_extractor @@ -164,7 +172,7 @@ def test_extractor_defaults_to_docling_for_pdf(): assert type(pdf_parser).__name__ == "DoclingPDFParser" -def test_extractor_uses_fast_path_when_requested(): +def test_extractor_uses_fast_path_when_requested(docling_engine): from application.parser.file.bulk import get_default_file_extractor from application.parser.file.pdfium_parser import PdfiumTextParser @@ -174,7 +182,20 @@ def test_extractor_uses_fast_path_when_requested(): assert type(pdf_parser.fallback_parser).__name__ == "DoclingPDFParser" -def test_fast_path_does_not_change_non_pdf_parsers(): +def test_anydoc_engine_ignores_fast_path(monkeypatch): + """anydoc already reads the text layer in milliseconds and keeps structure.""" + pytest.importorskip("anydoc") + from application.core.settings import settings + from application.parser.file.bulk import get_default_file_extractor + + monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "anydoc") + + pdf_parser = get_default_file_extractor(pdf_text_fast_path=True)[".pdf"] + + assert type(pdf_parser).__name__ == "AnydocParser" + + +def test_fast_path_does_not_change_non_pdf_parsers(docling_engine): from application.parser.file.bulk import get_default_file_extractor plain = get_default_file_extractor() diff --git a/tests/parser/test_document_reader.py b/tests/parser/test_document_reader.py index e73ef3a3..db6648af 100644 --- a/tests/parser/test_document_reader.py +++ b/tests/parser/test_document_reader.py @@ -50,6 +50,19 @@ def test_zip_bomb_declared_size_over_cap_is_rejected(monkeypatch): assert "error" in out and "too much data" in out["error"] +@pytest.mark.unit +@pytest.mark.parametrize( + "suffix", [".docm", ".xlsm", ".xlsb", ".pptm", ".ppsx", ".ppsm", ".odt", ".ods", ".odp"] +) +def test_zip_bomb_guard_covers_every_anydoc_zip_format(monkeypatch, suffix): + """Chat attachments have no extension whitelist, so every zip-packaged format + the anydoc map routes must be gated the same way as .docx.""" + monkeypatch.setattr(dr.settings, "DOCUMENT_MAX_DECOMPRESSED_BYTES", 1000, raising=False) + data = _make_zip({"content.xml": b"A" * 50_000}) + reason = dr._reject_zip_bomb(data, suffix) + assert reason is not None and "too much data" in reason + + @pytest.mark.unit def test_zip_too_many_entries_is_rejected(monkeypatch): monkeypatch.setattr(dr.settings, "DOCUMENT_MAX_ARCHIVE_ENTRIES", 3, raising=False) @@ -186,7 +199,7 @@ class _FakeParser: @pytest.mark.unit def test_engine_picks_parser_and_coerces_list(monkeypatch): fake = _FakeParser(["chunk A", "chunk B"]) - monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None: {".pdf": fake}) + monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": fake}) out = parse_document_bytes(b"%PDF-1.4", "doc.pdf", output="text", engine="docling", include_tables=False) assert out["content"] == "chunk A\n\nchunk B" @@ -213,6 +226,9 @@ def test_default_markdown_with_tables_converts_docling_once(monkeypatch): import sys import types + # engine=auto resolves to the server default; this test is about the docling path. + monkeypatch.setattr(dr.settings, "DOC_PARSER_ENGINE", "docling") + from application.parser.file.docling_parser import DoclingParser counter = {"instances": 0, "converts": 0} @@ -272,7 +288,7 @@ def test_default_markdown_with_tables_converts_docling_once(monkeypatch): # Only reached if the collapse path fails; the test asserts it doesn't. return "# parse-file content" - monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None: {".pdf": _FakeDoclingParser()}) + monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": _FakeDoclingParser()}) # Defaults: output=markdown, engine=auto, include_tables=True -> the double-parse path. out = parse_document_bytes(b"%PDF-1.4", "doc.pdf") @@ -293,7 +309,7 @@ def test_default_markdown_with_tables_converts_docling_once(monkeypatch): @pytest.mark.unit def test_pages_slices_form_feed_blob(monkeypatch): fake = _FakeParser("page1\fpage2\fpage3") - monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None: {".pdf": fake}) + monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": fake}) out = parse_document_bytes(b"%PDF", "doc.pdf", output="text", pages="2", engine="docling", include_tables=False) assert out["content"] == "page2" @@ -443,7 +459,7 @@ def test_temp_file_cleaned_up_on_parser_error(monkeypatch): fake = _FakeParser("x") fake.parse_file = lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")) - monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None: {".pdf": fake}) + monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": fake}) out = parse_document_bytes(b"%PDF", "doc.pdf", output="text", engine="docling", include_tables=False) assert "error" in out and "parsing failed" in out["error"] @@ -622,3 +638,72 @@ def test_structured_tables_render_blank_cells_as_strings(monkeypatch, tmp_path): assert all(isinstance(c, str) for c in cells), f"non-string cells: {cells!r}" assert "1001" in cells and "1001.0" not in cells assert "" in cells # blank cell renders as empty string, not "nan"/"None" + + +# --------------------------------------------------------------------------- +# engine=anydoc / auto: the default engine must never trigger a docling pass +# --------------------------------------------------------------------------- +@pytest.mark.unit +def test_anydoc_engine_is_accepted_and_passed_through(monkeypatch): + fake = _FakeParser("anydoc text") + seen = {} + + def _extractor(ocr_enabled=None, **kwargs): + seen.update(kwargs) + return {".pdf": fake} + + monkeypatch.setattr(dr, "get_default_file_extractor", _extractor) + out = parse_document_bytes(b"%PDF-1.4", "doc.pdf", output="text", engine="anydoc", include_tables=False) + assert out["content"] == "anydoc text" + assert seen == {"engine": "anydoc"} + + +@pytest.mark.unit +def test_auto_engine_under_anydoc_skips_docling_table_pass(monkeypatch): + """Default read (markdown, engine=auto, include_tables) must not run a docling conversion + just to harvest tables when the configured engine is anydoc — that pass would cost more + than the whole read and re-import the stack anydoc exists to avoid.""" + monkeypatch.setattr(dr.settings, "DOC_PARSER_ENGINE", "anydoc") + fake = _FakeParser("# md body") + monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": fake}) + + def _boom(*args, **kwargs): + raise AssertionError("docling conversion attempted under the anydoc engine") + + monkeypatch.setattr(dr, "_docling_structured", _boom) + + out = parse_document_bytes(b"%PDF-1.4", "doc.pdf") # markdown, auto, include_tables=True + + assert out["content"] == "# md body" + assert "tables" not in out + + +@pytest.mark.unit +def test_explicit_docling_engine_still_collects_tables(monkeypatch): + monkeypatch.setattr(dr.settings, "DOC_PARSER_ENGINE", "anydoc") + fake = _FakeParser("body") + monkeypatch.setattr(dr, "get_default_file_extractor", lambda ocr_enabled=None, **kwargs: {".pdf": fake}) + monkeypatch.setattr( + dr, + "_docling_structured", + lambda *a, **k: {"markdown": "body", "tables": [{"markdown": "| t |"}], "structured": {}, "page_count": 1}, + ) + + out = parse_document_bytes(b"%PDF-1.4", "doc.pdf", engine="docling") + + assert out["tables"] == [{"markdown": "| t |"}] + + +@pytest.mark.unit +def test_structured_output_stays_docling_under_anydoc(monkeypatch): + monkeypatch.setattr(dr.settings, "DOC_PARSER_ENGINE", "anydoc") + calls = [] + + def _fake_structured(path, *, ocr_enabled, include_tables, parser=None): + calls.append(path) + return {"markdown": "# s", "tables": [], "structured": {"texts": []}, "page_count": 1} + + monkeypatch.setattr(dr, "_docling_structured", _fake_structured) + out = parse_document_bytes(b"%PDF-1.4", "doc.pdf", output="structured") + assert out["output"] == "structured" + assert len(calls) == 1