Implement anydoc with docling

application/agents/tools/read_document.py

@@ -102,7 +102,8 @@ class ReadDocumentTool(Tool):
                             "enum": ["markdown", "text", "structured", "chunks"],
                             "description": "Shape of the parsed result (default: markdown). Note: "
                             "`structured` always uses the Docling engine regardless of `engine` "
-                            "(the `fast` engine is markdown/text only).",
+                            "(requires the optional docling install; the other engines are "
+                            "markdown/text only).",
                         },
                         "ocr": {
                             "type": "string",
@@ -115,9 +116,11 @@ class ReadDocumentTool(Tool):
                         },
                         "engine": {
                             "type": "string",
-                            "enum": ["auto", "docling", "fast"],
-                            "description": "Parser engine (default: auto). `fast` is a lighter "
-                            "markdown/text-only engine; it is ignored when `output='structured'`, "
+                            "enum": ["auto", "docling", "anydoc", "fast"],
+                            "description": "Parser engine (default: auto = the server's configured "
+                            "engine). `anydoc` is the fast Markdown converter, `docling` the "
+                            "layout-model engine (optional install, needed for tables/OCR), "
+                            "`fast` a plain-text legacy path. Ignored when `output='structured'`, "
                             "which always uses Docling.",
                         },
                         "max_chars": {

application/core/settings.py

@@ -106,6 +106,15 @@ class Settings(BaseSettings):
     UPLOAD_MAX_ARCHIVE_DEPTH: int = Field(default=3, ge=0)
     PARSE_PDF_AS_IMAGE: bool = False
     PARSE_IMAGE_REMOTE: bool = False
+    # Document parser for source ingestion, chat attachments and the
+    # read_document tool. "anydoc" (default): firecrawl-anydoc, a Rust
+    # converter with no ML models — milliseconds per file, ~100 MB peak RSS.
+    # "docling": the layout/table-model pipeline (optional install; needed
+    # for OCR and for read_document's structured output). Files anydoc
+    # cannot convert (scanned PDFs, malformed input) fall back to docling
+    # when it is installed, otherwise to the legacy parsers. Rollback to the
+    # previous behaviour is this one variable.
+    DOC_PARSER_ENGINE: str = "anydoc"
     DOCLING_OCR_ENABLED: bool = False  # Enable OCR for docling parsers (PDF, images)
     DOCLING_OCR_ATTACHMENTS_ENABLED: bool = False  # Enable OCR for docling when parsing attachments
     # Pages docling's threaded pipeline buffers in flight; the library
@@ -114,6 +123,12 @@ class Settings(BaseSettings):
     DOCLING_COMPILE_TORCH_MODELS: bool = False
     DOCLING_TABULAR_MAX_BYTES: int = 2_000_000
     DOCLING_MARKUP_MAX_BYTES: int = 8_000_000
+    # HTML/XHTML larger than this (bytes) are head-truncated before the
+    # markdownify parser runs (the anydoc engine's HTML path). The tree that
+    # path builds costs ~50x the input — 30 MB of HTML measured at 1.6 GB RSS —
+    # and the upload cap is 100 MB, so the gate is what keeps one upload from
+    # taking the ingest worker down. 0 disables it.
+    MARKUP_MAX_BYTES: int = 8_000_000
     # Chars-per-page floor below which an OCR'd PDF/image parse is treated as a docling
     # pipeline dropout (long-running workers were observed returning zero characters for
     # every scanned page after a long scanned PDF, with no error) rather than as content.

application/parser/document_reader.py

@@ -1,10 +1,11 @@
 """In-process document parsing for the ``read_document`` tool, run on the Celery parsing worker.
 
 ``parse_document_bytes`` turns untrusted document bytes into a bounded, shaped
-result (markdown/text/structured/chunks) using the BACKEND parsers (Docling by
-default). It applies the same untrusted-content safeguards as uploads — an
-extension whitelist, a byte cap, ``safe_filename`` staging into a temp file, and
-temp cleanup — so a hostile filename or document is treated as inert data.
+result (markdown/text/structured/chunks) using the BACKEND parsers (the engine
+selected by ``DOC_PARSER_ENGINE``; ``structured`` output always needs Docling).
+It applies the same untrusted-content safeguards as uploads — an extension
+whitelist, a byte cap, ``safe_filename`` staging into a temp file, and temp
+cleanup — so a hostile filename or document is treated as inert data.
 """
 
 from __future__ import annotations
@@ -40,7 +41,7 @@ _MAX_PAGE_SELECTOR_TOKENS = 20_000
 
 _VALID_OUTPUTS = ("markdown", "text", "structured", "chunks")
 _VALID_OCR = ("auto", "on", "off")
-_VALID_ENGINES = ("auto", "docling", "fast")
+_VALID_ENGINES = ("auto", "docling", "anydoc", "fast")
 
 
 def truncate_text_head_tail(text: str, max_bytes: Optional[int] = None) -> str:
@@ -102,7 +103,13 @@ def _max_input_bytes() -> int:
     return int(getattr(settings, "SANDBOX_MAX_INPUT_BYTES", 25 * 1024 * 1024))
 
 
-_ZIP_CONTAINER_EXTENSIONS = frozenset({".docx", ".xlsx", ".pptx", ".epub"})
+# Every zip-packaged format a parser map can route: OOXML and its macro/
+# binary variants, OpenDocument, EPUB. Chat attachments have no extension
+# whitelist, so the anydoc engine can receive any of these.
+_ZIP_CONTAINER_EXTENSIONS = frozenset({
+    ".docx", ".docm", ".xlsx", ".xlsm", ".xlsb", ".pptx", ".pptm", ".ppsx", ".ppsm",
+    ".odt", ".ods", ".odp", ".epub",
+})
 
 
 def _zip_bomb_reason(source: Union[bytes, str, Path], suffix: str) -> Optional[str]:
@@ -165,13 +172,24 @@ def _resolve_ocr_enabled(ocr: str) -> bool:
     return bool(getattr(settings, "DOCLING_OCR_ENABLED", False))
 
 
+def _effective_engine(engine: str) -> str:
+    """Resolve ``auto`` to the server's ``DOC_PARSER_ENGINE``; other values pass through."""
+    if engine != "auto":
+        return engine
+    configured = getattr(settings, "DOC_PARSER_ENGINE", None) or "anydoc"
+    return str(configured).strip().lower()
+
+
 def _pick_parser(suffix: str, *, ocr_enabled: bool, engine: str):
     """Select the parser for ``suffix`` honoring the requested engine; None when unsupported."""
     if engine == "fast":
         legacy = _legacy_parser_for(suffix)
         if legacy is not None:
             return legacy
-    extractor = get_default_file_extractor(ocr_enabled=ocr_enabled)
+    if engine in ("docling", "anydoc"):
+        extractor = get_default_file_extractor(ocr_enabled=ocr_enabled, engine=engine)
+    else:
+        extractor = get_default_file_extractor(ocr_enabled=ocr_enabled)
     return extractor.get(suffix)
 
 
@@ -491,7 +509,12 @@ def _shape(
         }
 
     parser = _pick_parser(suffix, ocr_enabled=ocr_enabled, engine=engine)
-    wants_tables = include_tables and engine != "fast" and output != "chunks"
+    # Tables come from a Docling conversion, so they are only collected when
+    # Docling is the engine in play: under anydoc/fast a table pass would be
+    # a second, full docling conversion that costs more than the whole read.
+    wants_tables = (
+        include_tables and output != "chunks" and _effective_engine(engine) == "docling"
+    )
 
     # A Docling-backed parser already converts the whole document to produce its text.
     # When tables are also requested, reuse that single conversion for both the markdown

application/parser/file/anydoc_parser.py

@@ -0,0 +1,164 @@
+"""anydoc parser.
+
+Converts documents to GitHub-Flavored Markdown with `firecrawl-anydoc
+<https://github.com/firecrawl/anydoc>`_: a single Rust extension with no ML
+models and no Python dependencies. Measured against docling on a 20-document
+corpus it was 20-160x faster per file (1.2 s vs 193 s on a 150-page annual
+report), imported in 6 ms instead of 3.9 s, and peaked at 107 MB RSS instead
+of 2.3 GB, with equivalent output on clean office files, CSV and text-layer
+PDFs.
+
+anydoc never OCRs. It *detects* a scanned or image-only PDF and raises
+``UnsupportedError`` ("OCR is required"), which is the routing point: a
+parser given a ``fallback_parser`` (docling when installed, otherwise the
+legacy parser for the suffix) delegates there; without one the file fails
+loudly as ``DocumentParseError`` rather than being stored empty.
+"""
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, Union
+
+from application.parser.file.base_parser import (
+    BaseParser,
+    DocumentParseError,
+    delegate_parse,
+    module_available,
+)
+
+logger = logging.getLogger(__name__)
+
+# Suffixes anydoc 0.2.3 converts, verified with ``anydoc.format_from_extension``.
+# ``.epub`` is deliberately absent: it stays on ``EpubParser``.
+ANYDOC_SUFFIXES: Tuple[str, ...] = (
+    # Word processing
+    ".pdf",
+    ".docx",
+    ".docm",
+    ".doc",
+    ".odt",
+    ".rtf",
+    # Presentations
+    ".pptx",
+    ".pptm",
+    ".ppsx",
+    ".ppsm",
+    ".ppt",
+    ".pps",
+    ".pot",
+    ".odp",
+    # Spreadsheets / tabular
+    ".xlsx",
+    ".xlsm",
+    ".xlsb",
+    ".xls",
+    ".ods",
+    ".csv",
+)
+
+
+def anydoc_available() -> bool:
+    """Whether the ``anydoc`` extension can be imported."""
+    return module_available("anydoc")
+
+
+class AnydocParser(BaseParser):
+    """Markdown conversion via ``anydoc.to_markdown`` with a typed fallback.
+
+    Attributes:
+        fallback_parser: Parser used when anydoc cannot convert the file
+            (scanned PDF, unsupported or malformed input). ``None`` means
+            such files raise ``DocumentParseError``.
+        last_engine: Name of the engine that produced the most recent parse:
+            ``"anydoc"``, or the fallback parser's ``last_engine`` / class
+            name when it was delegated to. Mirrors ``PdfiumTextParser`` so
+            the attachment worker records what actually ran.
+    """
+
+    def __init__(
+        self,
+        fallback_parser: Optional[BaseParser] = None,
+        parser_config: Optional[Dict] = None,
+    ) -> None:
+        super().__init__(parser_config)
+        self.fallback_parser = fallback_parser
+        self.last_engine: Optional[str] = None
+
+    def _init_parser(self) -> Dict:
+        # Import for real rather than trusting ``find_spec``: a wheel whose
+        # native extension fails to load (glibc/arch mismatch) has a spec but
+        # no importable module, and that must fail here — where the ingest
+        # task treats it as a setup error — not as a bare ImportError from
+        # ``parse_file`` mid-batch, which ``load_data`` does not catch.
+        try:
+            import anydoc  # noqa: F401
+        except ImportError as exc:
+            raise ImportError(
+                "firecrawl-anydoc is required for AnydocParser. "
+                "Install it with: pip install firecrawl-anydoc"
+            ) from exc
+        fallback = self.fallback_parser
+        return {"fallback_parser": type(fallback).__name__ if fallback else None}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Convert ``file`` to Markdown.
+
+        Args:
+            file: Path to the document.
+            errors: Decoding error policy; anydoc decodes internally, so this
+                is only forwarded to the fallback parser.
+
+        Returns:
+            The document as Markdown.
+
+        Raises:
+            DocumentParseError: When neither anydoc nor the fallback could
+                produce text for the file, or anydoc hit a resource limit.
+        """
+        import anydoc
+
+        path = Path(file)
+        try:
+            content = anydoc.to_markdown(str(path))
+        except anydoc.ResourceLimitError as exc:
+            # anydoc refused because the file is too expensive (declared
+            # decompression size, nesting depth, node count). A heavier
+            # engine would spend exactly what was just refused, so this is
+            # terminal rather than a reason to delegate.
+            self.last_engine = None
+            raise DocumentParseError(
+                f"Failed to parse {path.name}: {type(exc).__name__}: {exc}"
+            ) from exc
+        except anydoc.ConvertError as exc:
+            # Typed, per-document failures another engine may still handle:
+            # UnsupportedError (scanned PDF / unknown format), MalformedError,
+            # EncryptedError, MissingPartError.
+            return self._delegate(path, errors, f"{type(exc).__name__}: {exc}")
+        except OSError as exc:
+            raise DocumentParseError(
+                f"Failed to parse {path.name}: the file could not be read."
+            ) from exc
+        except Exception as exc:
+            logger.error(f"anydoc failed on {path.name}: {exc}", exc_info=True)
+            raise DocumentParseError(
+                f"Failed to parse {path.name} with anydoc: {exc}"
+            ) from exc
+
+        if not content or not content.strip():
+            return self._delegate(path, errors, "anydoc produced no text")
+
+        self.last_engine = "anydoc"
+        return content
+
+    def _delegate(self, path: Path, errors: str, reason: str) -> Union[str, List[str]]:
+        """Hand ``path`` to the fallback parser, or fail loudly without one."""
+        fallback = self.fallback_parser
+        if fallback is None:
+            self.last_engine = None
+            raise DocumentParseError(f"Failed to parse {path.name}: {reason}")
+        logger.warning(
+            f"anydoc could not convert {path.name} ({reason}); "
+            f"falling back to {type(fallback).__name__}"
+        )
+        result = delegate_parse(fallback, path, errors)
+        self.last_engine = getattr(fallback, "last_engine", None) or type(fallback).__name__
+        return result

application/parser/file/base_parser.py

@@ -1,9 +1,14 @@
 """Base parser and config class."""
 
+import importlib.util
+import logging
+import sys
 from abc import abstractmethod
 from pathlib import Path
 from typing import Dict, List, Optional, Union
 
+logger = logging.getLogger(__name__)
+
 
 class DocumentParseError(Exception):
     """A file could not be converted to text.
@@ -57,3 +62,67 @@ class BaseParser:
         """Return parser-specific metadata for the most recently parsed file."""
         _ = file
         return {}
+
+
+def module_available(name: str) -> bool:
+    """Whether top-level module ``name`` can be imported, without importing it.
+
+    Cheap enough to call while building a parser map. Honors ``sys.modules``
+    first so tests that stub a module in (or block one out with ``None``)
+    are seen the same way the import system would see them; a stub with no
+    ``__spec__`` counts as present rather than raising.
+
+    Args:
+        name: Top-level module name, e.g. ``"docling"``.
+
+    Returns:
+        True when an import of ``name`` would succeed.
+    """
+    if name in sys.modules:
+        return sys.modules[name] is not None
+    try:
+        return importlib.util.find_spec(name) is not None
+    except (ImportError, ValueError):
+        return False
+
+
+def delegate_parse(
+    parser: "BaseParser", file: Path, errors: str
+) -> Union[str, List[str]]:
+    """Run a fallback parser, normalizing its failures to ``DocumentParseError``.
+
+    Used wherever one parser hands a file to another (docling to the plain
+    tabular parsers for oversized sheets, anydoc to docling/legacy parsers
+    for files it cannot convert). Wrapping matters twice over: the Celery
+    tasks that own these paths list ``DocumentParseError`` in
+    ``dont_autoretry_for``, so a deterministic content error fails once
+    instead of retrying four times; and ``SimpleDirectoryReader.load_data``
+    catches only ``DocumentParseError``, so one poison file is skipped into
+    ``failed_files`` instead of aborting an entire multi-file ingest.
+
+    Args:
+        parser: The parser to delegate to; initialized here if needed.
+        file: Path to the file being parsed.
+        errors: Decoding error policy, forwarded to the parser.
+
+    Returns:
+        The parsed text, or list of row strings.
+
+    Raises:
+        DocumentParseError: If the fallback parser fails for any reason.
+    """
+    try:
+        if not parser.parser_config_set:
+            parser.init_parser()
+        return parser.parse_file(file, errors)
+    except DocumentParseError:
+        raise
+    except Exception as e:
+        logger.error(
+            f"Fallback parse of {Path(file).name} with "
+            f"{type(parser).__name__} failed: {e}",
+            exc_info=True,
+        )
+        raise DocumentParseError(
+            f"Failed to parse {Path(file).name}: the file could not be read."
+        ) from e

application/parser/file/bulk.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Callable, Dict, List, Optional, Tuple, Union
 
 from application.parser.file.base import BaseReader
-from application.parser.file.base_parser import BaseParser, DocumentParseError
+from application.parser.file.base_parser import BaseParser, DocumentParseError, module_available
 from application.parser.file.docs_parser import DocxParser, PDFParser
 from application.parser.file.epub_parser import EpubParser
 from application.parser.file.html_parser import HTMLParser
@@ -49,28 +49,66 @@ def _wrap_pdf_fast_path(pdf_parser: BaseParser) -> BaseParser:
     )
 
 
-def get_default_file_extractor(
-    ocr_enabled: Optional[bool] = None,
-    pdf_text_fast_path: bool = False,
-) -> Dict[str, BaseParser]:
-    """Get the default file extractor.
-
-    Uses docling parsers by default for advanced document processing.
-    Falls back to standard parsers if docling is not installed.
+def _legacy_file_extractor(pdf_text_fast_path: bool = False) -> Dict[str, BaseParser]:
+    """Parser map that needs neither docling nor anydoc.
 
     Args:
-        ocr_enabled: Enable OCR in the docling parsers. Defaults to
-            ``settings.DOCLING_OCR_ENABLED``.
-        pdf_text_fast_path: Read PDFs via their embedded text layer
-            (pypdfium2) instead of docling, falling back to docling for scans.
-            Off by default: it trades docling's structural markdown for speed,
-            which suits attachments (read into a prompt) but not source
-            ingestion (chunked and embedded for retrieval).
+        pdf_text_fast_path: Put the pypdfium2 text-layer parser in front of
+            the legacy PDF parser.
+
+    Returns:
+        Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
+    """
+    legacy_pdf: BaseParser = PDFParser()
+    if pdf_text_fast_path:
+        legacy_pdf = _wrap_pdf_fast_path(legacy_pdf)
+    return {
+        ".pdf": legacy_pdf,
+        ".docx": DocxParser(),
+        ".csv": PandasCSVParser(),
+        ".xlsx": ExcelParser(),
+        ".epub": EpubParser(),
+        ".md": MarkdownParser(),
+        ".rst": RstParser(),
+        ".html": HTMLParser(),
+        ".xhtml": HTMLParser(),
+        ".mdx": MarkdownParser(),
+        ".json": JSONParser(),
+        ".pptx": PPTXParser(),
+        ".png": ImageParser(),
+        ".jpg": ImageParser(),
+        ".jpeg": ImageParser(),
+        **_build_audio_parser_mapping(),
+    }
+
+
+def _docling_file_extractor(
+    ocr_enabled: bool,
+    pdf_text_fast_path: bool = False,
+    *,
+    missing_log_level: int = logging.WARNING,
+) -> Dict[str, BaseParser]:
+    """docling parser map, or the legacy map when docling is not installed.
+
+    The availability check has to happen here, not at parse time:
+    ``DoclingParser`` only imports docling inside ``_init_parser``, and the
+    ``ImportError`` that raises is not a ``DocumentParseError``, so a map
+    that handed out docling parsers on a docling-less install would abort
+    the whole ingest on its first file instead of degrading.
+
+    Args:
+        ocr_enabled: Enable OCR in the docling PDF/image parsers.
+        pdf_text_fast_path: Put the pypdfium2 text-layer parser in front of
+            the PDF parser (attachments only; see ``get_default_file_extractor``).
+        missing_log_level: Level at which a missing docling install is logged;
+            informational when docling is only the *fallback* engine.
 
     Returns:
         Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
     """
     try:
+        if not module_available("docling"):
+            raise ImportError("docling is not installed")
         from application.parser.file.docling_parser import (
             DoclingPDFParser,
             DoclingDocxParser,
@@ -83,71 +121,140 @@ def get_default_file_extractor(
             DoclingVTTParser,
             DoclingXMLParser,
         )
-        if ocr_enabled is None:
-            ocr_enabled = settings.DOCLING_OCR_ENABLED
-        pdf_parser: BaseParser = DoclingPDFParser(ocr_enabled=ocr_enabled)
-        if pdf_text_fast_path:
-            pdf_parser = _wrap_pdf_fast_path(pdf_parser)
-        return {
-            # Documents
-            ".pdf": pdf_parser,
-            ".docx": DoclingDocxParser(),
-            ".pptx": DoclingPPTXParser(),
-            ".xlsx": DoclingXLSXParser(),
-            # Web formats
-            ".html": DoclingHTMLParser(),
-            ".xhtml": DoclingHTMLParser(),
-            # Data formats
-            ".csv": DoclingCSVParser(),
-            ".json": JSONParser(),  # Keep JSON parser (specialized handling)
-            # Text/markup formats
-            ".md": MarkdownParser(),  # Keep markdown parser (specialized handling)
-            ".mdx": MarkdownParser(),
-            ".rst": RstParser(),
-            ".adoc": DoclingAsciiDocParser(),
-            ".asciidoc": DoclingAsciiDocParser(),
-            # Images (with OCR) - only use Docling when OCR is enabled
-            ".png": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".jpg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".jpeg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".tiff": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".tif": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".bmp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            ".webp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
-            # Media/subtitles
-            ".vtt": DoclingVTTParser(),
-            **_build_audio_parser_mapping(),
-            # Specialized XML formats
-            ".xml": DoclingXMLParser(),
-            # Formats docling doesn't support - use standard parsers
-            ".epub": EpubParser(),
-        }
     except ImportError:
-        logging.warning(
+        logging.log(
+            missing_log_level,
             "docling is not installed. Using standard parsers. "
-            "For advanced document parsing, install with: pip install docling"
+            "For OCR and layout-model parsing, install with: pip install docling",
         )
-        # Fallback to standard parsers
-        legacy_pdf: BaseParser = PDFParser()
-        if pdf_text_fast_path:
-            legacy_pdf = _wrap_pdf_fast_path(legacy_pdf)
-        return {
-            ".pdf": legacy_pdf,
-            ".docx": DocxParser(),
-            ".csv": PandasCSVParser(),
-            ".xlsx": ExcelParser(),
-            ".epub": EpubParser(),
-            ".md": MarkdownParser(),
-            ".rst": RstParser(),
-            ".html": HTMLParser(),
-            ".mdx": MarkdownParser(),
-            ".json": JSONParser(),
-            ".pptx": PPTXParser(),
-            ".png": ImageParser(),
-            ".jpg": ImageParser(),
-            ".jpeg": ImageParser(),
-            **_build_audio_parser_mapping(),
-        }
+        return _legacy_file_extractor(pdf_text_fast_path)
+
+    pdf_parser: BaseParser = DoclingPDFParser(ocr_enabled=ocr_enabled)
+    if pdf_text_fast_path:
+        pdf_parser = _wrap_pdf_fast_path(pdf_parser)
+    return {
+        # Documents
+        ".pdf": pdf_parser,
+        ".docx": DoclingDocxParser(),
+        ".pptx": DoclingPPTXParser(),
+        ".xlsx": DoclingXLSXParser(),
+        # Web formats
+        ".html": DoclingHTMLParser(),
+        ".xhtml": DoclingHTMLParser(),
+        # Data formats
+        ".csv": DoclingCSVParser(),
+        ".json": JSONParser(),  # Keep JSON parser (specialized handling)
+        # Text/markup formats
+        ".md": MarkdownParser(),  # Keep markdown parser (specialized handling)
+        ".mdx": MarkdownParser(),
+        ".rst": RstParser(),
+        ".adoc": DoclingAsciiDocParser(),
+        ".asciidoc": DoclingAsciiDocParser(),
+        # Images (with OCR) - only use Docling when OCR is enabled
+        ".png": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+        ".jpg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+        ".jpeg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+        ".tiff": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+        ".tif": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+        ".bmp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+        ".webp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+        # Media/subtitles
+        ".vtt": DoclingVTTParser(),
+        **_build_audio_parser_mapping(),
+        # Specialized XML formats
+        ".xml": DoclingXMLParser(),
+        # Formats docling doesn't support - use standard parsers
+        ".epub": EpubParser(),
+    }
+
+
+def _anydoc_file_extractor(ocr_enabled: bool, pdf_text_fast_path: bool = False) -> Dict[str, BaseParser]:
+    """anydoc parser map, layered over the docling (or legacy) map.
+
+    anydoc takes every format it converts; each of those parsers gets the
+    base map's parser for the same suffix as ``fallback_parser``, so a
+    scanned PDF still reaches docling+OCR when docling is installed and the
+    legacy parser otherwise. docling, when installed, also keeps serving
+    what anydoc cannot: image OCR and ``.adoc``/``.vtt``/``.xml``. HTML goes
+    to the markdownify-based ``HTMLMarkdownParser``.
+
+    ``pdf_text_fast_path`` is deliberately ignored here: anydoc already reads
+    the text layer in milliseconds *and* keeps headings/tables, and does its
+    own scanned-PDF detection, so putting pypdfium2 in front of it would
+    only lose structure.
+
+    Args:
+        ocr_enabled: Enable OCR in the docling fallback/image parsers.
+        pdf_text_fast_path: Honored only when anydoc is not installed and the
+            map degrades to the base engine.
+
+    Returns:
+        Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
+    """
+    from application.parser.file.anydoc_parser import (
+        ANYDOC_SUFFIXES,
+        AnydocParser,
+        anydoc_available,
+    )
+    from application.parser.file.html_parser import HTMLMarkdownParser
+
+    if not anydoc_available():
+        logging.warning(
+            "firecrawl-anydoc is not installed; DOC_PARSER_ENGINE=anydoc is using "
+            "the %s parsers instead. Install with: pip install firecrawl-anydoc",
+            "docling" if module_available("docling") else "standard",
+        )
+        return _docling_file_extractor(ocr_enabled, pdf_text_fast_path)
+
+    base = _docling_file_extractor(
+        ocr_enabled, pdf_text_fast_path=False, missing_log_level=logging.INFO
+    )
+    extractor = dict(base)
+    for suffix in ANYDOC_SUFFIXES:
+        extractor[suffix] = AnydocParser(fallback_parser=base.get(suffix))
+    extractor[".html"] = HTMLMarkdownParser()
+    extractor[".xhtml"] = HTMLMarkdownParser()
+    return extractor
+
+
+def get_default_file_extractor(
+    ocr_enabled: Optional[bool] = None,
+    pdf_text_fast_path: bool = False,
+    engine: Optional[str] = None,
+) -> Dict[str, BaseParser]:
+    """Get the default file extractor.
+
+    The engine is ``settings.DOC_PARSER_ENGINE`` unless overridden:
+
+    * ``anydoc`` (default): firecrawl-anydoc for office/PDF/tabular formats,
+      markdownify for HTML, docling (if installed) for image OCR and as the
+      fallback for files anydoc rejects; legacy parsers otherwise.
+    * ``docling``: the docling map, with the legacy parsers when docling is
+      not installed.
+
+    Args:
+        ocr_enabled: Enable OCR in the docling parsers. Defaults to
+            ``settings.DOCLING_OCR_ENABLED``.
+        pdf_text_fast_path: Read PDFs via their embedded text layer
+            (pypdfium2) instead of docling, falling back to docling for scans.
+            Off by default: it trades docling's structural markdown for speed,
+            which suits attachments (read into a prompt) but not source
+            ingestion (chunked and embedded for retrieval). A no-op under the
+            anydoc engine, which is already that fast and keeps structure.
+        engine: ``"anydoc"`` or ``"docling"``; ``None`` reads the setting.
+
+    Returns:
+        Dict[str, BaseParser]: Parser keyed by lower-case file suffix.
+    """
+    if ocr_enabled is None:
+        ocr_enabled = settings.DOCLING_OCR_ENABLED
+    selected = (engine or getattr(settings, "DOC_PARSER_ENGINE", None) or "anydoc")
+    selected = str(selected).strip().lower()
+    if selected == "docling":
+        return _docling_file_extractor(ocr_enabled, pdf_text_fast_path)
+    if selected != "anydoc":
+        logging.warning("Unknown DOC_PARSER_ENGINE %r; using anydoc", selected)
+    return _anydoc_file_extractor(ocr_enabled, pdf_text_fast_path)
 
 
 # For backwards compatibility

application/parser/file/docling_parser.py

@@ -15,7 +15,11 @@ import zipfile
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
-from application.parser.file.base_parser import BaseParser, DocumentParseError
+from application.parser.file.base_parser import (
+    BaseParser,
+    DocumentParseError,
+    delegate_parse as _delegate,
+)
 from application.utils import truncate_to_line_boundary
 
 logger = logging.getLogger(__name__)
@@ -118,46 +122,6 @@ def _exceeds_tabular_gate(file: Path) -> bool:
     return _tabular_content_size(file) > max_bytes
 
 
-def _delegate(
-    parser: BaseParser, file: Path, errors: str
-) -> Union[str, List[str]]:
-    """Run a lightweight fallback parser, normalizing its failures.
-
-    Anything the fallback raises is converted to ``DocumentParseError``,
-    which matters twice over. The Celery tasks that own these paths list
-    ``DocumentParseError`` in ``dont_autoretry_for``, so a deterministic
-    content error now fails once instead of retrying four times (each retry
-    re-downloading the file from S3); and ``SimpleDirectoryReader.load_data``
-    catches only ``DocumentParseError``, so one poison spreadsheet is skipped
-    into ``failed_files`` instead of aborting an entire multi-file ingest.
-
-    This restores the symmetry with ``DoclingParser.parse_file``, whose own
-    failures are already wrapped — the delegation branch bypassed that.
-
-    Args:
-        parser: The lightweight parser to delegate to.
-        file: Path to the file being parsed.
-        errors: Decoding error policy, forwarded to the parser.
-
-    Returns:
-        The parsed text, or list of row strings.
-
-    Raises:
-        DocumentParseError: If the fallback parser fails for any reason.
-    """
-    try:
-        return parser.parse_file(file, errors)
-    except DocumentParseError:
-        raise
-    except Exception as e:
-        logger.error(
-            f"Lightweight parse of {file.name} failed: {e}", exc_info=True
-        )
-        raise DocumentParseError(
-            f"Failed to parse {file.name}: the file could not be read."
-        ) from e
-
-
 def _capped_markup_copy(file: Path) -> Optional[str]:
     """Head-truncate an oversized markup file to a temp copy for docling.
 

application/parser/file/html_parser.py

@@ -1,12 +1,136 @@
-"""HTML parser.
-
-Contains parser for html files.
+"""HTML parsers.
 
+``HTMLParser`` returns the visible text of a page (the ``fast`` engine and
+the legacy no-docling map rely on that plain-text contract).
+``HTMLMarkdownParser`` returns Markdown instead and is what the anydoc
+engine maps ``.html``/``.xhtml`` to: anydoc has no HTML support, and
+markdownify — already a dependency of the web crawler — keeps links, GFM
+tables, headings and byte-exact code blocks at a few ms per page.
 """
+import logging
+import re
 from pathlib import Path
-from typing import Dict, Union
+from typing import Dict, Optional, Tuple, Union
 
+from application.core.settings import settings
 from application.parser.file.base_parser import BaseParser
+from application.utils import truncate_to_line_boundary
+
+logger = logging.getLogger(__name__)
+
+# The crawler's conventions (``crawler_markdown.py`` / ``read_webpage.py``),
+# so file and web ingestion produce the same Markdown shape.
+MARKDOWNIFY_OPTIONS = {"heading_style": "ATX", "newline_style": "BACKSLASH"}
+
+# Elements whose text is never document content. ``title`` is reported via
+# ``get_file_metadata`` instead of leaking in as a stray first line.
+_DROP_TAGS = ("title", "script", "style", "noscript", "template")
+
+
+def html_to_markdown(html: Union[str, bytes]) -> str:
+    """Convert an HTML/XHTML document to Markdown.
+
+    Args:
+        html: The markup. Pass the raw bytes when you have them: BeautifulSoup
+            then decodes by the document's own BOM / ``<meta charset>``
+            instead of the process locale, so a windows-1250 page keeps its
+            diacritics.
+
+    Returns:
+        Markdown with runs of blank lines collapsed to one.
+    """
+    from bs4 import BeautifulSoup
+
+    return soup_to_markdown(BeautifulSoup(html, "html.parser"))
+
+
+def soup_to_markdown(soup) -> str:
+    """Convert a parsed ``BeautifulSoup`` tree to Markdown.
+
+    Destructive: the non-content elements in ``_DROP_TAGS`` (including
+    ``<title>``) are removed from ``soup``; read the title first.
+
+    Args:
+        soup: The parsed document.
+
+    Returns:
+        Markdown with runs of blank lines collapsed to one.
+    """
+    from markdownify import MarkdownConverter
+
+    for tag in soup.find_all(_DROP_TAGS):
+        tag.decompose()
+    markdown = MarkdownConverter(**MARKDOWNIFY_OPTIONS).convert_soup(soup)
+    return re.sub(r"\n{3,}", "\n\n", markdown).strip()
+
+
+def _soup_title(soup) -> Optional[str]:
+    """The document's ``<title>`` text, or None."""
+    if soup.title and soup.title.string:
+        return str(soup.title.string)
+    return None
+
+
+def _trim_torn_utf8_tail(data: bytes) -> bytes:
+    """Drop a trailing partial multi-byte UTF-8 sequence left by a byte cut.
+
+    A cut that lands inside a multi-byte character matters more than one
+    lost glyph: BeautifulSoup decodes strictly and, when the declared UTF-8
+    fails on the torn tail, retries as windows-1252 — and if every other
+    non-ASCII byte happens to be cp1252-valid (German umlauts are), the
+    *whole* document comes back as mojibake. Only a UTF-8 tail is
+    recognised; a single-byte page loses at most the final byte.
+
+    Args:
+        data: A head window whose last bytes may be a torn sequence.
+
+    Returns:
+        ``data`` minus any trailing incomplete UTF-8 sequence.
+    """
+    end = len(data)
+    trailing = 0
+    while end > 0 and trailing < 3 and 0x80 <= data[end - 1] < 0xC0:
+        end -= 1
+        trailing += 1
+    if end == 0:
+        return data
+    lead = data[end - 1]
+    if lead < 0xC0:
+        return data
+    need = 2 if lead < 0xE0 else 3 if lead < 0xF0 else 4
+    if trailing + 1 < need:
+        return data[: end - 1]
+    return data
+
+
+def read_markup_head(file: Path, max_bytes: int) -> bytes:
+    """Read a markup file, head-truncated to ``max_bytes`` on a line boundary.
+
+    HTML has no lightweight fallback parser to bound it by other means (the
+    way oversized CSV/XLSX go to the plain tabular parsers), and the soup +
+    markdownify tree costs ~50x the input — 30 MB measured at 1.6 GB RSS,
+    against a 100 MB upload cap. Parsing only the head keeps one upload from
+    taking the ingest worker down; the cut lands on a line boundary when one
+    is reasonably close, and the lenient HTML parser copes with the torn tail.
+
+    Args:
+        file: Path to the markup file.
+        max_bytes: Size gate; ``<= 0`` reads the whole file.
+
+    Returns:
+        The file's bytes, or its first ``max_bytes`` when it is larger.
+    """
+    with open(file, "rb") as fh:
+        if max_bytes <= 0:
+            return fh.read()
+        head = fh.read(max_bytes + 1)
+    if len(head) <= max_bytes:
+        return head
+    logger.warning(
+        f"Markup {Path(file).name} exceeds MARKUP_MAX_BYTES ({max_bytes}); "
+        f"parsing the first {max_bytes} bytes to bound memory"
+    )
+    return _trim_torn_utf8_tail(truncate_to_line_boundary(head[:max_bytes]))
 
 
 class HTMLParser(BaseParser):
@@ -40,3 +164,51 @@ class HTMLParser(BaseParser):
         if soup.title and soup.title.string:
             return {"title": str(soup.title.string)}
         return {}
+
+
+class HTMLMarkdownParser(HTMLParser):
+    """HTML/XHTML to Markdown, for chunking and retrieval.
+
+    Differs from ``HTMLParser`` in three ways: the body comes back as
+    Markdown; the file is read as bytes so the document's own charset
+    declaration wins over the process locale; and input is head-truncated at
+    ``MARKUP_MAX_BYTES`` (the docling HTML path has the same gate under
+    ``DOCLING_MARKUP_MAX_BYTES``).
+    """
+
+    def __init__(self, parser_config: Optional[Dict] = None) -> None:
+        super().__init__(parser_config)
+        # (path, title) of the most recent ``parse_file``, so the metadata
+        # call that follows it does not build the soup a second time.
+        self._last_title: Optional[Tuple[Path, Optional[str]]] = None
+
+    @staticmethod
+    def _read(file: Path) -> bytes:
+        return read_markup_head(Path(file), int(settings.MARKUP_MAX_BYTES))
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        """Convert an HTML file to Markdown.
+
+        ``errors`` is accepted for the ``BaseParser`` contract but unused:
+        decoding is done by BeautifulSoup from the raw bytes.
+        """
+        from bs4 import BeautifulSoup
+
+        _ = errors
+        soup = BeautifulSoup(self._read(file), "html.parser")
+        self._last_title = (Path(file), _soup_title(soup))
+        return soup_to_markdown(soup)
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Return the document title, decoded the same way as the body."""
+        from bs4 import BeautifulSoup
+
+        last = self._last_title
+        if last is not None and last[0] == Path(file):
+            title = last[1]
+        else:
+            try:
+                title = _soup_title(BeautifulSoup(self._read(file), "html.parser"))
+            except OSError:
+                return {}
+        return {"title": title} if title else {}

application/parser/remote/crawler_markdown.py

@@ -110,8 +110,9 @@ class CrawlerLoader(BaseRemote):
         language = language_tag.get('lang', 'en') if language_tag else "en"
 
         markdownified = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH")
-        # Reduce sequences of more than two newlines to exactly three
-        markdownified = re.sub(r'\n{3,}', '\n\n\n', markdownified)
+        # Collapse runs of blank lines to a single one — the same shape
+        # ``html_to_markdown`` gives uploaded HTML files.
+        markdownified = re.sub(r'\n{3,}', '\n\n', markdownified)
         return title, language, markdownified
 
     def _extract_links(self, html_content, current_url):

application/requirements.txt

@@ -13,6 +13,9 @@ dataclasses-json==0.6.7
 daytona==0.205.1
 defusedxml==0.7.1
 docling==2.119.0
+# Default document converter (DOC_PARSER_ENGINE=anydoc); docling above is
+# the optional heavy engine for OCR/structured output.
+firecrawl-anydoc==0.2.3
 rapidocr==3.9.2
 onnxruntime==1.28.0
 docx2txt==0.9

application/worker.py

@@ -1739,10 +1739,11 @@ def attachment_worker(self, file_info, user):
             scope={"kind": "attachment", "id": str(attachment_id)},
         )
 
-        # Attachments only: PDFs are read via their text layer where one
-        # exists (see ``ATTACHMENT_PDF_TEXT_FAST_PATH``). Source ingestion
+        # Attachments only: under the docling engine PDFs are read via their
+        # text layer where one exists (``ATTACHMENT_PDF_TEXT_FAST_PATH``; a
+        # no-op under anydoc, which is already that fast). Source ingestion
         # calls SimpleDirectoryReader without a ``file_extractor`` and so keeps
-        # the docling default, which is what retrieval quality depends on.
+        # the ``DOC_PARSER_ENGINE`` default map, which retrieval quality depends on.
         file_extractor = get_default_file_extractor(
             ocr_enabled=settings.DOCLING_OCR_ATTACHMENTS_ENABLED,
             pdf_text_fast_path=settings.ATTACHMENT_PDF_TEXT_FAST_PATH,