Batch fixes 2

application/parser/document_reader.py

@@ -196,18 +196,27 @@ def _pick_parser(suffix: str, *, ocr_enabled: bool, engine: str):
 def _legacy_parser_for(suffix: str):
     """Return a non-Docling parser for ``suffix`` (the ``fast`` engine), or None."""
     from application.parser.file.docs_parser import DocxParser, PDFParser
+    from application.parser.file.epub_parser import EpubParser
     from application.parser.file.html_parser import HTMLParser
+    from application.parser.file.json_parser import JSONParser
     from application.parser.file.markdown_parser import MarkdownParser
+    from application.parser.file.pptx_parser import PPTXParser
+    from application.parser.file.rst_parser import RstParser
     from application.parser.file.tabular_parser import ExcelParser, PandasCSVParser
 
     legacy = {
         ".pdf": PDFParser,
         ".docx": DocxParser,
+        ".pptx": PPTXParser,
         ".csv": PandasCSVParser,
         ".xlsx": ExcelParser,
         ".html": HTMLParser,
+        ".xhtml": HTMLParser,
+        ".epub": EpubParser,
         ".md": MarkdownParser,
         ".mdx": MarkdownParser,
+        ".rst": RstParser,
+        ".json": JSONParser,
     }
     cls = legacy.get(suffix)
     return cls() if cls is not None else None
@@ -223,6 +232,17 @@ def _parse_to_text(parser: Any, path: Path) -> str:
     return str(parsed)
 
 
+def _is_native_ocr_parser(parser: Any) -> bool:
+    """True for the native OCR PDF parser (OCR_BACKEND=native): a docling table pass would OCR the scan again."""
+    if parser is None:
+        return False
+    try:
+        from application.parser.file.ocr_parser import NativeOcrPdfParser
+    except Exception:
+        return False
+    return isinstance(parser, NativeOcrPdfParser)
+
+
 def _is_docling_parser(parser: Any) -> bool:
     """True when ``parser`` is Docling-backed (collecting tables would otherwise re-convert)."""
     if parser is None:
@@ -510,10 +530,17 @@ def _shape(
 
     parser = _pick_parser(suffix, ocr_enabled=ocr_enabled, engine=engine)
     # Tables come from a Docling conversion, so they are only collected when
-    # Docling is the engine in play: under anydoc/fast a table pass would be
-    # a second, full docling conversion that costs more than the whole read.
+    # a Docling parser is actually in play: under anydoc/fast a table pass
+    # would be a second, full docling conversion that costs more than the
+    # whole read, and under OCR_BACKEND=native the PDF parser is the native
+    # OCR parser (docling only as its text_parser), where a vanilla
+    # DocumentConverter pass would OCR the scan a second time with docling's
+    # default engine, ignoring OCR_ENGINE.
     wants_tables = (
-        include_tables and output != "chunks" and _effective_engine(engine) == "docling"
+        include_tables
+        and output != "chunks"
+        and _effective_engine(engine) == "docling"
+        and not _is_native_ocr_parser(parser)
     )
 
     # A Docling-backed parser already converts the whole document to produce its text.
@@ -536,7 +563,16 @@ def _shape(
 
     if parser is None:
         # A whitelisted extension with no dedicated parser (e.g. .txt) reads as plain
-        # text, matching SimpleDirectoryReader's standard-read fallback.
+        # text, matching SimpleDirectoryReader's standard-read fallback. Binary
+        # office formats that only anydoc reads must not: without anydoc they
+        # would come back as OLE/zip bytes decoded as text.
+        from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
+
+        if suffix in ANYDOC_GAINED_SUFFIXES:
+            return {
+                "error": f"No parser is available for {suffix} files: firecrawl-anydoc "
+                "is not installed (pip install firecrawl-anydoc)"
+            }
         text = path.read_text(errors="ignore")
     else:
         text = _parse_to_text(parser, path)

application/parser/file/base_parser.py

@@ -117,6 +117,11 @@ def delegate_parse(
         return parser.parse_file(file, errors)
     except DocumentParseError:
         raise
+    except ImportError:
+        # A fallback whose dependency is missing is a deployment problem, not
+        # a bad document: let it reach the ingest task's setup-error path
+        # instead of blaming every file with "could not be read".
+        raise
     except Exception as e:
         logger.error(
             f"Fallback parse of {Path(file).name} with "

application/parser/file/bulk.py

@@ -53,9 +53,10 @@ def _gained_format_entries() -> Dict[str, BaseParser]:
     """Anydoc-only formats (legacy/macro Office, OpenDocument, RTF) for every map.
 
     anydoc is a core dependency, so these suffixes are parseable under both
-    engines. When anydoc is somehow missing the entries are omitted, and such
-    files fall to ``SimpleDirectoryReader``'s plain-text read — the
-    pre-existing behaviour for unmapped suffixes.
+    engines. When anydoc is somehow missing the entries are omitted and
+    ``SimpleDirectoryReader.load_data`` rejects such files with a
+    ``DocumentParseError`` naming the install, rather than reading OLE/zip
+    bytes as text.
     """
     from application.parser.file.anydoc_parser import (
         ANYDOC_GAINED_SUFFIXES,
@@ -511,6 +512,15 @@ class SimpleDirectoryReader(BaseReader):
                     data = parser.parse_file(input_file, errors=self.errors)
                     parser_metadata = parser.get_file_metadata(input_file)
                 else:
+                    from application.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
+
+                    if suffix_lower in ANYDOC_GAINED_SUFFIXES:
+                        # Binary office formats only anydoc reads; without it
+                        # the standard read would index OLE/zip bytes as text.
+                        raise DocumentParseError(
+                            f"No parser is available for {suffix_lower} files: "
+                            "firecrawl-anydoc is not installed (pip install firecrawl-anydoc)"
+                        )
                     # do standard read
                     with open(input_file, "r", errors=self.errors) as f:
                         data = f.read()

application/parser/file/docling_parser.py

@@ -148,11 +148,31 @@ def _build_ocr_options(
         if engine == "tesseract":
             from docling.datamodel.pipeline_options import TesseractCliOcrOptions
 
+            from application.parser.file.ocr_parser import tesseract_languages
+
             langs = (
                 languages
                 or [lang.strip() for lang in settings.OCR_LANGS.split("+") if lang.strip()]
                 or ["eng"]
             )
+            # docling swallows tesseract's per-region failures: a language list
+            # with no installed pack logs an error per region and exports ''
+            # which the dropout guard then indexes as "text-sparse". Drop the
+            # packs that are not installed up front and say so.
+            installed = tesseract_languages()
+            if installed is not None:
+                missing = [lang for lang in langs if lang not in installed]
+                if missing:
+                    kept = [lang for lang in langs if lang in installed]
+                    if not kept and "eng" in installed:
+                        kept = ["eng"]
+                    logger.warning(
+                        "OCR_LANGS lists tesseract language pack(s) that are not installed: %s; "
+                        "OCR-ing with %s (install tesseract-ocr-<lang> for each missing pack)",
+                        "+".join(missing),
+                        "+".join(kept) if kept else "the requested list anyway",
+                    )
+                    langs = kept or langs
             return TesseractCliOcrOptions(
                 lang=langs, force_full_page_ocr=force_full_page_ocr
             )

application/parser/file/html_parser.py

@@ -25,6 +25,10 @@ MARKDOWNIFY_OPTIONS = {"heading_style": "ATX", "newline_style": "BACKSLASH"}
 # Elements whose text is never document content. ``title`` is reported via
 # ``get_file_metadata`` instead of leaking in as a stray first line.
 _DROP_TAGS = ("title", "script", "style", "noscript", "template")
+# Attributes whose ``data:`` URIs would otherwise land in the Markdown as
+# link/image targets: one inline image is megabytes of base64 "text" for the
+# chunker and embedder (a 5 MB data-URI ``<img>`` measured 7 MB of output).
+_URI_ATTRIBUTES = ("src", "href", "srcset", "poster", "data")
 
 
 def html_to_markdown(html: Union[str, bytes]) -> str:
@@ -56,10 +60,22 @@ def soup_to_markdown(soup) -> str:
     Returns:
         Markdown with runs of blank lines collapsed to one.
     """
+    from bs4 import CData, Declaration, ProcessingInstruction
     from markdownify import MarkdownConverter
 
     for tag in soup.find_all(_DROP_TAGS):
         tag.decompose()
+    # ``<?xml ...?>`` (every XHTML file), CDATA and stray declarations are
+    # not text; markdownify would emit them as the document's first line.
+    for node in soup.find_all(string=lambda s: isinstance(s, (ProcessingInstruction, CData, Declaration))):
+        node.extract()
+    for tag in soup.find_all(True):
+        for attribute in _URI_ATTRIBUTES:
+            value = tag.get(attribute)
+            if isinstance(value, list):
+                value = " ".join(value)
+            if isinstance(value, str) and "data:" in value.lower():
+                del tag[attribute]
     markdown = MarkdownConverter(**MARKDOWNIFY_OPTIONS).convert_soup(soup)
     return re.sub(r"\n{3,}", "\n\n", markdown).strip()
 
@@ -130,7 +146,15 @@ def read_markup_head(file: Path, max_bytes: int) -> bytes:
         f"Markup {Path(file).name} exceeds MARKUP_MAX_BYTES ({max_bytes}); "
         f"parsing the first {max_bytes} bytes to bound memory"
     )
-    return _trim_torn_utf8_tail(truncate_to_line_boundary(head[:max_bytes]))
+    head = head[:max_bytes]
+    if head[:2] in (b"\xff\xfe", b"\xfe\xff"):
+        # UTF-16 (BOM-declared): a byte-level line cut lands between the two
+        # bytes of a code unit and the strict decode then fails outright, so
+        # keep an even byte count and let the lenient parser take the torn
+        # tail. (Only 2-byte units matter here; a torn surrogate pair is one
+        # lost character.)
+        return head[: len(head) - (len(head) % 2)]
+    return _trim_torn_utf8_tail(truncate_to_line_boundary(head))
 
 
 class HTMLParser(BaseParser):

application/parser/file/ocr_parser.py

@@ -23,6 +23,7 @@ tesseract binary alone, and one that does install it keeps today's
 behaviour unchanged.
 """
 import base64
+import functools
 import io
 import logging
 import re
@@ -63,6 +64,11 @@ _MIN_RENDER_DPI, _MAX_RENDER_DPI = 72, 600
 # usable ~125 dpi; anything larger renders at the scale that fits.
 _MAX_RENDER_PIXELS = 40_000_000
 _TESSERACT_TIMEOUT_SECONDS = 300
+# tesseract reports a missing language pack on stderr and, when at least one
+# other requested pack loads, exits 0 and silently OCRs with what it has —
+# so the exit code alone cannot catch OCR_LANGS=eng+chi_sim without chi_sim.
+_TESSERACT_LANG_ERROR_RE = re.compile(r"Error opening data file|Failed loading language")
+_TESSERACT_LANG_RE = re.compile(r"^[A-Za-z0-9_/\-]+$")
 # docling's DeepSeek-OCR prompt minus its ``<|grounding|>`` prefix: grounding
 # makes the model wrap every element in ref/det tags with bounding boxes,
 # which docling parses back into a layout tree. Plain Markdown is what the
@@ -91,7 +97,6 @@ class OcrEngine(Protocol):
 
     def ocr_image(self, image) -> str:  # pragma: no cover - protocol
         """Return the text (or Markdown) recognised in ``image`` (a PIL image)."""
-        ...
 
 
 # ---------------------------------------------------------------------------
@@ -181,15 +186,80 @@ def render_dpi() -> int:
     return max(_MIN_RENDER_DPI, min(_MAX_RENDER_DPI, dpi))
 
 
+def _has_alpha(image) -> bool:
+    return "A" in image.getbands() or (image.mode == "P" and "transparency" in image.info)
+
+
 def _png_bytes(image) -> bytes:
-    """Encode a PIL image as PNG, flattening modes the engines cannot take."""
-    if image.mode not in ("RGB", "L"):
+    """Encode a PIL image as PNG, flattening modes the engines cannot take.
+
+    Transparency is composited onto white: a plain ``convert("RGB")`` drops
+    the alpha channel and leaves transparent pixels black, which turns dark
+    text on a transparent background into an all-black page that OCRs to
+    nothing.
+    """
+    from PIL import Image
+
+    if _has_alpha(image):
+        rgba = image.convert("RGBA")
+        canvas = Image.new("RGB", rgba.size, "white")
+        canvas.paste(rgba, mask=rgba.getchannel("A"))
+        image = canvas
+    elif image.mode not in ("RGB", "L"):
         image = image.convert("RGB")
     buffer = io.BytesIO()
     image.save(buffer, format="PNG")
     return buffer.getvalue()
 
 
+def fit_to_pixel_budget(image):
+    """Downscale a decoded image to ``_MAX_RENDER_PIXELS`` before OCR.
+
+    The same budget the PDF renderer applies: a 28 KB PNG can decode to
+    81 MP (Pillow's bomb guard only trips at ~178 MP), and every extra
+    pixel is paid again in the PNG re-encode and inside tesseract.
+    """
+    from PIL import Image
+
+    width, height = image.size
+    scale = render_scale(width, height, 72)
+    if scale >= 1.0:
+        return image
+    if image.mode not in ("RGB", "L", "RGBA", "LA"):
+        image = image.convert("RGBA" if _has_alpha(image) else "RGB")
+    logger.warning(
+        "Image is %dx%d px; downscaling to %.0f%% to stay within %d MP before OCR",
+        width,
+        height,
+        scale * 100,
+        _MAX_RENDER_PIXELS // 1_000_000,
+    )
+    return image.resize(
+        (max(1, int(width * scale)), max(1, int(height * scale))), Image.Resampling.BILINEAR
+    )
+
+
+@functools.lru_cache(maxsize=1)
+def tesseract_languages() -> Optional[frozenset]:
+    """Language packs the ``tesseract`` binary reports (``--list-langs``), or None when unknown.
+
+    Cached for the process: packs are installed with the image, not at runtime.
+    """
+    if shutil.which("tesseract") is None:
+        return None
+    try:
+        completed = subprocess.run(
+            ["tesseract", "--list-langs"], capture_output=True, timeout=30, check=False
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    output = completed.stdout.decode("utf-8", "replace") + "\n" + completed.stderr.decode("utf-8", "replace")
+    langs = frozenset(
+        line.strip() for line in output.splitlines() if _TESSERACT_LANG_RE.match(line.strip())
+    )
+    return langs or None
+
+
 # ---------------------------------------------------------------------------
 # Engines
 # ---------------------------------------------------------------------------
@@ -261,9 +331,17 @@ class TesseractEngine:
             raise DocumentParseError(f"tesseract timed out after {self.timeout:.0f}s on a page") from exc
         except OSError as exc:
             raise DocumentParseError(f"tesseract could not be started: {exc}") from exc
+        stderr = completed.stderr.decode("utf-8", "replace").strip()
+        if _TESSERACT_LANG_ERROR_RE.search(stderr):
+            # Exit code 0 here means tesseract dropped the missing pack and
+            # OCR'd with the rest: Chinese scans would come out as garbage
+            # with no signal. Fail the file and name the fix instead.
+            raise OcrUnavailableError(
+                f"tesseract could not load a language pack for OCR_LANGS={self._language_arg()!r}: "
+                f"{stderr[:300]}. Install the tesseract-ocr-<lang> package for every language listed."
+            )
         if completed.returncode != 0:
-            detail = completed.stderr.decode("utf-8", "replace").strip()[:300]
-            raise DocumentParseError(f"tesseract failed (exit {completed.returncode}): {detail}")
+            raise DocumentParseError(f"tesseract failed (exit {completed.returncode}): {stderr[:300]}")
         return collapse_cjk_spaces(completed.stdout.decode("utf-8", "replace")).strip()
 
 
@@ -343,13 +421,15 @@ class DeepseekOcrEngine:
             response = requests.post(self.url, json=self.payload(image), timeout=self.timeout)
             response.raise_for_status()
             body = response.json()
+        except requests.exceptions.JSONDecodeError as exc:
+            # Subclasses RequestException too, so it must be caught first or a
+            # proxy's HTML error page is reported as a connection failure.
+            raise DocumentParseError(f"DeepSeek-OCR endpoint {self.url} returned a non-JSON body") from exc
         except requests.RequestException as exc:
             raise DocumentParseError(
                 f"DeepSeek-OCR request to {self.url} failed: {exc}. Check OCR_DEEPSEEK_URL "
                 f"and that model {self.model!r} is served (e.g. `ollama pull {self.model}`)."
             ) from exc
-        except ValueError as exc:
-            raise DocumentParseError(f"DeepSeek-OCR endpoint {self.url} returned a non-JSON body") from exc
         try:
             content = body["choices"][0]["message"]["content"]
         except (KeyError, IndexError, TypeError) as exc:
@@ -433,7 +513,10 @@ def _page_has_image(page) -> bool:
     import pypdfium2.raw as pdfium_c
 
     try:
-        return next(iter(page.get_objects(filter=(pdfium_c.FPDF_PAGEOBJ_IMAGE,))), None) is not None
+        # pypdfium2's default max_depth=2 misses images inside nested Form
+        # XObjects, which print drivers and InDesign produce routinely.
+        objects = page.get_objects(filter=(pdfium_c.FPDF_PAGEOBJ_IMAGE,), max_depth=16)
+        return next(iter(objects), None) is not None
     except Exception:  # noqa: BLE001 - treat an unreadable page as image-bearing
         return True
 
@@ -663,7 +746,7 @@ class NativeOcrPdfParser(BaseParser):
                     if counts[index] >= self.min_text_chars:
                         textpage = page.get_textpage()
                         try:
-                            text = textpage.get_text_range()
+                            text = textpage.get_text_bounded()
                         finally:
                             textpage.close()
                     else:
@@ -723,6 +806,12 @@ class NativeOcrImageParser(BaseParser):
     def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
         """OCR an image file.
 
+        Only TIFF frames are pages (multi-page faxes and scans); every other
+        multi-frame format is an animation (GIF, WebP, APNG) whose frames
+        repeat one picture, so only the first is OCR'd — a 50-frame WebP
+        would otherwise cost 50 tesseract runs and then be rejected as an
+        empty multi-page document.
+
         Raises:
             DocumentParseError: The image cannot be decoded, the engine failed,
                 or a multi-frame image OCR'd to nothing.
@@ -737,8 +826,9 @@ class NativeOcrImageParser(BaseParser):
         texts: List[str] = []
         try:
             with Image.open(path) as image:
-                for frame in ImageSequence.Iterator(image):
-                    texts.append((engine.ocr_image(frame.convert("RGB")) or "").strip())
+                frames = ImageSequence.Iterator(image) if path.suffix.lower() in (".tif", ".tiff") else [image]
+                for frame in frames:
+                    texts.append((engine.ocr_image(fit_to_pixel_budget(frame)) or "").strip())
         except DocumentParseError:
             raise
         except Exception as exc:

application/parser/file/tableize.py

@@ -37,14 +37,23 @@ def _merge_currency(tokens: List[str]) -> List[str]:
 
 
 def _parse_row(line: str) -> Optional[Tuple[str, List[str]]]:
-    """``(label, values)`` when the line looks like a typographic table row, else None."""
-    match = _LEADER.match(line) or _TRAILING.match(line)
+    """``(label, values)`` when the line looks like a typographic table row, else None.
+
+    Without dot leaders a row needs at least two numeric columns: a single
+    trailing number is what "Chapter 1", "Footnote 2", "ISO 9001" and
+    "Version 3.0" look like, and three of those in a row are a list, not a
+    table.
+    """
+    leader = _LEADER.match(line)
+    match = leader or _TRAILING.match(line)
     if not match:
         return None
     label, rest = match.group(1).strip(" ."), match.group(2)
     values = _merge_currency(rest.split())
     if not values or not all(_NUM_TOKEN.fullmatch(v.lstrip("$")) for v in values):
         return None
+    if leader is None and len(values) < 2:
+        return None
     if not label or _NUM_TOKEN.fullmatch(label):
         return None
     return label, values

application/parser/remote/s3_loader.py

@@ -5,6 +5,7 @@ import tempfile
 import mimetypes
 from typing import List, Optional
 from application.core.url_validation import SSRFError, validate_url
+from application.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
 from application.parser.remote.base import BaseRemote
 from application.parser.schema.base import Document
 
@@ -17,6 +18,61 @@ except ImportError:
 logger = logging.getLogger(__name__)
 
 
+# Plain-text suffixes the loader reads directly (no document parser).
+_TEXT_EXTENSIONS = {
+        ".txt",
+        ".md",
+        ".markdown",
+        ".rst",
+        ".json",
+        ".xml",
+        ".yaml",
+        ".yml",
+        ".py",
+        ".js",
+        ".ts",
+        ".jsx",
+        ".tsx",
+        ".java",
+        ".c",
+        ".cpp",
+        ".h",
+        ".hpp",
+        ".cs",
+        ".go",
+        ".rs",
+        ".rb",
+        ".php",
+        ".swift",
+        ".kt",
+        ".scala",
+        ".html",
+        ".css",
+        ".scss",
+        ".sass",
+        ".less",
+        ".sh",
+        ".bash",
+        ".zsh",
+        ".fish",
+        ".sql",
+        ".r",
+        ".m",
+        ".mat",
+        ".ini",
+        ".cfg",
+        ".conf",
+        ".config",
+        ".env",
+        ".gitignore",
+        ".dockerignore",
+        ".editorconfig",
+        ".log",
+        ".csv",
+        ".tsv",
+    }
+
+
 class S3Loader(BaseRemote):
     """Load documents from an AWS S3 bucket."""
 
@@ -128,58 +184,7 @@ class S3Loader(BaseRemote):
 
     def is_text_file(self, file_path: str) -> bool:
         """Determine if a file is a text file based on extension."""
-        text_extensions = {
-            ".txt",
-            ".md",
-            ".markdown",
-            ".rst",
-            ".json",
-            ".xml",
-            ".yaml",
-            ".yml",
-            ".py",
-            ".js",
-            ".ts",
-            ".jsx",
-            ".tsx",
-            ".java",
-            ".c",
-            ".cpp",
-            ".h",
-            ".hpp",
-            ".cs",
-            ".go",
-            ".rs",
-            ".rb",
-            ".php",
-            ".swift",
-            ".kt",
-            ".scala",
-            ".html",
-            ".css",
-            ".scss",
-            ".sass",
-            ".less",
-            ".sh",
-            ".bash",
-            ".zsh",
-            ".fish",
-            ".sql",
-            ".r",
-            ".m",
-            ".mat",
-            ".ini",
-            ".cfg",
-            ".conf",
-            ".config",
-            ".env",
-            ".gitignore",
-            ".dockerignore",
-            ".editorconfig",
-            ".log",
-            ".csv",
-            ".tsv",
-        }
+        text_extensions = _TEXT_EXTENSIONS
 
         file_lower = file_path.lower()
         for ext in text_extensions:
@@ -197,7 +202,9 @@ class S3Loader(BaseRemote):
 
     def is_supported_document(self, file_path: str) -> bool:
         """Check if file is a supported document type for parsing."""
-        document_extensions = {
+        # The upload whitelist plus the historical S3 list, so an object the
+        # upload API accepts is never silently skipped here.
+        document_extensions = (set(SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS) - _TEXT_EXTENSIONS) | {
             ".pdf",
             ".docx",
             ".doc",

application/requirements-docling.txt

@@ -2,7 +2,7 @@
 #
 # Needed only for: DOC_PARSER_ENGINE=docling, the docling OCR backend
 # (OCR_BACKEND=docling: layout-model hybrid OCR, ocrmac/rapidocr engines),
-# .adoc/.vtt/.xml source parsing, and the read_document tool's `structured`
+# .adoc/.vtt/.xml attachment parsing, and the read_document tool's `structured`
 # output. The default anydoc engine needs none of this, and OCR itself does
 # not either: OCR_ENABLED=true with the tesseract binary (or a DeepSeek-OCR
 # endpoint) runs through application/parser/file/ocr_parser.py.