feat: limit docling attachment file sizes explicitly

tests/parser/file/test_docling_parser.py

@@ -4,6 +4,7 @@ Covers: DoclingParser (init, _init_parser, _get_ocr_options, _export_content,
 parse_file), subclass initialization, error handling.
 """
 
+import os
 from pathlib import Path
 from unittest.mock import MagicMock, patch
 
@@ -503,3 +504,249 @@ class TestApplyPipelineCaps:
 
         assert not hasattr(opts, "queue_max_size")
         assert not hasattr(opts, "layout_batch_size")
+
+
+# =====================================================================
+# Tabular size gate (CSV / XLSX)
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestDoclingTabularSizeGate:
+    """Oversized tabular files must bypass docling.
+
+    Docling materializes a ``TableCell`` object per cell (measured ~11 KB of
+    RSS per 4-cell CSV row), so a multi-MB CSV balloons the worker by tens of
+    GB. Above ``DOCLING_TABULAR_MAX_BYTES`` the docling tabular parsers must
+    delegate to the lightweight parsers in ``tabular_parser``.
+    """
+
+    def _write_csv(self, tmp_path: Path, rows: int = 50) -> Path:
+        path = tmp_path / "data.csv"
+        path.write_text("\n".join(f"{i},{i * 2}" for i in range(rows)) + "\n")
+        return path
+
+    def test_oversized_csv_delegates_to_plain_csv_parser(self, tmp_path, monkeypatch):
+        from application.core.settings import settings
+        from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
+
+        monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
+        docling_parse = MagicMock(name="docling_parse")
+        monkeypatch.setattr(DoclingParser, "parse_file", docling_parse)
+
+        path = self._write_csv(tmp_path)
+        assert path.stat().st_size > 64
+
+        out = DoclingCSVParser().parse_file(path)
+
+        docling_parse.assert_not_called()
+        # Plain ``CSVParser`` output: rows joined with ", ", newline-separated.
+        assert out.startswith("0, 0\n1, 2\n")
+
+    def test_small_csv_still_uses_docling(self, tmp_path, monkeypatch):
+        from application.core.settings import settings
+        from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
+
+        monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 10_000_000)
+        docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
+        monkeypatch.setattr(DoclingParser, "parse_file", docling_parse)
+
+        out = DoclingCSVParser().parse_file(self._write_csv(tmp_path))
+
+        assert out == "DOCLING"
+        docling_parse.assert_called_once()
+
+    def test_gate_disabled_when_max_bytes_is_zero(self, tmp_path, monkeypatch):
+        from application.core.settings import settings
+        from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
+
+        monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 0)
+        docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
+        monkeypatch.setattr(DoclingParser, "parse_file", docling_parse)
+
+        out = DoclingCSVParser().parse_file(self._write_csv(tmp_path, rows=5000))
+
+        assert out == "DOCLING"
+        docling_parse.assert_called_once()
+
+    def test_oversized_xlsx_delegates_to_excel_parser(self, tmp_path, monkeypatch):
+        from application.core.settings import settings
+        from application.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
+        from application.parser.file.tabular_parser import ExcelParser
+
+        monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
+        docling_parse = MagicMock(name="docling_parse")
+        monkeypatch.setattr(DoclingParser, "parse_file", docling_parse)
+        excel_parse = MagicMock(name="excel_parse", return_value="PLAIN-XLSX")
+        monkeypatch.setattr(ExcelParser, "parse_file", excel_parse)
+
+        path = tmp_path / "big.xlsx"
+        path.write_bytes(b"x" * 200)
+
+        out = DoclingXLSXParser().parse_file(path)
+
+        assert out == "PLAIN-XLSX"
+        docling_parse.assert_not_called()
+        excel_parse.assert_called_once()
+
+    def test_small_xlsx_still_uses_docling(self, tmp_path, monkeypatch):
+        from application.core.settings import settings
+        from application.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
+
+        monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 10_000_000)
+        docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
+        monkeypatch.setattr(DoclingParser, "parse_file", docling_parse)
+
+        path = tmp_path / "small.xlsx"
+        path.write_bytes(b"x" * 200)
+
+        assert DoclingXLSXParser().parse_file(path) == "DOCLING"
+        docling_parse.assert_called_once()
+
+
+# =====================================================================
+# Tabular content-size gate — XLSX compression must not defeat it
+# =====================================================================
+
+
+def _make_xlsx(path: Path, rows: int) -> None:
+    from openpyxl import Workbook
+
+    wb = Workbook(write_only=True)
+    ws = wb.create_sheet()
+    for i in range(rows):
+        ws.append([i, i * 2, i % 7, i % 13, i * 3])
+    wb.save(str(path))
+
+
+@pytest.mark.unit
+class TestTabularContentSize:
+    """XLSX is zip-compressed, so the gate must measure inner-uncompressed
+    size, not on-disk bytes — otherwise a small-on-disk / many-cell xlsx
+    (2.44 GB in docling) slips under a byte gate."""
+
+    def test_xlsx_content_size_is_inner_not_ondisk(self, tmp_path):
+        from application.parser.file.docling_parser import _tabular_content_size
+
+        path = tmp_path / "data.xlsx"
+        _make_xlsx(path, rows=5000)
+        on_disk = path.stat().st_size
+        inner = _tabular_content_size(path)
+        # Repetitive numeric data compresses hard: inner XML >> zip on disk.
+        assert inner > on_disk
+
+    def test_csv_content_size_is_ondisk(self, tmp_path):
+        from application.parser.file.docling_parser import _tabular_content_size
+
+        path = tmp_path / "data.csv"
+        path.write_text("a,b\n1,2\n3,4\n")
+        assert _tabular_content_size(path) == path.stat().st_size
+
+    def test_compressed_xlsx_over_inner_gate_delegates(self, tmp_path, monkeypatch):
+        """The regression: on-disk < threshold < inner-uncompressed must gate."""
+        from application.core.settings import settings
+        from application.parser.file.docling_parser import (
+            DoclingParser,
+            DoclingXLSXParser,
+            _tabular_content_size,
+        )
+        from application.parser.file.tabular_parser import ExcelParser
+
+        path = tmp_path / "wide.xlsx"
+        _make_xlsx(path, rows=5000)
+        on_disk = path.stat().st_size
+        inner = _tabular_content_size(path)
+        assert inner > on_disk, "premise: compression hides cell count"
+
+        # A byte gate between the two would have sent this to docling; the
+        # inner-size gate must catch it.
+        monkeypatch.setattr(
+            settings, "DOCLING_TABULAR_MAX_BYTES", (on_disk + inner) // 2
+        )
+        docling_parse = MagicMock(name="docling_parse")
+        monkeypatch.setattr(DoclingParser, "parse_file", docling_parse)
+        excel_parse = MagicMock(name="excel_parse", return_value="PLAIN")
+        monkeypatch.setattr(ExcelParser, "parse_file", excel_parse)
+
+        out = DoclingXLSXParser().parse_file(path)
+
+        assert out == "PLAIN"
+        docling_parse.assert_not_called()
+        excel_parse.assert_called_once()
+
+
+# =====================================================================
+# Markup gate (HTML / VTT) — truncate oversized element-dense markup
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestDoclingMarkupGate:
+    def _write(self, tmp_path: Path, name: str, nbytes: int) -> Path:
+        path = tmp_path / name
+        # newline-terminated lines so the line-boundary trim has something to cut
+        path.write_text(("x" * 63 + "\n") * (nbytes // 64 + 1))
+        return path
+
+    @pytest.mark.parametrize("name", ["big.html", "big.vtt"])
+    def test_oversized_markup_parses_truncated_copy(self, tmp_path, monkeypatch, name):
+        from application.core.settings import settings
+        from application.parser.file import docling_parser as dp
+
+        monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 512)
+        path = self._write(tmp_path, name, 4096)
+        assert path.stat().st_size > 512
+
+        seen = {}
+
+        def fake_parse(self_parser, file, errors="ignore"):
+            p = str(file)
+            seen["path"] = p
+            seen["size"] = os.path.getsize(p)
+            return "PARSED"
+
+        monkeypatch.setattr(dp.DoclingParser, "parse_file", fake_parse)
+
+        cls = dp.DoclingHTMLParser if name.endswith(".html") else dp.DoclingVTTParser
+        out = cls().parse_file(path)
+
+        assert out == "PARSED"
+        assert seen["path"] != str(path), "must parse a temp copy, not the original"
+        assert seen["size"] <= 512, "temp copy must be truncated to the cap"
+        assert not os.path.exists(seen["path"]), "temp copy must be cleaned up"
+        assert path.stat().st_size > 512, "original must be untouched"
+
+    def test_small_markup_parses_original(self, tmp_path, monkeypatch):
+        from application.core.settings import settings
+        from application.parser.file import docling_parser as dp
+
+        monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 10_000_000)
+        path = self._write(tmp_path, "small.html", 1024)
+        seen = {}
+
+        def fake_parse(self_parser, file, errors="ignore"):
+            seen["path"] = str(file)
+            return "PARSED"
+
+        monkeypatch.setattr(dp.DoclingParser, "parse_file", fake_parse)
+        out = dp.DoclingHTMLParser().parse_file(path)
+
+        assert out == "PARSED"
+        assert seen["path"] == str(path)
+
+    def test_markup_gate_disabled_when_zero(self, tmp_path, monkeypatch):
+        from application.core.settings import settings
+        from application.parser.file import docling_parser as dp
+
+        monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 0)
+        path = self._write(tmp_path, "big.vtt", 8192)
+        seen = {}
+
+        def fake_parse(self_parser, file, errors="ignore"):
+            seen["path"] = str(file)
+            return "PARSED"
+
+        monkeypatch.setattr(dp.DoclingParser, "parse_file", fake_parse)
+        dp.DoclingVTTParser().parse_file(path)
+
+        assert seen["path"] == str(path), "disabled gate must parse the original"

tests/parser/file/test_tabular_parser.py

@@ -212,4 +212,24 @@ def test_excel_parser_import_error(excel_parser):
     import sys
     with patch.dict(sys.modules, {"pandas": None}):
         with pytest.raises(ValueError, match="pandas module is required to read Excel files"):
-            excel_parser.parse_file(Path("test.xlsx"))
+            excel_parser.parse_file(Path("test.xlsx"))
+
+def test_excel_numeric_headers_do_not_crash(tmp_path):
+    """Regression: a headerless/numeric xlsx gives integer column labels, and
+    ``", ".join(headers)`` used to raise TypeError. The XLSX size-gate delegates
+    to this parser, so it must handle numeric headers."""
+    from openpyxl import Workbook
+
+    from application.parser.file.tabular_parser import ExcelParser
+
+    wb = Workbook()
+    ws = wb.active
+    for i in range(5):
+        ws.append([i, i * 2, i * 3])
+    path = tmp_path / "numeric.xlsx"
+    wb.save(str(path))
+
+    out = ExcelParser().parse_file(path)
+
+    assert isinstance(out, str)
+    assert len(out) > 0