feat: improve tool call durability on long calls

tests/parser/file/test_docling_parser.py

@@ -788,3 +788,106 @@ class TestDoclingMarkupGate:
         dp.DoclingVTTParser().parse_file(path)
 
         assert seen["path"] == str(path), "disabled gate must parse the original"
+
+
+# =====================================================================
+# Gate seam: the oversized path through the REAL lightweight parser
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestTabularGateSeam:
+    """Exercise the gate and the lightweight parser together.
+
+    Every test in ``TestDoclingTabularSizeGate`` monkeypatches
+    ``ExcelParser.parse_file`` away, so the seam between the size gate and
+    the real fallback had zero coverage — which is how a crash on blank
+    cells reached production and silently destroyed an upload.
+    """
+
+    def _write_xlsx_with_hole(self, path: Path) -> Path:
+        openpyxl = pytest.importorskip("openpyxl")
+        wb = openpyxl.Workbook()
+        ws = wb.active
+        ws.append([f"col{i}" for i in range(14)])
+        ws.append(list(range(14)))
+        holed = list(range(14))
+        holed[11] = None
+        ws.append(holed)
+        wb.save(path)
+        return path
+
+    def test_oversized_xlsx_with_blank_cells_parses_end_to_end(
+        self, tmp_path, monkeypatch
+    ):
+        """The real incident, end to end: gate trips, blanks do not crash."""
+        from application.core.settings import settings
+        from application.parser.file.docling_parser import (
+            DoclingParser,
+            DoclingXLSXParser,
+        )
+
+        monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
+        docling_parse = MagicMock(name="docling_parse")
+        monkeypatch.setattr(DoclingParser, "parse_file", docling_parse)
+
+        path = self._write_xlsx_with_hole(tmp_path / "focus.xlsx")
+
+        out = DoclingXLSXParser().parse_file(path)
+
+        docling_parse.assert_not_called()
+        assert isinstance(out, str) and out
+        assert "nan" not in out
+
+    def test_lightweight_failure_becomes_document_parse_error(
+        self, tmp_path, monkeypatch
+    ):
+        """A fallback crash must be non-retryable and batch-skippable.
+
+        ``DocumentParseError`` is listed in ``dont_autoretry_for`` on both
+        ``ingest`` and ``store_attachment``, and is the only exception
+        ``SimpleDirectoryReader.load_data`` skips rather than propagating.
+        """
+        from application.core.settings import settings
+        from application.parser.file.base_parser import DocumentParseError
+        from application.parser.file.docling_parser import (
+            DoclingParser,
+            DoclingXLSXParser,
+        )
+        from application.parser.file.tabular_parser import ExcelParser
+
+        monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
+        monkeypatch.setattr(DoclingParser, "parse_file", MagicMock())
+        monkeypatch.setattr(
+            ExcelParser,
+            "parse_file",
+            MagicMock(side_effect=TypeError("sequence item 11")),
+        )
+
+        path = self._write_xlsx_with_hole(tmp_path / "boom.xlsx")
+
+        with pytest.raises(DocumentParseError):
+            DoclingXLSXParser().parse_file(path)
+
+    def test_oversized_csv_failure_becomes_document_parse_error(
+        self, tmp_path, monkeypatch
+    ):
+        from application.core.settings import settings
+        from application.parser.file.base_parser import DocumentParseError
+        from application.parser.file.docling_parser import (
+            DoclingCSVParser,
+            DoclingParser,
+        )
+        from application.parser.file.tabular_parser import CSVParser
+
+        monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 8)
+        monkeypatch.setattr(DoclingParser, "parse_file", MagicMock())
+        monkeypatch.setattr(
+            CSVParser, "parse_file", MagicMock(side_effect=ValueError("bad"))
+        )
+
+        path = tmp_path / "data.csv"
+        path.write_text("a,b\n1,2\n" * 20)
+
+        with pytest.raises(DocumentParseError):
+            DoclingCSVParser().parse_file(path)

tests/parser/file/test_tabular_parser.py

@@ -66,8 +66,8 @@ def test_pandas_csv_parser_concat_rows(pandas_csv_parser):
     mock_df = MagicMock()
     mock_df.columns.tolist.return_value = ["col1", "col2"]
     mock_df.iterrows.return_value = [
-        (0, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["value1", "value2"]))),
-        (1, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["value3", "value4"])))
+        (0, MagicMock(tolist=lambda: ["value1", "value2"])),
+        (1, MagicMock(tolist=lambda: ["value3", "value4"]))
     ]
 
     with patch("pandas.read_csv", return_value=mock_df):
@@ -92,9 +92,9 @@ def test_pandas_csv_parser_header_period(pandas_csv_parser):
     mock_df = MagicMock()
     mock_df.columns.tolist.return_value = ["col1", "col2"]
     mock_df.iterrows.return_value = [
-        (0, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["value1", "value2"]))),
-        (1, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["value3", "value4"]))),
-        (2, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["value5", "value6"])))
+        (0, MagicMock(tolist=lambda: ["value1", "value2"])),
+        (1, MagicMock(tolist=lambda: ["value3", "value4"])),
+        (2, MagicMock(tolist=lambda: ["value5", "value6"]))
     ]
     mock_df.__len__.return_value = 3
 
@@ -108,8 +108,8 @@ def test_excel_parser_concat_rows(excel_parser):
     mock_df = MagicMock()
     mock_df.columns.tolist.return_value = ["col1", "col2"]
     mock_df.iterrows.return_value = [
-        (0, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["value1", "value2"]))),
-        (1, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["value3", "value4"])))
+        (0, MagicMock(tolist=lambda: ["value1", "value2"])),
+        (1, MagicMock(tolist=lambda: ["value3", "value4"]))
     ]
 
     with patch("pandas.read_excel", return_value=mock_df):
@@ -134,8 +134,8 @@ def test_excel_parser_header_period(excel_parser):
     mock_df = MagicMock()
     mock_df.columns.tolist.return_value = ["col1", "col2"]
     mock_df.iterrows.return_value = [
-        (0, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["value1", "value2"]))),
-        (1, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["value3", "value4"])))
+        (0, MagicMock(tolist=lambda: ["value1", "value2"])),
+        (1, MagicMock(tolist=lambda: ["value3", "value4"]))
     ]
     mock_df.__len__.return_value = 2
 
@@ -163,8 +163,8 @@ def test_pandas_csv_parser_header_period_zero(pandas_csv_parser):
     mock_df = MagicMock()
     mock_df.columns.tolist.return_value = ["c1", "c2"]
     mock_df.iterrows.return_value = [
-        (0, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["v1", "v2"]))),
-        (1, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["v3", "v4"]))),
+        (0, MagicMock(tolist=lambda: ["v1", "v2"])),
+        (1, MagicMock(tolist=lambda: ["v3", "v4"])),
     ]
     with patch("pandas.read_csv", return_value=mock_df):
         result = pandas_csv_parser.parse_file(Path("f.csv"))
@@ -176,8 +176,8 @@ def test_pandas_csv_parser_header_period_one(pandas_csv_parser):
     mock_df = MagicMock()
     mock_df.columns.tolist.return_value = ["a", "b"]
     mock_df.iterrows.return_value = [
-        (0, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["x", "y"]))),
-        (1, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["m", "n"]))),
+        (0, MagicMock(tolist=lambda: ["x", "y"])),
+        (1, MagicMock(tolist=lambda: ["m", "n"])),
     ]
     mock_df.__len__.return_value = 2
     with patch("pandas.read_csv", return_value=mock_df):
@@ -202,7 +202,7 @@ def test_excel_parser_custom_joiners_and_prefix(excel_parser):
     mock_df = MagicMock()
     mock_df.columns.tolist.return_value = ["A", "B"]
     mock_df.iterrows.return_value = [
-        (0, MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["x", "y"]))),
+        (0, MagicMock(tolist=lambda: ["x", "y"])),
     ]
     with patch("pandas.read_excel", return_value=mock_df):
         result = excel_parser.parse_file(Path("t.xlsx"))

tests/parser/file/test_tabular_parser_real_fixtures.py

@@ -0,0 +1,189 @@
+"""Real-fixture tests for the tabular parsers.
+
+The mock-based tests in ``test_tabular_parser.py`` stub pandas out entirely
+(``MagicMock(astype=lambda _: MagicMock(tolist=lambda: ["value1", "value2"]))``),
+so real pandas semantics were never exercised. That is why the pandas 3.0
+change to ``Series.astype(str)`` — which now *preserves* missing values instead
+of rendering them as the string ``"nan"`` — reached production as a
+``TypeError: sequence item N: expected str instance, float found`` that
+destroyed an entire spreadsheet upload.
+
+These tests drive the parsers with real files.
+"""
+
+import datetime
+from pathlib import Path
+
+import pytest
+
+from application.parser.file.tabular_parser import (
+    ExcelParser,
+    PandasCSVParser,
+    cell_to_text,
+)
+
+openpyxl = pytest.importorskip("openpyxl")
+
+
+def _write_xlsx(path: Path, rows: list[list], sheets: dict[str, list[list]] | None = None) -> Path:
+    """Write ``rows`` to ``path`` as the first sheet, plus any extra ``sheets``."""
+    wb = openpyxl.Workbook()
+    ws = wb.active
+    for row in rows:
+        ws.append(row)
+    for name, extra_rows in (sheets or {}).items():
+        extra = wb.create_sheet(title=name)
+        for row in extra_rows:
+            extra.append(row)
+    wb.save(path)
+    return path
+
+
+def test_excel_blank_cell_in_row_does_not_crash(tmp_path):
+    """A single blank cell must not destroy the whole document.
+
+    Reproduces the production failure verbatim: a 14-column sheet whose
+    second data row is blank at column index 11 raises
+    ``sequence item 11: expected str instance, float found``.
+    """
+    headers = [f"col{i}" for i in range(14)]
+    full_row = list(range(14))
+    holed_row = list(range(14))
+    holed_row[11] = None
+
+    path = _write_xlsx(tmp_path / "focus.xlsx", [headers, full_row, holed_row])
+
+    text = ExcelParser().parse_file(path)
+
+    assert isinstance(text, str)
+    assert "nan" not in text
+    # The blank renders as an empty field between two joiners, not as a crash.
+    assert ", , " in text
+
+
+def test_excel_all_blank_row_does_not_crash(tmp_path):
+    headers = ["a", "b", "c"]
+    path = _write_xlsx(
+        tmp_path / "blanks.xlsx", [headers, [1, 2, 3], [None, None, None]]
+    )
+
+    text = ExcelParser().parse_file(path)
+
+    assert isinstance(text, str)
+    assert "nan" not in text
+
+
+def test_excel_mixed_types_render_readably(tmp_path):
+    """Ints stay ints, dates are ISO, blanks are empty, nothing is ``nan``."""
+    headers = ["id", "name", "ratio", "when", "flag", "blank"]
+    path = _write_xlsx(
+        tmp_path / "mixed.xlsx",
+        [
+            headers,
+            [1001, "alpha", 1.5, datetime.datetime(2024, 1, 2, 3, 4), True, "x"],
+            [1002, "beta", 2.25, datetime.datetime(2024, 5, 6, 7, 8), False, None],
+        ],
+    )
+
+    text = ExcelParser().parse_file(path)
+
+    assert "nan" not in text
+    assert "NaT" not in text
+    # A blank in the last column upcasts nothing: ids must not become 1001.0.
+    assert "1001" in text and "1001.0" not in text
+    assert "2024-01-02T03:04:00" in text
+
+
+def test_excel_uncalculated_formula_cells_do_not_crash(tmp_path):
+    """openpyxl never calculates formulas, so cached values are absent.
+
+    Any workbook written by a non-Excel tool has all-NaN formula columns,
+    which makes this a guaranteed crash on the pre-fix code.
+    """
+    path = _write_xlsx(
+        tmp_path / "formulas.xlsx",
+        [["a", "b", "total"], [1, 2, "=A2+B2"], [3, 4, "=A3+B3"]],
+    )
+
+    text = ExcelParser().parse_file(path)
+
+    assert isinstance(text, str)
+    assert "nan" not in text
+
+
+def test_excel_error_values_do_not_crash(tmp_path):
+    path = _write_xlsx(
+        tmp_path / "errors.xlsx",
+        [["a", "b"], [1, "#REF!"], [2, "#N/A"]],
+    )
+
+    text = ExcelParser().parse_file(path)
+
+    assert isinstance(text, str)
+    assert "nan" not in text
+
+
+def test_excel_non_concat_rows_blank_cell_does_not_crash(tmp_path):
+    """Covers the ``concat_rows=False`` branch, a separate join site."""
+    path = _write_xlsx(
+        tmp_path / "rows.xlsx", [["a", "b", "c"], [1, None, 3]]
+    )
+
+    rows = ExcelParser(concat_rows=False).parse_file(path)
+
+    assert isinstance(rows, list)
+    assert all(isinstance(r, str) for r in rows)
+    assert not any("nan" in r for r in rows)
+
+
+def test_nul_byte_in_cell_is_stripped(tmp_path):
+    """NUL bytes crash the Postgres text write further downstream.
+
+    openpyxl refuses to *write* a NUL into a worksheet
+    (``IllegalCharacterError``), so the reachable route is CSV — plus the
+    normalizer itself, which is what a hand-crafted or corrupt workbook
+    would hit.
+    """
+    assert cell_to_text("di\x00rty") == "dirty"
+
+    path = tmp_path / "nul.csv"
+    path.write_bytes(b"a,b\nclean,di\x00rty\n")
+
+    text = PandasCSVParser().parse_file(path)
+
+    # pandas' C parser truncates the field at the NUL, so the tail is lost
+    # before the normalizer sees it. What matters is that no NUL survives
+    # into the text we hand to Postgres.
+    assert "\x00" not in text
+    assert "clean" in text
+
+
+def test_pandas_csv_blank_cell_does_not_crash(tmp_path):
+    path = tmp_path / "holed.csv"
+    path.write_text("a,b,c\n1,,3\n4,5,6\n")
+
+    text = PandasCSVParser().parse_file(path)
+
+    assert isinstance(text, str)
+    assert "nan" not in text
+
+
+def test_pandas_csv_non_string_headers_do_not_crash(tmp_path):
+    """``header=None`` yields integer column labels; the header join was never hardened."""
+    path = tmp_path / "noheader.csv"
+    path.write_text("1,2,3\n4,5,6\n")
+
+    text = PandasCSVParser(pandas_config={"header": None}).parse_file(path)
+
+    assert isinstance(text, str)
+    assert text.startswith("HEADERS: ")
+
+
+def test_pandas_csv_non_concat_rows_blank_cell_does_not_crash(tmp_path):
+    path = tmp_path / "holed2.csv"
+    path.write_text("a,b,c\n1,,3\n")
+
+    rows = PandasCSVParser(concat_rows=False).parse_file(path)
+
+    assert isinstance(rows, list)
+    assert not any("nan" in r for r in rows)