feat: attachment provenance

tests/parser/file/test_markdown_parser.py

@@ -11,6 +11,9 @@ class _Enc:
     def encode(self, s: str):
         return list(s)
 
+    def encode_ordinary(self, s: str):
+        return list(s)
+
 
 @pytest.fixture(autouse=True)
 def _patch_tokenizer(monkeypatch):

tests/parser/test_chunking.py

@@ -281,3 +281,24 @@ class TestChunkerIntegration:
             assert doc.extra_info is not None
             assert "token_count" in doc.extra_info
             assert doc.extra_info["token_count"] > 0
+
+
+# =====================================================================
+# Special-token text must chunk as ordinary text, not raise
+# =====================================================================
+
+
+@pytest.mark.unit
+class TestSpecialTokenText:
+
+    def test_chunking_document_containing_special_token_markers(self):
+        # A document ABOUT LLMs contains literal <|endoftext|>; plain
+        # ``encode()`` raises ValueError on it and destroyed the ingest.
+        chunker = Chunker(chunking_strategy="classic_chunk", max_tokens=50, min_tokens=0)
+        doc = Document(
+            text="The <|endoftext|> marker separates documents. " * 30,
+            doc_id="d1",
+        )
+        chunks = chunker.chunk([doc])
+        assert chunks
+        assert all("<|endoftext|>" in c.text for c in chunks[:1])

tests/parser/test_document_reader.py

@@ -564,3 +564,61 @@ def test_vanilla_converter_applies_inference_settings(monkeypatch, tmp_path):
 
     assert out["markdown"] == "# md"
     assert called == [True]
+
+
+# ---------------------------------------------------------------------------
+# _docling_structured tables: blank cells must not leak NaN or mangle ints
+# ---------------------------------------------------------------------------
+@pytest.mark.unit
+def test_structured_tables_render_blank_cells_as_strings(monkeypatch, tmp_path):
+    """pandas 3 preserves missing values through ``astype(str)``, so a blank
+    spreadsheet cell leaked a float NaN into the tables payload and a blank
+    in an integer column mangled every id to ``1001.0``. Cells must come out
+    as join-safe strings (same contract as the tabular parser's
+    ``cell_to_text``)."""
+    import sys
+    import types
+
+    import pandas as pd
+
+    df = pd.DataFrame({"id": [1001, None, 1003], "name": ["a", "b", None]})
+
+    class _Table:
+        def export_to_dataframe(self):
+            return df
+
+    class _Doc:
+        tables = [_Table()]
+        pages = {"1": {}}
+
+        def export_to_markdown(self):
+            return "# md"
+
+        def export_to_dict(self):
+            return {"texts": [], "tables": [{}], "pages": {"1": {}}}
+
+    class _Converter:
+        def convert(self, *a, **k):
+            return types.SimpleNamespace(document=_Doc())
+
+    monkeypatch.setattr(
+        "application.parser.file.docling_parser._apply_inference_settings",
+        lambda: None,
+    )
+    fake_docling = types.ModuleType("docling")
+    fake_dc_module = types.ModuleType("docling.document_converter")
+    fake_dc_module.DocumentConverter = _Converter
+    fake_docling.document_converter = fake_dc_module
+    monkeypatch.setitem(sys.modules, "docling", fake_docling)
+    monkeypatch.setitem(sys.modules, "docling.document_converter", fake_dc_module)
+
+    path = tmp_path / "doc.xlsx"
+    path.write_bytes(b"stub")
+
+    out = dr._docling_structured(path, ocr_enabled=False, include_tables=True, parser=None)
+
+    [table] = out["tables"]
+    cells = [c for row in table["rows"] for c in row]
+    assert all(isinstance(c, str) for c in cells), f"non-string cells: {cells!r}"
+    assert "1001" in cells and "1001.0" not in cells
+    assert "" in cells  # blank cell renders as empty string, not "nan"/"None"