Parse documents on a Celery parsing worker via a read_document tool

Replace the sandbox Docling extractor with read_document, backed by the in-process
backend parser (the same one ingestion uses) and offloaded to a dedicated
'parsing' Celery queue so it can run on GPU-capable workers with predictable RAM.
The tool resolves the input ref under the run-scoped gate, enqueues the parse,
and awaits it with a timeout (degrading to an error rather than hanging); the
worker independently re-resolves the artifact through the same gate and never
trusts a raw path. Untrusted files get the upload path's safeguards (extension
whitelist, size cap, sanitized temp file, cleanup). Options: output
(markdown/text/structured/chunks), ocr, pages, engine, max_chars, include_tables,
persist, json_schema. The workflow native-file 'extract' fallback now uses the
same worker path, so document parsing no longer needs the sandbox and works on
every backend.

Also fixes the branch's periodic-task test (the sandbox reaper made it 12) and
points the dev and e2e Celery workers at the parsing queue.
This commit is contained in:
Alex committed 2026-06-25 13:24:12 +01:00
1 parent 273c830b55
commit 37d93cbd86
25 files changed
+1611 -1275

No files matched your search

+7 -18
View File
@@ -225,18 +225,13 @@ def test_native_on_unsupported_mime_raises(monkeypatch):
@pytest.mark.unit
def test_extract_non_text_uses_docling(monkeypatch):
"""A non-text mime under extract routes through the Docling helper."""
def test_extract_non_text_uses_parse_worker(monkeypatch):
"""A non-text mime under extract routes through the parsing-worker path (no sandbox)."""
aid, rec = _artifact(RUN_ID, "application/vnd.openxmlformats", filename="r.docx")
_patch_repo(monkeypatch, {aid: rec})
eng = _engine(monkeypatch)
monkeypatch.setattr(
"application.storage.storage_creator.StorageCreator.get_storage",
staticmethod(lambda: _FakeStorage(b"\x00binary")),
)
monkeypatch.setattr(
"application.agents.tools.document_extractor.extract_markdown_from_bytes",
lambda data, filename, session_id, **kw: "EXTRACTED MD",
WorkflowEngine, "_parse_document_text", lambda self, artifact_id: "EXTRACTED MD"
)
cfg = _node_config(input_documents=[aid], file_passing="extract")
@@ -298,10 +293,9 @@ def test_native_file_cap_bounds_native_then_extracts(monkeypatch):
"application.storage.storage_creator.StorageCreator.get_storage",
staticmethod(lambda: _FakeStorage(b"img-bytes")),
)
# extract of a non-text image falls back to Docling; stub it so it returns text.
# extract of a non-text image routes through the parsing worker; stub it so it returns text.
monkeypatch.setattr(
"application.agents.tools.document_extractor.extract_markdown_from_bytes",
lambda data, filename, session_id, **kw: "fallback text",
WorkflowEngine, "_parse_document_text", lambda self, artifact_id: "fallback text"
)
cfg = _node_config(input_documents=ids, file_passing="auto")
@@ -354,7 +348,7 @@ def test_oversize_text_skipped_by_post_read_guard_when_size_missing(monkeypatch)
@pytest.mark.unit
def test_large_under_cap_text_is_windowed_not_inlined_whole(monkeypatch):
"""A large-but-under-cap text file is bounded to a head+tail window, not inlined whole."""
from application.agents.tools.document_extractor import _MARKDOWN_MAX_BYTES
from application.parser.document_reader import _TEXT_MAX_BYTES as _MARKDOWN_MAX_BYTES
big_text = ("A" * (_MARKDOWN_MAX_BYTES * 3)).encode("utf-8")
aid, rec = _artifact(RUN_ID, "text/plain", filename="notes.txt", size=len(big_text))
@@ -490,12 +484,7 @@ def test_execute_agent_node_native_decision_tracks_provider_types(monkeypatch):
_patch_repo(monkeypatch, {aid: rec})
_patch_capabilities(monkeypatch)
monkeypatch.setattr(
"application.storage.storage_creator.StorageCreator.get_storage",
staticmethod(lambda: _FakeStorage(b"image-bytes")),
)
monkeypatch.setattr(
"application.agents.tools.document_extractor.extract_markdown_from_bytes",
lambda data, filename, session_id, **kw: "EXTRACTED",
WorkflowEngine, "_parse_document_text", lambda self, artifact_id: "EXTRACTED"
)
eng = _engine(monkeypatch)