mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-15 00:16:46 +00:00
Parse documents on a Celery parsing worker via a read_document tool
Replace the sandbox Docling extractor with read_document, backed by the in-process backend parser (the same one ingestion uses) and offloaded to a dedicated 'parsing' Celery queue so it can run on GPU-capable workers with predictable RAM. The tool resolves the input ref under the run-scoped gate, enqueues the parse, and awaits it with a timeout (degrading to an error rather than hanging); the worker independently re-resolves the artifact through the same gate and never trusts a raw path. Untrusted files get the upload path's safeguards (extension whitelist, size cap, sanitized temp file, cleanup). Options: output (markdown/text/structured/chunks), ocr, pages, engine, max_chars, include_tables, persist, json_schema. The workflow native-file 'extract' fallback now uses the same worker path, so document parsing no longer needs the sandbox and works on every backend. Also fixes the branch's periodic-task test (the sandbox reaper made it 12) and points the dev and e2e Celery workers at the parsing queue.
This commit is contained in:
1 parent
273c830b55
commit
37d93cbd86
25 files changed
+1611
-1275
No files matched your search
@@ -225,18 +225,13 @@ def test_native_on_unsupported_mime_raises(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_extract_non_text_uses_docling(monkeypatch):
|
||||
"""A non-text mime under extract routes through the Docling helper."""
|
||||
def test_extract_non_text_uses_parse_worker(monkeypatch):
|
||||
"""A non-text mime under extract routes through the parsing-worker path (no sandbox)."""
|
||||
aid, rec = _artifact(RUN_ID, "application/vnd.openxmlformats", filename="r.docx")
|
||||
_patch_repo(monkeypatch, {aid: rec})
|
||||
eng = _engine(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"application.storage.storage_creator.StorageCreator.get_storage",
|
||||
staticmethod(lambda: _FakeStorage(b"\x00binary")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"application.agents.tools.document_extractor.extract_markdown_from_bytes",
|
||||
lambda data, filename, session_id, **kw: "EXTRACTED MD",
|
||||
WorkflowEngine, "_parse_document_text", lambda self, artifact_id: "EXTRACTED MD"
|
||||
)
|
||||
cfg = _node_config(input_documents=[aid], file_passing="extract")
|
||||
|
||||
@@ -298,10 +293,9 @@ def test_native_file_cap_bounds_native_then_extracts(monkeypatch):
|
||||
"application.storage.storage_creator.StorageCreator.get_storage",
|
||||
staticmethod(lambda: _FakeStorage(b"img-bytes")),
|
||||
)
|
||||
# extract of a non-text image falls back to Docling; stub it so it returns text.
|
||||
# extract of a non-text image routes through the parsing worker; stub it so it returns text.
|
||||
monkeypatch.setattr(
|
||||
"application.agents.tools.document_extractor.extract_markdown_from_bytes",
|
||||
lambda data, filename, session_id, **kw: "fallback text",
|
||||
WorkflowEngine, "_parse_document_text", lambda self, artifact_id: "fallback text"
|
||||
)
|
||||
cfg = _node_config(input_documents=ids, file_passing="auto")
|
||||
|
||||
@@ -354,7 +348,7 @@ def test_oversize_text_skipped_by_post_read_guard_when_size_missing(monkeypatch)
|
||||
@pytest.mark.unit
|
||||
def test_large_under_cap_text_is_windowed_not_inlined_whole(monkeypatch):
|
||||
"""A large-but-under-cap text file is bounded to a head+tail window, not inlined whole."""
|
||||
from application.agents.tools.document_extractor import _MARKDOWN_MAX_BYTES
|
||||
from application.parser.document_reader import _TEXT_MAX_BYTES as _MARKDOWN_MAX_BYTES
|
||||
|
||||
big_text = ("A" * (_MARKDOWN_MAX_BYTES * 3)).encode("utf-8")
|
||||
aid, rec = _artifact(RUN_ID, "text/plain", filename="notes.txt", size=len(big_text))
|
||||
@@ -490,12 +484,7 @@ def test_execute_agent_node_native_decision_tracks_provider_types(monkeypatch):
|
||||
_patch_repo(monkeypatch, {aid: rec})
|
||||
_patch_capabilities(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
"application.storage.storage_creator.StorageCreator.get_storage",
|
||||
staticmethod(lambda: _FakeStorage(b"image-bytes")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"application.agents.tools.document_extractor.extract_markdown_from_bytes",
|
||||
lambda data, filename, session_id, **kw: "EXTRACTED",
|
||||
WorkflowEngine, "_parse_document_text", lambda self, artifact_id: "EXTRACTED"
|
||||
)
|
||||
eng = _engine(monkeypatch)
|
||||
|
||||
|
||||
Reference in new issue
Block a user