mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 08:17:34 +00:00
refactor: rename the application package to docsgpt
The backend import package is now docsgpt, the name it will carry on PyPI; application was far too generic to install into anyone's site-packages. git mv plus a mechanical rewrite of every import, dotted string and path reference: 734 Python files, the compose files, Dockerfile, workflows, docs, setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage config, .gitignore. Behaviour is unchanged. Kept for one release: - A top-level application package whose meta-path finder resolves application.x.y to the already-imported docsgpt.x.y object, so old imports and entry points (celery -A application.app.celery, uvicorn application.asgi:asgi_app) keep working with a FutureWarning. - Celery registers every application.* task name as an alias of its docsgpt.* task on start-up, so messages queued by the previous release still run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries the previous release wrote are left unread instead of firing twice. The backend image builds from the repository root (docker build -f docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore allow-lists docsgpt/ and application/ and keeps caches, local data, .env files, the sample index files and the Dockerfile out. Compose and the image workflows point at the new context.
This commit is contained in:
1 parent
3e7f1f91b4
commit
574f96341e
984 files changed
+9847
-9682
No files matched your search
@@ -1,4 +1,4 @@
|
||||
"""Comprehensive tests for application/parser/file/docling_parser.py
|
||||
"""Comprehensive tests for docsgpt/parser/file/docling_parser.py
|
||||
|
||||
Covers: DoclingParser (init, _init_parser, OCR engine selection, _export_content,
|
||||
parse_file), subclass initialization, error handling.
|
||||
@@ -21,7 +21,7 @@ import pytest
|
||||
class TestDoclingParserInit:
|
||||
|
||||
def test_default_init(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser()
|
||||
assert parser.ocr_enabled is True
|
||||
@@ -33,7 +33,7 @@ class TestDoclingParserInit:
|
||||
assert parser._converter is None
|
||||
|
||||
def test_custom_init(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser(
|
||||
ocr_enabled=False,
|
||||
@@ -60,7 +60,7 @@ class TestDoclingParserInit:
|
||||
class TestDoclingParserInitParser:
|
||||
|
||||
def test_init_parser_raises_without_docling(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser()
|
||||
|
||||
@@ -72,7 +72,7 @@ class TestDoclingParserInitParser:
|
||||
"""A missing parent package makes find_spec raise; the hint must still show."""
|
||||
import sys
|
||||
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
for name in [m for m in sys.modules if m == "docling" or m.startswith("docling.")]:
|
||||
monkeypatch.delitem(sys.modules, name)
|
||||
@@ -82,7 +82,7 @@ class TestDoclingParserInitParser:
|
||||
DoclingParser()._init_parser()
|
||||
|
||||
def test_init_parser_success(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser()
|
||||
|
||||
@@ -108,14 +108,14 @@ class TestOcrEngineSelection:
|
||||
|
||||
@pytest.fixture
|
||||
def settings(self):
|
||||
from application.core.settings import settings
|
||||
from docsgpt.core.settings import settings
|
||||
|
||||
return settings
|
||||
|
||||
def test_default_setting_is_tesseract(self):
|
||||
# Field defaults, not the live settings: a developer's ``.env`` may
|
||||
# legitimately set other engines/languages.
|
||||
from application.core.settings import Settings
|
||||
from docsgpt.core.settings import Settings
|
||||
|
||||
defaults = Settings.model_construct()
|
||||
assert defaults.OCR_ENGINE == "tesseract"
|
||||
@@ -123,30 +123,30 @@ class TestOcrEngineSelection:
|
||||
assert defaults.OCR_BACKEND == "auto"
|
||||
|
||||
def test_none_reads_setting(self, settings, monkeypatch):
|
||||
from application.parser.file.docling_parser import _resolve_ocr_engine
|
||||
from docsgpt.parser.file.docling_parser import _resolve_ocr_engine
|
||||
|
||||
monkeypatch.setattr(settings, "OCR_ENGINE", "auto")
|
||||
assert _resolve_ocr_engine(None) == "auto"
|
||||
|
||||
def test_unknown_engine_degrades_to_auto(self):
|
||||
from application.parser.file.docling_parser import _resolve_ocr_engine
|
||||
from docsgpt.parser.file.docling_parser import _resolve_ocr_engine
|
||||
|
||||
assert _resolve_ocr_engine("easyocr") == "auto"
|
||||
|
||||
def test_tesseract_without_binary_degrades_to_auto(self, monkeypatch):
|
||||
from application.parser.file import docling_parser as dp
|
||||
from docsgpt.parser.file import docling_parser as dp
|
||||
|
||||
monkeypatch.setattr(dp.shutil, "which", lambda name: None)
|
||||
assert dp._resolve_ocr_engine("tesseract") == "auto"
|
||||
|
||||
def test_tesseract_with_binary_selected(self, monkeypatch):
|
||||
from application.parser.file import docling_parser as dp
|
||||
from docsgpt.parser.file import docling_parser as dp
|
||||
|
||||
monkeypatch.setattr(dp.shutil, "which", lambda name: "/usr/bin/tesseract")
|
||||
assert dp._resolve_ocr_engine("tesseract") == "tesseract"
|
||||
|
||||
def test_ocrmac_off_darwin_degrades_to_auto(self, monkeypatch):
|
||||
from application.parser.file import docling_parser as dp
|
||||
from docsgpt.parser.file import docling_parser as dp
|
||||
|
||||
monkeypatch.setattr(dp.sys, "platform", "linux")
|
||||
assert dp._resolve_ocr_engine("ocrmac") == "auto"
|
||||
@@ -154,25 +154,25 @@ class TestOcrEngineSelection:
|
||||
def test_rapidocr_missing_degrades_to_auto(self, monkeypatch):
|
||||
import sys
|
||||
|
||||
from application.parser.file import docling_parser as dp
|
||||
from docsgpt.parser.file import docling_parser as dp
|
||||
|
||||
monkeypatch.setitem(sys.modules, "rapidocr", None)
|
||||
assert dp._resolve_ocr_engine("rapidocr") == "auto"
|
||||
|
||||
def test_deepseek_passes_through(self):
|
||||
from application.parser.file.docling_parser import _resolve_ocr_engine
|
||||
from docsgpt.parser.file.docling_parser import _resolve_ocr_engine
|
||||
|
||||
assert _resolve_ocr_engine("deepseek") == "deepseek"
|
||||
|
||||
def test_build_auto_returns_none(self):
|
||||
from application.parser.file.docling_parser import _build_ocr_options
|
||||
from docsgpt.parser.file.docling_parser import _build_ocr_options
|
||||
|
||||
assert _build_ocr_options("auto", None, True) is None
|
||||
|
||||
def test_build_tesseract_reads_ocr_langs(self, settings, monkeypatch):
|
||||
pytest.importorskip("docling")
|
||||
import application.parser.file.ocr_parser as op
|
||||
from application.parser.file.docling_parser import _build_ocr_options
|
||||
import docsgpt.parser.file.ocr_parser as op
|
||||
from docsgpt.parser.file.docling_parser import _build_ocr_options
|
||||
|
||||
# Pack inventory unknown: the resolved list is passed through untouched
|
||||
# (a host with tesseract but no chi_sim pack would otherwise drop it).
|
||||
@@ -186,8 +186,8 @@ class TestOcrEngineSelection:
|
||||
|
||||
def test_build_tesseract_explicit_languages_win(self, monkeypatch):
|
||||
pytest.importorskip("docling")
|
||||
import application.parser.file.ocr_parser as op
|
||||
from application.parser.file.docling_parser import _build_ocr_options
|
||||
import docsgpt.parser.file.ocr_parser as op
|
||||
from docsgpt.parser.file.docling_parser import _build_ocr_options
|
||||
|
||||
# Pack inventory unknown: the requested list is passed through untouched.
|
||||
monkeypatch.setattr(op, "tesseract_languages", lambda: None)
|
||||
@@ -196,7 +196,7 @@ class TestOcrEngineSelection:
|
||||
|
||||
def test_build_rapidocr(self):
|
||||
pytest.importorskip("docling")
|
||||
from application.parser.file.docling_parser import _build_ocr_options
|
||||
from docsgpt.parser.file.docling_parser import _build_ocr_options
|
||||
|
||||
options = _build_ocr_options("rapidocr", None, False)
|
||||
assert type(options).__name__ == "RapidOcrOptions"
|
||||
@@ -205,7 +205,7 @@ class TestOcrEngineSelection:
|
||||
def test_build_import_failure_returns_none(self, monkeypatch):
|
||||
import sys
|
||||
|
||||
from application.parser.file.docling_parser import _build_ocr_options
|
||||
from docsgpt.parser.file.docling_parser import _build_ocr_options
|
||||
|
||||
monkeypatch.setitem(sys.modules, "docling.datamodel.pipeline_options", None)
|
||||
assert _build_ocr_options("tesseract", ["eng"], False) is None
|
||||
@@ -215,7 +215,7 @@ class TestOcrEngineSelection:
|
||||
import sys
|
||||
import types
|
||||
|
||||
from application.parser.file.docling_parser import _build_ocr_options
|
||||
from docsgpt.parser.file.docling_parser import _build_ocr_options
|
||||
|
||||
fake = types.ModuleType("docling.datamodel.pipeline_options")
|
||||
|
||||
@@ -236,8 +236,8 @@ class TestDeepseekVlmConverter:
|
||||
pytest.importorskip("docling")
|
||||
|
||||
def test_deepseek_builds_vlm_converter(self, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
monkeypatch.setattr(settings, "OCR_DEEPSEEK_URL", "http://gpu-host:8000/v1/chat/completions")
|
||||
monkeypatch.setattr(settings, "OCR_DEEPSEEK_MODEL", "deepseek-ocr-x")
|
||||
@@ -267,7 +267,7 @@ class TestDeepseekVlmConverter:
|
||||
assert pdf_option.pipeline_options.enable_remote_services is True
|
||||
|
||||
def test_deepseek_ignored_when_ocr_disabled(self, monkeypatch):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
vlm_called = []
|
||||
monkeypatch.setattr(
|
||||
@@ -290,7 +290,7 @@ class TestDeepseekVlmConverter:
|
||||
class TestExportContent:
|
||||
|
||||
def test_export_markdown(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser(export_format="markdown")
|
||||
mock_doc = MagicMock()
|
||||
@@ -302,7 +302,7 @@ class TestExportContent:
|
||||
mock_doc.export_to_markdown.assert_called_once()
|
||||
|
||||
def test_export_html(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser(export_format="html")
|
||||
mock_doc = MagicMock()
|
||||
@@ -313,7 +313,7 @@ class TestExportContent:
|
||||
assert "<h1>" in result
|
||||
|
||||
def test_export_text(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser(export_format="text")
|
||||
mock_doc = MagicMock()
|
||||
@@ -324,7 +324,7 @@ class TestExportContent:
|
||||
assert "Plain text" in result
|
||||
|
||||
def test_fallback_to_texts_on_minimal_content(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser(export_format="markdown")
|
||||
mock_doc = MagicMock()
|
||||
@@ -341,7 +341,7 @@ class TestExportContent:
|
||||
assert "OCR extracted text 2" in result
|
||||
|
||||
def test_no_fallback_for_substantial_content(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser(export_format="markdown")
|
||||
mock_doc = MagicMock()
|
||||
@@ -352,7 +352,7 @@ class TestExportContent:
|
||||
assert result == "A" * 100
|
||||
|
||||
def test_fallback_skipped_when_no_texts(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser(export_format="markdown")
|
||||
mock_doc = MagicMock()
|
||||
@@ -363,7 +363,7 @@ class TestExportContent:
|
||||
assert result == "short"
|
||||
|
||||
def test_fallback_skips_empty_texts(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser(export_format="markdown")
|
||||
mock_doc = MagicMock()
|
||||
@@ -386,7 +386,7 @@ class TestExportContent:
|
||||
class TestDoclingParserParseFile:
|
||||
|
||||
def test_parse_file_success(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser()
|
||||
|
||||
@@ -403,7 +403,7 @@ class TestDoclingParserParseFile:
|
||||
assert "Parsed document content" in result
|
||||
|
||||
def test_parse_file_inits_converter_on_first_call(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser()
|
||||
parser._converter = None
|
||||
@@ -433,8 +433,8 @@ class TestDoclingParserParseFile:
|
||||
as if it were the PDF. ``errors`` controls *decoding* leniency, not
|
||||
"substitute the traceback for the document".
|
||||
"""
|
||||
from application.parser.file.base_parser import DocumentParseError
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.base_parser import DocumentParseError
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser()
|
||||
mock_converter = MagicMock()
|
||||
@@ -453,8 +453,8 @@ class TestDoclingParserParseFile:
|
||||
|
||||
def test_parse_file_error_ignore_never_returns_a_string(self):
|
||||
"""Belt-and-braces: no code path may hand back error text as content."""
|
||||
from application.parser.file.base_parser import DocumentParseError
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.base_parser import DocumentParseError
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser()
|
||||
mock_converter = MagicMock()
|
||||
@@ -474,7 +474,7 @@ class TestDoclingParserParseFile:
|
||||
)
|
||||
|
||||
def test_parse_file_error_raise(self):
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
parser = DoclingParser()
|
||||
mock_converter = MagicMock()
|
||||
@@ -494,82 +494,82 @@ class TestDoclingParserParseFile:
|
||||
class TestDoclingSubclasses:
|
||||
|
||||
def test_pdf_parser_init(self):
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser()
|
||||
assert parser.ocr_enabled is True
|
||||
assert parser.export_format == "markdown"
|
||||
|
||||
def test_pdf_parser_custom_ocr(self):
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=False, force_full_page_ocr=True)
|
||||
assert parser.ocr_enabled is False
|
||||
assert parser.force_full_page_ocr is True
|
||||
|
||||
def test_docx_parser_init(self):
|
||||
from application.parser.file.docling_parser import DoclingDocxParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingDocxParser
|
||||
|
||||
parser = DoclingDocxParser()
|
||||
assert parser.export_format == "markdown"
|
||||
|
||||
def test_pptx_parser_init(self):
|
||||
from application.parser.file.docling_parser import DoclingPPTXParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPPTXParser
|
||||
|
||||
parser = DoclingPPTXParser()
|
||||
assert parser.export_format == "markdown"
|
||||
|
||||
def test_xlsx_parser_init(self):
|
||||
from application.parser.file.docling_parser import DoclingXLSXParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingXLSXParser
|
||||
|
||||
parser = DoclingXLSXParser()
|
||||
assert parser.table_structure is True
|
||||
|
||||
def test_html_parser_init(self):
|
||||
from application.parser.file.docling_parser import DoclingHTMLParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingHTMLParser
|
||||
|
||||
parser = DoclingHTMLParser()
|
||||
assert parser.export_format == "markdown"
|
||||
|
||||
def test_image_parser_init(self):
|
||||
from application.parser.file.docling_parser import DoclingImageParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingImageParser
|
||||
|
||||
parser = DoclingImageParser()
|
||||
assert parser.ocr_enabled is True
|
||||
assert parser.force_full_page_ocr is True
|
||||
|
||||
def test_image_parser_custom(self):
|
||||
from application.parser.file.docling_parser import DoclingImageParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingImageParser
|
||||
|
||||
parser = DoclingImageParser(ocr_enabled=False)
|
||||
assert parser.ocr_enabled is False
|
||||
|
||||
def test_csv_parser_init(self):
|
||||
from application.parser.file.docling_parser import DoclingCSVParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingCSVParser
|
||||
|
||||
parser = DoclingCSVParser()
|
||||
assert parser.table_structure is True
|
||||
|
||||
def test_markdown_parser_init(self):
|
||||
from application.parser.file.docling_parser import DoclingMarkdownParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingMarkdownParser
|
||||
|
||||
parser = DoclingMarkdownParser()
|
||||
assert parser.export_format == "markdown"
|
||||
|
||||
def test_asciidoc_parser_init(self):
|
||||
from application.parser.file.docling_parser import DoclingAsciiDocParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingAsciiDocParser
|
||||
|
||||
parser = DoclingAsciiDocParser()
|
||||
assert parser.export_format == "markdown"
|
||||
|
||||
def test_vtt_parser_init(self):
|
||||
from application.parser.file.docling_parser import DoclingVTTParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingVTTParser
|
||||
|
||||
parser = DoclingVTTParser()
|
||||
assert parser.export_format == "markdown"
|
||||
|
||||
def test_xml_parser_init(self):
|
||||
from application.parser.file.docling_parser import DoclingXMLParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingXMLParser
|
||||
|
||||
parser = DoclingXMLParser()
|
||||
assert parser.export_format == "markdown"
|
||||
@@ -584,7 +584,7 @@ class TestDoclingSubclasses:
|
||||
class TestDoclingParserGaps:
|
||||
def test_csv_parser_init(self):
|
||||
"""Cover line 289: DoclingCSVParser.__init__ calls super."""
|
||||
from application.parser.file.docling_parser import DoclingCSVParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingCSVParser
|
||||
|
||||
parser = DoclingCSVParser()
|
||||
assert parser.export_format == "markdown"
|
||||
@@ -611,13 +611,13 @@ class TestNonOcrParsersLeaveTextAlone:
|
||||
],
|
||||
)
|
||||
def test_ocr_is_off_by_construction(self, name):
|
||||
from application.parser.file import docling_parser as dp
|
||||
from docsgpt.parser.file import docling_parser as dp
|
||||
|
||||
assert getattr(dp, name)().ocr_enabled is False
|
||||
|
||||
def test_cjk_spaces_survive_in_a_docx_export(self, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import DoclingDocxParser
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import DoclingDocxParser
|
||||
|
||||
monkeypatch.setattr(settings, "OCR_ENGINE", "tesseract")
|
||||
parser = DoclingDocxParser()
|
||||
@@ -628,7 +628,7 @@ class TestNonOcrParsersLeaveTextAlone:
|
||||
assert parser._postprocess_ocr_text(text) == text
|
||||
|
||||
def test_pdf_parser_with_tesseract_still_collapses(self):
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
parser._active_ocr_engine = "tesseract"
|
||||
@@ -645,8 +645,8 @@ class TestApplyPipelineCaps:
|
||||
"""_apply_pipeline_caps bounds docling's threaded-pipeline buffering."""
|
||||
|
||||
def test_caps_threaded_pipeline_knobs(self, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import _apply_pipeline_caps
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "DOCLING_PIPELINE_QUEUE_MAX_SIZE", 2, raising=False
|
||||
@@ -668,8 +668,8 @@ class TestApplyPipelineCaps:
|
||||
assert opts.ocr_batch_size == 1
|
||||
|
||||
def test_queue_size_is_settings_driven(self, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import _apply_pipeline_caps
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "DOCLING_PIPELINE_QUEUE_MAX_SIZE", 6, raising=False
|
||||
@@ -684,8 +684,8 @@ class TestApplyPipelineCaps:
|
||||
|
||||
def test_misconfigured_zero_floors_to_one(self, monkeypatch):
|
||||
"""A 0 queue depth could deadlock the threaded pipeline — floor it."""
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import _apply_pipeline_caps
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "DOCLING_PIPELINE_QUEUE_MAX_SIZE", 0, raising=False
|
||||
@@ -701,7 +701,7 @@ class TestApplyPipelineCaps:
|
||||
def test_noop_on_docling_without_threaded_pipeline(self):
|
||||
"""Builds predating the threaded pipeline lack the knobs — the cap
|
||||
must be a silent no-op, not an AttributeError."""
|
||||
from application.parser.file.docling_parser import _apply_pipeline_caps
|
||||
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
|
||||
|
||||
class LegacyOpts:
|
||||
__slots__ = ("do_ocr", "do_table_structure")
|
||||
@@ -738,8 +738,8 @@ class TestDoclingTabularSizeGate:
|
||||
return path
|
||||
|
||||
def test_oversized_csv_delegates_to_plain_csv_parser(self, tmp_path, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import DoclingCSVParser, DoclingParser
|
||||
|
||||
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
|
||||
docling_parse = MagicMock(name="docling_parse")
|
||||
@@ -755,8 +755,8 @@ class TestDoclingTabularSizeGate:
|
||||
assert out.startswith("0, 0\n1, 2\n")
|
||||
|
||||
def test_small_csv_still_uses_docling(self, tmp_path, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import DoclingCSVParser, DoclingParser
|
||||
|
||||
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 10_000_000)
|
||||
docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
|
||||
@@ -768,8 +768,8 @@ class TestDoclingTabularSizeGate:
|
||||
docling_parse.assert_called_once()
|
||||
|
||||
def test_gate_disabled_when_max_bytes_is_zero(self, tmp_path, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import DoclingCSVParser, DoclingParser
|
||||
|
||||
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 0)
|
||||
docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
|
||||
@@ -781,9 +781,9 @@ class TestDoclingTabularSizeGate:
|
||||
docling_parse.assert_called_once()
|
||||
|
||||
def test_oversized_xlsx_delegates_to_excel_parser(self, tmp_path, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
|
||||
from application.parser.file.tabular_parser import ExcelParser
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
|
||||
from docsgpt.parser.file.tabular_parser import ExcelParser
|
||||
|
||||
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
|
||||
docling_parse = MagicMock(name="docling_parse")
|
||||
@@ -801,8 +801,8 @@ class TestDoclingTabularSizeGate:
|
||||
excel_parse.assert_called_once()
|
||||
|
||||
def test_small_xlsx_still_uses_docling(self, tmp_path, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
|
||||
|
||||
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 10_000_000)
|
||||
docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
|
||||
@@ -837,7 +837,7 @@ class TestTabularContentSize:
|
||||
(2.44 GB in docling) slips under a byte gate."""
|
||||
|
||||
def test_xlsx_content_size_is_inner_not_ondisk(self, tmp_path):
|
||||
from application.parser.file.docling_parser import _tabular_content_size
|
||||
from docsgpt.parser.file.docling_parser import _tabular_content_size
|
||||
|
||||
path = tmp_path / "data.xlsx"
|
||||
_make_xlsx(path, rows=5000)
|
||||
@@ -847,7 +847,7 @@ class TestTabularContentSize:
|
||||
assert inner > on_disk
|
||||
|
||||
def test_csv_content_size_is_ondisk(self, tmp_path):
|
||||
from application.parser.file.docling_parser import _tabular_content_size
|
||||
from docsgpt.parser.file.docling_parser import _tabular_content_size
|
||||
|
||||
path = tmp_path / "data.csv"
|
||||
path.write_text("a,b\n1,2\n3,4\n")
|
||||
@@ -855,13 +855,13 @@ class TestTabularContentSize:
|
||||
|
||||
def test_compressed_xlsx_over_inner_gate_delegates(self, tmp_path, monkeypatch):
|
||||
"""The regression: on-disk < threshold < inner-uncompressed must gate."""
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import (
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import (
|
||||
DoclingParser,
|
||||
DoclingXLSXParser,
|
||||
_tabular_content_size,
|
||||
)
|
||||
from application.parser.file.tabular_parser import ExcelParser
|
||||
from docsgpt.parser.file.tabular_parser import ExcelParser
|
||||
|
||||
path = tmp_path / "wide.xlsx"
|
||||
_make_xlsx(path, rows=5000)
|
||||
@@ -901,8 +901,8 @@ class TestDoclingMarkupGate:
|
||||
|
||||
@pytest.mark.parametrize("name", ["big.html", "big.vtt"])
|
||||
def test_oversized_markup_parses_truncated_copy(self, tmp_path, monkeypatch, name):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file import docling_parser as dp
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file import docling_parser as dp
|
||||
|
||||
monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 512)
|
||||
path = self._write(tmp_path, name, 4096)
|
||||
@@ -928,8 +928,8 @@ class TestDoclingMarkupGate:
|
||||
assert path.stat().st_size > 512, "original must be untouched"
|
||||
|
||||
def test_small_markup_parses_original(self, tmp_path, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file import docling_parser as dp
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file import docling_parser as dp
|
||||
|
||||
monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 10_000_000)
|
||||
path = self._write(tmp_path, "small.html", 1024)
|
||||
@@ -946,8 +946,8 @@ class TestDoclingMarkupGate:
|
||||
assert seen["path"] == str(path)
|
||||
|
||||
def test_markup_gate_disabled_when_zero(self, tmp_path, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file import docling_parser as dp
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file import docling_parser as dp
|
||||
|
||||
monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 0)
|
||||
path = self._write(tmp_path, "big.vtt", 8192)
|
||||
@@ -994,8 +994,8 @@ class TestTabularGateSeam:
|
||||
self, tmp_path, monkeypatch
|
||||
):
|
||||
"""The real incident, end to end: gate trips, blanks do not crash."""
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import (
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import (
|
||||
DoclingParser,
|
||||
DoclingXLSXParser,
|
||||
)
|
||||
@@ -1021,13 +1021,13 @@ class TestTabularGateSeam:
|
||||
``ingest`` and ``store_attachment``, and is the only exception
|
||||
``SimpleDirectoryReader.load_data`` skips rather than propagating.
|
||||
"""
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.base_parser import DocumentParseError
|
||||
from application.parser.file.docling_parser import (
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.base_parser import DocumentParseError
|
||||
from docsgpt.parser.file.docling_parser import (
|
||||
DoclingParser,
|
||||
DoclingXLSXParser,
|
||||
)
|
||||
from application.parser.file.tabular_parser import ExcelParser
|
||||
from docsgpt.parser.file.tabular_parser import ExcelParser
|
||||
|
||||
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
|
||||
monkeypatch.setattr(DoclingParser, "parse_file", MagicMock())
|
||||
@@ -1045,13 +1045,13 @@ class TestTabularGateSeam:
|
||||
def test_oversized_csv_failure_becomes_document_parse_error(
|
||||
self, tmp_path, monkeypatch
|
||||
):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.base_parser import DocumentParseError
|
||||
from application.parser.file.docling_parser import (
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.base_parser import DocumentParseError
|
||||
from docsgpt.parser.file.docling_parser import (
|
||||
DoclingCSVParser,
|
||||
DoclingParser,
|
||||
)
|
||||
from application.parser.file.tabular_parser import CSVParser
|
||||
from docsgpt.parser.file.tabular_parser import CSVParser
|
||||
|
||||
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 8)
|
||||
monkeypatch.setattr(DoclingParser, "parse_file", MagicMock())
|
||||
@@ -1087,7 +1087,7 @@ class TestApplyInferenceSettings:
|
||||
pytest.importorskip("docling")
|
||||
|
||||
def test_disables_torch_compile_by_default(self, monkeypatch):
|
||||
from application.parser.file.docling_parser import _apply_inference_settings
|
||||
from docsgpt.parser.file.docling_parser import _apply_inference_settings
|
||||
|
||||
class Inference:
|
||||
compile_torch_models = True
|
||||
@@ -1105,8 +1105,8 @@ class TestApplyInferenceSettings:
|
||||
assert docling_settings.inference.compile_torch_models is False
|
||||
|
||||
def test_opt_in_reenables_torch_compile(self, monkeypatch):
|
||||
from application.core.settings import settings
|
||||
from application.parser.file.docling_parser import _apply_inference_settings
|
||||
from docsgpt.core.settings import settings
|
||||
from docsgpt.parser.file.docling_parser import _apply_inference_settings
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings, "DOCLING_COMPILE_TORCH_MODELS", True, raising=False
|
||||
@@ -1129,7 +1129,7 @@ class TestApplyInferenceSettings:
|
||||
|
||||
def test_noop_on_docling_without_inference_settings(self, monkeypatch):
|
||||
"""Builds predating the inference settings must be a silent no-op."""
|
||||
from application.parser.file.docling_parser import _apply_inference_settings
|
||||
from docsgpt.parser.file.docling_parser import _apply_inference_settings
|
||||
|
||||
class DoclingSettings:
|
||||
pass
|
||||
@@ -1142,11 +1142,11 @@ class TestApplyInferenceSettings:
|
||||
|
||||
def test_create_converter_applies_inference_settings(self, monkeypatch):
|
||||
"""The cap is worthless unless the converter path actually calls it."""
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
called = []
|
||||
monkeypatch.setattr(
|
||||
"application.parser.file.docling_parser._apply_inference_settings",
|
||||
"docsgpt.parser.file.docling_parser._apply_inference_settings",
|
||||
lambda: called.append(True),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
@@ -1167,11 +1167,11 @@ class TestApplyInferenceSettings:
|
||||
"""
|
||||
import docling.datamodel.pipeline_options as dpo
|
||||
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
events = []
|
||||
monkeypatch.setattr(
|
||||
"application.parser.file.docling_parser._apply_inference_settings",
|
||||
"docsgpt.parser.file.docling_parser._apply_inference_settings",
|
||||
lambda: events.append("settings"),
|
||||
)
|
||||
|
||||
@@ -1210,7 +1210,7 @@ def _mock_conversion(markdown: str, pages: int = 1) -> MagicMock:
|
||||
|
||||
def _set_threshold(monkeypatch, value: int) -> None:
|
||||
"""Point the OCR dropout guard at a specific chars-per-page floor."""
|
||||
from application.core.settings import settings as real_settings
|
||||
from docsgpt.core.settings import settings as real_settings
|
||||
|
||||
class _Stub:
|
||||
def __getattr__(self, name):
|
||||
@@ -1218,7 +1218,7 @@ def _set_threshold(monkeypatch, value: int) -> None:
|
||||
|
||||
stub = _Stub()
|
||||
stub.OCR_MIN_CHARS_PER_PAGE = value
|
||||
monkeypatch.setattr("application.core.settings.settings", stub)
|
||||
monkeypatch.setattr("docsgpt.core.settings.settings", stub)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -1232,7 +1232,7 @@ class TestOCRDropoutGuard:
|
||||
"""
|
||||
|
||||
def test_near_empty_first_pass_recovers_on_fresh_converter(self, caplog):
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
degraded = MagicMock()
|
||||
@@ -1274,8 +1274,8 @@ class TestOCRDropoutGuard:
|
||||
assert "Recovered scan.pdf on retry" in messages
|
||||
|
||||
def test_near_empty_both_passes_raises_document_parse_error(self):
|
||||
from application.parser.file.base_parser import DocumentParseError
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.base_parser import DocumentParseError
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
degraded = MagicMock()
|
||||
@@ -1301,11 +1301,11 @@ class TestOCRDropoutGuard:
|
||||
|
||||
def test_dropout_error_reports_an_unread_text_layer(self, monkeypatch):
|
||||
"""A text layer docling ignored points at the pipeline, not the scan."""
|
||||
from application.parser.file.base_parser import DocumentParseError
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.base_parser import DocumentParseError
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
monkeypatch.setattr(
|
||||
"application.parser.file.docling_parser._pdf_text_layer_probe",
|
||||
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
|
||||
lambda file: (12, 48_000),
|
||||
)
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
@@ -1319,7 +1319,7 @@ class TestOCRDropoutGuard:
|
||||
assert "The PDF carries a 48000-char text layer" in str(excinfo.value)
|
||||
|
||||
def test_ocr_disabled_returns_near_empty_content_untouched(self):
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=False)
|
||||
converter = MagicMock()
|
||||
@@ -1334,7 +1334,7 @@ class TestOCRDropoutGuard:
|
||||
assert parser._converter is converter
|
||||
|
||||
def test_threshold_zero_disables_the_guard(self, monkeypatch):
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
_set_threshold(monkeypatch, 0)
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
@@ -1349,7 +1349,7 @@ class TestOCRDropoutGuard:
|
||||
assert parser._converter is converter
|
||||
|
||||
def test_healthy_ocr_parse_converts_once(self):
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
converter = MagicMock()
|
||||
@@ -1368,7 +1368,7 @@ class TestOCRDropoutGuard:
|
||||
|
||||
def test_threshold_is_per_page_not_per_document(self):
|
||||
"""200 chars is healthy for one page and suspicious for a hundred."""
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
content = "x" * 200
|
||||
|
||||
@@ -1388,7 +1388,7 @@ class TestOCRDropoutGuard:
|
||||
with patch.object(
|
||||
long_doc, "_create_converter", return_value=long_doc._converter
|
||||
) as create, patch(
|
||||
"application.parser.file.docling_parser._pdf_text_layer_probe",
|
||||
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
|
||||
return_value=(100, 0),
|
||||
):
|
||||
assert long_doc.parse_file(Path("hundred-page.pdf")) == content
|
||||
@@ -1396,7 +1396,7 @@ class TestOCRDropoutGuard:
|
||||
|
||||
def test_guard_does_not_apply_to_non_ocr_formats(self):
|
||||
"""DOCX parsers construct with OCR off: they never OCR anything."""
|
||||
from application.parser.file.docling_parser import DoclingDocxParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingDocxParser
|
||||
|
||||
parser = DoclingDocxParser()
|
||||
assert parser.ocr_enabled is False
|
||||
@@ -1411,7 +1411,7 @@ class TestOCRDropoutGuard:
|
||||
|
||||
def test_guard_applies_to_images(self):
|
||||
"""The retry covers images -- a degraded converter may still recover one."""
|
||||
from application.parser.file.docling_parser import DoclingImageParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingImageParser
|
||||
|
||||
parser = DoclingImageParser(ocr_enabled=True)
|
||||
degraded = MagicMock()
|
||||
@@ -1432,7 +1432,7 @@ class TestOCRDropoutGuard:
|
||||
DocumentParseError is in ``dont_autoretry_for``, so raising here fails a
|
||||
single-file upload permanently.
|
||||
"""
|
||||
from application.parser.file.docling_parser import DoclingImageParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingImageParser
|
||||
|
||||
parser = DoclingImageParser(ocr_enabled=True)
|
||||
converter = MagicMock()
|
||||
@@ -1447,7 +1447,7 @@ class TestOCRDropoutGuard:
|
||||
|
||||
def test_a_text_sparse_scan_indexes_rather_than_failing_the_upload(self):
|
||||
"""20 pages of pictures with a few captions is the document, not a fault."""
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
content = "Fig 1. Fig 2. Fig 3."
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
@@ -1458,15 +1458,15 @@ class TestOCRDropoutGuard:
|
||||
with patch.object(
|
||||
parser, "_create_converter", return_value=converter
|
||||
), patch(
|
||||
"application.parser.file.docling_parser._pdf_text_layer_probe",
|
||||
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
|
||||
return_value=(20, 0),
|
||||
):
|
||||
assert parser.parse_file(Path("catalog.pdf")) == content
|
||||
|
||||
def test_a_multi_page_zero_char_parse_is_still_a_dropout(self):
|
||||
"""The incident this guard exists for: every page OCR'd to nothing."""
|
||||
from application.parser.file.base_parser import DocumentParseError
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.base_parser import DocumentParseError
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
converter = MagicMock()
|
||||
@@ -1476,7 +1476,7 @@ class TestOCRDropoutGuard:
|
||||
with patch.object(
|
||||
parser, "_create_converter", return_value=converter
|
||||
), patch(
|
||||
"application.parser.file.docling_parser._pdf_text_layer_probe",
|
||||
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
|
||||
return_value=(40, 0),
|
||||
):
|
||||
with pytest.raises(DocumentParseError, match="scan.pdf"):
|
||||
@@ -1484,8 +1484,8 @@ class TestOCRDropoutGuard:
|
||||
|
||||
def test_a_pdf_whose_text_layer_was_missed_is_still_a_dropout(self):
|
||||
"""Text docling should have read without OCR at all -- positive evidence."""
|
||||
from application.parser.file.base_parser import DocumentParseError
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.base_parser import DocumentParseError
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
converter = MagicMock()
|
||||
@@ -1495,15 +1495,15 @@ class TestOCRDropoutGuard:
|
||||
with patch.object(
|
||||
parser, "_create_converter", return_value=converter
|
||||
), patch(
|
||||
"application.parser.file.docling_parser._pdf_text_layer_probe",
|
||||
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
|
||||
return_value=(20, 5000),
|
||||
):
|
||||
with pytest.raises(DocumentParseError, match="text layer"):
|
||||
parser.parse_file(Path("report.pdf"))
|
||||
|
||||
def test_retry_conversion_failure_is_still_a_parse_error(self):
|
||||
from application.parser.file.base_parser import DocumentParseError
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.base_parser import DocumentParseError
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
parser._converter = MagicMock()
|
||||
@@ -1524,7 +1524,7 @@ class TestOCRDropoutGuard:
|
||||
class TestOCRDropoutHelpers:
|
||||
|
||||
def test_image_placeholders_are_not_text(self):
|
||||
from application.parser.file.docling_parser import _text_char_count
|
||||
from docsgpt.parser.file.docling_parser import _text_char_count
|
||||
|
||||
assert _text_char_count(None) == 0
|
||||
assert _text_char_count("") == 0
|
||||
@@ -1532,20 +1532,20 @@ class TestOCRDropoutHelpers:
|
||||
assert _text_char_count(" <!-- image --> abc ") == 3
|
||||
|
||||
def test_page_count_prefers_document_pages(self):
|
||||
from application.parser.file.docling_parser import _result_page_count
|
||||
from docsgpt.parser.file.docling_parser import _result_page_count
|
||||
|
||||
result = _mock_conversion("x", pages=7)
|
||||
assert _result_page_count(result, Path("a.pdf")) == 7
|
||||
|
||||
def test_page_count_falls_back_to_num_pages(self):
|
||||
from application.parser.file.docling_parser import _result_page_count
|
||||
from docsgpt.parser.file.docling_parser import _result_page_count
|
||||
|
||||
result = _mock_conversion("x", pages=0)
|
||||
result.document.num_pages.return_value = 5
|
||||
assert _result_page_count(result, Path("a.pdf")) == 5
|
||||
|
||||
def test_page_count_falls_back_to_result_pages(self):
|
||||
from application.parser.file.docling_parser import _result_page_count
|
||||
from docsgpt.parser.file.docling_parser import _result_page_count
|
||||
|
||||
result = _mock_conversion("x", pages=0)
|
||||
result.document.num_pages.return_value = None
|
||||
@@ -1553,7 +1553,7 @@ class TestOCRDropoutHelpers:
|
||||
assert _result_page_count(result, Path("a.pdf")) == 2
|
||||
|
||||
def test_page_count_defaults_to_one(self):
|
||||
from application.parser.file.docling_parser import _result_page_count
|
||||
from docsgpt.parser.file.docling_parser import _result_page_count
|
||||
|
||||
result = _mock_conversion("x", pages=0)
|
||||
result.document.num_pages.return_value = None
|
||||
@@ -1561,13 +1561,13 @@ class TestOCRDropoutHelpers:
|
||||
assert _result_page_count(result, Path("a.pdf")) == 1
|
||||
|
||||
def test_images_are_always_one_page(self):
|
||||
from application.parser.file.docling_parser import _result_page_count
|
||||
from docsgpt.parser.file.docling_parser import _result_page_count
|
||||
|
||||
result = _mock_conversion("x", pages=9)
|
||||
assert _result_page_count(result, Path("a.png")) == 1
|
||||
|
||||
def test_threshold_reads_settings_with_a_default(self, monkeypatch):
|
||||
from application.parser.file.docling_parser import _ocr_min_chars_per_page
|
||||
from docsgpt.parser.file.docling_parser import _ocr_min_chars_per_page
|
||||
|
||||
assert _ocr_min_chars_per_page() == 20
|
||||
_set_threshold(monkeypatch, 5)
|
||||
@@ -1576,7 +1576,7 @@ class TestOCRDropoutHelpers:
|
||||
assert _ocr_min_chars_per_page() == 20
|
||||
|
||||
def test_text_layer_probe_survives_an_unreadable_pdf(self, tmp_path):
|
||||
from application.parser.file.docling_parser import _pdf_text_layer_probe
|
||||
from docsgpt.parser.file.docling_parser import _pdf_text_layer_probe
|
||||
|
||||
assert _pdf_text_layer_probe(tmp_path / "missing.pdf") == (0, 0)
|
||||
broken = tmp_path / "broken.pdf"
|
||||
@@ -1597,7 +1597,7 @@ class TestForceFullPageOCRWiring:
|
||||
def _pipeline_options(self, monkeypatch, **kwargs):
|
||||
import docling.datamodel.pipeline_options as dpo
|
||||
|
||||
from application.parser.file.docling_parser import DoclingParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingParser
|
||||
|
||||
built = []
|
||||
real_options = dpo.PdfPipelineOptions
|
||||
@@ -1636,7 +1636,7 @@ class TestDoclingTesseractPostprocessing:
|
||||
"""docling's tesseract path gets the same CJK glyph-space cleanup as the native engine."""
|
||||
|
||||
def _parser_with_export(self, engine, ocr_enabled=True):
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=ocr_enabled)
|
||||
parser._active_ocr_engine = engine
|
||||
@@ -1660,7 +1660,7 @@ class TestDoclingTesseractPostprocessing:
|
||||
@pytest.mark.unit
|
||||
class TestDoclingOcrPages:
|
||||
def test_converts_each_requested_page_with_page_range(self, tmp_path):
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
converter = MagicMock()
|
||||
@@ -1683,8 +1683,8 @@ class TestDoclingOcrPages:
|
||||
assert [c.kwargs["page_range"] for c in converter.convert.call_args_list] == [(3, 3), (1, 1)]
|
||||
|
||||
def test_failure_is_a_parse_error(self, tmp_path):
|
||||
from application.parser.file.base_parser import DocumentParseError
|
||||
from application.parser.file.docling_parser import DoclingPDFParser
|
||||
from docsgpt.parser.file.base_parser import DocumentParseError
|
||||
from docsgpt.parser.file.docling_parser import DoclingPDFParser
|
||||
|
||||
parser = DoclingPDFParser(ocr_enabled=True)
|
||||
parser._converter = MagicMock()
|
||||
@@ -1699,8 +1699,8 @@ class TestDoclingOcrPages:
|
||||
class TestTesseractLanguageFilter:
|
||||
def test_uninstalled_packs_are_dropped_with_a_warning(self, monkeypatch, caplog):
|
||||
pytest.importorskip("docling")
|
||||
import application.parser.file.ocr_parser as op
|
||||
from application.parser.file.docling_parser import _build_ocr_options
|
||||
import docsgpt.parser.file.ocr_parser as op
|
||||
from docsgpt.parser.file.docling_parser import _build_ocr_options
|
||||
|
||||
monkeypatch.setattr(op, "tesseract_languages", lambda: frozenset({"eng", "osd"}))
|
||||
with caplog.at_level("WARNING"):
|
||||
@@ -1710,16 +1710,16 @@ class TestTesseractLanguageFilter:
|
||||
|
||||
def test_all_packs_missing_falls_back_to_eng(self, monkeypatch):
|
||||
pytest.importorskip("docling")
|
||||
import application.parser.file.ocr_parser as op
|
||||
from application.parser.file.docling_parser import _build_ocr_options
|
||||
import docsgpt.parser.file.ocr_parser as op
|
||||
from docsgpt.parser.file.docling_parser import _build_ocr_options
|
||||
|
||||
monkeypatch.setattr(op, "tesseract_languages", lambda: frozenset({"eng"}))
|
||||
assert _build_ocr_options("tesseract", ["xyz"], False).lang == ["eng"]
|
||||
|
||||
def test_unknown_inventory_keeps_the_list(self, monkeypatch):
|
||||
pytest.importorskip("docling")
|
||||
import application.parser.file.ocr_parser as op
|
||||
from application.parser.file.docling_parser import _build_ocr_options
|
||||
import docsgpt.parser.file.ocr_parser as op
|
||||
from docsgpt.parser.file.docling_parser import _build_ocr_options
|
||||
|
||||
monkeypatch.setattr(op, "tesseract_languages", lambda: None)
|
||||
assert _build_ocr_options("tesseract", ["eng", "deu"], False).lang == ["eng", "deu"]
|
||||
Reference in new issue
Block a user