tiennm99/DocsGPT
1
0 Fork 0
mirror of https://github.com/tiennm99/DocsGPT.git synced 2026-10-15 14:16:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: rename the application package to docsgpt

Browse Source 
The backend import package is now docsgpt, the name it will carry on PyPI;
application was far too generic to install into anyone's site-packages.
git mv plus a mechanical rewrite of every import, dotted string and path
reference: 734 Python files, the compose files, Dockerfile, workflows, docs,
setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage
config, .gitignore. Behaviour is unchanged.

Kept for one release:
- A top-level application package whose meta-path finder resolves
  application.x.y to the already-imported docsgpt.x.y object, so old imports
  and entry points (celery -A application.app.celery,
  uvicorn application.asgi:asgi_app) keep working with a FutureWarning.
- Celery registers every application.* task name as an alias of its
  docsgpt.* task on start-up, so messages queued by the previous release still
  run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries
  the previous release wrote are left unread instead of firing twice.

The backend image builds from the repository root (docker build -f
docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore
allow-lists docsgpt/ and application/ and keeps caches, local data, .env
files, the sample index files and the Dockerfile out. Compose and the image
workflows point at the new context.
This commit is contained in:
Alex committed 2026-09-07 10:20:43 +01:00
1 parent 3e7f1f91b4
commit 574f96341e
984 files changed
+9847 -9682
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/parser/file/test_anydoc_parser.py
+9 -9
View File
@@ -17,11 +17,11 @@ from pathlib import Path
import pytest
from application.parser.file.base_parser import BaseParser, DocumentParseError
from docsgpt.parser.file.base_parser import BaseParser, DocumentParseError
anydoc = pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import ( # noqa: E402 — after importorskip
from docsgpt.parser.file.anydoc_parser import ( # noqa: E402 — after importorskip
ANYDOC_SUFFIXES,
AnydocParser,
anydoc_available,
@@ -309,7 +309,7 @@ def test_init_parser_imports_for_real_not_just_find_spec(monkeypatch):
"""A wheel whose native extension fails to load has a spec but no importable
module; that must surface at init, not as a bare ImportError from parse_file
mid-ingest (which load_data does not catch)."""
from application.parser.file import anydoc_parser as mod
from docsgpt.parser.file import anydoc_parser as mod
monkeypatch.setattr(mod, "anydoc_available", lambda: True)
monkeypatch.setitem(sys.modules, "anydoc", None)
@@ -332,7 +332,7 @@ class _FakeDoclingFallback:
"""Registered as a DoclingParser subclass so ``_is_docling_backed`` is True."""
def __new__(cls):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
class _Inner(DoclingParser):
def __init__(self):
@@ -374,7 +374,7 @@ def test_trust_flagged_pdf_without_docling_keeps_output_and_warns():
def test_trust_check_disabled_stamps_nothing(monkeypatch):
from application.parser.file import anydoc_parser as ap
from docsgpt.parser.file import anydoc_parser as ap
monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
parser = AnydocParser()
@@ -431,7 +431,7 @@ def test_trust_check_errors_never_fail_the_parse(monkeypatch, tmp_path):
def _explode(path, markdown):
raise RuntimeError("scanner bug")
import application.parser.file.pdf_trust as pt
import docsgpt.parser.file.pdf_trust as pt
monkeypatch.setattr(pt, "verify_pdf_file", _explode)
_fake_anydoc(monkeypatch, lambda path: "# converted fine")
@@ -442,7 +442,7 @@ def test_trust_check_errors_never_fail_the_parse(monkeypatch, tmp_path):
def test_tableize_applied_when_enabled(monkeypatch, tmp_path):
from application.parser.file import anydoc_parser as ap
from docsgpt.parser.file import anydoc_parser as ap
monkeypatch.setattr(ap.settings, "ANYDOC_TABLEIZE", True)
monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
@@ -459,7 +459,7 @@ def test_tableize_applied_when_enabled(monkeypatch, tmp_path):
def test_tableize_disabled_by_default(monkeypatch, tmp_path):
from application.parser.file import anydoc_parser as ap
from docsgpt.parser.file import anydoc_parser as ap
monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
flat = "Cash ..... 1,234 900\nDebt ..... 2,000 1,500\nEquity ..... 900 800"
@@ -471,7 +471,7 @@ def test_tableize_disabled_by_default(monkeypatch, tmp_path):
def test_tableize_never_touches_docling_reroute(monkeypatch):
from application.parser.file import anydoc_parser as ap
from docsgpt.parser.file import anydoc_parser as ap
monkeypatch.setattr(ap.settings, "ANYDOC_TABLEIZE", True)
fallback = _FakeDoclingFallback()
tests/parser/file/test_audio_parser.py
+7 -7
View File
@@ -1,8 +1,8 @@
from unittest.mock import MagicMock, patch
from application.parser.file.audio_parser import AudioParser
from application.parser.file.bulk import get_default_file_extractor
from application.stt.upload_limits import AudioFileTooLargeError
from docsgpt.parser.file.audio_parser import AudioParser
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.stt.upload_limits import AudioFileTooLargeError
def test_audio_init_parser():
@@ -13,9 +13,9 @@ def test_audio_init_parser():
assert parser.parser_config_set
@patch("application.stt.upload_limits.settings")
@patch("application.parser.file.audio_parser.STTCreator.create_stt")
@patch("application.parser.file.audio_parser.settings")
@patch("docsgpt.stt.upload_limits.settings")
@patch("docsgpt.parser.file.audio_parser.STTCreator.create_stt")
@patch("docsgpt.parser.file.audio_parser.settings")
def test_audio_parser_transcribes_file(
mock_settings, mock_create_stt, mock_limit_settings, tmp_path
):
@@ -44,7 +44,7 @@ def test_audio_parser_transcribes_file(
)
@patch("application.stt.upload_limits.settings")
@patch("docsgpt.stt.upload_limits.settings")
def test_audio_parser_rejects_oversized_files(mock_limit_settings, tmp_path):
mock_limit_settings.STT_MAX_FILE_SIZE_MB = 1
tests/parser/file/test_bulk.py
+61 -61
View File
@@ -1,4 +1,4 @@
"""Comprehensive tests for application/parser/file/bulk.py
"""Comprehensive tests for docsgpt/parser/file/bulk.py
Covers: SimpleDirectoryReader (init, file discovery, load_data, directory
structure building), get_default_file_extractor.
@@ -9,7 +9,7 @@ from unittest.mock import MagicMock, patch
import pytest
from application.parser.schema.base import Document
from docsgpt.parser.schema.base import Document
# =====================================================================
@@ -47,54 +47,54 @@ def temp_dir_with_types(tmp_path):
class TestSimpleDirectoryReaderInit:
def test_init_with_dir(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(input_dir=str(temp_dir))
assert len(reader.input_files) >= 2
def test_init_with_files(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
files = [str(temp_dir / "file1.md")]
reader = SimpleDirectoryReader(input_files=files)
assert len(reader.input_files) == 1
def test_init_requires_input(self):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
with pytest.raises(ValueError, match="Must provide"):
SimpleDirectoryReader()
def test_exclude_hidden(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(input_dir=str(temp_dir), exclude_hidden=True)
filenames = [f.name for f in reader.input_files]
assert ".hidden" not in filenames
def test_include_hidden(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(input_dir=str(temp_dir), exclude_hidden=False)
filenames = [f.name for f in reader.input_files]
assert ".hidden" in filenames
def test_recursive(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(input_dir=str(temp_dir), recursive=True)
filenames = [f.name for f in reader.input_files]
assert "file3.md" in filenames
def test_non_recursive(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(input_dir=str(temp_dir), recursive=False)
filenames = [f.name for f in reader.input_files]
assert "file3.md" not in filenames
def test_required_exts(self, temp_dir_with_types):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(
input_dir=str(temp_dir_with_types), required_exts=[".md"]
@@ -105,7 +105,7 @@ class TestSimpleDirectoryReaderInit:
assert "notes.txt" not in filenames
def test_required_exts_case_insensitive(self, tmp_path):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "FILE.MD").write_text("content")
reader = SimpleDirectoryReader(
@@ -114,7 +114,7 @@ class TestSimpleDirectoryReaderInit:
assert len(reader.input_files) == 1
def test_num_files_limit(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(
input_dir=str(temp_dir), num_files_limit=1, recursive=False
@@ -122,7 +122,7 @@ class TestSimpleDirectoryReaderInit:
assert len(reader.input_files) <= 1
def test_custom_file_extractor(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
reader = SimpleDirectoryReader(
@@ -141,7 +141,7 @@ class TestSimpleDirectoryReaderInit:
class TestSimpleDirectoryReaderLoadData:
def test_load_data_returns_documents(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = True
@@ -166,8 +166,8 @@ class TestSimpleDirectoryReaderLoadData:
traceback as the document text, so an unguarded loop turns a single
corrupt PDF into a failed ingest for the whole zip/folder/sync.
"""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "good1.md").write_text("first")
(tmp_path / "bad.md").write_text("corrupt")
@@ -203,8 +203,8 @@ class TestSimpleDirectoryReaderLoadData:
Skipping there would hand ``load_data()[0]`` an empty list and turn a
clear "this PDF could not be read" into an IndexError.
"""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "bad.pdf").write_text("not really a pdf")
@@ -224,8 +224,8 @@ class TestSimpleDirectoryReaderLoadData:
def test_all_files_unparseable_raises(self, tmp_path):
"""Nothing parsed is a failed ingest, not an empty success."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "a.md").write_text("x")
(tmp_path / "b.md").write_text("y")
@@ -246,8 +246,8 @@ class TestSimpleDirectoryReaderLoadData:
def test_skipped_file_still_advances_progress(self, tmp_path):
"""Progress must not stall on a skipped file."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "good.md").write_text("ok")
(tmp_path / "bad.md").write_text("bad")
@@ -273,7 +273,7 @@ class TestSimpleDirectoryReaderLoadData:
assert [c[0] for c in calls] == [1, 2]
def test_load_data_progress_callback_fires_per_file(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(
input_dir=str(temp_dir), recursive=False, exclude_hidden=True,
@@ -288,7 +288,7 @@ class TestSimpleDirectoryReaderLoadData:
assert all(c[1] == total_files for c in calls)
def test_load_data_progress_callback_errors_swallowed(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(
input_dir=str(temp_dir), recursive=False, exclude_hidden=True,
@@ -302,7 +302,7 @@ class TestSimpleDirectoryReaderLoadData:
assert len(docs) >= 1
def test_load_data_concatenate(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = True
@@ -319,7 +319,7 @@ class TestSimpleDirectoryReaderLoadData:
assert len(docs) == 1
def test_load_data_with_file_metadata(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
def custom_metadata(filename):
return {"custom_key": f"meta_{filename}"}
@@ -343,7 +343,7 @@ class TestSimpleDirectoryReaderLoadData:
assert "custom_key" in doc.extra_info
def test_load_data_inits_parser_if_not_set(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = False
@@ -360,7 +360,7 @@ class TestSimpleDirectoryReaderLoadData:
mock_parser.init_parser.assert_called()
def test_load_data_standard_read_for_unknown_ext(self, tmp_path):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "file.xyz").write_text("xyz content")
reader = SimpleDirectoryReader(
@@ -372,7 +372,7 @@ class TestSimpleDirectoryReaderLoadData:
assert "xyz content" in docs[0].text
def test_load_data_list_return_from_parser(self, tmp_path):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "multi.md").write_text("content")
mock_parser = MagicMock()
@@ -388,7 +388,7 @@ class TestSimpleDirectoryReaderLoadData:
assert len(docs) == 2
def test_load_data_tracks_token_counts(self, tmp_path):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "test.md").write_text("hello world")
mock_parser = MagicMock()
@@ -414,7 +414,7 @@ class TestSimpleDirectoryReaderLoadData:
class TestBuildDirectoryStructure:
def test_builds_structure(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = True
@@ -431,7 +431,7 @@ class TestBuildDirectoryStructure:
assert isinstance(reader.directory_structure, dict)
def test_structure_contains_files_and_dirs(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = True
@@ -453,7 +453,7 @@ class TestBuildDirectoryStructure:
assert "size_bytes" in val
def test_structure_excludes_hidden(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = True
@@ -469,7 +469,7 @@ class TestBuildDirectoryStructure:
assert ".hidden" not in reader.directory_structure
def test_no_structure_without_input_dir(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
files = [str(temp_dir / "file1.md")]
mock_parser = MagicMock()
@@ -494,7 +494,7 @@ class TestBuildDirectoryStructure:
class TestGetDefaultFileExtractor:
def test_returns_dict(self):
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
with patch.dict("sys.modules", {"docling": None, "docling.document_converter": None}):
result = get_default_file_extractor()
@@ -503,7 +503,7 @@ class TestGetDefaultFileExtractor:
def test_fallback_parsers_on_import_error(self):
with patch(
"application.parser.file.bulk.get_default_file_extractor"
"docsgpt.parser.file.bulk.get_default_file_extractor"
) as mock_fn:
mock_fn.return_value = {".pdf": MagicMock(), ".md": MagicMock()}
result = mock_fn()
@@ -521,7 +521,7 @@ class TestParserEngineSwitch:
@pytest.fixture
def settings(self):
from application.core.settings import settings
from docsgpt.core.settings import settings
return settings
@@ -529,16 +529,16 @@ class TestParserEngineSwitch:
def _default_ocr_settings(self, monkeypatch):
"""Pin the OCR knobs to their defaults so a developer's ``.env`` (OCR on,
native backend) cannot change which parsers the maps hand out here."""
from application.core.settings import settings
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "OCR_ENABLED", False)
monkeypatch.setattr(settings, "OCR_BACKEND", "auto")
def test_default_engine_is_anydoc(self, settings, monkeypatch):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import ANYDOC_SUFFIXES, AnydocParser
from application.parser.file.bulk import get_default_file_extractor
from application.parser.file.html_parser import HTMLMarkdownParser
from docsgpt.parser.file.anydoc_parser import ANYDOC_SUFFIXES, AnydocParser
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.html_parser import HTMLMarkdownParser
monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "anydoc")
extractor = get_default_file_extractor()
@@ -555,7 +555,7 @@ class TestParserEngineSwitch:
def test_anydoc_falls_back_to_docling_when_installed(self):
pytest.importorskip("anydoc")
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
extractor = get_default_file_extractor(engine="anydoc")
@@ -568,7 +568,7 @@ class TestParserEngineSwitch:
def test_anydoc_fallback_honours_ocr_flag(self):
pytest.importorskip("anydoc")
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
extractor = get_default_file_extractor(engine="anydoc", ocr_enabled=True)
@@ -579,7 +579,7 @@ class TestParserEngineSwitch:
pytest.importorskip("anydoc")
import sys
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "docling", None)
extractor = get_default_file_extractor(engine="anydoc")
@@ -590,7 +590,7 @@ class TestParserEngineSwitch:
def test_docling_engine_keeps_docling_map(self):
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
extractor = get_default_file_extractor(engine="docling")
@@ -599,22 +599,22 @@ class TestParserEngineSwitch:
def test_setting_selects_docling(self, settings, monkeypatch):
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "docling")
assert type(get_default_file_extractor()[".pdf"]).__name__ == "DoclingPDFParser"
def test_unknown_engine_uses_anydoc(self):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.bulk import get_default_file_extractor
assert isinstance(get_default_file_extractor(engine="ghost")[".pdf"], AnydocParser)
def test_missing_anydoc_degrades_to_base_engine(self, monkeypatch):
import sys
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "anydoc", None)
extractor = get_default_file_extractor(engine="anydoc")
@@ -628,7 +628,7 @@ class TestParserEngineSwitch:
the first file aborted the whole ingest instead of degrading."""
import sys
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "docling", None)
extractor = get_default_file_extractor(engine="docling")
@@ -655,8 +655,8 @@ class TestGainedFormats:
(`.txt` is the one deliberate plain-text read.)
"""
pytest.importorskip("anydoc")
from application.parser.file.bulk import get_default_file_extractor
from application.parser.file.constants import (
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.constants import (
SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS,
)
@@ -671,11 +671,11 @@ class TestGainedFormats:
def test_gained_formats_map_to_anydoc_under_both_engines(self):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import (
from docsgpt.parser.file.anydoc_parser import (
ANYDOC_GAINED_SUFFIXES,
AnydocParser,
)
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
for engine in ("anydoc", "docling"):
extractor = get_default_file_extractor(engine=engine)
@@ -684,8 +684,8 @@ class TestGainedFormats:
def test_gained_formats_never_fall_back_to_anydoc_itself(self):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.bulk import get_default_file_extractor
extractor = get_default_file_extractor(engine="anydoc")
assert extractor[".doc"].fallback_parser is None
@@ -697,7 +697,7 @@ class TestGainedFormats:
def test_gained_entries_absent_without_anydoc(self, monkeypatch):
import sys
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "anydoc", None)
extractor = get_default_file_extractor(engine="docling")
@@ -705,7 +705,7 @@ class TestGainedFormats:
def test_rtf_converts_end_to_end(self, tmp_path):
pytest.importorskip("anydoc")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
path = tmp_path / "note.rtf"
path.write_text(r"{\rtf1\ansi Hello {\b bold} world.\par Second paragraph.}")
@@ -721,7 +721,7 @@ class TestGainedFormats:
pytest.importorskip("anydoc")
import zipfile
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
path = tmp_path / "doc.odt"
with zipfile.ZipFile(path, "w") as z:
@@ -753,8 +753,8 @@ class TestGainedFormats:
def test_gained_suffix_without_a_parser_is_rejected_not_read_as_text(tmp_path):
"""Without anydoc an OLE .doc must not be indexed as decoded binary garbage."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.bulk import SimpleDirectoryReader
path = tmp_path / "legacy.doc"
path.write_bytes(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 64)
tests/parser/file/test_constants.py
+2 -2
View File
@@ -6,8 +6,8 @@ from pathlib import Path
import pytest
from application.parser.file.bulk import get_default_file_extractor
from application.parser.file.constants import (
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.constants import (
ATTACHMENT_PARSER_EXTENSIONS,
attachment_extension,
has_attachment_parser,
tests/parser/file/test_docling_parser.py
+150 -150
View File
@@ -1,4 +1,4 @@
"""Comprehensive tests for application/parser/file/docling_parser.py
"""Comprehensive tests for docsgpt/parser/file/docling_parser.py
Covers: DoclingParser (init, _init_parser, OCR engine selection, _export_content,
parse_file), subclass initialization, error handling.
@@ -21,7 +21,7 @@ import pytest
class TestDoclingParserInit:
def test_default_init(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
assert parser.ocr_enabled is True
@@ -33,7 +33,7 @@ class TestDoclingParserInit:
assert parser._converter is None
def test_custom_init(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(
ocr_enabled=False,
@@ -60,7 +60,7 @@ class TestDoclingParserInit:
class TestDoclingParserInitParser:
def test_init_parser_raises_without_docling(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
@@ -72,7 +72,7 @@ class TestDoclingParserInitParser:
"""A missing parent package makes find_spec raise; the hint must still show."""
import sys
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
for name in [m for m in sys.modules if m == "docling" or m.startswith("docling.")]:
monkeypatch.delitem(sys.modules, name)
@@ -82,7 +82,7 @@ class TestDoclingParserInitParser:
DoclingParser()._init_parser()
def test_init_parser_success(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
@@ -108,14 +108,14 @@ class TestOcrEngineSelection:
@pytest.fixture
def settings(self):
from application.core.settings import settings
from docsgpt.core.settings import settings
return settings
def test_default_setting_is_tesseract(self):
# Field defaults, not the live settings: a developer's ``.env`` may
# legitimately set other engines/languages.
from application.core.settings import Settings
from docsgpt.core.settings import Settings
defaults = Settings.model_construct()
assert defaults.OCR_ENGINE == "tesseract"
@@ -123,30 +123,30 @@ class TestOcrEngineSelection:
assert defaults.OCR_BACKEND == "auto"
def test_none_reads_setting(self, settings, monkeypatch):
from application.parser.file.docling_parser import _resolve_ocr_engine
from docsgpt.parser.file.docling_parser import _resolve_ocr_engine
monkeypatch.setattr(settings, "OCR_ENGINE", "auto")
assert _resolve_ocr_engine(None) == "auto"
def test_unknown_engine_degrades_to_auto(self):
from application.parser.file.docling_parser import _resolve_ocr_engine
from docsgpt.parser.file.docling_parser import _resolve_ocr_engine
assert _resolve_ocr_engine("easyocr") == "auto"
def test_tesseract_without_binary_degrades_to_auto(self, monkeypatch):
from application.parser.file import docling_parser as dp
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(dp.shutil, "which", lambda name: None)
assert dp._resolve_ocr_engine("tesseract") == "auto"
def test_tesseract_with_binary_selected(self, monkeypatch):
from application.parser.file import docling_parser as dp
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(dp.shutil, "which", lambda name: "/usr/bin/tesseract")
assert dp._resolve_ocr_engine("tesseract") == "tesseract"
def test_ocrmac_off_darwin_degrades_to_auto(self, monkeypatch):
from application.parser.file import docling_parser as dp
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(dp.sys, "platform", "linux")
assert dp._resolve_ocr_engine("ocrmac") == "auto"
@@ -154,25 +154,25 @@ class TestOcrEngineSelection:
def test_rapidocr_missing_degrades_to_auto(self, monkeypatch):
import sys
from application.parser.file import docling_parser as dp
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setitem(sys.modules, "rapidocr", None)
assert dp._resolve_ocr_engine("rapidocr") == "auto"
def test_deepseek_passes_through(self):
from application.parser.file.docling_parser import _resolve_ocr_engine
from docsgpt.parser.file.docling_parser import _resolve_ocr_engine
assert _resolve_ocr_engine("deepseek") == "deepseek"
def test_build_auto_returns_none(self):
from application.parser.file.docling_parser import _build_ocr_options
from docsgpt.parser.file.docling_parser import _build_ocr_options
assert _build_ocr_options("auto", None, True) is None
def test_build_tesseract_reads_ocr_langs(self, settings, monkeypatch):
pytest.importorskip("docling")
import application.parser.file.ocr_parser as op
from application.parser.file.docling_parser import _build_ocr_options
import docsgpt.parser.file.ocr_parser as op
from docsgpt.parser.file.docling_parser import _build_ocr_options
# Pack inventory unknown: the resolved list is passed through untouched
# (a host with tesseract but no chi_sim pack would otherwise drop it).
@@ -186,8 +186,8 @@ class TestOcrEngineSelection:
def test_build_tesseract_explicit_languages_win(self, monkeypatch):
pytest.importorskip("docling")
import application.parser.file.ocr_parser as op
from application.parser.file.docling_parser import _build_ocr_options
import docsgpt.parser.file.ocr_parser as op
from docsgpt.parser.file.docling_parser import _build_ocr_options
# Pack inventory unknown: the requested list is passed through untouched.
monkeypatch.setattr(op, "tesseract_languages", lambda: None)
@@ -196,7 +196,7 @@ class TestOcrEngineSelection:
def test_build_rapidocr(self):
pytest.importorskip("docling")
from application.parser.file.docling_parser import _build_ocr_options
from docsgpt.parser.file.docling_parser import _build_ocr_options
options = _build_ocr_options("rapidocr", None, False)
assert type(options).__name__ == "RapidOcrOptions"
@@ -205,7 +205,7 @@ class TestOcrEngineSelection:
def test_build_import_failure_returns_none(self, monkeypatch):
import sys
from application.parser.file.docling_parser import _build_ocr_options
from docsgpt.parser.file.docling_parser import _build_ocr_options
monkeypatch.setitem(sys.modules, "docling.datamodel.pipeline_options", None)
assert _build_ocr_options("tesseract", ["eng"], False) is None
@@ -215,7 +215,7 @@ class TestOcrEngineSelection:
import sys
import types
from application.parser.file.docling_parser import _build_ocr_options
from docsgpt.parser.file.docling_parser import _build_ocr_options
fake = types.ModuleType("docling.datamodel.pipeline_options")
@@ -236,8 +236,8 @@ class TestDeepseekVlmConverter:
pytest.importorskip("docling")
def test_deepseek_builds_vlm_converter(self, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingParser
monkeypatch.setattr(settings, "OCR_DEEPSEEK_URL", "http://gpu-host:8000/v1/chat/completions")
monkeypatch.setattr(settings, "OCR_DEEPSEEK_MODEL", "deepseek-ocr-x")
@@ -267,7 +267,7 @@ class TestDeepseekVlmConverter:
assert pdf_option.pipeline_options.enable_remote_services is True
def test_deepseek_ignored_when_ocr_disabled(self, monkeypatch):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
vlm_called = []
monkeypatch.setattr(
@@ -290,7 +290,7 @@ class TestDeepseekVlmConverter:
class TestExportContent:
def test_export_markdown(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="markdown")
mock_doc = MagicMock()
@@ -302,7 +302,7 @@ class TestExportContent:
mock_doc.export_to_markdown.assert_called_once()
def test_export_html(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="html")
mock_doc = MagicMock()
@@ -313,7 +313,7 @@ class TestExportContent:
assert "<h1>" in result
def test_export_text(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="text")
mock_doc = MagicMock()
@@ -324,7 +324,7 @@ class TestExportContent:
assert "Plain text" in result
def test_fallback_to_texts_on_minimal_content(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="markdown")
mock_doc = MagicMock()
@@ -341,7 +341,7 @@ class TestExportContent:
assert "OCR extracted text 2" in result
def test_no_fallback_for_substantial_content(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="markdown")
mock_doc = MagicMock()
@@ -352,7 +352,7 @@ class TestExportContent:
assert result == "A" * 100
def test_fallback_skipped_when_no_texts(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="markdown")
mock_doc = MagicMock()
@@ -363,7 +363,7 @@ class TestExportContent:
assert result == "short"
def test_fallback_skips_empty_texts(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="markdown")
mock_doc = MagicMock()
@@ -386,7 +386,7 @@ class TestExportContent:
class TestDoclingParserParseFile:
def test_parse_file_success(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
@@ -403,7 +403,7 @@ class TestDoclingParserParseFile:
assert "Parsed document content" in result
def test_parse_file_inits_converter_on_first_call(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
parser._converter = None
@@ -433,8 +433,8 @@ class TestDoclingParserParseFile:
as if it were the PDF. ``errors`` controls *decoding* leniency, not
"substitute the traceback for the document".
"""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
mock_converter = MagicMock()
@@ -453,8 +453,8 @@ class TestDoclingParserParseFile:
def test_parse_file_error_ignore_never_returns_a_string(self):
"""Belt-and-braces: no code path may hand back error text as content."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
mock_converter = MagicMock()
@@ -474,7 +474,7 @@ class TestDoclingParserParseFile:
)
def test_parse_file_error_raise(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
mock_converter = MagicMock()
@@ -494,82 +494,82 @@ class TestDoclingParserParseFile:
class TestDoclingSubclasses:
def test_pdf_parser_init(self):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser()
assert parser.ocr_enabled is True
assert parser.export_format == "markdown"
def test_pdf_parser_custom_ocr(self):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=False, force_full_page_ocr=True)
assert parser.ocr_enabled is False
assert parser.force_full_page_ocr is True
def test_docx_parser_init(self):
from application.parser.file.docling_parser import DoclingDocxParser
from docsgpt.parser.file.docling_parser import DoclingDocxParser
parser = DoclingDocxParser()
assert parser.export_format == "markdown"
def test_pptx_parser_init(self):
from application.parser.file.docling_parser import DoclingPPTXParser
from docsgpt.parser.file.docling_parser import DoclingPPTXParser
parser = DoclingPPTXParser()
assert parser.export_format == "markdown"
def test_xlsx_parser_init(self):
from application.parser.file.docling_parser import DoclingXLSXParser
from docsgpt.parser.file.docling_parser import DoclingXLSXParser
parser = DoclingXLSXParser()
assert parser.table_structure is True
def test_html_parser_init(self):
from application.parser.file.docling_parser import DoclingHTMLParser
from docsgpt.parser.file.docling_parser import DoclingHTMLParser
parser = DoclingHTMLParser()
assert parser.export_format == "markdown"
def test_image_parser_init(self):
from application.parser.file.docling_parser import DoclingImageParser
from docsgpt.parser.file.docling_parser import DoclingImageParser
parser = DoclingImageParser()
assert parser.ocr_enabled is True
assert parser.force_full_page_ocr is True
def test_image_parser_custom(self):
from application.parser.file.docling_parser import DoclingImageParser
from docsgpt.parser.file.docling_parser import DoclingImageParser
parser = DoclingImageParser(ocr_enabled=False)
assert parser.ocr_enabled is False
def test_csv_parser_init(self):
from application.parser.file.docling_parser import DoclingCSVParser
from docsgpt.parser.file.docling_parser import DoclingCSVParser
parser = DoclingCSVParser()
assert parser.table_structure is True
def test_markdown_parser_init(self):
from application.parser.file.docling_parser import DoclingMarkdownParser
from docsgpt.parser.file.docling_parser import DoclingMarkdownParser
parser = DoclingMarkdownParser()
assert parser.export_format == "markdown"
def test_asciidoc_parser_init(self):
from application.parser.file.docling_parser import DoclingAsciiDocParser
from docsgpt.parser.file.docling_parser import DoclingAsciiDocParser
parser = DoclingAsciiDocParser()
assert parser.export_format == "markdown"
def test_vtt_parser_init(self):
from application.parser.file.docling_parser import DoclingVTTParser
from docsgpt.parser.file.docling_parser import DoclingVTTParser
parser = DoclingVTTParser()
assert parser.export_format == "markdown"
def test_xml_parser_init(self):
from application.parser.file.docling_parser import DoclingXMLParser
from docsgpt.parser.file.docling_parser import DoclingXMLParser
parser = DoclingXMLParser()
assert parser.export_format == "markdown"
@@ -584,7 +584,7 @@ class TestDoclingSubclasses:
class TestDoclingParserGaps:
def test_csv_parser_init(self):
"""Cover line 289: DoclingCSVParser.__init__ calls super."""
from application.parser.file.docling_parser import DoclingCSVParser
from docsgpt.parser.file.docling_parser import DoclingCSVParser
parser = DoclingCSVParser()
assert parser.export_format == "markdown"
@@ -611,13 +611,13 @@ class TestNonOcrParsersLeaveTextAlone:
],
)
def test_ocr_is_off_by_construction(self, name):
from application.parser.file import docling_parser as dp
from docsgpt.parser.file import docling_parser as dp
assert getattr(dp, name)().ocr_enabled is False
def test_cjk_spaces_survive_in_a_docx_export(self, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingDocxParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingDocxParser
monkeypatch.setattr(settings, "OCR_ENGINE", "tesseract")
parser = DoclingDocxParser()
@@ -628,7 +628,7 @@ class TestNonOcrParsersLeaveTextAlone:
assert parser._postprocess_ocr_text(text) == text
def test_pdf_parser_with_tesseract_still_collapses(self):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
parser._active_ocr_engine = "tesseract"
@@ -645,8 +645,8 @@ class TestApplyPipelineCaps:
"""_apply_pipeline_caps bounds docling's threaded-pipeline buffering."""
def test_caps_threaded_pipeline_knobs(self, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import _apply_pipeline_caps
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
monkeypatch.setattr(
settings, "DOCLING_PIPELINE_QUEUE_MAX_SIZE", 2, raising=False
@@ -668,8 +668,8 @@ class TestApplyPipelineCaps:
assert opts.ocr_batch_size == 1
def test_queue_size_is_settings_driven(self, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import _apply_pipeline_caps
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
monkeypatch.setattr(
settings, "DOCLING_PIPELINE_QUEUE_MAX_SIZE", 6, raising=False
@@ -684,8 +684,8 @@ class TestApplyPipelineCaps:
def test_misconfigured_zero_floors_to_one(self, monkeypatch):
"""A 0 queue depth could deadlock the threaded pipeline — floor it."""
from application.core.settings import settings
from application.parser.file.docling_parser import _apply_pipeline_caps
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
monkeypatch.setattr(
settings, "DOCLING_PIPELINE_QUEUE_MAX_SIZE", 0, raising=False
@@ -701,7 +701,7 @@ class TestApplyPipelineCaps:
def test_noop_on_docling_without_threaded_pipeline(self):
"""Builds predating the threaded pipeline lack the knobs — the cap
must be a silent no-op, not an AttributeError."""
from application.parser.file.docling_parser import _apply_pipeline_caps
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
class LegacyOpts:
__slots__ = ("do_ocr", "do_table_structure")
@@ -738,8 +738,8 @@ class TestDoclingTabularSizeGate:
return path
def test_oversized_csv_delegates_to_plain_csv_parser(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingCSVParser, DoclingParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
docling_parse = MagicMock(name="docling_parse")
@@ -755,8 +755,8 @@ class TestDoclingTabularSizeGate:
assert out.startswith("0, 0\n1, 2\n")
def test_small_csv_still_uses_docling(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingCSVParser, DoclingParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 10_000_000)
docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
@@ -768,8 +768,8 @@ class TestDoclingTabularSizeGate:
docling_parse.assert_called_once()
def test_gate_disabled_when_max_bytes_is_zero(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingCSVParser, DoclingParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 0)
docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
@@ -781,9 +781,9 @@ class TestDoclingTabularSizeGate:
docling_parse.assert_called_once()
def test_oversized_xlsx_delegates_to_excel_parser(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
from application.parser.file.tabular_parser import ExcelParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
from docsgpt.parser.file.tabular_parser import ExcelParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
docling_parse = MagicMock(name="docling_parse")
@@ -801,8 +801,8 @@ class TestDoclingTabularSizeGate:
excel_parse.assert_called_once()
def test_small_xlsx_still_uses_docling(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 10_000_000)
docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
@@ -837,7 +837,7 @@ class TestTabularContentSize:
(2.44 GB in docling) slips under a byte gate."""
def test_xlsx_content_size_is_inner_not_ondisk(self, tmp_path):
from application.parser.file.docling_parser import _tabular_content_size
from docsgpt.parser.file.docling_parser import _tabular_content_size
path = tmp_path / "data.xlsx"
_make_xlsx(path, rows=5000)
@@ -847,7 +847,7 @@ class TestTabularContentSize:
assert inner > on_disk
def test_csv_content_size_is_ondisk(self, tmp_path):
from application.parser.file.docling_parser import _tabular_content_size
from docsgpt.parser.file.docling_parser import _tabular_content_size
path = tmp_path / "data.csv"
path.write_text("a,b\n1,2\n3,4\n")
@@ -855,13 +855,13 @@ class TestTabularContentSize:
def test_compressed_xlsx_over_inner_gate_delegates(self, tmp_path, monkeypatch):
"""The regression: on-disk < threshold < inner-uncompressed must gate."""
from application.core.settings import settings
from application.parser.file.docling_parser import (
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import (
DoclingParser,
DoclingXLSXParser,
_tabular_content_size,
)
from application.parser.file.tabular_parser import ExcelParser
from docsgpt.parser.file.tabular_parser import ExcelParser
path = tmp_path / "wide.xlsx"
_make_xlsx(path, rows=5000)
@@ -901,8 +901,8 @@ class TestDoclingMarkupGate:
@pytest.mark.parametrize("name", ["big.html", "big.vtt"])
def test_oversized_markup_parses_truncated_copy(self, tmp_path, monkeypatch, name):
from application.core.settings import settings
from application.parser.file import docling_parser as dp
from docsgpt.core.settings import settings
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 512)
path = self._write(tmp_path, name, 4096)
@@ -928,8 +928,8 @@ class TestDoclingMarkupGate:
assert path.stat().st_size > 512, "original must be untouched"
def test_small_markup_parses_original(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file import docling_parser as dp
from docsgpt.core.settings import settings
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 10_000_000)
path = self._write(tmp_path, "small.html", 1024)
@@ -946,8 +946,8 @@ class TestDoclingMarkupGate:
assert seen["path"] == str(path)
def test_markup_gate_disabled_when_zero(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file import docling_parser as dp
from docsgpt.core.settings import settings
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 0)
path = self._write(tmp_path, "big.vtt", 8192)
@@ -994,8 +994,8 @@ class TestTabularGateSeam:
self, tmp_path, monkeypatch
):
"""The real incident, end to end: gate trips, blanks do not crash."""
from application.core.settings import settings
from application.parser.file.docling_parser import (
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import (
DoclingParser,
DoclingXLSXParser,
)
@@ -1021,13 +1021,13 @@ class TestTabularGateSeam:
``ingest`` and ``store_attachment``, and is the only exception
``SimpleDirectoryReader.load_data`` skips rather than propagating.
"""
from application.core.settings import settings
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import (
from docsgpt.core.settings import settings
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import (
DoclingParser,
DoclingXLSXParser,
)
from application.parser.file.tabular_parser import ExcelParser
from docsgpt.parser.file.tabular_parser import ExcelParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
monkeypatch.setattr(DoclingParser, "parse_file", MagicMock())
@@ -1045,13 +1045,13 @@ class TestTabularGateSeam:
def test_oversized_csv_failure_becomes_document_parse_error(
self, tmp_path, monkeypatch
):
from application.core.settings import settings
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import (
from docsgpt.core.settings import settings
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import (
DoclingCSVParser,
DoclingParser,
)
from application.parser.file.tabular_parser import CSVParser
from docsgpt.parser.file.tabular_parser import CSVParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 8)
monkeypatch.setattr(DoclingParser, "parse_file", MagicMock())
@@ -1087,7 +1087,7 @@ class TestApplyInferenceSettings:
pytest.importorskip("docling")
def test_disables_torch_compile_by_default(self, monkeypatch):
from application.parser.file.docling_parser import _apply_inference_settings
from docsgpt.parser.file.docling_parser import _apply_inference_settings
class Inference:
compile_torch_models = True
@@ -1105,8 +1105,8 @@ class TestApplyInferenceSettings:
assert docling_settings.inference.compile_torch_models is False
def test_opt_in_reenables_torch_compile(self, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import _apply_inference_settings
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import _apply_inference_settings
monkeypatch.setattr(
settings, "DOCLING_COMPILE_TORCH_MODELS", True, raising=False
@@ -1129,7 +1129,7 @@ class TestApplyInferenceSettings:
def test_noop_on_docling_without_inference_settings(self, monkeypatch):
"""Builds predating the inference settings must be a silent no-op."""
from application.parser.file.docling_parser import _apply_inference_settings
from docsgpt.parser.file.docling_parser import _apply_inference_settings
class DoclingSettings:
pass
@@ -1142,11 +1142,11 @@ class TestApplyInferenceSettings:
def test_create_converter_applies_inference_settings(self, monkeypatch):
"""The cap is worthless unless the converter path actually calls it."""
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
called = []
monkeypatch.setattr(
"application.parser.file.docling_parser._apply_inference_settings",
"docsgpt.parser.file.docling_parser._apply_inference_settings",
lambda: called.append(True),
)
monkeypatch.setattr(
@@ -1167,11 +1167,11 @@ class TestApplyInferenceSettings:
"""
import docling.datamodel.pipeline_options as dpo
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
events = []
monkeypatch.setattr(
"application.parser.file.docling_parser._apply_inference_settings",
"docsgpt.parser.file.docling_parser._apply_inference_settings",
lambda: events.append("settings"),
)
@@ -1210,7 +1210,7 @@ def _mock_conversion(markdown: str, pages: int = 1) -> MagicMock:
def _set_threshold(monkeypatch, value: int) -> None:
"""Point the OCR dropout guard at a specific chars-per-page floor."""
from application.core.settings import settings as real_settings
from docsgpt.core.settings import settings as real_settings
class _Stub:
def __getattr__(self, name):
@@ -1218,7 +1218,7 @@ def _set_threshold(monkeypatch, value: int) -> None:
stub = _Stub()
stub.OCR_MIN_CHARS_PER_PAGE = value
monkeypatch.setattr("application.core.settings.settings", stub)
monkeypatch.setattr("docsgpt.core.settings.settings", stub)
@pytest.mark.unit
@@ -1232,7 +1232,7 @@ class TestOCRDropoutGuard:
"""
def test_near_empty_first_pass_recovers_on_fresh_converter(self, caplog):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
degraded = MagicMock()
@@ -1274,8 +1274,8 @@ class TestOCRDropoutGuard:
assert "Recovered scan.pdf on retry" in messages
def test_near_empty_both_passes_raises_document_parse_error(self):
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
degraded = MagicMock()
@@ -1301,11 +1301,11 @@ class TestOCRDropoutGuard:
def test_dropout_error_reports_an_unread_text_layer(self, monkeypatch):
"""A text layer docling ignored points at the pipeline, not the scan."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
monkeypatch.setattr(
"application.parser.file.docling_parser._pdf_text_layer_probe",
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
lambda file: (12, 48_000),
)
parser = DoclingPDFParser(ocr_enabled=True)
@@ -1319,7 +1319,7 @@ class TestOCRDropoutGuard:
assert "The PDF carries a 48000-char text layer" in str(excinfo.value)
def test_ocr_disabled_returns_near_empty_content_untouched(self):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=False)
converter = MagicMock()
@@ -1334,7 +1334,7 @@ class TestOCRDropoutGuard:
assert parser._converter is converter
def test_threshold_zero_disables_the_guard(self, monkeypatch):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
_set_threshold(monkeypatch, 0)
parser = DoclingPDFParser(ocr_enabled=True)
@@ -1349,7 +1349,7 @@ class TestOCRDropoutGuard:
assert parser._converter is converter
def test_healthy_ocr_parse_converts_once(self):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
converter = MagicMock()
@@ -1368,7 +1368,7 @@ class TestOCRDropoutGuard:
def test_threshold_is_per_page_not_per_document(self):
"""200 chars is healthy for one page and suspicious for a hundred."""
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
content = "x" * 200
@@ -1388,7 +1388,7 @@ class TestOCRDropoutGuard:
with patch.object(
long_doc, "_create_converter", return_value=long_doc._converter
) as create, patch(
"application.parser.file.docling_parser._pdf_text_layer_probe",
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
return_value=(100, 0),
):
assert long_doc.parse_file(Path("hundred-page.pdf")) == content
@@ -1396,7 +1396,7 @@ class TestOCRDropoutGuard:
def test_guard_does_not_apply_to_non_ocr_formats(self):
"""DOCX parsers construct with OCR off: they never OCR anything."""
from application.parser.file.docling_parser import DoclingDocxParser
from docsgpt.parser.file.docling_parser import DoclingDocxParser
parser = DoclingDocxParser()
assert parser.ocr_enabled is False
@@ -1411,7 +1411,7 @@ class TestOCRDropoutGuard:
def test_guard_applies_to_images(self):
"""The retry covers images -- a degraded converter may still recover one."""
from application.parser.file.docling_parser import DoclingImageParser
from docsgpt.parser.file.docling_parser import DoclingImageParser
parser = DoclingImageParser(ocr_enabled=True)
degraded = MagicMock()
@@ -1432,7 +1432,7 @@ class TestOCRDropoutGuard:
DocumentParseError is in ``dont_autoretry_for``, so raising here fails a
single-file upload permanently.
"""
from application.parser.file.docling_parser import DoclingImageParser
from docsgpt.parser.file.docling_parser import DoclingImageParser
parser = DoclingImageParser(ocr_enabled=True)
converter = MagicMock()
@@ -1447,7 +1447,7 @@ class TestOCRDropoutGuard:
def test_a_text_sparse_scan_indexes_rather_than_failing_the_upload(self):
"""20 pages of pictures with a few captions is the document, not a fault."""
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
content = "Fig 1. Fig 2. Fig 3."
parser = DoclingPDFParser(ocr_enabled=True)
@@ -1458,15 +1458,15 @@ class TestOCRDropoutGuard:
with patch.object(
parser, "_create_converter", return_value=converter
), patch(
"application.parser.file.docling_parser._pdf_text_layer_probe",
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
return_value=(20, 0),
):
assert parser.parse_file(Path("catalog.pdf")) == content
def test_a_multi_page_zero_char_parse_is_still_a_dropout(self):
"""The incident this guard exists for: every page OCR'd to nothing."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
converter = MagicMock()
@@ -1476,7 +1476,7 @@ class TestOCRDropoutGuard:
with patch.object(
parser, "_create_converter", return_value=converter
), patch(
"application.parser.file.docling_parser._pdf_text_layer_probe",
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
return_value=(40, 0),
):
with pytest.raises(DocumentParseError, match="scan.pdf"):
@@ -1484,8 +1484,8 @@ class TestOCRDropoutGuard:
def test_a_pdf_whose_text_layer_was_missed_is_still_a_dropout(self):
"""Text docling should have read without OCR at all -- positive evidence."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
converter = MagicMock()
@@ -1495,15 +1495,15 @@ class TestOCRDropoutGuard:
with patch.object(
parser, "_create_converter", return_value=converter
), patch(
"application.parser.file.docling_parser._pdf_text_layer_probe",
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
return_value=(20, 5000),
):
with pytest.raises(DocumentParseError, match="text layer"):
parser.parse_file(Path("report.pdf"))
def test_retry_conversion_failure_is_still_a_parse_error(self):
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
parser._converter = MagicMock()
@@ -1524,7 +1524,7 @@ class TestOCRDropoutGuard:
class TestOCRDropoutHelpers:
def test_image_placeholders_are_not_text(self):
from application.parser.file.docling_parser import _text_char_count
from docsgpt.parser.file.docling_parser import _text_char_count
assert _text_char_count(None) == 0
assert _text_char_count("") == 0
@@ -1532,20 +1532,20 @@ class TestOCRDropoutHelpers:
assert _text_char_count(" <!-- image --> abc ") == 3
def test_page_count_prefers_document_pages(self):
from application.parser.file.docling_parser import _result_page_count
from docsgpt.parser.file.docling_parser import _result_page_count
result = _mock_conversion("x", pages=7)
assert _result_page_count(result, Path("a.pdf")) == 7
def test_page_count_falls_back_to_num_pages(self):
from application.parser.file.docling_parser import _result_page_count
from docsgpt.parser.file.docling_parser import _result_page_count
result = _mock_conversion("x", pages=0)
result.document.num_pages.return_value = 5
assert _result_page_count(result, Path("a.pdf")) == 5
def test_page_count_falls_back_to_result_pages(self):
from application.parser.file.docling_parser import _result_page_count
from docsgpt.parser.file.docling_parser import _result_page_count
result = _mock_conversion("x", pages=0)
result.document.num_pages.return_value = None
@@ -1553,7 +1553,7 @@ class TestOCRDropoutHelpers:
assert _result_page_count(result, Path("a.pdf")) == 2
def test_page_count_defaults_to_one(self):
from application.parser.file.docling_parser import _result_page_count
from docsgpt.parser.file.docling_parser import _result_page_count
result = _mock_conversion("x", pages=0)
result.document.num_pages.return_value = None
@@ -1561,13 +1561,13 @@ class TestOCRDropoutHelpers:
assert _result_page_count(result, Path("a.pdf")) == 1
def test_images_are_always_one_page(self):
from application.parser.file.docling_parser import _result_page_count
from docsgpt.parser.file.docling_parser import _result_page_count
result = _mock_conversion("x", pages=9)
assert _result_page_count(result, Path("a.png")) == 1
def test_threshold_reads_settings_with_a_default(self, monkeypatch):
from application.parser.file.docling_parser import _ocr_min_chars_per_page
from docsgpt.parser.file.docling_parser import _ocr_min_chars_per_page
assert _ocr_min_chars_per_page() == 20
_set_threshold(monkeypatch, 5)
@@ -1576,7 +1576,7 @@ class TestOCRDropoutHelpers:
assert _ocr_min_chars_per_page() == 20
def test_text_layer_probe_survives_an_unreadable_pdf(self, tmp_path):
from application.parser.file.docling_parser import _pdf_text_layer_probe
from docsgpt.parser.file.docling_parser import _pdf_text_layer_probe
assert _pdf_text_layer_probe(tmp_path / "missing.pdf") == (0, 0)
broken = tmp_path / "broken.pdf"
@@ -1597,7 +1597,7 @@ class TestForceFullPageOCRWiring:
def _pipeline_options(self, monkeypatch, **kwargs):
import docling.datamodel.pipeline_options as dpo
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
built = []
real_options = dpo.PdfPipelineOptions
@@ -1636,7 +1636,7 @@ class TestDoclingTesseractPostprocessing:
"""docling's tesseract path gets the same CJK glyph-space cleanup as the native engine."""
def _parser_with_export(self, engine, ocr_enabled=True):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=ocr_enabled)
parser._active_ocr_engine = engine
@@ -1660,7 +1660,7 @@ class TestDoclingTesseractPostprocessing:
@pytest.mark.unit
class TestDoclingOcrPages:
def test_converts_each_requested_page_with_page_range(self, tmp_path):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
converter = MagicMock()
@@ -1683,8 +1683,8 @@ class TestDoclingOcrPages:
assert [c.kwargs["page_range"] for c in converter.convert.call_args_list] == [(3, 3), (1, 1)]
def test_failure_is_a_parse_error(self, tmp_path):
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
parser._converter = MagicMock()
@@ -1699,8 +1699,8 @@ class TestDoclingOcrPages:
class TestTesseractLanguageFilter:
def test_uninstalled_packs_are_dropped_with_a_warning(self, monkeypatch, caplog):
pytest.importorskip("docling")
import application.parser.file.ocr_parser as op
from application.parser.file.docling_parser import _build_ocr_options
import docsgpt.parser.file.ocr_parser as op
from docsgpt.parser.file.docling_parser import _build_ocr_options
monkeypatch.setattr(op, "tesseract_languages", lambda: frozenset({"eng", "osd"}))
with caplog.at_level("WARNING"):
@@ -1710,16 +1710,16 @@ class TestTesseractLanguageFilter:
def test_all_packs_missing_falls_back_to_eng(self, monkeypatch):
pytest.importorskip("docling")
import application.parser.file.ocr_parser as op
from application.parser.file.docling_parser import _build_ocr_options
import docsgpt.parser.file.ocr_parser as op
from docsgpt.parser.file.docling_parser import _build_ocr_options
monkeypatch.setattr(op, "tesseract_languages", lambda: frozenset({"eng"}))
assert _build_ocr_options("tesseract", ["xyz"], False).lang == ["eng"]
def test_unknown_inventory_keeps_the_list(self, monkeypatch):
pytest.importorskip("docling")
import application.parser.file.ocr_parser as op
from application.parser.file.docling_parser import _build_ocr_options
import docsgpt.parser.file.ocr_parser as op
from docsgpt.parser.file.docling_parser import _build_ocr_options
monkeypatch.setattr(op, "tesseract_languages", lambda: None)
assert _build_ocr_options("tesseract", ["eng", "deu"], False).lang == ["eng", "deu"]
tests/parser/file/test_docs_parser.py
+12 -12
View File
@@ -1,4 +1,4 @@
"""Comprehensive tests for application/parser/file/docs_parser.py
"""Comprehensive tests for docsgpt/parser/file/docs_parser.py
Covers: PDFParser (init, parse with pypdf, parse as image, import error),
DocxParser (init, parse, import error).
@@ -9,7 +9,7 @@ from unittest.mock import MagicMock, patch, mock_open
import pytest
from application.parser.file.docs_parser import PDFParser, DocxParser
from docsgpt.parser.file.docs_parser import PDFParser, DocxParser
# =====================================================================
@@ -44,7 +44,7 @@ class TestPDFParserInit:
@pytest.mark.unit
class TestPDFParserParse:
@patch("application.parser.file.docs_parser.settings")
@patch("docsgpt.parser.file.docs_parser.settings")
def test_parse_with_pypdf(self, mock_settings):
mock_settings.PARSE_PDF_AS_IMAGE = False
@@ -58,7 +58,7 @@ class TestPDFParserParse:
mock_reader = MagicMock()
mock_reader.pages = [mock_page1, mock_page2]
with patch("application.parser.file.docs_parser.PdfReader",
with patch("docsgpt.parser.file.docs_parser.PdfReader",
create=True), \
patch("builtins.open", mock_open()):
# Need to patch the import inside the function
@@ -74,8 +74,8 @@ class TestPDFParserParse:
finally:
del sys.modules["pypdf"]
@patch("application.parser.file.docs_parser.settings")
@patch("application.parser.file.docs_parser.requests")
@patch("docsgpt.parser.file.docs_parser.settings")
@patch("docsgpt.parser.file.docs_parser.requests")
def test_parse_as_image(self, mock_requests, mock_settings):
mock_settings.PARSE_PDF_AS_IMAGE = True
@@ -89,7 +89,7 @@ class TestPDFParserParse:
result = parser.parse_file(Path("test.pdf"))
assert result == "# OCR Result"
@patch("application.parser.file.docs_parser.settings")
@patch("docsgpt.parser.file.docs_parser.settings")
def test_parse_raises_on_missing_pypdf(self, mock_settings):
mock_settings.PARSE_PDF_AS_IMAGE = False
@@ -196,17 +196,17 @@ class TestBaseParserProperties:
class TestDocsParserGaps:
def test_pdf_parser_parse_as_image(self, tmp_path):
"""Cover lines 33-34: PARSE_PDF_AS_IMAGE sends to external service."""
from application.parser.file.docs_parser import PDFParser
from docsgpt.parser.file.docs_parser import PDFParser
pdf_file = tmp_path / "test.pdf"
pdf_file.write_bytes(b"%PDF-1.4 fake content")
with patch(
"application.parser.file.docs_parser.settings"
"docsgpt.parser.file.docs_parser.settings"
) as mock_settings:
mock_settings.PARSE_PDF_AS_IMAGE = True
with patch(
"application.parser.file.docs_parser.requests.post"
"docsgpt.parser.file.docs_parser.requests.post"
) as mock_post:
mock_post.return_value = MagicMock(
json=MagicMock(return_value={"markdown": "# Parsed Content"})
@@ -218,7 +218,7 @@ class TestDocsParserGaps:
def test_docx_parser_init_parser(self):
"""Cover line 59: DocxParser._init_parser returns empty dict."""
from application.parser.file.docs_parser import DocxParser
from docsgpt.parser.file.docs_parser import DocxParser
parser = DocxParser()
config = parser._init_parser()
@@ -226,7 +226,7 @@ class TestDocsParserGaps:
def test_docx_parser_import_error(self):
"""Cover line 63: ImportError when docx2txt not installed."""
from application.parser.file.docs_parser import DocxParser
from docsgpt.parser.file.docs_parser import DocxParser
parser = DocxParser()
with patch.dict("sys.modules", {"docx2txt": None}):
tests/parser/file/test_embedding_pipeline.py
+17 -17
View File
@@ -2,7 +2,7 @@ import pytest
import logging
from unittest.mock import patch, MagicMock
from application.parser.embedding_pipeline import (
from docsgpt.parser.embedding_pipeline import (
DEFAULT_EMBEDDINGS_BATCH_SIZE,
EmbeddingPipelineError,
_resolve_batch_size,
@@ -45,7 +45,7 @@ def test_add_text_to_store_with_retry_success():
def mock_settings(monkeypatch):
mock_settings = MagicMock()
monkeypatch.setattr(
"application.parser.embedding_pipeline.settings", mock_settings
"docsgpt.parser.embedding_pipeline.settings", mock_settings
)
return mock_settings
@@ -54,7 +54,7 @@ def mock_settings(monkeypatch):
def mock_vector_creator(monkeypatch):
mock_creator = MagicMock()
monkeypatch.setattr(
"application.parser.embedding_pipeline.VectorCreator", mock_creator
"docsgpt.parser.embedding_pipeline.VectorCreator", mock_creator
)
return mock_creator
@@ -126,7 +126,7 @@ def test_embed_and_store_documents_progress_band(
assert currents == sorted(currents)
@patch("application.parser.embedding_pipeline.add_texts_to_store_with_retry")
@patch("docsgpt.parser.embedding_pipeline.add_texts_to_store_with_retry")
def test_embed_and_store_documents_partial_failure_raises(
mock_add_retry, tmp_path, mock_settings, mock_vector_creator, caplog
):
@@ -170,7 +170,7 @@ def test_embed_and_store_documents_partial_failure_raises(
mock_store.save_local.assert_called()
@patch("application.parser.embedding_pipeline.add_texts_to_store_with_retry")
@patch("docsgpt.parser.embedding_pipeline.add_texts_to_store_with_retry")
def test_embed_and_store_documents_all_chunks_succeed_no_raise(
mock_add_retry, tmp_path, mock_settings, mock_vector_creator,
):
@@ -200,7 +200,7 @@ def test_assert_index_complete_raises_on_partial(monkeypatch):
"embedded_chunks": 4, "total_chunks": 10,
}
monkeypatch.setattr(
"application.parser.embedding_pipeline.IngestChunkProgressRepository",
"docsgpt.parser.embedding_pipeline.IngestChunkProgressRepository",
lambda conn: fake_repo,
)
from contextlib import contextmanager
@@ -210,7 +210,7 @@ def test_assert_index_complete_raises_on_partial(monkeypatch):
yield None
monkeypatch.setattr(
"application.parser.embedding_pipeline.db_session", _fake_session,
"docsgpt.parser.embedding_pipeline.db_session", _fake_session,
)
with pytest.raises(EmbeddingPipelineError, match=r"4/10"):
assert_index_complete("src-partial")
@@ -222,7 +222,7 @@ def test_assert_index_complete_passes_on_full(monkeypatch):
"embedded_chunks": 10, "total_chunks": 10,
}
monkeypatch.setattr(
"application.parser.embedding_pipeline.IngestChunkProgressRepository",
"docsgpt.parser.embedding_pipeline.IngestChunkProgressRepository",
lambda conn: fake_repo,
)
from contextlib import contextmanager
@@ -232,7 +232,7 @@ def test_assert_index_complete_passes_on_full(monkeypatch):
yield None
monkeypatch.setattr(
"application.parser.embedding_pipeline.db_session", _fake_session,
"docsgpt.parser.embedding_pipeline.db_session", _fake_session,
)
assert_index_complete("src-full") # no raise
@@ -242,7 +242,7 @@ def test_assert_index_complete_no_op_when_no_progress_row(monkeypatch):
fake_repo = MagicMock()
fake_repo.get_progress.return_value = None
monkeypatch.setattr(
"application.parser.embedding_pipeline.IngestChunkProgressRepository",
"docsgpt.parser.embedding_pipeline.IngestChunkProgressRepository",
lambda conn: fake_repo,
)
from contextlib import contextmanager
@@ -252,7 +252,7 @@ def test_assert_index_complete_no_op_when_no_progress_row(monkeypatch):
yield None
monkeypatch.setattr(
"application.parser.embedding_pipeline.db_session", _fake_session,
"docsgpt.parser.embedding_pipeline.db_session", _fake_session,
)
assert_index_complete("src-missing")
@@ -270,7 +270,7 @@ def test_assert_index_complete_no_op_when_lookup_fails(monkeypatch, caplog):
yield # pragma: no cover
monkeypatch.setattr(
"application.parser.embedding_pipeline.db_session", _broken_session,
"docsgpt.parser.embedding_pipeline.db_session", _broken_session,
)
with caplog.at_level(logging.WARNING, logger="root"):
assert_index_complete("src-db-down") # no raise
@@ -328,7 +328,7 @@ def test_add_texts_to_store_with_retry_sanitizes_and_skips_empty():
def test_resolve_batch_size_falls_back_on_bad_setting(monkeypatch):
fake = MagicMock() # attribute access yields a MagicMock, not an int
monkeypatch.setattr("application.parser.embedding_pipeline.settings", fake)
monkeypatch.setattr("docsgpt.parser.embedding_pipeline.settings", fake)
assert _resolve_batch_size() == DEFAULT_EMBEDDINGS_BATCH_SIZE
fake.EMBEDDINGS_BATCH_SIZE = 0
@@ -346,7 +346,7 @@ def test_embed_loop_batches_chunks(tmp_path, mock_settings, mock_vector_creator)
store = MagicMock()
mock_vector_creator.create_vectorstore.return_value = store
with patch("application.parser.embedding_pipeline._record_progress") as rec:
with patch("docsgpt.parser.embedding_pipeline._record_progress") as rec:
embed_and_store_documents(
docs, str(tmp_path / "s"), "sid", MagicMock(),
)
@@ -395,10 +395,10 @@ def test_poison_chunk_isolated_by_per_chunk_fallback(
raise RuntimeError("input too large")
with patch(
"application.parser.embedding_pipeline.add_texts_to_store_with_retry",
"docsgpt.parser.embedding_pipeline.add_texts_to_store_with_retry",
side_effect=fake_add,
):
with patch("application.parser.embedding_pipeline._record_progress") as rec:
with patch("docsgpt.parser.embedding_pipeline._record_progress") as rec:
with pytest.raises(EmbeddingPipelineError) as exc:
embed_and_store_documents(
docs, str(tmp_path / "s"), "sid", MagicMock(),
@@ -429,7 +429,7 @@ def test_batch_only_failure_recovers_via_fallback(
raise RuntimeError("payload too large")
with patch(
"application.parser.embedding_pipeline.add_texts_to_store_with_retry",
"docsgpt.parser.embedding_pipeline.add_texts_to_store_with_retry",
side_effect=fake_add,
):
embed_and_store_documents(docs, str(tmp_path / "s"), "sid", MagicMock())
tests/parser/file/test_epub_parser.py
+1 -1
View File
@@ -4,7 +4,7 @@ from unittest.mock import patch, MagicMock
import sys
import types
from application.parser.file.epub_parser import EpubParser
from docsgpt.parser.file.epub_parser import EpubParser
@pytest.fixture
tests/parser/file/test_html_parser.py
+10 -10
View File
@@ -1,6 +1,6 @@
import pytest
from application.parser.file.html_parser import HTMLParser
from docsgpt.parser.file.html_parser import HTMLParser
HTML = (
@@ -58,7 +58,7 @@ def test_html_parser_metadata_unreadable_file(tmp_path):
# --- HTMLMarkdownParser: the anydoc engine's HTML path ---------------------------
from application.parser.file.html_parser import HTMLMarkdownParser, html_to_markdown # noqa: E402
from docsgpt.parser.file.html_parser import HTMLMarkdownParser, html_to_markdown # noqa: E402
RICH_HTML = """<html><head><title>Doc Title</title>
<style>.x { color: red }</style><script>var secret = 1;</script></head>
@@ -139,7 +139,7 @@ def test_html_to_markdown_collapses_blank_runs():
def test_markdown_parser_head_truncates_oversized_markup(tmp_path, monkeypatch):
"""Markup past MARKUP_MAX_BYTES is not parsed: the soup+markdownify tree costs
~50x the input, and the upload cap is 100 MB."""
from application.core.settings import settings
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", 600)
path = tmp_path / "big.html"
@@ -155,7 +155,7 @@ def test_markdown_parser_head_truncates_oversized_markup(tmp_path, monkeypatch):
def test_markdown_parser_gate_disabled_reads_everything(tmp_path, monkeypatch):
from application.core.settings import settings
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", 0)
path = tmp_path / "big.html"
@@ -187,7 +187,7 @@ def test_markdown_parser_metadata_unreadable_file(tmp_path):
def test_markdown_parser_cut_inside_utf8_char_does_not_mojibake_the_page(tmp_path, monkeypatch):
"""A byte cut mid-character makes strict UTF-8 fail; BeautifulSoup would then
retry as windows-1252, which *succeeds* on umlauts and garbles everything."""
from application.core.settings import settings
from docsgpt.core.settings import settings
path = tmp_path / "minified.html"
body = "<p>Größe Übermaß schön für</p>" * 400 # no newlines: the cut is arbitrary
@@ -203,7 +203,7 @@ def test_markdown_parser_cut_inside_utf8_char_does_not_mojibake_the_page(tmp_pat
def test_trim_torn_utf8_tail():
from application.parser.file.html_parser import _trim_torn_utf8_tail as trim
from docsgpt.parser.file.html_parser import _trim_torn_utf8_tail as trim
e_acute, snowman, emoji = "é".encode(), "☃".encode(), "😀".encode()
assert trim(b"abc") == b"abc"
@@ -218,7 +218,7 @@ def test_trim_torn_utf8_tail():
def test_markdown_parser_metadata_reuses_last_parse(tmp_path, rich_html_file, monkeypatch):
"""The metadata call after parse_file must not build the soup again."""
from application.parser.file import html_parser as mod
from docsgpt.parser.file import html_parser as mod
parser = HTMLMarkdownParser()
parser.parse_file(rich_html_file)
@@ -237,7 +237,7 @@ def test_markdown_parser_metadata_reuses_last_parse(tmp_path, rich_html_file, mo
def test_xml_prolog_does_not_leak_into_markdown(tmp_path):
from application.parser.file.html_parser import HTMLMarkdownParser
from docsgpt.parser.file.html_parser import HTMLMarkdownParser
path = tmp_path / "doc.xhtml"
path.write_bytes(
@@ -264,8 +264,8 @@ def test_data_uris_are_stripped_from_images_and_links():
def test_utf16_head_keeps_an_even_byte_count(tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.html_parser import HTMLMarkdownParser, read_markup_head
from docsgpt.core.settings import settings
from docsgpt.parser.file.html_parser import HTMLMarkdownParser, read_markup_head
body = "".join(f"<p>Zeile {i} Über Größe</p>\n" for i in range(200))
path = tmp_path / "wide.html"
tests/parser/file/test_image_parser.py
+5 -5
View File
@@ -1,7 +1,7 @@
from pathlib import Path
from unittest.mock import patch, MagicMock, mock_open
from application.parser.file.image_parser import ImageParser
from docsgpt.parser.file.image_parser import ImageParser
def test_image_init_parser():
@@ -12,7 +12,7 @@ def test_image_init_parser():
assert parser.parser_config_set
@patch("application.parser.file.image_parser.settings")
@patch("docsgpt.parser.file.image_parser.settings")
def test_image_parser_remote_true(mock_settings):
mock_settings.PARSE_IMAGE_REMOTE = True
parser = ImageParser()
@@ -20,7 +20,7 @@ def test_image_parser_remote_true(mock_settings):
mock_response = MagicMock()
mock_response.json.return_value = {"markdown": "# From Image"}
with patch("application.parser.file.image_parser.requests.post", return_value=mock_response) as mock_post:
with patch("docsgpt.parser.file.image_parser.requests.post", return_value=mock_response) as mock_post:
with patch("builtins.open", mock_open()):
result = parser.parse_file(Path("img.png"))
@@ -28,12 +28,12 @@ def test_image_parser_remote_true(mock_settings):
mock_post.assert_called_once()
@patch("application.parser.file.image_parser.settings")
@patch("docsgpt.parser.file.image_parser.settings")
def test_image_parser_remote_false(mock_settings):
mock_settings.PARSE_IMAGE_REMOTE = False
parser = ImageParser()
with patch("application.parser.file.image_parser.requests.post") as mock_post:
with patch("docsgpt.parser.file.image_parser.requests.post") as mock_post:
result = parser.parse_file(Path("img.png"))
assert result == ""
tests/parser/file/test_json_parser.py
+1 -1
View File
@@ -1,7 +1,7 @@
from pathlib import Path
from unittest.mock import patch, mock_open
from application.parser.file.json_parser import JSONParser
from docsgpt.parser.file.json_parser import JSONParser
def test_json_init_parser():
tests/parser/file/test_markdown_parser.py
+2 -2
View File
@@ -3,8 +3,8 @@ from unittest.mock import mock_open, patch
import pytest
from application.parser.file.markdown_parser import MarkdownParser
from application import utils
from docsgpt.parser.file.markdown_parser import MarkdownParser
from docsgpt import utils
class _Enc:
tests/parser/file/test_ocr_parser.py
+26 -26
View File
@@ -18,8 +18,8 @@ pytest.importorskip("pypdfium2")
PIL = pytest.importorskip("PIL")
from PIL import Image, ImageDraw # noqa: E402
from application.parser.file.base_parser import DocumentParseError # noqa: E402
from application.parser.file import ocr_parser as op # noqa: E402
from docsgpt.parser.file.base_parser import DocumentParseError # noqa: E402
from docsgpt.parser.file import ocr_parser as op # noqa: E402
# ---------------------------------------------------------------------------
@@ -83,7 +83,7 @@ def _pdf_page_count(path: Path) -> int:
@pytest.fixture
def settings():
from application.core.settings import settings
from docsgpt.core.settings import settings
return settings
@@ -96,7 +96,7 @@ def settings():
@pytest.mark.unit
class TestSettingsAliases:
def test_legacy_docling_names_still_configure_ocr(self, monkeypatch):
from application.core.settings import Settings
from docsgpt.core.settings import Settings
monkeypatch.setenv("DOCLING_OCR_ENABLED", "true")
monkeypatch.setenv("DOCLING_OCR_ATTACHMENTS_ENABLED", "true")
@@ -107,7 +107,7 @@ class TestSettingsAliases:
assert loaded.OCR_MIN_CHARS_PER_PAGE == 7
def test_new_names_and_defaults(self, monkeypatch):
from application.core.settings import Settings
from docsgpt.core.settings import Settings
for name in (
"OCR_ENABLED",
@@ -537,7 +537,7 @@ class TestNativeOcrImageParser:
@pytest.mark.unit
class TestExtractorWiring:
def test_legacy_map_without_ocr_is_unchanged(self):
from application.parser.file.bulk import _legacy_file_extractor
from docsgpt.parser.file.bulk import _legacy_file_extractor
extractor = _legacy_file_extractor()
assert type(extractor[".pdf"]).__name__ == "PDFParser"
@@ -545,7 +545,7 @@ class TestExtractorWiring:
assert ".tiff" not in extractor
def test_legacy_map_with_ocr_uses_native_parsers(self):
from application.parser.file.bulk import _legacy_file_extractor
from docsgpt.parser.file.bulk import _legacy_file_extractor
extractor = _legacy_file_extractor(ocr_enabled=True)
assert isinstance(extractor[".pdf"], op.NativeOcrPdfParser)
@@ -555,8 +555,8 @@ class TestExtractorWiring:
def test_ocr_without_docling_reaches_native_under_anydoc(self, monkeypatch):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "docling", None)
extractor = get_default_file_extractor(engine="anydoc", ocr_enabled=True)
@@ -566,7 +566,7 @@ class TestExtractorWiring:
assert isinstance(extractor[".png"], op.NativeOcrImageParser)
def test_ocr_without_docling_reaches_native_under_docling_engine(self, monkeypatch):
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "docling", None)
extractor = get_default_file_extractor(engine="docling", ocr_enabled=True)
@@ -574,7 +574,7 @@ class TestExtractorWiring:
assert isinstance(extractor[".jpg"], op.NativeOcrImageParser)
def test_ocr_off_without_docling_keeps_legacy(self, monkeypatch):
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "docling", None)
extractor = get_default_file_extractor(engine="docling", ocr_enabled=False)
@@ -583,8 +583,8 @@ class TestExtractorWiring:
def test_native_backend_with_docling_installed_wraps_docling_for_text_pdfs(self, monkeypatch, settings):
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.docling_parser import DoclingPDFParser
monkeypatch.setattr(settings, "OCR_BACKEND", "native")
extractor = get_default_file_extractor(engine="docling", ocr_enabled=True)
@@ -600,8 +600,8 @@ class TestExtractorWiring:
def test_native_backend_under_anydoc_keeps_docling_reroute_for_trust_check(self, monkeypatch, settings):
pytest.importorskip("anydoc")
pytest.importorskip("docling")
from application.parser.file.anydoc_parser import _is_docling_backed
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.anydoc_parser import _is_docling_backed
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setattr(settings, "OCR_BACKEND", "native")
extractor = get_default_file_extractor(engine="anydoc", ocr_enabled=True)
@@ -612,7 +612,7 @@ class TestExtractorWiring:
def test_auto_backend_with_docling_installed_is_unchanged(self, monkeypatch, settings):
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setattr(settings, "OCR_BACKEND", "auto")
extractor = get_default_file_extractor(engine="docling", ocr_enabled=True)
@@ -622,7 +622,7 @@ class TestExtractorWiring:
def test_anydoc_scan_hint_names_new_setting(self, tmp_path):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
fallback = MagicMock()
fallback.parser_config_set = True
@@ -700,7 +700,7 @@ class TestAnydocMixedDocuments:
def test_scanned_page_is_ocrd_and_appended(self, mixed_pdf, monkeypatch, settings):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
monkeypatch.setattr(settings, "OCR_RENDER_DPI", 72)
engine = FakeEngine(["SCANNED PAGE WORDS"])
@@ -717,8 +717,8 @@ class TestAnydocMixedDocuments:
def test_no_ocr_fallback_keeps_text_pages_only(self, mixed_pdf):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from application.parser.file.docs_parser import PDFParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.docs_parser import PDFParser
parser = AnydocParser(fallback_parser=PDFParser())
content = parser.parse_file(mixed_pdf)
@@ -728,7 +728,7 @@ class TestAnydocMixedDocuments:
def test_fallback_with_ocr_off_is_not_used(self, mixed_pdf):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
fallback = MagicMock()
fallback.ocr_enabled = False
@@ -738,7 +738,7 @@ class TestAnydocMixedDocuments:
def test_ocr_failure_keeps_text_pages(self, mixed_pdf, caplog):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
fallback = MagicMock()
fallback.ocr_enabled = True
@@ -755,7 +755,7 @@ class TestAnydocMixedDocuments:
def test_all_text_document_never_probes_ocr(self, tmp_path):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
pdf = _text_pdf(tmp_path / "t.pdf", pages=2, text="Digital text page with enough characters", lines=25)
fallback = MagicMock()
@@ -797,7 +797,7 @@ class TestRenderBudget:
try:
page = pdf[0]
try:
with caplog.at_level(logging.WARNING, logger="application.parser.file.ocr_parser"):
with caplog.at_level(logging.WARNING, logger="docsgpt.parser.file.ocr_parser"):
image = op._render_page(page, 200)
finally:
page.close()
@@ -906,7 +906,7 @@ class TestImageNormalisation:
def test_oversized_image_is_downscaled_to_the_budget(self, caplog):
big = Image.new("L", (9000, 9000), 255)
with caplog.at_level(logging.WARNING, logger="application.parser.file.ocr_parser"):
with caplog.at_level(logging.WARNING, logger="docsgpt.parser.file.ocr_parser"):
small = op.fit_to_pixel_budget(big)
assert small.width * small.height <= op._MAX_RENDER_PIXELS
assert "downscaling" in caplog.text
@@ -953,7 +953,7 @@ class TestDeepseekErrorShapes:
@pytest.mark.unit
def test_delegate_parse_lets_setup_errors_through():
"""A fallback whose dependency is missing is a deployment problem, not a bad file."""
from application.parser.file.base_parser import BaseParser, delegate_parse
from docsgpt.parser.file.base_parser import BaseParser, delegate_parse
class _NeedsLib(BaseParser):
def _init_parser(self):
tests/parser/file/test_openapi3_parser.py
+9 -9
View File
@@ -1,4 +1,4 @@
"""Tests for application.parser.file.openapi3_parser covering lines 7-8, 45."""
"""Tests for docsgpt.parser.file.openapi3_parser covering lines 7-8, 45."""
import pytest
from unittest.mock import MagicMock, patch
@@ -9,16 +9,16 @@ class TestOpenAPI3ParserImportFallback:
def test_import_fallback_to_base_parser(self):
"""Cover lines 7-8: try/except ModuleNotFoundError import fallback."""
# The fallback import is a module-level concern. Just verify the class works.
with patch("application.parser.file.openapi3_parser.parse"):
from application.parser.file.openapi3_parser import OpenAPI3Parser
with patch("docsgpt.parser.file.openapi3_parser.parse"):
from docsgpt.parser.file.openapi3_parser import OpenAPI3Parser
parser = OpenAPI3Parser()
assert parser is not None
def test_get_base_urls(self):
"""Cover basic URL extraction."""
with patch("application.parser.file.openapi3_parser.parse"):
from application.parser.file.openapi3_parser import OpenAPI3Parser
with patch("docsgpt.parser.file.openapi3_parser.parse"):
from docsgpt.parser.file.openapi3_parser import OpenAPI3Parser
parser = OpenAPI3Parser()
urls = parser.get_base_urls([
@@ -32,8 +32,8 @@ class TestOpenAPI3ParserImportFallback:
def test_get_info_from_paths_empty(self):
"""Cover path with no operations."""
with patch("application.parser.file.openapi3_parser.parse"):
from application.parser.file.openapi3_parser import OpenAPI3Parser
with patch("docsgpt.parser.file.openapi3_parser.parse"):
from docsgpt.parser.file.openapi3_parser import OpenAPI3Parser
parser = OpenAPI3Parser()
mock_path = MagicMock()
@@ -43,8 +43,8 @@ class TestOpenAPI3ParserImportFallback:
def test_parse_file_writes_results(self, tmp_path):
"""Cover line 45: parse_file writes to results.txt."""
with patch("application.parser.file.openapi3_parser.parse") as mock_parse:
from application.parser.file.openapi3_parser import OpenAPI3Parser
with patch("docsgpt.parser.file.openapi3_parser.parse") as mock_parse:
from docsgpt.parser.file.openapi3_parser import OpenAPI3Parser
mock_server = MagicMock()
mock_server.url = "https://api.example.com"
tests/parser/file/test_pdf_trust.py
+3 -3
View File
@@ -10,7 +10,7 @@ its whole Chinese column in anydoc with no error.
import zlib
from pathlib import Path
from application.parser.file.pdf_trust import (
from docsgpt.parser.file.pdf_trust import (
check_pdf_fonts,
verify_extraction,
verify_pdf_file,
@@ -134,7 +134,7 @@ class TestRealFixture:
def test_flate_bomb_streams_are_capped():
"""Flate reaches ~1000:1, so per-stream inflation must be capped — an
uncapped decompress of a crafted PDF would OOM the ingest worker."""
from application.parser.file.pdf_trust import _decompressed_streams, _STREAM_INFLATE_CAP
from docsgpt.parser.file.pdf_trust import _decompressed_streams, _STREAM_INFLATE_CAP
bomb = _stream(b"\0" * (_STREAM_INFLATE_CAP * 4))
chunks = list(_decompressed_streams(bomb))
@@ -187,7 +187,7 @@ class TestObjectWalk:
"""The forward walk yields exactly what ``\\d+\\s+\\d+\\s+obj(.*?)endobj`` did."""
import re
from application.parser.file.pdf_trust import _object_bodies
from docsgpt.parser.file.pdf_trust import _object_bodies
reference = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL)
data = (
tests/parser/file/test_pdfium_parser.py
+9 -9
View File
@@ -13,7 +13,7 @@ the fast path must only appear when a caller explicitly asks for it.
import pytest
from application.parser.file.base_parser import BaseParser, DocumentParseError
from docsgpt.parser.file.base_parser import BaseParser, DocumentParseError
pypdfium2 = pytest.importorskip("pypdfium2")
@@ -66,7 +66,7 @@ class _RecordingFallback(BaseParser):
@pytest.fixture
def parser_cls():
from application.parser.file.pdfium_parser import PdfiumTextParser
from docsgpt.parser.file.pdfium_parser import PdfiumTextParser
return PdfiumTextParser
@@ -162,7 +162,7 @@ def docling_engine(monkeypatch):
# (docling is an optional extra) the engine degrades to the legacy
# parsers and these tests do not apply.
pytest.importorskip("docling")
from application.core.settings import settings
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "docling")
# A developer .env with OCR on (and OCR_BACKEND=native) would otherwise
@@ -172,7 +172,7 @@ def docling_engine(monkeypatch):
def test_extractor_defaults_to_docling_for_pdf(docling_engine):
"""Sources must be unaffected: no fast path unless explicitly requested."""
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
pdf_parser = get_default_file_extractor()[".pdf"]
@@ -180,8 +180,8 @@ def test_extractor_defaults_to_docling_for_pdf(docling_engine):
def test_extractor_uses_fast_path_when_requested(docling_engine):
from application.parser.file.bulk import get_default_file_extractor
from application.parser.file.pdfium_parser import PdfiumTextParser
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.pdfium_parser import PdfiumTextParser
pdf_parser = get_default_file_extractor(pdf_text_fast_path=True)[".pdf"]
@@ -192,8 +192,8 @@ def test_extractor_uses_fast_path_when_requested(docling_engine):
def test_anydoc_engine_ignores_fast_path(monkeypatch):
"""anydoc already reads the text layer in milliseconds and keeps structure."""
pytest.importorskip("anydoc")
from application.core.settings import settings
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.core.settings import settings
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "anydoc")
@@ -203,7 +203,7 @@ def test_anydoc_engine_ignores_fast_path(monkeypatch):
def test_fast_path_does_not_change_non_pdf_parsers(docling_engine):
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
plain = get_default_file_extractor()
fast = get_default_file_extractor(pdf_text_fast_path=True)
tests/parser/file/test_pptx_parser.py
+1 -1
View File
@@ -2,7 +2,7 @@ import pytest
from pathlib import Path
from unittest.mock import patch
from application.parser.file.pptx_parser import PPTXParser
from docsgpt.parser.file.pptx_parser import PPTXParser
def test_pptx_init_parser():
tests/parser/file/test_rst_parser.py
+1 -1
View File
@@ -2,7 +2,7 @@ import pytest
from pathlib import Path
from unittest.mock import patch, mock_open
from application.parser.file.rst_parser import RstParser
from docsgpt.parser.file.rst_parser import RstParser
@pytest.fixture
tests/parser/file/test_tableize.py
+3 -3
View File
@@ -1,5 +1,5 @@
"""Tests for the dot-leader/whitespace table reconstruction (``ANYDOC_TABLEIZE``)."""
from application.parser.file.tableize import tableize
from docsgpt.parser.file.tableize import tableize
DOT_LEADER = """Revenues
Insurance premiums ............ 83,431 77,731
@@ -76,7 +76,7 @@ def test_non_table_text_passes_through_verbatim():
def test_single_trailing_number_lines_are_not_a_table():
"""Headings, footnotes and version lists look like 'word number' rows; leave them alone."""
from application.parser.file.tableize import tableize
from docsgpt.parser.file.tableize import tableize
for block in (
"Chapter 1\nChapter 2\nChapter 3",
@@ -88,7 +88,7 @@ def test_single_trailing_number_lines_are_not_a_table():
def test_leader_rows_with_one_value_still_convert():
from application.parser.file.tableize import tableize
from docsgpt.parser.file.tableize import tableize
block = "Revenue ...... 1,234\nCosts ...... 567\nProfit ...... 667"
assert "| --- |" in tableize(block)
tests/parser/file/test_tabular_parser.py
+2 -2
View File
@@ -2,7 +2,7 @@ import pytest
from pathlib import Path
from unittest.mock import patch, MagicMock, mock_open
from application.parser.file.tabular_parser import CSVParser, PandasCSVParser, ExcelParser
from docsgpt.parser.file.tabular_parser import CSVParser, PandasCSVParser, ExcelParser
@pytest.fixture
@@ -220,7 +220,7 @@ def test_excel_numeric_headers_do_not_crash(tmp_path):
to this parser, so it must handle numeric headers."""
from openpyxl import Workbook
from application.parser.file.tabular_parser import ExcelParser
from docsgpt.parser.file.tabular_parser import ExcelParser
wb = Workbook()
ws = wb.active
tests/parser/file/test_tabular_parser_real_fixtures.py
+1 -1
View File
@@ -16,7 +16,7 @@ from pathlib import Path
import pytest
from application.parser.file.tabular_parser import (
from docsgpt.parser.file.tabular_parser import (
ExcelParser,
PandasCSVParser,
cell_to_text,