tiennm99/DocsGPT
1
0 Fork 0
mirror of https://github.com/tiennm99/DocsGPT.git synced 2026-10-14 16:16:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: rename the application package to docsgpt

Browse Source 
The backend import package is now docsgpt, the name it will carry on PyPI;
application was far too generic to install into anyone's site-packages.
git mv plus a mechanical rewrite of every import, dotted string and path
reference: 734 Python files, the compose files, Dockerfile, workflows, docs,
setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage
config, .gitignore. Behaviour is unchanged.

Kept for one release:
- A top-level application package whose meta-path finder resolves
  application.x.y to the already-imported docsgpt.x.y object, so old imports
  and entry points (celery -A application.app.celery,
  uvicorn application.asgi:asgi_app) keep working with a FutureWarning.
- Celery registers every application.* task name as an alias of its
  docsgpt.* task on start-up, so messages queued by the previous release still
  run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries
  the previous release wrote are left unread instead of firing twice.

The backend image builds from the repository root (docker build -f
docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore
allow-lists docsgpt/ and application/ and keeps caches, local data, .env
files, the sample index files and the Dockerfile out. Compose and the image
workflows point at the new context.
This commit is contained in:
Alex committed 2026-09-07 10:20:43 +01:00
1 parent 3e7f1f91b4
commit 574f96341e
984 files changed
+9847 -9682
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/parser/conftest.py
+1 -1
View File
@@ -18,6 +18,6 @@ def _mpnet_tokenizer():
@pytest.fixture
def hf_counter(_mpnet_tokenizer):
from application.parser.tokenization import HuggingFaceCounter
from docsgpt.parser.tokenization import HuggingFaceCounter
return HuggingFaceCounter(_mpnet_tokenizer, "sentence-transformers/all-mpnet-base-v2")
tests/parser/connectors/confluence/test_auth.py
+9 -9
View File
@@ -1,4 +1,4 @@
"""Tests for application/parser/connectors/confluence/auth.py"""
"""Tests for docsgpt/parser/connectors/confluence/auth.py"""
import datetime
from unittest.mock import MagicMock, patch
@@ -24,8 +24,8 @@ def mock_settings():
@pytest.fixture
def auth(mock_settings):
with patch("application.parser.connectors.confluence.auth.settings", mock_settings):
from application.parser.connectors.confluence.auth import ConfluenceAuth
with patch("docsgpt.parser.connectors.confluence.auth.settings", mock_settings):
from docsgpt.parser.connectors.confluence.auth import ConfluenceAuth
return ConfluenceAuth()
@@ -46,16 +46,16 @@ class TestConfluenceAuthInit:
@pytest.mark.unit
def test_init_missing_client_id_raises(self, mock_settings):
mock_settings.CONFLUENCE_CLIENT_ID = None
with patch("application.parser.connectors.confluence.auth.settings", mock_settings):
from application.parser.connectors.confluence.auth import ConfluenceAuth
with patch("docsgpt.parser.connectors.confluence.auth.settings", mock_settings):
from docsgpt.parser.connectors.confluence.auth import ConfluenceAuth
with pytest.raises(ValueError, match="CONFLUENCE_CLIENT_ID"):
ConfluenceAuth()
@pytest.mark.unit
def test_init_missing_client_secret_raises(self, mock_settings):
mock_settings.CONFLUENCE_CLIENT_SECRET = None
with patch("application.parser.connectors.confluence.auth.settings", mock_settings):
from application.parser.connectors.confluence.auth import ConfluenceAuth
with patch("docsgpt.parser.connectors.confluence.auth.settings", mock_settings):
from docsgpt.parser.connectors.confluence.auth import ConfluenceAuth
with pytest.raises(ValueError, match="CONFLUENCE_CLIENT_SECRET"):
ConfluenceAuth()
@@ -63,8 +63,8 @@ class TestConfluenceAuthInit:
def test_init_both_missing_raises(self, mock_settings):
mock_settings.CONFLUENCE_CLIENT_ID = None
mock_settings.CONFLUENCE_CLIENT_SECRET = None
with patch("application.parser.connectors.confluence.auth.settings", mock_settings):
from application.parser.connectors.confluence.auth import ConfluenceAuth
with patch("docsgpt.parser.connectors.confluence.auth.settings", mock_settings):
from docsgpt.parser.connectors.confluence.auth import ConfluenceAuth
with pytest.raises(ValueError):
ConfluenceAuth()
tests/parser/connectors/confluence/test_auth_session.py
+8 -8
View File
@@ -13,14 +13,14 @@ def _patch_db(conn):
yield conn
with patch(
"application.storage.db.session.db_readonly", _yield
"docsgpt.storage.db.session.db_readonly", _yield
):
yield
class TestGetTokenInfoFromSession:
def test_invalid_session_token_raises(self, pg_conn):
from application.parser.connectors.confluence.auth import (
from docsgpt.parser.connectors.confluence.auth import (
ConfluenceAuth,
)
@@ -29,10 +29,10 @@ class TestGetTokenInfoFromSession:
auth.get_token_info_from_session("no-such-token")
def test_missing_token_info_raises(self, pg_conn):
from application.parser.connectors.confluence.auth import (
from docsgpt.parser.connectors.confluence.auth import (
ConfluenceAuth,
)
from application.storage.db.repositories.connector_sessions import (
from docsgpt.storage.db.repositories.connector_sessions import (
ConnectorSessionsRepository,
)
@@ -47,10 +47,10 @@ class TestGetTokenInfoFromSession:
auth.get_token_info_from_session("tok-no-info")
def test_missing_required_fields_raises(self, pg_conn):
from application.parser.connectors.confluence.auth import (
from docsgpt.parser.connectors.confluence.auth import (
ConfluenceAuth,
)
from application.storage.db.repositories.connector_sessions import (
from docsgpt.storage.db.repositories.connector_sessions import (
ConnectorSessionsRepository,
)
@@ -70,10 +70,10 @@ class TestGetTokenInfoFromSession:
auth.get_token_info_from_session("tok-partial")
def test_complete_token_info_returned(self, pg_conn):
from application.parser.connectors.confluence.auth import (
from docsgpt.parser.connectors.confluence.auth import (
ConfluenceAuth,
)
from application.storage.db.repositories.connector_sessions import (
from docsgpt.storage.db.repositories.connector_sessions import (
ConnectorSessionsRepository,
)
tests/parser/connectors/confluence/test_loader.py
+7 -7
View File
@@ -1,4 +1,4 @@
"""Tests for application/parser/connectors/confluence/loader.py"""
"""Tests for docsgpt/parser/connectors/confluence/loader.py"""
import os
import tempfile
@@ -23,12 +23,12 @@ def _make_loader(token_info=None):
"cloud_id": "test_cloud",
}
with patch("application.parser.connectors.confluence.loader.ConfluenceAuth") as MockAuth:
with patch("docsgpt.parser.connectors.confluence.loader.ConfluenceAuth") as MockAuth:
mock_auth = MagicMock()
mock_auth.get_token_info_from_session.return_value = token_info
MockAuth.return_value = mock_auth
from application.parser.connectors.confluence.loader import ConfluenceLoader
from docsgpt.parser.connectors.confluence.loader import ConfluenceLoader
loader = ConfluenceLoader("session_tok")
loader.auth = mock_auth
@@ -469,25 +469,25 @@ class TestExtractCursor:
@pytest.mark.unit
def test_extracts_cursor_from_link(self):
from application.parser.connectors.confluence.loader import ConfluenceLoader
from docsgpt.parser.connectors.confluence.loader import ConfluenceLoader
link = "/wiki/api/v2/spaces?limit=10&cursor=abc123"
result = ConfluenceLoader._extract_cursor(link)
assert result == "abc123"
@pytest.mark.unit
def test_returns_none_for_no_link(self):
from application.parser.connectors.confluence.loader import ConfluenceLoader
from docsgpt.parser.connectors.confluence.loader import ConfluenceLoader
assert ConfluenceLoader._extract_cursor(None) is None
@pytest.mark.unit
def test_returns_none_for_link_without_cursor(self):
from application.parser.connectors.confluence.loader import ConfluenceLoader
from docsgpt.parser.connectors.confluence.loader import ConfluenceLoader
link = "/wiki/api/v2/spaces?limit=10"
assert ConfluenceLoader._extract_cursor(link) is None
@pytest.mark.unit
def test_returns_first_cursor_value(self):
from application.parser.connectors.confluence.loader import ConfluenceLoader
from docsgpt.parser.connectors.confluence.loader import ConfluenceLoader
link = "/wiki/api/v2/spaces?cursor=val1&cursor=val2"
result = ConfluenceLoader._extract_cursor(link)
assert result == "val1"
tests/parser/connectors/test_auth_token_leakage.py
+6 -6
View File
@@ -15,7 +15,7 @@ from unittest.mock import MagicMock, patch
import pytest
from application.parser.connectors._auth_utils import session_token_fingerprint
from docsgpt.parser.connectors._auth_utils import session_token_fingerprint
SECRET_TOKEN = "super-secret-session-token-ABCDEF1234567890"
@@ -51,12 +51,12 @@ def _patches(session_return: Optional[Dict[str, Any]]):
)
return (
patch(
"application.storage.db.repositories.connector_sessions."
"docsgpt.storage.db.repositories.connector_sessions."
"ConnectorSessionsRepository",
fake_repo_cls,
),
patch(
"application.storage.db.session.db_readonly",
"docsgpt.storage.db.session.db_readonly",
lambda: _FakeReadonlyCtx(),
),
)
@@ -92,7 +92,7 @@ class TestConfluenceAuthDoesNotLeakToken:
@pytest.mark.unit
def test_invalid_session_does_not_interpolate_token(self) -> None:
from application.parser.connectors.confluence.auth import ConfluenceAuth
from docsgpt.parser.connectors.confluence.auth import ConfluenceAuth
auth = ConfluenceAuth.__new__(ConfluenceAuth)
repo_patch, ctx_patch = _patches(None)
@@ -109,7 +109,7 @@ class TestGoogleDriveAuthDoesNotLeakToken:
@pytest.mark.unit
def test_invalid_session_does_not_interpolate_token(self) -> None:
from application.parser.connectors.google_drive.auth import GoogleDriveAuth
from docsgpt.parser.connectors.google_drive.auth import GoogleDriveAuth
auth = GoogleDriveAuth.__new__(GoogleDriveAuth)
repo_patch, ctx_patch = _patches(None)
@@ -130,7 +130,7 @@ class TestSharePointAuthDoesNotLeakToken:
@pytest.mark.unit
def test_invalid_session_does_not_interpolate_token(self) -> None:
from application.parser.connectors.share_point.auth import SharePointAuth
from docsgpt.parser.connectors.share_point.auth import SharePointAuth
auth = SharePointAuth.__new__(SharePointAuth)
repo_patch, ctx_patch = _patches(None)
tests/parser/connectors/test_base.py
+2 -2
View File
@@ -3,8 +3,8 @@
import pytest
from application.parser.connectors.base import BaseConnectorAuth, BaseConnectorLoader
from application.parser.schema.base import Document
from docsgpt.parser.connectors.base import BaseConnectorAuth, BaseConnectorLoader
from docsgpt.parser.schema.base import Document
class ConcreteAuth(BaseConnectorAuth):
tests/parser/connectors/test_connector_creator.py
+11 -11
View File
@@ -19,11 +19,11 @@ class TestConnectorCreator:
mock_settings.MICROSOFT_TENANT_ID = "tid"
mock_settings.MONGO_DB_NAME = "test_db"
with patch("application.core.settings.settings", mock_settings), \
patch("application.parser.connectors.share_point.auth.settings", mock_settings), \
patch("application.parser.connectors.google_drive.auth.settings", mock_settings), \
patch("application.parser.connectors.share_point.auth.ConfidentialClientApplication"):
from application.parser.connectors.connector_creator import ConnectorCreator
with patch("docsgpt.core.settings.settings", mock_settings), \
patch("docsgpt.parser.connectors.share_point.auth.settings", mock_settings), \
patch("docsgpt.parser.connectors.google_drive.auth.settings", mock_settings), \
patch("docsgpt.parser.connectors.share_point.auth.ConfidentialClientApplication"):
from docsgpt.parser.connectors.connector_creator import ConnectorCreator
self.ConnectorCreator = ConnectorCreator
yield
@@ -51,13 +51,13 @@ class TestConnectorCreator:
@pytest.mark.unit
def test_create_auth_google_drive(self):
auth = self.ConnectorCreator.create_auth("google_drive")
from application.parser.connectors.google_drive.auth import GoogleDriveAuth
from docsgpt.parser.connectors.google_drive.auth import GoogleDriveAuth
assert isinstance(auth, GoogleDriveAuth)
@pytest.mark.unit
def test_create_auth_share_point(self):
auth = self.ConnectorCreator.create_auth("share_point")
from application.parser.connectors.share_point.auth import SharePointAuth
from docsgpt.parser.connectors.share_point.auth import SharePointAuth
assert isinstance(auth, SharePointAuth)
@pytest.mark.unit
@@ -72,7 +72,7 @@ class TestConnectorCreator:
@pytest.mark.unit
def test_create_connector_google_drive(self):
with patch("application.parser.connectors.google_drive.loader.GoogleDriveAuth") as MockAuth:
with patch("docsgpt.parser.connectors.google_drive.loader.GoogleDriveAuth") as MockAuth:
mock_auth_instance = MagicMock()
mock_auth_instance.get_token_info_from_session.return_value = {
"access_token": "at", "refresh_token": "rt"
@@ -85,12 +85,12 @@ class TestConnectorCreator:
MockAuth.return_value = mock_auth_instance
loader = self.ConnectorCreator.create_connector("google_drive", "session_tok")
from application.parser.connectors.google_drive.loader import GoogleDriveLoader
from docsgpt.parser.connectors.google_drive.loader import GoogleDriveLoader
assert isinstance(loader, GoogleDriveLoader)
@pytest.mark.unit
def test_create_connector_share_point(self):
with patch("application.parser.connectors.share_point.loader.SharePointAuth") as MockAuth:
with patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth") as MockAuth:
mock_auth_instance = MagicMock()
mock_auth_instance.get_token_info_from_session.return_value = {
"access_token": "at", "refresh_token": "rt"
@@ -98,5 +98,5 @@ class TestConnectorCreator:
MockAuth.return_value = mock_auth_instance
loader = self.ConnectorCreator.create_connector("share_point", "session_tok")
from application.parser.connectors.share_point.loader import SharePointLoader
from docsgpt.parser.connectors.share_point.loader import SharePointLoader
assert isinstance(loader, SharePointLoader)
tests/parser/connectors/test_google_drive_auth.py
+24 -24
View File
@@ -18,8 +18,8 @@ def mock_settings():
@pytest.fixture
def auth(mock_settings):
with patch("application.parser.connectors.google_drive.auth.settings", mock_settings):
from application.parser.connectors.google_drive.auth import GoogleDriveAuth
with patch("docsgpt.parser.connectors.google_drive.auth.settings", mock_settings):
from docsgpt.parser.connectors.google_drive.auth import GoogleDriveAuth
return GoogleDriveAuth()
@@ -34,16 +34,16 @@ class TestGoogleDriveAuthInit:
@pytest.mark.unit
def test_init_missing_client_id_raises(self, mock_settings):
mock_settings.GOOGLE_CLIENT_ID = None
with patch("application.parser.connectors.google_drive.auth.settings", mock_settings):
from application.parser.connectors.google_drive.auth import GoogleDriveAuth
with patch("docsgpt.parser.connectors.google_drive.auth.settings", mock_settings):
from docsgpt.parser.connectors.google_drive.auth import GoogleDriveAuth
with pytest.raises(ValueError, match="Google OAuth credentials not configured"):
GoogleDriveAuth()
@pytest.mark.unit
def test_init_missing_client_secret_raises(self, mock_settings):
mock_settings.GOOGLE_CLIENT_SECRET = None
with patch("application.parser.connectors.google_drive.auth.settings", mock_settings):
from application.parser.connectors.google_drive.auth import GoogleDriveAuth
with patch("docsgpt.parser.connectors.google_drive.auth.settings", mock_settings):
from docsgpt.parser.connectors.google_drive.auth import GoogleDriveAuth
with pytest.raises(ValueError, match="Google OAuth credentials not configured"):
GoogleDriveAuth()
@@ -55,7 +55,7 @@ class TestGetAuthorizationUrl:
mock_flow = MagicMock()
mock_flow.authorization_url.return_value = ("https://accounts.google.com/auth?state=s1", "s1")
with patch("application.parser.connectors.google_drive.auth.Flow") as MockFlow:
with patch("docsgpt.parser.connectors.google_drive.auth.Flow") as MockFlow:
MockFlow.from_client_config.return_value = mock_flow
url = auth.get_authorization_url(state="s1")
@@ -69,7 +69,7 @@ class TestGetAuthorizationUrl:
@pytest.mark.unit
def test_raises_on_flow_error(self, auth):
with patch("application.parser.connectors.google_drive.auth.Flow") as MockFlow:
with patch("docsgpt.parser.connectors.google_drive.auth.Flow") as MockFlow:
MockFlow.from_client_config.side_effect = Exception("flow error")
with pytest.raises(Exception, match="flow error"):
auth.get_authorization_url()
@@ -91,7 +91,7 @@ class TestExchangeCodeForTokens:
mock_flow = MagicMock()
mock_flow.credentials = mock_creds
with patch("application.parser.connectors.google_drive.auth.Flow") as MockFlow:
with patch("docsgpt.parser.connectors.google_drive.auth.Flow") as MockFlow:
MockFlow.from_client_config.return_value = mock_flow
result = auth.exchange_code_for_tokens("auth_code_123")
@@ -115,7 +115,7 @@ class TestExchangeCodeForTokens:
mock_flow = MagicMock()
mock_flow.credentials = mock_creds
with patch("application.parser.connectors.google_drive.auth.Flow") as MockFlow:
with patch("docsgpt.parser.connectors.google_drive.auth.Flow") as MockFlow:
MockFlow.from_client_config.return_value = mock_flow
with pytest.raises(ValueError, match="did not return an access token"):
auth.exchange_code_for_tokens("code")
@@ -128,7 +128,7 @@ class TestExchangeCodeForTokens:
mock_flow = MagicMock()
mock_flow.credentials = mock_creds
with patch("application.parser.connectors.google_drive.auth.Flow") as MockFlow:
with patch("docsgpt.parser.connectors.google_drive.auth.Flow") as MockFlow:
MockFlow.from_client_config.return_value = mock_flow
with pytest.raises(ValueError, match="No refresh token received"):
auth.exchange_code_for_tokens("code")
@@ -146,7 +146,7 @@ class TestExchangeCodeForTokens:
mock_flow = MagicMock()
mock_flow.credentials = mock_creds
with patch("application.parser.connectors.google_drive.auth.Flow") as MockFlow:
with patch("docsgpt.parser.connectors.google_drive.auth.Flow") as MockFlow:
MockFlow.from_client_config.return_value = mock_flow
result = auth.exchange_code_for_tokens("code")
@@ -160,7 +160,7 @@ class TestRefreshAccessToken:
@pytest.mark.unit
def test_successful_refresh(self, auth):
mock_request_cls = MagicMock()
with patch("application.parser.connectors.google_drive.auth.Credentials") as MockCreds, \
with patch("docsgpt.parser.connectors.google_drive.auth.Credentials") as MockCreds, \
patch("google.auth.transport.requests.Request", mock_request_cls):
mock_cred_instance = MagicMock()
mock_cred_instance.token = "new_access"
@@ -184,7 +184,7 @@ class TestRefreshAccessToken:
@pytest.mark.unit
def test_refresh_failure_raises(self, auth):
with patch("application.parser.connectors.google_drive.auth.Credentials") as MockCreds, \
with patch("docsgpt.parser.connectors.google_drive.auth.Credentials") as MockCreds, \
patch("google.auth.transport.requests.Request"):
mock_cred_instance = MagicMock()
mock_cred_instance.refresh.side_effect = Exception("refresh failed")
@@ -198,8 +198,8 @@ class TestCreateCredentialsFromTokenInfo:
@pytest.mark.unit
def test_creates_credentials(self, auth, mock_settings):
with patch("application.parser.connectors.google_drive.auth.Credentials") as MockCreds, \
patch("application.parser.connectors.google_drive.auth.settings", mock_settings):
with patch("docsgpt.parser.connectors.google_drive.auth.Credentials") as MockCreds, \
patch("docsgpt.parser.connectors.google_drive.auth.settings", mock_settings):
mock_cred = MagicMock()
mock_cred.token = "at"
MockCreds.return_value = mock_cred
@@ -213,14 +213,14 @@ class TestCreateCredentialsFromTokenInfo:
@pytest.mark.unit
def test_missing_access_token_raises(self, auth, mock_settings):
with patch("application.parser.connectors.google_drive.auth.settings", mock_settings):
with patch("docsgpt.parser.connectors.google_drive.auth.settings", mock_settings):
with pytest.raises(ValueError, match="No access token found"):
auth.create_credentials_from_token_info({})
@pytest.mark.unit
def test_credentials_without_valid_token_raises(self, auth, mock_settings):
with patch("application.parser.connectors.google_drive.auth.Credentials") as MockCreds, \
patch("application.parser.connectors.google_drive.auth.settings", mock_settings):
with patch("docsgpt.parser.connectors.google_drive.auth.Credentials") as MockCreds, \
patch("docsgpt.parser.connectors.google_drive.auth.settings", mock_settings):
mock_cred = MagicMock()
mock_cred.token = None
MockCreds.return_value = mock_cred
@@ -238,7 +238,7 @@ class TestBuildDriveService:
mock_creds.refresh_token = "rt"
mock_creds.expired = False
with patch("application.parser.connectors.google_drive.auth.build") as mock_build:
with patch("docsgpt.parser.connectors.google_drive.auth.build") as mock_build:
mock_build.return_value = MagicMock()
service = auth.build_drive_service(mock_creds)
mock_build.assert_called_once_with('drive', 'v3', credentials=mock_creds)
@@ -264,7 +264,7 @@ class TestBuildDriveService:
mock_creds.refresh_token = "rt"
mock_creds.expired = True
with patch("application.parser.connectors.google_drive.auth.build") as mock_build, \
with patch("docsgpt.parser.connectors.google_drive.auth.build") as mock_build, \
patch("google.auth.transport.requests.Request"):
mock_build.return_value = MagicMock()
auth.build_drive_service(mock_creds)
@@ -302,7 +302,7 @@ class TestBuildDriveService:
mock_resp = MagicMock()
mock_resp.status = 500
with patch("application.parser.connectors.google_drive.auth.build") as mock_build:
with patch("docsgpt.parser.connectors.google_drive.auth.build") as mock_build:
mock_build.side_effect = HttpError(mock_resp, b"error")
with pytest.raises(ValueError, match="HTTP 500"):
auth.build_drive_service(mock_creds)
@@ -374,11 +374,11 @@ class TestGetTokenInfoFromSession:
)
return (
patch(
"application.storage.db.repositories.connector_sessions.ConnectorSessionsRepository",
"docsgpt.storage.db.repositories.connector_sessions.ConnectorSessionsRepository",
fake_repo_cls,
),
patch(
"application.storage.db.session.db_readonly",
"docsgpt.storage.db.session.db_readonly",
lambda: _FakeReadonlyCtx(),
),
)
tests/parser/connectors/test_google_drive_loader.py
+14 -14
View File
@@ -4,12 +4,12 @@ from unittest.mock import MagicMock, patch
import pytest
from application.parser.schema.base import Document
from docsgpt.parser.schema.base import Document
def _make_loader(service=None):
"""Create a GoogleDriveLoader with mocked dependencies."""
with patch("application.parser.connectors.google_drive.loader.GoogleDriveAuth") as MockAuth:
with patch("docsgpt.parser.connectors.google_drive.loader.GoogleDriveAuth") as MockAuth:
mock_auth = MagicMock()
mock_auth.get_token_info_from_session.return_value = {
"access_token": "at",
@@ -23,7 +23,7 @@ def _make_loader(service=None):
mock_auth.build_drive_service.return_value = service or MagicMock()
MockAuth.return_value = mock_auth
from application.parser.connectors.google_drive.loader import GoogleDriveLoader
from docsgpt.parser.connectors.google_drive.loader import GoogleDriveLoader
loader = GoogleDriveLoader("session_tok")
return loader
@@ -49,7 +49,7 @@ class TestGoogleDriveLoaderInit:
@pytest.mark.unit
def test_init_service_failure_sets_none(self):
with patch("application.parser.connectors.google_drive.loader.GoogleDriveAuth") as MockAuth:
with patch("docsgpt.parser.connectors.google_drive.loader.GoogleDriveAuth") as MockAuth:
mock_auth = MagicMock()
mock_auth.get_token_info_from_session.return_value = {
"access_token": "at", "refresh_token": "rt"
@@ -60,7 +60,7 @@ class TestGoogleDriveLoaderInit:
mock_auth.build_drive_service.side_effect = Exception("service fail")
MockAuth.return_value = mock_auth
from application.parser.connectors.google_drive.loader import GoogleDriveLoader
from docsgpt.parser.connectors.google_drive.loader import GoogleDriveLoader
loader = GoogleDriveLoader("st")
assert loader.service is None
@@ -386,7 +386,7 @@ class TestDownloadFileContent:
mock_request = MagicMock()
mock_service.files.return_value.get_media.return_value = mock_request
with patch("application.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
with patch("docsgpt.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
mock_dl = MagicMock()
mock_dl.next_chunk.side_effect = [(None, False), (None, True)]
MockDownload.return_value = mock_dl
@@ -405,7 +405,7 @@ class TestDownloadFileContent:
mock_request = MagicMock()
mock_service.files.return_value.export_media.return_value = mock_request
with patch("application.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
with patch("docsgpt.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
mock_dl = MagicMock()
mock_dl.next_chunk.return_value = (None, True)
MockDownload.return_value = mock_dl
@@ -425,7 +425,7 @@ class TestDownloadFileContent:
mock_request = MagicMock()
mock_service.files.return_value.get_media.return_value = mock_request
with patch("application.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
with patch("docsgpt.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
mock_dl = MagicMock()
mock_dl.next_chunk.return_value = (None, True)
MockDownload.return_value = mock_dl
@@ -454,7 +454,7 @@ class TestDownloadFileContent:
loader._ensure_service = MagicMock()
loader.service.files.return_value.get_media.return_value = MagicMock()
with patch("application.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
with patch("docsgpt.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
mock_dl = MagicMock()
mock_dl.next_chunk.return_value = (None, True)
MockDownload.return_value = mock_dl
@@ -496,7 +496,7 @@ class TestDownloadFileContent:
loader._ensure_service = MagicMock()
loader.service.files.return_value.get_media.return_value = MagicMock()
with patch("application.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
with patch("docsgpt.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
mock_dl = MagicMock()
mock_dl.next_chunk.return_value = (None, True)
MockDownload.return_value = mock_dl
@@ -535,7 +535,7 @@ class TestDownloadFileContent:
resp = MagicMock()
resp.status = 500
with patch("application.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
with patch("docsgpt.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
mock_dl = MagicMock()
mock_dl.next_chunk.side_effect = HttpError(resp, b"server error")
MockDownload.return_value = mock_dl
@@ -549,7 +549,7 @@ class TestDownloadFileContent:
mock_request = MagicMock()
mock_service.files.return_value.get_media.return_value = mock_request
with patch("application.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
with patch("docsgpt.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDownload:
mock_dl = MagicMock()
mock_dl.next_chunk.side_effect = RuntimeError("chunk fail")
MockDownload.return_value = mock_dl
@@ -674,7 +674,7 @@ class TestDownloadSingleFile:
mock_request = MagicMock()
mock_service.files.return_value.get_media.return_value = mock_request
with patch("application.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDl:
with patch("docsgpt.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDl:
mock_dl = MagicMock()
mock_dl.next_chunk.return_value = (None, True)
MockDl.return_value = mock_dl
@@ -698,7 +698,7 @@ class TestDownloadSingleFile:
mock_request = MagicMock()
mock_service.files.return_value.export_media.return_value = mock_request
with patch("application.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDl:
with patch("docsgpt.parser.connectors.google_drive.loader.MediaIoBaseDownload") as MockDl:
mock_dl = MagicMock()
mock_dl.next_chunk.return_value = (None, True)
MockDl.return_value = mock_dl
tests/parser/connectors/test_share_point_auth.py
+11 -11
View File
@@ -21,7 +21,7 @@ def mock_settings():
@pytest.fixture
def mock_msal():
with patch("application.parser.connectors.share_point.auth.ConfidentialClientApplication") as MockMSAL:
with patch("docsgpt.parser.connectors.share_point.auth.ConfidentialClientApplication") as MockMSAL:
mock_app = MagicMock()
MockMSAL.return_value = mock_app
yield mock_app
@@ -29,8 +29,8 @@ def mock_msal():
@pytest.fixture
def auth(mock_settings, mock_msal):
with patch("application.parser.connectors.share_point.auth.settings", mock_settings):
from application.parser.connectors.share_point.auth import SharePointAuth
with patch("docsgpt.parser.connectors.share_point.auth.settings", mock_settings):
from docsgpt.parser.connectors.share_point.auth import SharePointAuth
return SharePointAuth()
@@ -46,18 +46,18 @@ class TestSharePointAuthInit:
@pytest.mark.unit
def test_missing_client_id_raises(self, mock_settings):
mock_settings.MICROSOFT_CLIENT_ID = None
with patch("application.parser.connectors.share_point.auth.settings", mock_settings), \
patch("application.parser.connectors.share_point.auth.ConfidentialClientApplication"):
from application.parser.connectors.share_point.auth import SharePointAuth
with patch("docsgpt.parser.connectors.share_point.auth.settings", mock_settings), \
patch("docsgpt.parser.connectors.share_point.auth.ConfidentialClientApplication"):
from docsgpt.parser.connectors.share_point.auth import SharePointAuth
with pytest.raises(ValueError, match="MICROSOFT_CLIENT_ID"):
SharePointAuth()
@pytest.mark.unit
def test_missing_client_secret_raises(self, mock_settings):
mock_settings.MICROSOFT_CLIENT_SECRET = None
with patch("application.parser.connectors.share_point.auth.settings", mock_settings), \
patch("application.parser.connectors.share_point.auth.ConfidentialClientApplication"):
from application.parser.connectors.share_point.auth import SharePointAuth
with patch("docsgpt.parser.connectors.share_point.auth.settings", mock_settings), \
patch("docsgpt.parser.connectors.share_point.auth.ConfidentialClientApplication"):
from docsgpt.parser.connectors.share_point.auth import SharePointAuth
with pytest.raises(ValueError, match="MICROSOFT_CLIENT_SECRET"):
SharePointAuth()
@@ -295,11 +295,11 @@ class TestGetTokenInfoFromSession:
)
return (
patch(
"application.storage.db.repositories.connector_sessions.ConnectorSessionsRepository",
"docsgpt.storage.db.repositories.connector_sessions.ConnectorSessionsRepository",
fake_repo_cls,
),
patch(
"application.storage.db.session.db_readonly",
"docsgpt.storage.db.session.db_readonly",
lambda: _FakeReadonlyCtx(),
),
)
tests/parser/connectors/test_share_point_loader.py
+50 -50
View File
@@ -6,12 +6,12 @@ from unittest.mock import MagicMock, patch
import pytest
import requests as real_requests
from application.parser.schema.base import Document
from docsgpt.parser.schema.base import Document
def _make_loader(access_token="at", refresh_token="rt", allows_shared=False):
"""Create a SharePointLoader with mocked dependencies."""
with patch("application.parser.connectors.share_point.loader.SharePointAuth") as MockAuth:
with patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth") as MockAuth:
mock_auth = MagicMock()
mock_auth.get_token_info_from_session.return_value = {
"access_token": access_token,
@@ -21,7 +21,7 @@ def _make_loader(access_token="at", refresh_token="rt", allows_shared=False):
mock_auth.is_token_expired.return_value = False
MockAuth.return_value = mock_auth
from application.parser.connectors.share_point.loader import SharePointLoader
from docsgpt.parser.connectors.share_point.loader import SharePointLoader
loader = SharePointLoader("session_tok")
return loader
@@ -48,7 +48,7 @@ class TestSharePointLoaderInit:
@pytest.mark.unit
def test_no_access_token_raises(self):
with patch("application.parser.connectors.share_point.loader.SharePointAuth") as MockAuth:
with patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth") as MockAuth:
mock_auth = MagicMock()
mock_auth.get_token_info_from_session.return_value = {
"access_token": None,
@@ -56,7 +56,7 @@ class TestSharePointLoaderInit:
}
MockAuth.return_value = mock_auth
from application.parser.connectors.share_point.loader import SharePointLoader
from docsgpt.parser.connectors.share_point.loader import SharePointLoader
with pytest.raises(ValueError, match="No access token"):
SharePointLoader("st")
@@ -282,7 +282,7 @@ class TestLoadFileById:
loader._process_file = MagicMock(return_value=Document(text="", doc_id="f1", extra_info={}))
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
doc = loader._load_file_by_id("f1")
assert doc is not None
@@ -307,13 +307,13 @@ class TestLoadFileById:
loader._process_file = MagicMock(return_value=Document(text="", doc_id="f1", extra_info={}))
loader.auth.refresh_access_token.return_value = {"access_token": "new_at"}
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect):
doc = loader._load_file_by_id("f1")
assert doc is not None
@pytest.mark.unit
def test_general_exception_returns_none(self, loader):
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
doc = loader._load_file_by_id("f1")
assert doc is None
@@ -333,7 +333,7 @@ class TestListItemsInParent:
loader._process_file = MagicMock(return_value=Document(text="", doc_id="file1", extra_info={}))
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
docs = loader._list_items_in_parent("root")
assert len(docs) == 2
assert docs[0].extra_info["is_folder"] is True
@@ -344,7 +344,7 @@ class TestListItemsInParent:
mock_response.json.return_value = {"value": []}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response) as mock_get:
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response) as mock_get:
loader._list_items_in_parent("root", search_query="report")
call_url = mock_get.call_args[0][0]
assert "search" in call_url
@@ -355,7 +355,7 @@ class TestListItemsInParent:
mock_response.json.return_value = {"value": []}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response) as mock_get:
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response) as mock_get:
loader._list_items_in_parent("drive1:folder1", search_query="test")
call_url = mock_get.call_args[0][0]
assert "drives/drive1" in call_url
@@ -372,7 +372,7 @@ class TestListItemsInParent:
mock_response.raise_for_status = MagicMock()
loader._process_file = MagicMock(return_value=Document(text="", doc_id="f1", extra_info={}))
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
loader._list_items_in_parent("root")
assert loader.next_page_token == "abc123"
@@ -382,7 +382,7 @@ class TestListItemsInParent:
mock_response.json.return_value = {"value": []}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
loader._list_items_in_parent("root")
assert loader.next_page_token is None
@@ -397,7 +397,7 @@ class TestListItemsInParent:
mock_response.raise_for_status = MagicMock()
loader._process_file = MagicMock(side_effect=lambda m, **kw: Document(text="", doc_id=m["id"], extra_info={}))
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
docs = loader._list_items_in_parent("root", limit=3)
assert len(docs) == 3
@@ -407,14 +407,14 @@ class TestListItemsInParent:
mock_response.json.return_value = {"value": []}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response) as mock_get:
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response) as mock_get:
loader._list_items_in_parent("root", page_token="tok123")
call_kwargs = mock_get.call_args[1]
assert call_kwargs["params"]["$skipToken"] == "tok123"
@pytest.mark.unit
def test_exception_returns_partial(self, loader):
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
docs = loader._list_items_in_parent("root")
assert docs == []
@@ -427,7 +427,7 @@ class TestListItemsInParent:
}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
loader._list_items_in_parent("root")
assert loader.next_page_token is None
@@ -440,7 +440,7 @@ class TestDownloadFileContent:
mock_response.content = b"file content"
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
content = loader._download_file_content("f1")
assert content == "file content"
@@ -451,7 +451,7 @@ class TestDownloadFileContent:
mock_response.content.decode.side_effect = UnicodeDecodeError("utf-8", b"", 0, 1, "bad")
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
result = loader._download_file_content("f1")
assert result is None
@@ -460,7 +460,7 @@ class TestDownloadFileContent:
mock_response = MagicMock()
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
result = loader._download_file_content("f1")
assert result is None
@@ -483,7 +483,7 @@ class TestDownloadFileContent:
loader.auth.refresh_access_token.return_value = {"access_token": "new_at"}
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect):
content = loader._download_file_content("f1")
assert content == "content after refresh"
@@ -523,7 +523,7 @@ class TestListSharedItems:
}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
docs = loader_shared._list_shared_items(limit=100, load_content=False)
assert len(docs) == 2
@@ -545,7 +545,7 @@ class TestListSharedItems:
}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
docs = loader_shared._list_shared_items(limit=100, load_content=False)
assert len(docs) == 1
@@ -563,7 +563,7 @@ class TestListSharedItems:
}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
docs = loader_shared._list_shared_items(limit=2, page_token="2", load_content=False)
assert len(docs) == 2
assert loader_shared.next_page_token == "4"
@@ -580,7 +580,7 @@ class TestListSharedItems:
}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
docs = loader_shared._list_shared_items(page_token="999")
assert len(docs) == 0
assert loader_shared.next_page_token is None
@@ -597,7 +597,7 @@ class TestListSharedItems:
}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
docs = loader_shared._list_shared_items(page_token="invalid", load_content=False)
assert len(docs) == 1
@@ -613,7 +613,7 @@ class TestListSharedItems:
}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
docs = loader_shared._list_shared_items(page_token="-5", load_content=False)
assert len(docs) == 1
@@ -625,7 +625,7 @@ class TestListSharedItems:
mock_response.json.return_value = {"value": []}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
docs = loader_shared._list_shared_items()
assert len(docs) == 0
@@ -639,7 +639,7 @@ class TestListSharedItems:
}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
docs = loader_shared._list_shared_items()
assert len(docs) == 0
@@ -655,7 +655,7 @@ class TestListSharedItems:
}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
docs = loader_shared._list_shared_items(load_content=False)
assert len(docs) == 0
@@ -672,7 +672,7 @@ class TestListSharedItems:
}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
docs = loader_shared._list_shared_items(load_content=True)
assert len(docs) == 1
assert docs[0].text == "content"
@@ -681,7 +681,7 @@ class TestListSharedItems:
def test_exception_returns_empty(self, loader_shared):
loader_shared._get_user_drive_web_url = MagicMock(return_value=None)
with patch("application.parser.connectors.share_point.loader.requests.post", side_effect=Exception("fail")):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", side_effect=Exception("fail")):
docs = loader_shared._list_shared_items()
assert docs == []
@@ -697,7 +697,7 @@ class TestListSharedItems:
}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.post", return_value=mock_response):
loader_shared._list_shared_items(limit=100, load_content=False)
assert loader_shared.next_page_token is None
@@ -710,13 +710,13 @@ class TestGetUserDriveWebUrl:
mock_response.json.return_value = {"webUrl": "https://user.sharepoint.com/drive"}
mock_response.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
url = loader._get_user_drive_web_url()
assert url == "https://user.sharepoint.com/drive"
@pytest.mark.unit
def test_exception_returns_none(self, loader):
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
url = loader._get_user_drive_web_url()
assert url is None
@@ -767,7 +767,7 @@ class TestDownloadToDirectory:
loader._download_folder_recursive = MagicMock(return_value=3)
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
result = loader.download_to_directory(str(tmp_path), {"folder_ids": ["folder1"]})
assert result["files_downloaded"] == 3
@@ -779,7 +779,7 @@ class TestDownloadToDirectory:
loader._download_folder_recursive = MagicMock(return_value=1)
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
result = loader.download_to_directory(str(tmp_path), {"folder_ids": "single_folder"})
assert result["files_downloaded"] == 1
@@ -800,7 +800,7 @@ class TestDownloadToDirectory:
loader._download_file_to_directory = MagicMock(return_value=True)
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
result = loader.download_to_directory(str(tmp_path), {"file_ids": ["f1"], "folder_ids": ["bad"]})
assert result["files_downloaded"] == 1
@@ -819,7 +819,7 @@ class TestDownloadSingleFile:
mock_dl_resp.content = b"pdf content"
mock_dl_resp.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=[mock_meta_resp, mock_dl_resp]):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=[mock_meta_resp, mock_dl_resp]):
result = loader._download_single_file("f1", str(tmp_path))
assert result is True
assert os.path.exists(os.path.join(str(tmp_path), "test.pdf"))
@@ -832,13 +832,13 @@ class TestDownloadSingleFile:
}
mock_resp.raise_for_status = MagicMock()
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_resp):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_resp):
result = loader._download_single_file("f1", str(tmp_path))
assert result is False
@pytest.mark.unit
def test_exception_returns_false(self, loader, tmp_path):
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
result = loader._download_single_file("f1", str(tmp_path))
assert result is False
@@ -857,7 +857,7 @@ class TestDownloadFolderRecursive:
loader._download_single_file = MagicMock(return_value=True)
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
count = loader._download_folder_recursive("folder1", str(tmp_path))
assert count == 1
@@ -885,7 +885,7 @@ class TestDownloadFolderRecursive:
loader._download_single_file = MagicMock(return_value=True)
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect):
count = loader._download_folder_recursive("folder1", str(tmp_path), recursive=True)
assert count == 2
@@ -902,7 +902,7 @@ class TestDownloadFolderRecursive:
loader._download_single_file = MagicMock(return_value=True)
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
count = loader._download_folder_recursive("folder1", str(tmp_path), recursive=False)
assert count == 1
@@ -919,7 +919,7 @@ class TestDownloadFolderRecursive:
loader._download_single_file = MagicMock(side_effect=[False, True])
with patch("application.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", return_value=mock_response):
count = loader._download_folder_recursive("folder1", str(tmp_path))
assert count == 1
@@ -943,13 +943,13 @@ class TestDownloadFolderRecursive:
loader._download_single_file = MagicMock(return_value=True)
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect):
count = loader._download_folder_recursive("folder1", str(tmp_path))
assert count == 2
@pytest.mark.unit
def test_exception_returns_partial(self, loader, tmp_path):
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=Exception("fail")):
count = loader._download_folder_recursive("folder1", str(tmp_path))
assert count == 0
@@ -1006,7 +1006,7 @@ class TestRetryOnAuthFailureDecorator:
loader.auth.refresh_access_token.return_value = {"access_token": "new_at"}
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=get_side_effect):
content = loader._download_file_content("f1")
assert content == "ok"
assert loader.access_token == "new_at"
@@ -1020,7 +1020,7 @@ class TestRetryOnAuthFailureDecorator:
loader.auth.refresh_access_token.side_effect = Exception("refresh fail")
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=http_error):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=http_error):
with pytest.raises(ValueError, match="could not be refreshed"):
loader._download_file_content("f1")
@@ -1031,6 +1031,6 @@ class TestRetryOnAuthFailureDecorator:
mock_resp_500.status_code = 500
http_error = real_requests.exceptions.HTTPError(response=mock_resp_500)
with patch("application.parser.connectors.share_point.loader.requests.get", side_effect=http_error):
with patch("docsgpt.parser.connectors.share_point.loader.requests.get", side_effect=http_error):
with pytest.raises(real_requests.exceptions.HTTPError):
loader._download_file_content("f1")
tests/parser/file/test_anydoc_parser.py
+9 -9
View File
@@ -17,11 +17,11 @@ from pathlib import Path
import pytest
from application.parser.file.base_parser import BaseParser, DocumentParseError
from docsgpt.parser.file.base_parser import BaseParser, DocumentParseError
anydoc = pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import ( # noqa: E402 — after importorskip
from docsgpt.parser.file.anydoc_parser import ( # noqa: E402 — after importorskip
ANYDOC_SUFFIXES,
AnydocParser,
anydoc_available,
@@ -309,7 +309,7 @@ def test_init_parser_imports_for_real_not_just_find_spec(monkeypatch):
"""A wheel whose native extension fails to load has a spec but no importable
module; that must surface at init, not as a bare ImportError from parse_file
mid-ingest (which load_data does not catch)."""
from application.parser.file import anydoc_parser as mod
from docsgpt.parser.file import anydoc_parser as mod
monkeypatch.setattr(mod, "anydoc_available", lambda: True)
monkeypatch.setitem(sys.modules, "anydoc", None)
@@ -332,7 +332,7 @@ class _FakeDoclingFallback:
"""Registered as a DoclingParser subclass so ``_is_docling_backed`` is True."""
def __new__(cls):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
class _Inner(DoclingParser):
def __init__(self):
@@ -374,7 +374,7 @@ def test_trust_flagged_pdf_without_docling_keeps_output_and_warns():
def test_trust_check_disabled_stamps_nothing(monkeypatch):
from application.parser.file import anydoc_parser as ap
from docsgpt.parser.file import anydoc_parser as ap
monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
parser = AnydocParser()
@@ -431,7 +431,7 @@ def test_trust_check_errors_never_fail_the_parse(monkeypatch, tmp_path):
def _explode(path, markdown):
raise RuntimeError("scanner bug")
import application.parser.file.pdf_trust as pt
import docsgpt.parser.file.pdf_trust as pt
monkeypatch.setattr(pt, "verify_pdf_file", _explode)
_fake_anydoc(monkeypatch, lambda path: "# converted fine")
@@ -442,7 +442,7 @@ def test_trust_check_errors_never_fail_the_parse(monkeypatch, tmp_path):
def test_tableize_applied_when_enabled(monkeypatch, tmp_path):
from application.parser.file import anydoc_parser as ap
from docsgpt.parser.file import anydoc_parser as ap
monkeypatch.setattr(ap.settings, "ANYDOC_TABLEIZE", True)
monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
@@ -459,7 +459,7 @@ def test_tableize_applied_when_enabled(monkeypatch, tmp_path):
def test_tableize_disabled_by_default(monkeypatch, tmp_path):
from application.parser.file import anydoc_parser as ap
from docsgpt.parser.file import anydoc_parser as ap
monkeypatch.setattr(ap.settings, "PDF_TRUST_CHECK", False)
flat = "Cash ..... 1,234 900\nDebt ..... 2,000 1,500\nEquity ..... 900 800"
@@ -471,7 +471,7 @@ def test_tableize_disabled_by_default(monkeypatch, tmp_path):
def test_tableize_never_touches_docling_reroute(monkeypatch):
from application.parser.file import anydoc_parser as ap
from docsgpt.parser.file import anydoc_parser as ap
monkeypatch.setattr(ap.settings, "ANYDOC_TABLEIZE", True)
fallback = _FakeDoclingFallback()
tests/parser/file/test_audio_parser.py
+7 -7
View File
@@ -1,8 +1,8 @@
from unittest.mock import MagicMock, patch
from application.parser.file.audio_parser import AudioParser
from application.parser.file.bulk import get_default_file_extractor
from application.stt.upload_limits import AudioFileTooLargeError
from docsgpt.parser.file.audio_parser import AudioParser
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.stt.upload_limits import AudioFileTooLargeError
def test_audio_init_parser():
@@ -13,9 +13,9 @@ def test_audio_init_parser():
assert parser.parser_config_set
@patch("application.stt.upload_limits.settings")
@patch("application.parser.file.audio_parser.STTCreator.create_stt")
@patch("application.parser.file.audio_parser.settings")
@patch("docsgpt.stt.upload_limits.settings")
@patch("docsgpt.parser.file.audio_parser.STTCreator.create_stt")
@patch("docsgpt.parser.file.audio_parser.settings")
def test_audio_parser_transcribes_file(
mock_settings, mock_create_stt, mock_limit_settings, tmp_path
):
@@ -44,7 +44,7 @@ def test_audio_parser_transcribes_file(
)
@patch("application.stt.upload_limits.settings")
@patch("docsgpt.stt.upload_limits.settings")
def test_audio_parser_rejects_oversized_files(mock_limit_settings, tmp_path):
mock_limit_settings.STT_MAX_FILE_SIZE_MB = 1
tests/parser/file/test_bulk.py
+61 -61
View File
@@ -1,4 +1,4 @@
"""Comprehensive tests for application/parser/file/bulk.py
"""Comprehensive tests for docsgpt/parser/file/bulk.py
Covers: SimpleDirectoryReader (init, file discovery, load_data, directory
structure building), get_default_file_extractor.
@@ -9,7 +9,7 @@ from unittest.mock import MagicMock, patch
import pytest
from application.parser.schema.base import Document
from docsgpt.parser.schema.base import Document
# =====================================================================
@@ -47,54 +47,54 @@ def temp_dir_with_types(tmp_path):
class TestSimpleDirectoryReaderInit:
def test_init_with_dir(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(input_dir=str(temp_dir))
assert len(reader.input_files) >= 2
def test_init_with_files(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
files = [str(temp_dir / "file1.md")]
reader = SimpleDirectoryReader(input_files=files)
assert len(reader.input_files) == 1
def test_init_requires_input(self):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
with pytest.raises(ValueError, match="Must provide"):
SimpleDirectoryReader()
def test_exclude_hidden(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(input_dir=str(temp_dir), exclude_hidden=True)
filenames = [f.name for f in reader.input_files]
assert ".hidden" not in filenames
def test_include_hidden(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(input_dir=str(temp_dir), exclude_hidden=False)
filenames = [f.name for f in reader.input_files]
assert ".hidden" in filenames
def test_recursive(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(input_dir=str(temp_dir), recursive=True)
filenames = [f.name for f in reader.input_files]
assert "file3.md" in filenames
def test_non_recursive(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(input_dir=str(temp_dir), recursive=False)
filenames = [f.name for f in reader.input_files]
assert "file3.md" not in filenames
def test_required_exts(self, temp_dir_with_types):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(
input_dir=str(temp_dir_with_types), required_exts=[".md"]
@@ -105,7 +105,7 @@ class TestSimpleDirectoryReaderInit:
assert "notes.txt" not in filenames
def test_required_exts_case_insensitive(self, tmp_path):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "FILE.MD").write_text("content")
reader = SimpleDirectoryReader(
@@ -114,7 +114,7 @@ class TestSimpleDirectoryReaderInit:
assert len(reader.input_files) == 1
def test_num_files_limit(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(
input_dir=str(temp_dir), num_files_limit=1, recursive=False
@@ -122,7 +122,7 @@ class TestSimpleDirectoryReaderInit:
assert len(reader.input_files) <= 1
def test_custom_file_extractor(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
reader = SimpleDirectoryReader(
@@ -141,7 +141,7 @@ class TestSimpleDirectoryReaderInit:
class TestSimpleDirectoryReaderLoadData:
def test_load_data_returns_documents(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = True
@@ -166,8 +166,8 @@ class TestSimpleDirectoryReaderLoadData:
traceback as the document text, so an unguarded loop turns a single
corrupt PDF into a failed ingest for the whole zip/folder/sync.
"""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "good1.md").write_text("first")
(tmp_path / "bad.md").write_text("corrupt")
@@ -203,8 +203,8 @@ class TestSimpleDirectoryReaderLoadData:
Skipping there would hand ``load_data()[0]`` an empty list and turn a
clear "this PDF could not be read" into an IndexError.
"""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "bad.pdf").write_text("not really a pdf")
@@ -224,8 +224,8 @@ class TestSimpleDirectoryReaderLoadData:
def test_all_files_unparseable_raises(self, tmp_path):
"""Nothing parsed is a failed ingest, not an empty success."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "a.md").write_text("x")
(tmp_path / "b.md").write_text("y")
@@ -246,8 +246,8 @@ class TestSimpleDirectoryReaderLoadData:
def test_skipped_file_still_advances_progress(self, tmp_path):
"""Progress must not stall on a skipped file."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "good.md").write_text("ok")
(tmp_path / "bad.md").write_text("bad")
@@ -273,7 +273,7 @@ class TestSimpleDirectoryReaderLoadData:
assert [c[0] for c in calls] == [1, 2]
def test_load_data_progress_callback_fires_per_file(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(
input_dir=str(temp_dir), recursive=False, exclude_hidden=True,
@@ -288,7 +288,7 @@ class TestSimpleDirectoryReaderLoadData:
assert all(c[1] == total_files for c in calls)
def test_load_data_progress_callback_errors_swallowed(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
reader = SimpleDirectoryReader(
input_dir=str(temp_dir), recursive=False, exclude_hidden=True,
@@ -302,7 +302,7 @@ class TestSimpleDirectoryReaderLoadData:
assert len(docs) >= 1
def test_load_data_concatenate(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = True
@@ -319,7 +319,7 @@ class TestSimpleDirectoryReaderLoadData:
assert len(docs) == 1
def test_load_data_with_file_metadata(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
def custom_metadata(filename):
return {"custom_key": f"meta_{filename}"}
@@ -343,7 +343,7 @@ class TestSimpleDirectoryReaderLoadData:
assert "custom_key" in doc.extra_info
def test_load_data_inits_parser_if_not_set(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = False
@@ -360,7 +360,7 @@ class TestSimpleDirectoryReaderLoadData:
mock_parser.init_parser.assert_called()
def test_load_data_standard_read_for_unknown_ext(self, tmp_path):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "file.xyz").write_text("xyz content")
reader = SimpleDirectoryReader(
@@ -372,7 +372,7 @@ class TestSimpleDirectoryReaderLoadData:
assert "xyz content" in docs[0].text
def test_load_data_list_return_from_parser(self, tmp_path):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "multi.md").write_text("content")
mock_parser = MagicMock()
@@ -388,7 +388,7 @@ class TestSimpleDirectoryReaderLoadData:
assert len(docs) == 2
def test_load_data_tracks_token_counts(self, tmp_path):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
(tmp_path / "test.md").write_text("hello world")
mock_parser = MagicMock()
@@ -414,7 +414,7 @@ class TestSimpleDirectoryReaderLoadData:
class TestBuildDirectoryStructure:
def test_builds_structure(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = True
@@ -431,7 +431,7 @@ class TestBuildDirectoryStructure:
assert isinstance(reader.directory_structure, dict)
def test_structure_contains_files_and_dirs(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = True
@@ -453,7 +453,7 @@ class TestBuildDirectoryStructure:
assert "size_bytes" in val
def test_structure_excludes_hidden(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
mock_parser = MagicMock()
mock_parser.parser_config_set = True
@@ -469,7 +469,7 @@ class TestBuildDirectoryStructure:
assert ".hidden" not in reader.directory_structure
def test_no_structure_without_input_dir(self, temp_dir):
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.bulk import SimpleDirectoryReader
files = [str(temp_dir / "file1.md")]
mock_parser = MagicMock()
@@ -494,7 +494,7 @@ class TestBuildDirectoryStructure:
class TestGetDefaultFileExtractor:
def test_returns_dict(self):
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
with patch.dict("sys.modules", {"docling": None, "docling.document_converter": None}):
result = get_default_file_extractor()
@@ -503,7 +503,7 @@ class TestGetDefaultFileExtractor:
def test_fallback_parsers_on_import_error(self):
with patch(
"application.parser.file.bulk.get_default_file_extractor"
"docsgpt.parser.file.bulk.get_default_file_extractor"
) as mock_fn:
mock_fn.return_value = {".pdf": MagicMock(), ".md": MagicMock()}
result = mock_fn()
@@ -521,7 +521,7 @@ class TestParserEngineSwitch:
@pytest.fixture
def settings(self):
from application.core.settings import settings
from docsgpt.core.settings import settings
return settings
@@ -529,16 +529,16 @@ class TestParserEngineSwitch:
def _default_ocr_settings(self, monkeypatch):
"""Pin the OCR knobs to their defaults so a developer's ``.env`` (OCR on,
native backend) cannot change which parsers the maps hand out here."""
from application.core.settings import settings
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "OCR_ENABLED", False)
monkeypatch.setattr(settings, "OCR_BACKEND", "auto")
def test_default_engine_is_anydoc(self, settings, monkeypatch):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import ANYDOC_SUFFIXES, AnydocParser
from application.parser.file.bulk import get_default_file_extractor
from application.parser.file.html_parser import HTMLMarkdownParser
from docsgpt.parser.file.anydoc_parser import ANYDOC_SUFFIXES, AnydocParser
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.html_parser import HTMLMarkdownParser
monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "anydoc")
extractor = get_default_file_extractor()
@@ -555,7 +555,7 @@ class TestParserEngineSwitch:
def test_anydoc_falls_back_to_docling_when_installed(self):
pytest.importorskip("anydoc")
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
extractor = get_default_file_extractor(engine="anydoc")
@@ -568,7 +568,7 @@ class TestParserEngineSwitch:
def test_anydoc_fallback_honours_ocr_flag(self):
pytest.importorskip("anydoc")
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
extractor = get_default_file_extractor(engine="anydoc", ocr_enabled=True)
@@ -579,7 +579,7 @@ class TestParserEngineSwitch:
pytest.importorskip("anydoc")
import sys
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "docling", None)
extractor = get_default_file_extractor(engine="anydoc")
@@ -590,7 +590,7 @@ class TestParserEngineSwitch:
def test_docling_engine_keeps_docling_map(self):
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
extractor = get_default_file_extractor(engine="docling")
@@ -599,22 +599,22 @@ class TestParserEngineSwitch:
def test_setting_selects_docling(self, settings, monkeypatch):
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "docling")
assert type(get_default_file_extractor()[".pdf"]).__name__ == "DoclingPDFParser"
def test_unknown_engine_uses_anydoc(self):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.bulk import get_default_file_extractor
assert isinstance(get_default_file_extractor(engine="ghost")[".pdf"], AnydocParser)
def test_missing_anydoc_degrades_to_base_engine(self, monkeypatch):
import sys
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "anydoc", None)
extractor = get_default_file_extractor(engine="anydoc")
@@ -628,7 +628,7 @@ class TestParserEngineSwitch:
the first file aborted the whole ingest instead of degrading."""
import sys
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "docling", None)
extractor = get_default_file_extractor(engine="docling")
@@ -655,8 +655,8 @@ class TestGainedFormats:
(`.txt` is the one deliberate plain-text read.)
"""
pytest.importorskip("anydoc")
from application.parser.file.bulk import get_default_file_extractor
from application.parser.file.constants import (
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.constants import (
SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS,
)
@@ -671,11 +671,11 @@ class TestGainedFormats:
def test_gained_formats_map_to_anydoc_under_both_engines(self):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import (
from docsgpt.parser.file.anydoc_parser import (
ANYDOC_GAINED_SUFFIXES,
AnydocParser,
)
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
for engine in ("anydoc", "docling"):
extractor = get_default_file_extractor(engine=engine)
@@ -684,8 +684,8 @@ class TestGainedFormats:
def test_gained_formats_never_fall_back_to_anydoc_itself(self):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.bulk import get_default_file_extractor
extractor = get_default_file_extractor(engine="anydoc")
assert extractor[".doc"].fallback_parser is None
@@ -697,7 +697,7 @@ class TestGainedFormats:
def test_gained_entries_absent_without_anydoc(self, monkeypatch):
import sys
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "anydoc", None)
extractor = get_default_file_extractor(engine="docling")
@@ -705,7 +705,7 @@ class TestGainedFormats:
def test_rtf_converts_end_to_end(self, tmp_path):
pytest.importorskip("anydoc")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
path = tmp_path / "note.rtf"
path.write_text(r"{\rtf1\ansi Hello {\b bold} world.\par Second paragraph.}")
@@ -721,7 +721,7 @@ class TestGainedFormats:
pytest.importorskip("anydoc")
import zipfile
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
path = tmp_path / "doc.odt"
with zipfile.ZipFile(path, "w") as z:
@@ -753,8 +753,8 @@ class TestGainedFormats:
def test_gained_suffix_without_a_parser_is_rejected_not_read_as_text(tmp_path):
"""Without anydoc an OLE .doc must not be indexed as decoded binary garbage."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.bulk import SimpleDirectoryReader
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.bulk import SimpleDirectoryReader
path = tmp_path / "legacy.doc"
path.write_bytes(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 64)
tests/parser/file/test_constants.py
+2 -2
View File
@@ -6,8 +6,8 @@ from pathlib import Path
import pytest
from application.parser.file.bulk import get_default_file_extractor
from application.parser.file.constants import (
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.constants import (
ATTACHMENT_PARSER_EXTENSIONS,
attachment_extension,
has_attachment_parser,
tests/parser/file/test_docling_parser.py
+150 -150
View File
@@ -1,4 +1,4 @@
"""Comprehensive tests for application/parser/file/docling_parser.py
"""Comprehensive tests for docsgpt/parser/file/docling_parser.py
Covers: DoclingParser (init, _init_parser, OCR engine selection, _export_content,
parse_file), subclass initialization, error handling.
@@ -21,7 +21,7 @@ import pytest
class TestDoclingParserInit:
def test_default_init(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
assert parser.ocr_enabled is True
@@ -33,7 +33,7 @@ class TestDoclingParserInit:
assert parser._converter is None
def test_custom_init(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(
ocr_enabled=False,
@@ -60,7 +60,7 @@ class TestDoclingParserInit:
class TestDoclingParserInitParser:
def test_init_parser_raises_without_docling(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
@@ -72,7 +72,7 @@ class TestDoclingParserInitParser:
"""A missing parent package makes find_spec raise; the hint must still show."""
import sys
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
for name in [m for m in sys.modules if m == "docling" or m.startswith("docling.")]:
monkeypatch.delitem(sys.modules, name)
@@ -82,7 +82,7 @@ class TestDoclingParserInitParser:
DoclingParser()._init_parser()
def test_init_parser_success(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
@@ -108,14 +108,14 @@ class TestOcrEngineSelection:
@pytest.fixture
def settings(self):
from application.core.settings import settings
from docsgpt.core.settings import settings
return settings
def test_default_setting_is_tesseract(self):
# Field defaults, not the live settings: a developer's ``.env`` may
# legitimately set other engines/languages.
from application.core.settings import Settings
from docsgpt.core.settings import Settings
defaults = Settings.model_construct()
assert defaults.OCR_ENGINE == "tesseract"
@@ -123,30 +123,30 @@ class TestOcrEngineSelection:
assert defaults.OCR_BACKEND == "auto"
def test_none_reads_setting(self, settings, monkeypatch):
from application.parser.file.docling_parser import _resolve_ocr_engine
from docsgpt.parser.file.docling_parser import _resolve_ocr_engine
monkeypatch.setattr(settings, "OCR_ENGINE", "auto")
assert _resolve_ocr_engine(None) == "auto"
def test_unknown_engine_degrades_to_auto(self):
from application.parser.file.docling_parser import _resolve_ocr_engine
from docsgpt.parser.file.docling_parser import _resolve_ocr_engine
assert _resolve_ocr_engine("easyocr") == "auto"
def test_tesseract_without_binary_degrades_to_auto(self, monkeypatch):
from application.parser.file import docling_parser as dp
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(dp.shutil, "which", lambda name: None)
assert dp._resolve_ocr_engine("tesseract") == "auto"
def test_tesseract_with_binary_selected(self, monkeypatch):
from application.parser.file import docling_parser as dp
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(dp.shutil, "which", lambda name: "/usr/bin/tesseract")
assert dp._resolve_ocr_engine("tesseract") == "tesseract"
def test_ocrmac_off_darwin_degrades_to_auto(self, monkeypatch):
from application.parser.file import docling_parser as dp
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(dp.sys, "platform", "linux")
assert dp._resolve_ocr_engine("ocrmac") == "auto"
@@ -154,25 +154,25 @@ class TestOcrEngineSelection:
def test_rapidocr_missing_degrades_to_auto(self, monkeypatch):
import sys
from application.parser.file import docling_parser as dp
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setitem(sys.modules, "rapidocr", None)
assert dp._resolve_ocr_engine("rapidocr") == "auto"
def test_deepseek_passes_through(self):
from application.parser.file.docling_parser import _resolve_ocr_engine
from docsgpt.parser.file.docling_parser import _resolve_ocr_engine
assert _resolve_ocr_engine("deepseek") == "deepseek"
def test_build_auto_returns_none(self):
from application.parser.file.docling_parser import _build_ocr_options
from docsgpt.parser.file.docling_parser import _build_ocr_options
assert _build_ocr_options("auto", None, True) is None
def test_build_tesseract_reads_ocr_langs(self, settings, monkeypatch):
pytest.importorskip("docling")
import application.parser.file.ocr_parser as op
from application.parser.file.docling_parser import _build_ocr_options
import docsgpt.parser.file.ocr_parser as op
from docsgpt.parser.file.docling_parser import _build_ocr_options
# Pack inventory unknown: the resolved list is passed through untouched
# (a host with tesseract but no chi_sim pack would otherwise drop it).
@@ -186,8 +186,8 @@ class TestOcrEngineSelection:
def test_build_tesseract_explicit_languages_win(self, monkeypatch):
pytest.importorskip("docling")
import application.parser.file.ocr_parser as op
from application.parser.file.docling_parser import _build_ocr_options
import docsgpt.parser.file.ocr_parser as op
from docsgpt.parser.file.docling_parser import _build_ocr_options
# Pack inventory unknown: the requested list is passed through untouched.
monkeypatch.setattr(op, "tesseract_languages", lambda: None)
@@ -196,7 +196,7 @@ class TestOcrEngineSelection:
def test_build_rapidocr(self):
pytest.importorskip("docling")
from application.parser.file.docling_parser import _build_ocr_options
from docsgpt.parser.file.docling_parser import _build_ocr_options
options = _build_ocr_options("rapidocr", None, False)
assert type(options).__name__ == "RapidOcrOptions"
@@ -205,7 +205,7 @@ class TestOcrEngineSelection:
def test_build_import_failure_returns_none(self, monkeypatch):
import sys
from application.parser.file.docling_parser import _build_ocr_options
from docsgpt.parser.file.docling_parser import _build_ocr_options
monkeypatch.setitem(sys.modules, "docling.datamodel.pipeline_options", None)
assert _build_ocr_options("tesseract", ["eng"], False) is None
@@ -215,7 +215,7 @@ class TestOcrEngineSelection:
import sys
import types
from application.parser.file.docling_parser import _build_ocr_options
from docsgpt.parser.file.docling_parser import _build_ocr_options
fake = types.ModuleType("docling.datamodel.pipeline_options")
@@ -236,8 +236,8 @@ class TestDeepseekVlmConverter:
pytest.importorskip("docling")
def test_deepseek_builds_vlm_converter(self, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingParser
monkeypatch.setattr(settings, "OCR_DEEPSEEK_URL", "http://gpu-host:8000/v1/chat/completions")
monkeypatch.setattr(settings, "OCR_DEEPSEEK_MODEL", "deepseek-ocr-x")
@@ -267,7 +267,7 @@ class TestDeepseekVlmConverter:
assert pdf_option.pipeline_options.enable_remote_services is True
def test_deepseek_ignored_when_ocr_disabled(self, monkeypatch):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
vlm_called = []
monkeypatch.setattr(
@@ -290,7 +290,7 @@ class TestDeepseekVlmConverter:
class TestExportContent:
def test_export_markdown(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="markdown")
mock_doc = MagicMock()
@@ -302,7 +302,7 @@ class TestExportContent:
mock_doc.export_to_markdown.assert_called_once()
def test_export_html(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="html")
mock_doc = MagicMock()
@@ -313,7 +313,7 @@ class TestExportContent:
assert "<h1>" in result
def test_export_text(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="text")
mock_doc = MagicMock()
@@ -324,7 +324,7 @@ class TestExportContent:
assert "Plain text" in result
def test_fallback_to_texts_on_minimal_content(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="markdown")
mock_doc = MagicMock()
@@ -341,7 +341,7 @@ class TestExportContent:
assert "OCR extracted text 2" in result
def test_no_fallback_for_substantial_content(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="markdown")
mock_doc = MagicMock()
@@ -352,7 +352,7 @@ class TestExportContent:
assert result == "A" * 100
def test_fallback_skipped_when_no_texts(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="markdown")
mock_doc = MagicMock()
@@ -363,7 +363,7 @@ class TestExportContent:
assert result == "short"
def test_fallback_skips_empty_texts(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser(export_format="markdown")
mock_doc = MagicMock()
@@ -386,7 +386,7 @@ class TestExportContent:
class TestDoclingParserParseFile:
def test_parse_file_success(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
@@ -403,7 +403,7 @@ class TestDoclingParserParseFile:
assert "Parsed document content" in result
def test_parse_file_inits_converter_on_first_call(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
parser._converter = None
@@ -433,8 +433,8 @@ class TestDoclingParserParseFile:
as if it were the PDF. ``errors`` controls *decoding* leniency, not
"substitute the traceback for the document".
"""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
mock_converter = MagicMock()
@@ -453,8 +453,8 @@ class TestDoclingParserParseFile:
def test_parse_file_error_ignore_never_returns_a_string(self):
"""Belt-and-braces: no code path may hand back error text as content."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
mock_converter = MagicMock()
@@ -474,7 +474,7 @@ class TestDoclingParserParseFile:
)
def test_parse_file_error_raise(self):
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
parser = DoclingParser()
mock_converter = MagicMock()
@@ -494,82 +494,82 @@ class TestDoclingParserParseFile:
class TestDoclingSubclasses:
def test_pdf_parser_init(self):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser()
assert parser.ocr_enabled is True
assert parser.export_format == "markdown"
def test_pdf_parser_custom_ocr(self):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=False, force_full_page_ocr=True)
assert parser.ocr_enabled is False
assert parser.force_full_page_ocr is True
def test_docx_parser_init(self):
from application.parser.file.docling_parser import DoclingDocxParser
from docsgpt.parser.file.docling_parser import DoclingDocxParser
parser = DoclingDocxParser()
assert parser.export_format == "markdown"
def test_pptx_parser_init(self):
from application.parser.file.docling_parser import DoclingPPTXParser
from docsgpt.parser.file.docling_parser import DoclingPPTXParser
parser = DoclingPPTXParser()
assert parser.export_format == "markdown"
def test_xlsx_parser_init(self):
from application.parser.file.docling_parser import DoclingXLSXParser
from docsgpt.parser.file.docling_parser import DoclingXLSXParser
parser = DoclingXLSXParser()
assert parser.table_structure is True
def test_html_parser_init(self):
from application.parser.file.docling_parser import DoclingHTMLParser
from docsgpt.parser.file.docling_parser import DoclingHTMLParser
parser = DoclingHTMLParser()
assert parser.export_format == "markdown"
def test_image_parser_init(self):
from application.parser.file.docling_parser import DoclingImageParser
from docsgpt.parser.file.docling_parser import DoclingImageParser
parser = DoclingImageParser()
assert parser.ocr_enabled is True
assert parser.force_full_page_ocr is True
def test_image_parser_custom(self):
from application.parser.file.docling_parser import DoclingImageParser
from docsgpt.parser.file.docling_parser import DoclingImageParser
parser = DoclingImageParser(ocr_enabled=False)
assert parser.ocr_enabled is False
def test_csv_parser_init(self):
from application.parser.file.docling_parser import DoclingCSVParser
from docsgpt.parser.file.docling_parser import DoclingCSVParser
parser = DoclingCSVParser()
assert parser.table_structure is True
def test_markdown_parser_init(self):
from application.parser.file.docling_parser import DoclingMarkdownParser
from docsgpt.parser.file.docling_parser import DoclingMarkdownParser
parser = DoclingMarkdownParser()
assert parser.export_format == "markdown"
def test_asciidoc_parser_init(self):
from application.parser.file.docling_parser import DoclingAsciiDocParser
from docsgpt.parser.file.docling_parser import DoclingAsciiDocParser
parser = DoclingAsciiDocParser()
assert parser.export_format == "markdown"
def test_vtt_parser_init(self):
from application.parser.file.docling_parser import DoclingVTTParser
from docsgpt.parser.file.docling_parser import DoclingVTTParser
parser = DoclingVTTParser()
assert parser.export_format == "markdown"
def test_xml_parser_init(self):
from application.parser.file.docling_parser import DoclingXMLParser
from docsgpt.parser.file.docling_parser import DoclingXMLParser
parser = DoclingXMLParser()
assert parser.export_format == "markdown"
@@ -584,7 +584,7 @@ class TestDoclingSubclasses:
class TestDoclingParserGaps:
def test_csv_parser_init(self):
"""Cover line 289: DoclingCSVParser.__init__ calls super."""
from application.parser.file.docling_parser import DoclingCSVParser
from docsgpt.parser.file.docling_parser import DoclingCSVParser
parser = DoclingCSVParser()
assert parser.export_format == "markdown"
@@ -611,13 +611,13 @@ class TestNonOcrParsersLeaveTextAlone:
],
)
def test_ocr_is_off_by_construction(self, name):
from application.parser.file import docling_parser as dp
from docsgpt.parser.file import docling_parser as dp
assert getattr(dp, name)().ocr_enabled is False
def test_cjk_spaces_survive_in_a_docx_export(self, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingDocxParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingDocxParser
monkeypatch.setattr(settings, "OCR_ENGINE", "tesseract")
parser = DoclingDocxParser()
@@ -628,7 +628,7 @@ class TestNonOcrParsersLeaveTextAlone:
assert parser._postprocess_ocr_text(text) == text
def test_pdf_parser_with_tesseract_still_collapses(self):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
parser._active_ocr_engine = "tesseract"
@@ -645,8 +645,8 @@ class TestApplyPipelineCaps:
"""_apply_pipeline_caps bounds docling's threaded-pipeline buffering."""
def test_caps_threaded_pipeline_knobs(self, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import _apply_pipeline_caps
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
monkeypatch.setattr(
settings, "DOCLING_PIPELINE_QUEUE_MAX_SIZE", 2, raising=False
@@ -668,8 +668,8 @@ class TestApplyPipelineCaps:
assert opts.ocr_batch_size == 1
def test_queue_size_is_settings_driven(self, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import _apply_pipeline_caps
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
monkeypatch.setattr(
settings, "DOCLING_PIPELINE_QUEUE_MAX_SIZE", 6, raising=False
@@ -684,8 +684,8 @@ class TestApplyPipelineCaps:
def test_misconfigured_zero_floors_to_one(self, monkeypatch):
"""A 0 queue depth could deadlock the threaded pipeline — floor it."""
from application.core.settings import settings
from application.parser.file.docling_parser import _apply_pipeline_caps
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
monkeypatch.setattr(
settings, "DOCLING_PIPELINE_QUEUE_MAX_SIZE", 0, raising=False
@@ -701,7 +701,7 @@ class TestApplyPipelineCaps:
def test_noop_on_docling_without_threaded_pipeline(self):
"""Builds predating the threaded pipeline lack the knobs — the cap
must be a silent no-op, not an AttributeError."""
from application.parser.file.docling_parser import _apply_pipeline_caps
from docsgpt.parser.file.docling_parser import _apply_pipeline_caps
class LegacyOpts:
__slots__ = ("do_ocr", "do_table_structure")
@@ -738,8 +738,8 @@ class TestDoclingTabularSizeGate:
return path
def test_oversized_csv_delegates_to_plain_csv_parser(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingCSVParser, DoclingParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
docling_parse = MagicMock(name="docling_parse")
@@ -755,8 +755,8 @@ class TestDoclingTabularSizeGate:
assert out.startswith("0, 0\n1, 2\n")
def test_small_csv_still_uses_docling(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingCSVParser, DoclingParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 10_000_000)
docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
@@ -768,8 +768,8 @@ class TestDoclingTabularSizeGate:
docling_parse.assert_called_once()
def test_gate_disabled_when_max_bytes_is_zero(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingCSVParser, DoclingParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingCSVParser, DoclingParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 0)
docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
@@ -781,9 +781,9 @@ class TestDoclingTabularSizeGate:
docling_parse.assert_called_once()
def test_oversized_xlsx_delegates_to_excel_parser(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
from application.parser.file.tabular_parser import ExcelParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
from docsgpt.parser.file.tabular_parser import ExcelParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
docling_parse = MagicMock(name="docling_parse")
@@ -801,8 +801,8 @@ class TestDoclingTabularSizeGate:
excel_parse.assert_called_once()
def test_small_xlsx_still_uses_docling(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import DoclingParser, DoclingXLSXParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 10_000_000)
docling_parse = MagicMock(name="docling_parse", return_value="DOCLING")
@@ -837,7 +837,7 @@ class TestTabularContentSize:
(2.44 GB in docling) slips under a byte gate."""
def test_xlsx_content_size_is_inner_not_ondisk(self, tmp_path):
from application.parser.file.docling_parser import _tabular_content_size
from docsgpt.parser.file.docling_parser import _tabular_content_size
path = tmp_path / "data.xlsx"
_make_xlsx(path, rows=5000)
@@ -847,7 +847,7 @@ class TestTabularContentSize:
assert inner > on_disk
def test_csv_content_size_is_ondisk(self, tmp_path):
from application.parser.file.docling_parser import _tabular_content_size
from docsgpt.parser.file.docling_parser import _tabular_content_size
path = tmp_path / "data.csv"
path.write_text("a,b\n1,2\n3,4\n")
@@ -855,13 +855,13 @@ class TestTabularContentSize:
def test_compressed_xlsx_over_inner_gate_delegates(self, tmp_path, monkeypatch):
"""The regression: on-disk < threshold < inner-uncompressed must gate."""
from application.core.settings import settings
from application.parser.file.docling_parser import (
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import (
DoclingParser,
DoclingXLSXParser,
_tabular_content_size,
)
from application.parser.file.tabular_parser import ExcelParser
from docsgpt.parser.file.tabular_parser import ExcelParser
path = tmp_path / "wide.xlsx"
_make_xlsx(path, rows=5000)
@@ -901,8 +901,8 @@ class TestDoclingMarkupGate:
@pytest.mark.parametrize("name", ["big.html", "big.vtt"])
def test_oversized_markup_parses_truncated_copy(self, tmp_path, monkeypatch, name):
from application.core.settings import settings
from application.parser.file import docling_parser as dp
from docsgpt.core.settings import settings
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 512)
path = self._write(tmp_path, name, 4096)
@@ -928,8 +928,8 @@ class TestDoclingMarkupGate:
assert path.stat().st_size > 512, "original must be untouched"
def test_small_markup_parses_original(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file import docling_parser as dp
from docsgpt.core.settings import settings
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 10_000_000)
path = self._write(tmp_path, "small.html", 1024)
@@ -946,8 +946,8 @@ class TestDoclingMarkupGate:
assert seen["path"] == str(path)
def test_markup_gate_disabled_when_zero(self, tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file import docling_parser as dp
from docsgpt.core.settings import settings
from docsgpt.parser.file import docling_parser as dp
monkeypatch.setattr(settings, "DOCLING_MARKUP_MAX_BYTES", 0)
path = self._write(tmp_path, "big.vtt", 8192)
@@ -994,8 +994,8 @@ class TestTabularGateSeam:
self, tmp_path, monkeypatch
):
"""The real incident, end to end: gate trips, blanks do not crash."""
from application.core.settings import settings
from application.parser.file.docling_parser import (
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import (
DoclingParser,
DoclingXLSXParser,
)
@@ -1021,13 +1021,13 @@ class TestTabularGateSeam:
``ingest`` and ``store_attachment``, and is the only exception
``SimpleDirectoryReader.load_data`` skips rather than propagating.
"""
from application.core.settings import settings
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import (
from docsgpt.core.settings import settings
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import (
DoclingParser,
DoclingXLSXParser,
)
from application.parser.file.tabular_parser import ExcelParser
from docsgpt.parser.file.tabular_parser import ExcelParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 64)
monkeypatch.setattr(DoclingParser, "parse_file", MagicMock())
@@ -1045,13 +1045,13 @@ class TestTabularGateSeam:
def test_oversized_csv_failure_becomes_document_parse_error(
self, tmp_path, monkeypatch
):
from application.core.settings import settings
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import (
from docsgpt.core.settings import settings
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import (
DoclingCSVParser,
DoclingParser,
)
from application.parser.file.tabular_parser import CSVParser
from docsgpt.parser.file.tabular_parser import CSVParser
monkeypatch.setattr(settings, "DOCLING_TABULAR_MAX_BYTES", 8)
monkeypatch.setattr(DoclingParser, "parse_file", MagicMock())
@@ -1087,7 +1087,7 @@ class TestApplyInferenceSettings:
pytest.importorskip("docling")
def test_disables_torch_compile_by_default(self, monkeypatch):
from application.parser.file.docling_parser import _apply_inference_settings
from docsgpt.parser.file.docling_parser import _apply_inference_settings
class Inference:
compile_torch_models = True
@@ -1105,8 +1105,8 @@ class TestApplyInferenceSettings:
assert docling_settings.inference.compile_torch_models is False
def test_opt_in_reenables_torch_compile(self, monkeypatch):
from application.core.settings import settings
from application.parser.file.docling_parser import _apply_inference_settings
from docsgpt.core.settings import settings
from docsgpt.parser.file.docling_parser import _apply_inference_settings
monkeypatch.setattr(
settings, "DOCLING_COMPILE_TORCH_MODELS", True, raising=False
@@ -1129,7 +1129,7 @@ class TestApplyInferenceSettings:
def test_noop_on_docling_without_inference_settings(self, monkeypatch):
"""Builds predating the inference settings must be a silent no-op."""
from application.parser.file.docling_parser import _apply_inference_settings
from docsgpt.parser.file.docling_parser import _apply_inference_settings
class DoclingSettings:
pass
@@ -1142,11 +1142,11 @@ class TestApplyInferenceSettings:
def test_create_converter_applies_inference_settings(self, monkeypatch):
"""The cap is worthless unless the converter path actually calls it."""
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
called = []
monkeypatch.setattr(
"application.parser.file.docling_parser._apply_inference_settings",
"docsgpt.parser.file.docling_parser._apply_inference_settings",
lambda: called.append(True),
)
monkeypatch.setattr(
@@ -1167,11 +1167,11 @@ class TestApplyInferenceSettings:
"""
import docling.datamodel.pipeline_options as dpo
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
events = []
monkeypatch.setattr(
"application.parser.file.docling_parser._apply_inference_settings",
"docsgpt.parser.file.docling_parser._apply_inference_settings",
lambda: events.append("settings"),
)
@@ -1210,7 +1210,7 @@ def _mock_conversion(markdown: str, pages: int = 1) -> MagicMock:
def _set_threshold(monkeypatch, value: int) -> None:
"""Point the OCR dropout guard at a specific chars-per-page floor."""
from application.core.settings import settings as real_settings
from docsgpt.core.settings import settings as real_settings
class _Stub:
def __getattr__(self, name):
@@ -1218,7 +1218,7 @@ def _set_threshold(monkeypatch, value: int) -> None:
stub = _Stub()
stub.OCR_MIN_CHARS_PER_PAGE = value
monkeypatch.setattr("application.core.settings.settings", stub)
monkeypatch.setattr("docsgpt.core.settings.settings", stub)
@pytest.mark.unit
@@ -1232,7 +1232,7 @@ class TestOCRDropoutGuard:
"""
def test_near_empty_first_pass_recovers_on_fresh_converter(self, caplog):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
degraded = MagicMock()
@@ -1274,8 +1274,8 @@ class TestOCRDropoutGuard:
assert "Recovered scan.pdf on retry" in messages
def test_near_empty_both_passes_raises_document_parse_error(self):
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
degraded = MagicMock()
@@ -1301,11 +1301,11 @@ class TestOCRDropoutGuard:
def test_dropout_error_reports_an_unread_text_layer(self, monkeypatch):
"""A text layer docling ignored points at the pipeline, not the scan."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
monkeypatch.setattr(
"application.parser.file.docling_parser._pdf_text_layer_probe",
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
lambda file: (12, 48_000),
)
parser = DoclingPDFParser(ocr_enabled=True)
@@ -1319,7 +1319,7 @@ class TestOCRDropoutGuard:
assert "The PDF carries a 48000-char text layer" in str(excinfo.value)
def test_ocr_disabled_returns_near_empty_content_untouched(self):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=False)
converter = MagicMock()
@@ -1334,7 +1334,7 @@ class TestOCRDropoutGuard:
assert parser._converter is converter
def test_threshold_zero_disables_the_guard(self, monkeypatch):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
_set_threshold(monkeypatch, 0)
parser = DoclingPDFParser(ocr_enabled=True)
@@ -1349,7 +1349,7 @@ class TestOCRDropoutGuard:
assert parser._converter is converter
def test_healthy_ocr_parse_converts_once(self):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
converter = MagicMock()
@@ -1368,7 +1368,7 @@ class TestOCRDropoutGuard:
def test_threshold_is_per_page_not_per_document(self):
"""200 chars is healthy for one page and suspicious for a hundred."""
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
content = "x" * 200
@@ -1388,7 +1388,7 @@ class TestOCRDropoutGuard:
with patch.object(
long_doc, "_create_converter", return_value=long_doc._converter
) as create, patch(
"application.parser.file.docling_parser._pdf_text_layer_probe",
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
return_value=(100, 0),
):
assert long_doc.parse_file(Path("hundred-page.pdf")) == content
@@ -1396,7 +1396,7 @@ class TestOCRDropoutGuard:
def test_guard_does_not_apply_to_non_ocr_formats(self):
"""DOCX parsers construct with OCR off: they never OCR anything."""
from application.parser.file.docling_parser import DoclingDocxParser
from docsgpt.parser.file.docling_parser import DoclingDocxParser
parser = DoclingDocxParser()
assert parser.ocr_enabled is False
@@ -1411,7 +1411,7 @@ class TestOCRDropoutGuard:
def test_guard_applies_to_images(self):
"""The retry covers images -- a degraded converter may still recover one."""
from application.parser.file.docling_parser import DoclingImageParser
from docsgpt.parser.file.docling_parser import DoclingImageParser
parser = DoclingImageParser(ocr_enabled=True)
degraded = MagicMock()
@@ -1432,7 +1432,7 @@ class TestOCRDropoutGuard:
DocumentParseError is in ``dont_autoretry_for``, so raising here fails a
single-file upload permanently.
"""
from application.parser.file.docling_parser import DoclingImageParser
from docsgpt.parser.file.docling_parser import DoclingImageParser
parser = DoclingImageParser(ocr_enabled=True)
converter = MagicMock()
@@ -1447,7 +1447,7 @@ class TestOCRDropoutGuard:
def test_a_text_sparse_scan_indexes_rather_than_failing_the_upload(self):
"""20 pages of pictures with a few captions is the document, not a fault."""
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
content = "Fig 1. Fig 2. Fig 3."
parser = DoclingPDFParser(ocr_enabled=True)
@@ -1458,15 +1458,15 @@ class TestOCRDropoutGuard:
with patch.object(
parser, "_create_converter", return_value=converter
), patch(
"application.parser.file.docling_parser._pdf_text_layer_probe",
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
return_value=(20, 0),
):
assert parser.parse_file(Path("catalog.pdf")) == content
def test_a_multi_page_zero_char_parse_is_still_a_dropout(self):
"""The incident this guard exists for: every page OCR'd to nothing."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
converter = MagicMock()
@@ -1476,7 +1476,7 @@ class TestOCRDropoutGuard:
with patch.object(
parser, "_create_converter", return_value=converter
), patch(
"application.parser.file.docling_parser._pdf_text_layer_probe",
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
return_value=(40, 0),
):
with pytest.raises(DocumentParseError, match="scan.pdf"):
@@ -1484,8 +1484,8 @@ class TestOCRDropoutGuard:
def test_a_pdf_whose_text_layer_was_missed_is_still_a_dropout(self):
"""Text docling should have read without OCR at all -- positive evidence."""
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
converter = MagicMock()
@@ -1495,15 +1495,15 @@ class TestOCRDropoutGuard:
with patch.object(
parser, "_create_converter", return_value=converter
), patch(
"application.parser.file.docling_parser._pdf_text_layer_probe",
"docsgpt.parser.file.docling_parser._pdf_text_layer_probe",
return_value=(20, 5000),
):
with pytest.raises(DocumentParseError, match="text layer"):
parser.parse_file(Path("report.pdf"))
def test_retry_conversion_failure_is_still_a_parse_error(self):
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
parser._converter = MagicMock()
@@ -1524,7 +1524,7 @@ class TestOCRDropoutGuard:
class TestOCRDropoutHelpers:
def test_image_placeholders_are_not_text(self):
from application.parser.file.docling_parser import _text_char_count
from docsgpt.parser.file.docling_parser import _text_char_count
assert _text_char_count(None) == 0
assert _text_char_count("") == 0
@@ -1532,20 +1532,20 @@ class TestOCRDropoutHelpers:
assert _text_char_count(" <!-- image --> abc ") == 3
def test_page_count_prefers_document_pages(self):
from application.parser.file.docling_parser import _result_page_count
from docsgpt.parser.file.docling_parser import _result_page_count
result = _mock_conversion("x", pages=7)
assert _result_page_count(result, Path("a.pdf")) == 7
def test_page_count_falls_back_to_num_pages(self):
from application.parser.file.docling_parser import _result_page_count
from docsgpt.parser.file.docling_parser import _result_page_count
result = _mock_conversion("x", pages=0)
result.document.num_pages.return_value = 5
assert _result_page_count(result, Path("a.pdf")) == 5
def test_page_count_falls_back_to_result_pages(self):
from application.parser.file.docling_parser import _result_page_count
from docsgpt.parser.file.docling_parser import _result_page_count
result = _mock_conversion("x", pages=0)
result.document.num_pages.return_value = None
@@ -1553,7 +1553,7 @@ class TestOCRDropoutHelpers:
assert _result_page_count(result, Path("a.pdf")) == 2
def test_page_count_defaults_to_one(self):
from application.parser.file.docling_parser import _result_page_count
from docsgpt.parser.file.docling_parser import _result_page_count
result = _mock_conversion("x", pages=0)
result.document.num_pages.return_value = None
@@ -1561,13 +1561,13 @@ class TestOCRDropoutHelpers:
assert _result_page_count(result, Path("a.pdf")) == 1
def test_images_are_always_one_page(self):
from application.parser.file.docling_parser import _result_page_count
from docsgpt.parser.file.docling_parser import _result_page_count
result = _mock_conversion("x", pages=9)
assert _result_page_count(result, Path("a.png")) == 1
def test_threshold_reads_settings_with_a_default(self, monkeypatch):
from application.parser.file.docling_parser import _ocr_min_chars_per_page
from docsgpt.parser.file.docling_parser import _ocr_min_chars_per_page
assert _ocr_min_chars_per_page() == 20
_set_threshold(monkeypatch, 5)
@@ -1576,7 +1576,7 @@ class TestOCRDropoutHelpers:
assert _ocr_min_chars_per_page() == 20
def test_text_layer_probe_survives_an_unreadable_pdf(self, tmp_path):
from application.parser.file.docling_parser import _pdf_text_layer_probe
from docsgpt.parser.file.docling_parser import _pdf_text_layer_probe
assert _pdf_text_layer_probe(tmp_path / "missing.pdf") == (0, 0)
broken = tmp_path / "broken.pdf"
@@ -1597,7 +1597,7 @@ class TestForceFullPageOCRWiring:
def _pipeline_options(self, monkeypatch, **kwargs):
import docling.datamodel.pipeline_options as dpo
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
built = []
real_options = dpo.PdfPipelineOptions
@@ -1636,7 +1636,7 @@ class TestDoclingTesseractPostprocessing:
"""docling's tesseract path gets the same CJK glyph-space cleanup as the native engine."""
def _parser_with_export(self, engine, ocr_enabled=True):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=ocr_enabled)
parser._active_ocr_engine = engine
@@ -1660,7 +1660,7 @@ class TestDoclingTesseractPostprocessing:
@pytest.mark.unit
class TestDoclingOcrPages:
def test_converts_each_requested_page_with_page_range(self, tmp_path):
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
converter = MagicMock()
@@ -1683,8 +1683,8 @@ class TestDoclingOcrPages:
assert [c.kwargs["page_range"] for c in converter.convert.call_args_list] == [(3, 3), (1, 1)]
def test_failure_is_a_parse_error(self, tmp_path):
from application.parser.file.base_parser import DocumentParseError
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.parser.file.docling_parser import DoclingPDFParser
parser = DoclingPDFParser(ocr_enabled=True)
parser._converter = MagicMock()
@@ -1699,8 +1699,8 @@ class TestDoclingOcrPages:
class TestTesseractLanguageFilter:
def test_uninstalled_packs_are_dropped_with_a_warning(self, monkeypatch, caplog):
pytest.importorskip("docling")
import application.parser.file.ocr_parser as op
from application.parser.file.docling_parser import _build_ocr_options
import docsgpt.parser.file.ocr_parser as op
from docsgpt.parser.file.docling_parser import _build_ocr_options
monkeypatch.setattr(op, "tesseract_languages", lambda: frozenset({"eng", "osd"}))
with caplog.at_level("WARNING"):
@@ -1710,16 +1710,16 @@ class TestTesseractLanguageFilter:
def test_all_packs_missing_falls_back_to_eng(self, monkeypatch):
pytest.importorskip("docling")
import application.parser.file.ocr_parser as op
from application.parser.file.docling_parser import _build_ocr_options
import docsgpt.parser.file.ocr_parser as op
from docsgpt.parser.file.docling_parser import _build_ocr_options
monkeypatch.setattr(op, "tesseract_languages", lambda: frozenset({"eng"}))
assert _build_ocr_options("tesseract", ["xyz"], False).lang == ["eng"]
def test_unknown_inventory_keeps_the_list(self, monkeypatch):
pytest.importorskip("docling")
import application.parser.file.ocr_parser as op
from application.parser.file.docling_parser import _build_ocr_options
import docsgpt.parser.file.ocr_parser as op
from docsgpt.parser.file.docling_parser import _build_ocr_options
monkeypatch.setattr(op, "tesseract_languages", lambda: None)
assert _build_ocr_options("tesseract", ["eng", "deu"], False).lang == ["eng", "deu"]
tests/parser/file/test_docs_parser.py
+12 -12
View File
@@ -1,4 +1,4 @@
"""Comprehensive tests for application/parser/file/docs_parser.py
"""Comprehensive tests for docsgpt/parser/file/docs_parser.py
Covers: PDFParser (init, parse with pypdf, parse as image, import error),
DocxParser (init, parse, import error).
@@ -9,7 +9,7 @@ from unittest.mock import MagicMock, patch, mock_open
import pytest
from application.parser.file.docs_parser import PDFParser, DocxParser
from docsgpt.parser.file.docs_parser import PDFParser, DocxParser
# =====================================================================
@@ -44,7 +44,7 @@ class TestPDFParserInit:
@pytest.mark.unit
class TestPDFParserParse:
@patch("application.parser.file.docs_parser.settings")
@patch("docsgpt.parser.file.docs_parser.settings")
def test_parse_with_pypdf(self, mock_settings):
mock_settings.PARSE_PDF_AS_IMAGE = False
@@ -58,7 +58,7 @@ class TestPDFParserParse:
mock_reader = MagicMock()
mock_reader.pages = [mock_page1, mock_page2]
with patch("application.parser.file.docs_parser.PdfReader",
with patch("docsgpt.parser.file.docs_parser.PdfReader",
create=True), \
patch("builtins.open", mock_open()):
# Need to patch the import inside the function
@@ -74,8 +74,8 @@ class TestPDFParserParse:
finally:
del sys.modules["pypdf"]
@patch("application.parser.file.docs_parser.settings")
@patch("application.parser.file.docs_parser.requests")
@patch("docsgpt.parser.file.docs_parser.settings")
@patch("docsgpt.parser.file.docs_parser.requests")
def test_parse_as_image(self, mock_requests, mock_settings):
mock_settings.PARSE_PDF_AS_IMAGE = True
@@ -89,7 +89,7 @@ class TestPDFParserParse:
result = parser.parse_file(Path("test.pdf"))
assert result == "# OCR Result"
@patch("application.parser.file.docs_parser.settings")
@patch("docsgpt.parser.file.docs_parser.settings")
def test_parse_raises_on_missing_pypdf(self, mock_settings):
mock_settings.PARSE_PDF_AS_IMAGE = False
@@ -196,17 +196,17 @@ class TestBaseParserProperties:
class TestDocsParserGaps:
def test_pdf_parser_parse_as_image(self, tmp_path):
"""Cover lines 33-34: PARSE_PDF_AS_IMAGE sends to external service."""
from application.parser.file.docs_parser import PDFParser
from docsgpt.parser.file.docs_parser import PDFParser
pdf_file = tmp_path / "test.pdf"
pdf_file.write_bytes(b"%PDF-1.4 fake content")
with patch(
"application.parser.file.docs_parser.settings"
"docsgpt.parser.file.docs_parser.settings"
) as mock_settings:
mock_settings.PARSE_PDF_AS_IMAGE = True
with patch(
"application.parser.file.docs_parser.requests.post"
"docsgpt.parser.file.docs_parser.requests.post"
) as mock_post:
mock_post.return_value = MagicMock(
json=MagicMock(return_value={"markdown": "# Parsed Content"})
@@ -218,7 +218,7 @@ class TestDocsParserGaps:
def test_docx_parser_init_parser(self):
"""Cover line 59: DocxParser._init_parser returns empty dict."""
from application.parser.file.docs_parser import DocxParser
from docsgpt.parser.file.docs_parser import DocxParser
parser = DocxParser()
config = parser._init_parser()
@@ -226,7 +226,7 @@ class TestDocsParserGaps:
def test_docx_parser_import_error(self):
"""Cover line 63: ImportError when docx2txt not installed."""
from application.parser.file.docs_parser import DocxParser
from docsgpt.parser.file.docs_parser import DocxParser
parser = DocxParser()
with patch.dict("sys.modules", {"docx2txt": None}):
tests/parser/file/test_embedding_pipeline.py
+17 -17
View File
@@ -2,7 +2,7 @@ import pytest
import logging
from unittest.mock import patch, MagicMock
from application.parser.embedding_pipeline import (
from docsgpt.parser.embedding_pipeline import (
DEFAULT_EMBEDDINGS_BATCH_SIZE,
EmbeddingPipelineError,
_resolve_batch_size,
@@ -45,7 +45,7 @@ def test_add_text_to_store_with_retry_success():
def mock_settings(monkeypatch):
mock_settings = MagicMock()
monkeypatch.setattr(
"application.parser.embedding_pipeline.settings", mock_settings
"docsgpt.parser.embedding_pipeline.settings", mock_settings
)
return mock_settings
@@ -54,7 +54,7 @@ def mock_settings(monkeypatch):
def mock_vector_creator(monkeypatch):
mock_creator = MagicMock()
monkeypatch.setattr(
"application.parser.embedding_pipeline.VectorCreator", mock_creator
"docsgpt.parser.embedding_pipeline.VectorCreator", mock_creator
)
return mock_creator
@@ -126,7 +126,7 @@ def test_embed_and_store_documents_progress_band(
assert currents == sorted(currents)
@patch("application.parser.embedding_pipeline.add_texts_to_store_with_retry")
@patch("docsgpt.parser.embedding_pipeline.add_texts_to_store_with_retry")
def test_embed_and_store_documents_partial_failure_raises(
mock_add_retry, tmp_path, mock_settings, mock_vector_creator, caplog
):
@@ -170,7 +170,7 @@ def test_embed_and_store_documents_partial_failure_raises(
mock_store.save_local.assert_called()
@patch("application.parser.embedding_pipeline.add_texts_to_store_with_retry")
@patch("docsgpt.parser.embedding_pipeline.add_texts_to_store_with_retry")
def test_embed_and_store_documents_all_chunks_succeed_no_raise(
mock_add_retry, tmp_path, mock_settings, mock_vector_creator,
):
@@ -200,7 +200,7 @@ def test_assert_index_complete_raises_on_partial(monkeypatch):
"embedded_chunks": 4, "total_chunks": 10,
}
monkeypatch.setattr(
"application.parser.embedding_pipeline.IngestChunkProgressRepository",
"docsgpt.parser.embedding_pipeline.IngestChunkProgressRepository",
lambda conn: fake_repo,
)
from contextlib import contextmanager
@@ -210,7 +210,7 @@ def test_assert_index_complete_raises_on_partial(monkeypatch):
yield None
monkeypatch.setattr(
"application.parser.embedding_pipeline.db_session", _fake_session,
"docsgpt.parser.embedding_pipeline.db_session", _fake_session,
)
with pytest.raises(EmbeddingPipelineError, match=r"4/10"):
assert_index_complete("src-partial")
@@ -222,7 +222,7 @@ def test_assert_index_complete_passes_on_full(monkeypatch):
"embedded_chunks": 10, "total_chunks": 10,
}
monkeypatch.setattr(
"application.parser.embedding_pipeline.IngestChunkProgressRepository",
"docsgpt.parser.embedding_pipeline.IngestChunkProgressRepository",
lambda conn: fake_repo,
)
from contextlib import contextmanager
@@ -232,7 +232,7 @@ def test_assert_index_complete_passes_on_full(monkeypatch):
yield None
monkeypatch.setattr(
"application.parser.embedding_pipeline.db_session", _fake_session,
"docsgpt.parser.embedding_pipeline.db_session", _fake_session,
)
assert_index_complete("src-full") # no raise
@@ -242,7 +242,7 @@ def test_assert_index_complete_no_op_when_no_progress_row(monkeypatch):
fake_repo = MagicMock()
fake_repo.get_progress.return_value = None
monkeypatch.setattr(
"application.parser.embedding_pipeline.IngestChunkProgressRepository",
"docsgpt.parser.embedding_pipeline.IngestChunkProgressRepository",
lambda conn: fake_repo,
)
from contextlib import contextmanager
@@ -252,7 +252,7 @@ def test_assert_index_complete_no_op_when_no_progress_row(monkeypatch):
yield None
monkeypatch.setattr(
"application.parser.embedding_pipeline.db_session", _fake_session,
"docsgpt.parser.embedding_pipeline.db_session", _fake_session,
)
assert_index_complete("src-missing")
@@ -270,7 +270,7 @@ def test_assert_index_complete_no_op_when_lookup_fails(monkeypatch, caplog):
yield # pragma: no cover
monkeypatch.setattr(
"application.parser.embedding_pipeline.db_session", _broken_session,
"docsgpt.parser.embedding_pipeline.db_session", _broken_session,
)
with caplog.at_level(logging.WARNING, logger="root"):
assert_index_complete("src-db-down") # no raise
@@ -328,7 +328,7 @@ def test_add_texts_to_store_with_retry_sanitizes_and_skips_empty():
def test_resolve_batch_size_falls_back_on_bad_setting(monkeypatch):
fake = MagicMock() # attribute access yields a MagicMock, not an int
monkeypatch.setattr("application.parser.embedding_pipeline.settings", fake)
monkeypatch.setattr("docsgpt.parser.embedding_pipeline.settings", fake)
assert _resolve_batch_size() == DEFAULT_EMBEDDINGS_BATCH_SIZE
fake.EMBEDDINGS_BATCH_SIZE = 0
@@ -346,7 +346,7 @@ def test_embed_loop_batches_chunks(tmp_path, mock_settings, mock_vector_creator)
store = MagicMock()
mock_vector_creator.create_vectorstore.return_value = store
with patch("application.parser.embedding_pipeline._record_progress") as rec:
with patch("docsgpt.parser.embedding_pipeline._record_progress") as rec:
embed_and_store_documents(
docs, str(tmp_path / "s"), "sid", MagicMock(),
)
@@ -395,10 +395,10 @@ def test_poison_chunk_isolated_by_per_chunk_fallback(
raise RuntimeError("input too large")
with patch(
"application.parser.embedding_pipeline.add_texts_to_store_with_retry",
"docsgpt.parser.embedding_pipeline.add_texts_to_store_with_retry",
side_effect=fake_add,
):
with patch("application.parser.embedding_pipeline._record_progress") as rec:
with patch("docsgpt.parser.embedding_pipeline._record_progress") as rec:
with pytest.raises(EmbeddingPipelineError) as exc:
embed_and_store_documents(
docs, str(tmp_path / "s"), "sid", MagicMock(),
@@ -429,7 +429,7 @@ def test_batch_only_failure_recovers_via_fallback(
raise RuntimeError("payload too large")
with patch(
"application.parser.embedding_pipeline.add_texts_to_store_with_retry",
"docsgpt.parser.embedding_pipeline.add_texts_to_store_with_retry",
side_effect=fake_add,
):
embed_and_store_documents(docs, str(tmp_path / "s"), "sid", MagicMock())
tests/parser/file/test_epub_parser.py
+1 -1
View File
@@ -4,7 +4,7 @@ from unittest.mock import patch, MagicMock
import sys
import types
from application.parser.file.epub_parser import EpubParser
from docsgpt.parser.file.epub_parser import EpubParser
@pytest.fixture
tests/parser/file/test_html_parser.py
+10 -10
View File
@@ -1,6 +1,6 @@
import pytest
from application.parser.file.html_parser import HTMLParser
from docsgpt.parser.file.html_parser import HTMLParser
HTML = (
@@ -58,7 +58,7 @@ def test_html_parser_metadata_unreadable_file(tmp_path):
# --- HTMLMarkdownParser: the anydoc engine's HTML path ---------------------------
from application.parser.file.html_parser import HTMLMarkdownParser, html_to_markdown # noqa: E402
from docsgpt.parser.file.html_parser import HTMLMarkdownParser, html_to_markdown # noqa: E402
RICH_HTML = """<html><head><title>Doc Title</title>
<style>.x { color: red }</style><script>var secret = 1;</script></head>
@@ -139,7 +139,7 @@ def test_html_to_markdown_collapses_blank_runs():
def test_markdown_parser_head_truncates_oversized_markup(tmp_path, monkeypatch):
"""Markup past MARKUP_MAX_BYTES is not parsed: the soup+markdownify tree costs
~50x the input, and the upload cap is 100 MB."""
from application.core.settings import settings
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", 600)
path = tmp_path / "big.html"
@@ -155,7 +155,7 @@ def test_markdown_parser_head_truncates_oversized_markup(tmp_path, monkeypatch):
def test_markdown_parser_gate_disabled_reads_everything(tmp_path, monkeypatch):
from application.core.settings import settings
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "MARKUP_MAX_BYTES", 0)
path = tmp_path / "big.html"
@@ -187,7 +187,7 @@ def test_markdown_parser_metadata_unreadable_file(tmp_path):
def test_markdown_parser_cut_inside_utf8_char_does_not_mojibake_the_page(tmp_path, monkeypatch):
"""A byte cut mid-character makes strict UTF-8 fail; BeautifulSoup would then
retry as windows-1252, which *succeeds* on umlauts and garbles everything."""
from application.core.settings import settings
from docsgpt.core.settings import settings
path = tmp_path / "minified.html"
body = "<p>Größe Übermaß schön für</p>" * 400 # no newlines: the cut is arbitrary
@@ -203,7 +203,7 @@ def test_markdown_parser_cut_inside_utf8_char_does_not_mojibake_the_page(tmp_pat
def test_trim_torn_utf8_tail():
from application.parser.file.html_parser import _trim_torn_utf8_tail as trim
from docsgpt.parser.file.html_parser import _trim_torn_utf8_tail as trim
e_acute, snowman, emoji = "é".encode(), "☃".encode(), "😀".encode()
assert trim(b"abc") == b"abc"
@@ -218,7 +218,7 @@ def test_trim_torn_utf8_tail():
def test_markdown_parser_metadata_reuses_last_parse(tmp_path, rich_html_file, monkeypatch):
"""The metadata call after parse_file must not build the soup again."""
from application.parser.file import html_parser as mod
from docsgpt.parser.file import html_parser as mod
parser = HTMLMarkdownParser()
parser.parse_file(rich_html_file)
@@ -237,7 +237,7 @@ def test_markdown_parser_metadata_reuses_last_parse(tmp_path, rich_html_file, mo
def test_xml_prolog_does_not_leak_into_markdown(tmp_path):
from application.parser.file.html_parser import HTMLMarkdownParser
from docsgpt.parser.file.html_parser import HTMLMarkdownParser
path = tmp_path / "doc.xhtml"
path.write_bytes(
@@ -264,8 +264,8 @@ def test_data_uris_are_stripped_from_images_and_links():
def test_utf16_head_keeps_an_even_byte_count(tmp_path, monkeypatch):
from application.core.settings import settings
from application.parser.file.html_parser import HTMLMarkdownParser, read_markup_head
from docsgpt.core.settings import settings
from docsgpt.parser.file.html_parser import HTMLMarkdownParser, read_markup_head
body = "".join(f"<p>Zeile {i} Über Größe</p>\n" for i in range(200))
path = tmp_path / "wide.html"
tests/parser/file/test_image_parser.py
+5 -5
View File
@@ -1,7 +1,7 @@
from pathlib import Path
from unittest.mock import patch, MagicMock, mock_open
from application.parser.file.image_parser import ImageParser
from docsgpt.parser.file.image_parser import ImageParser
def test_image_init_parser():
@@ -12,7 +12,7 @@ def test_image_init_parser():
assert parser.parser_config_set
@patch("application.parser.file.image_parser.settings")
@patch("docsgpt.parser.file.image_parser.settings")
def test_image_parser_remote_true(mock_settings):
mock_settings.PARSE_IMAGE_REMOTE = True
parser = ImageParser()
@@ -20,7 +20,7 @@ def test_image_parser_remote_true(mock_settings):
mock_response = MagicMock()
mock_response.json.return_value = {"markdown": "# From Image"}
with patch("application.parser.file.image_parser.requests.post", return_value=mock_response) as mock_post:
with patch("docsgpt.parser.file.image_parser.requests.post", return_value=mock_response) as mock_post:
with patch("builtins.open", mock_open()):
result = parser.parse_file(Path("img.png"))
@@ -28,12 +28,12 @@ def test_image_parser_remote_true(mock_settings):
mock_post.assert_called_once()
@patch("application.parser.file.image_parser.settings")
@patch("docsgpt.parser.file.image_parser.settings")
def test_image_parser_remote_false(mock_settings):
mock_settings.PARSE_IMAGE_REMOTE = False
parser = ImageParser()
with patch("application.parser.file.image_parser.requests.post") as mock_post:
with patch("docsgpt.parser.file.image_parser.requests.post") as mock_post:
result = parser.parse_file(Path("img.png"))
assert result == ""
tests/parser/file/test_json_parser.py
+1 -1
View File
@@ -1,7 +1,7 @@
from pathlib import Path
from unittest.mock import patch, mock_open
from application.parser.file.json_parser import JSONParser
from docsgpt.parser.file.json_parser import JSONParser
def test_json_init_parser():
tests/parser/file/test_markdown_parser.py
+2 -2
View File
@@ -3,8 +3,8 @@ from unittest.mock import mock_open, patch
import pytest
from application.parser.file.markdown_parser import MarkdownParser
from application import utils
from docsgpt.parser.file.markdown_parser import MarkdownParser
from docsgpt import utils
class _Enc:
tests/parser/file/test_ocr_parser.py
+26 -26
View File
@@ -18,8 +18,8 @@ pytest.importorskip("pypdfium2")
PIL = pytest.importorskip("PIL")
from PIL import Image, ImageDraw # noqa: E402
from application.parser.file.base_parser import DocumentParseError # noqa: E402
from application.parser.file import ocr_parser as op # noqa: E402
from docsgpt.parser.file.base_parser import DocumentParseError # noqa: E402
from docsgpt.parser.file import ocr_parser as op # noqa: E402
# ---------------------------------------------------------------------------
@@ -83,7 +83,7 @@ def _pdf_page_count(path: Path) -> int:
@pytest.fixture
def settings():
from application.core.settings import settings
from docsgpt.core.settings import settings
return settings
@@ -96,7 +96,7 @@ def settings():
@pytest.mark.unit
class TestSettingsAliases:
def test_legacy_docling_names_still_configure_ocr(self, monkeypatch):
from application.core.settings import Settings
from docsgpt.core.settings import Settings
monkeypatch.setenv("DOCLING_OCR_ENABLED", "true")
monkeypatch.setenv("DOCLING_OCR_ATTACHMENTS_ENABLED", "true")
@@ -107,7 +107,7 @@ class TestSettingsAliases:
assert loaded.OCR_MIN_CHARS_PER_PAGE == 7
def test_new_names_and_defaults(self, monkeypatch):
from application.core.settings import Settings
from docsgpt.core.settings import Settings
for name in (
"OCR_ENABLED",
@@ -537,7 +537,7 @@ class TestNativeOcrImageParser:
@pytest.mark.unit
class TestExtractorWiring:
def test_legacy_map_without_ocr_is_unchanged(self):
from application.parser.file.bulk import _legacy_file_extractor
from docsgpt.parser.file.bulk import _legacy_file_extractor
extractor = _legacy_file_extractor()
assert type(extractor[".pdf"]).__name__ == "PDFParser"
@@ -545,7 +545,7 @@ class TestExtractorWiring:
assert ".tiff" not in extractor
def test_legacy_map_with_ocr_uses_native_parsers(self):
from application.parser.file.bulk import _legacy_file_extractor
from docsgpt.parser.file.bulk import _legacy_file_extractor
extractor = _legacy_file_extractor(ocr_enabled=True)
assert isinstance(extractor[".pdf"], op.NativeOcrPdfParser)
@@ -555,8 +555,8 @@ class TestExtractorWiring:
def test_ocr_without_docling_reaches_native_under_anydoc(self, monkeypatch):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "docling", None)
extractor = get_default_file_extractor(engine="anydoc", ocr_enabled=True)
@@ -566,7 +566,7 @@ class TestExtractorWiring:
assert isinstance(extractor[".png"], op.NativeOcrImageParser)
def test_ocr_without_docling_reaches_native_under_docling_engine(self, monkeypatch):
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "docling", None)
extractor = get_default_file_extractor(engine="docling", ocr_enabled=True)
@@ -574,7 +574,7 @@ class TestExtractorWiring:
assert isinstance(extractor[".jpg"], op.NativeOcrImageParser)
def test_ocr_off_without_docling_keeps_legacy(self, monkeypatch):
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setitem(sys.modules, "docling", None)
extractor = get_default_file_extractor(engine="docling", ocr_enabled=False)
@@ -583,8 +583,8 @@ class TestExtractorWiring:
def test_native_backend_with_docling_installed_wraps_docling_for_text_pdfs(self, monkeypatch, settings):
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from application.parser.file.docling_parser import DoclingPDFParser
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.docling_parser import DoclingPDFParser
monkeypatch.setattr(settings, "OCR_BACKEND", "native")
extractor = get_default_file_extractor(engine="docling", ocr_enabled=True)
@@ -600,8 +600,8 @@ class TestExtractorWiring:
def test_native_backend_under_anydoc_keeps_docling_reroute_for_trust_check(self, monkeypatch, settings):
pytest.importorskip("anydoc")
pytest.importorskip("docling")
from application.parser.file.anydoc_parser import _is_docling_backed
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.anydoc_parser import _is_docling_backed
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setattr(settings, "OCR_BACKEND", "native")
extractor = get_default_file_extractor(engine="anydoc", ocr_enabled=True)
@@ -612,7 +612,7 @@ class TestExtractorWiring:
def test_auto_backend_with_docling_installed_is_unchanged(self, monkeypatch, settings):
pytest.importorskip("docling")
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setattr(settings, "OCR_BACKEND", "auto")
extractor = get_default_file_extractor(engine="docling", ocr_enabled=True)
@@ -622,7 +622,7 @@ class TestExtractorWiring:
def test_anydoc_scan_hint_names_new_setting(self, tmp_path):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
fallback = MagicMock()
fallback.parser_config_set = True
@@ -700,7 +700,7 @@ class TestAnydocMixedDocuments:
def test_scanned_page_is_ocrd_and_appended(self, mixed_pdf, monkeypatch, settings):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
monkeypatch.setattr(settings, "OCR_RENDER_DPI", 72)
engine = FakeEngine(["SCANNED PAGE WORDS"])
@@ -717,8 +717,8 @@ class TestAnydocMixedDocuments:
def test_no_ocr_fallback_keeps_text_pages_only(self, mixed_pdf):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from application.parser.file.docs_parser import PDFParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.docs_parser import PDFParser
parser = AnydocParser(fallback_parser=PDFParser())
content = parser.parse_file(mixed_pdf)
@@ -728,7 +728,7 @@ class TestAnydocMixedDocuments:
def test_fallback_with_ocr_off_is_not_used(self, mixed_pdf):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
fallback = MagicMock()
fallback.ocr_enabled = False
@@ -738,7 +738,7 @@ class TestAnydocMixedDocuments:
def test_ocr_failure_keeps_text_pages(self, mixed_pdf, caplog):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
fallback = MagicMock()
fallback.ocr_enabled = True
@@ -755,7 +755,7 @@ class TestAnydocMixedDocuments:
def test_all_text_document_never_probes_ocr(self, tmp_path):
pytest.importorskip("anydoc")
from application.parser.file.anydoc_parser import AnydocParser
from docsgpt.parser.file.anydoc_parser import AnydocParser
pdf = _text_pdf(tmp_path / "t.pdf", pages=2, text="Digital text page with enough characters", lines=25)
fallback = MagicMock()
@@ -797,7 +797,7 @@ class TestRenderBudget:
try:
page = pdf[0]
try:
with caplog.at_level(logging.WARNING, logger="application.parser.file.ocr_parser"):
with caplog.at_level(logging.WARNING, logger="docsgpt.parser.file.ocr_parser"):
image = op._render_page(page, 200)
finally:
page.close()
@@ -906,7 +906,7 @@ class TestImageNormalisation:
def test_oversized_image_is_downscaled_to_the_budget(self, caplog):
big = Image.new("L", (9000, 9000), 255)
with caplog.at_level(logging.WARNING, logger="application.parser.file.ocr_parser"):
with caplog.at_level(logging.WARNING, logger="docsgpt.parser.file.ocr_parser"):
small = op.fit_to_pixel_budget(big)
assert small.width * small.height <= op._MAX_RENDER_PIXELS
assert "downscaling" in caplog.text
@@ -953,7 +953,7 @@ class TestDeepseekErrorShapes:
@pytest.mark.unit
def test_delegate_parse_lets_setup_errors_through():
"""A fallback whose dependency is missing is a deployment problem, not a bad file."""
from application.parser.file.base_parser import BaseParser, delegate_parse
from docsgpt.parser.file.base_parser import BaseParser, delegate_parse
class _NeedsLib(BaseParser):
def _init_parser(self):
tests/parser/file/test_openapi3_parser.py
+9 -9
View File
@@ -1,4 +1,4 @@
"""Tests for application.parser.file.openapi3_parser covering lines 7-8, 45."""
"""Tests for docsgpt.parser.file.openapi3_parser covering lines 7-8, 45."""
import pytest
from unittest.mock import MagicMock, patch
@@ -9,16 +9,16 @@ class TestOpenAPI3ParserImportFallback:
def test_import_fallback_to_base_parser(self):
"""Cover lines 7-8: try/except ModuleNotFoundError import fallback."""
# The fallback import is a module-level concern. Just verify the class works.
with patch("application.parser.file.openapi3_parser.parse"):
from application.parser.file.openapi3_parser import OpenAPI3Parser
with patch("docsgpt.parser.file.openapi3_parser.parse"):
from docsgpt.parser.file.openapi3_parser import OpenAPI3Parser
parser = OpenAPI3Parser()
assert parser is not None
def test_get_base_urls(self):
"""Cover basic URL extraction."""
with patch("application.parser.file.openapi3_parser.parse"):
from application.parser.file.openapi3_parser import OpenAPI3Parser
with patch("docsgpt.parser.file.openapi3_parser.parse"):
from docsgpt.parser.file.openapi3_parser import OpenAPI3Parser
parser = OpenAPI3Parser()
urls = parser.get_base_urls([
@@ -32,8 +32,8 @@ class TestOpenAPI3ParserImportFallback:
def test_get_info_from_paths_empty(self):
"""Cover path with no operations."""
with patch("application.parser.file.openapi3_parser.parse"):
from application.parser.file.openapi3_parser import OpenAPI3Parser
with patch("docsgpt.parser.file.openapi3_parser.parse"):
from docsgpt.parser.file.openapi3_parser import OpenAPI3Parser
parser = OpenAPI3Parser()
mock_path = MagicMock()
@@ -43,8 +43,8 @@ class TestOpenAPI3ParserImportFallback:
def test_parse_file_writes_results(self, tmp_path):
"""Cover line 45: parse_file writes to results.txt."""
with patch("application.parser.file.openapi3_parser.parse") as mock_parse:
from application.parser.file.openapi3_parser import OpenAPI3Parser
with patch("docsgpt.parser.file.openapi3_parser.parse") as mock_parse:
from docsgpt.parser.file.openapi3_parser import OpenAPI3Parser
mock_server = MagicMock()
mock_server.url = "https://api.example.com"
tests/parser/file/test_pdf_trust.py
+3 -3
View File
@@ -10,7 +10,7 @@ its whole Chinese column in anydoc with no error.
import zlib
from pathlib import Path
from application.parser.file.pdf_trust import (
from docsgpt.parser.file.pdf_trust import (
check_pdf_fonts,
verify_extraction,
verify_pdf_file,
@@ -134,7 +134,7 @@ class TestRealFixture:
def test_flate_bomb_streams_are_capped():
"""Flate reaches ~1000:1, so per-stream inflation must be capped — an
uncapped decompress of a crafted PDF would OOM the ingest worker."""
from application.parser.file.pdf_trust import _decompressed_streams, _STREAM_INFLATE_CAP
from docsgpt.parser.file.pdf_trust import _decompressed_streams, _STREAM_INFLATE_CAP
bomb = _stream(b"\0" * (_STREAM_INFLATE_CAP * 4))
chunks = list(_decompressed_streams(bomb))
@@ -187,7 +187,7 @@ class TestObjectWalk:
"""The forward walk yields exactly what ``\\d+\\s+\\d+\\s+obj(.*?)endobj`` did."""
import re
from application.parser.file.pdf_trust import _object_bodies
from docsgpt.parser.file.pdf_trust import _object_bodies
reference = re.compile(rb"\d+\s+\d+\s+obj(.*?)endobj", re.DOTALL)
data = (
tests/parser/file/test_pdfium_parser.py
+9 -9
View File
@@ -13,7 +13,7 @@ the fast path must only appear when a caller explicitly asks for it.
import pytest
from application.parser.file.base_parser import BaseParser, DocumentParseError
from docsgpt.parser.file.base_parser import BaseParser, DocumentParseError
pypdfium2 = pytest.importorskip("pypdfium2")
@@ -66,7 +66,7 @@ class _RecordingFallback(BaseParser):
@pytest.fixture
def parser_cls():
from application.parser.file.pdfium_parser import PdfiumTextParser
from docsgpt.parser.file.pdfium_parser import PdfiumTextParser
return PdfiumTextParser
@@ -162,7 +162,7 @@ def docling_engine(monkeypatch):
# (docling is an optional extra) the engine degrades to the legacy
# parsers and these tests do not apply.
pytest.importorskip("docling")
from application.core.settings import settings
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "docling")
# A developer .env with OCR on (and OCR_BACKEND=native) would otherwise
@@ -172,7 +172,7 @@ def docling_engine(monkeypatch):
def test_extractor_defaults_to_docling_for_pdf(docling_engine):
"""Sources must be unaffected: no fast path unless explicitly requested."""
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
pdf_parser = get_default_file_extractor()[".pdf"]
@@ -180,8 +180,8 @@ def test_extractor_defaults_to_docling_for_pdf(docling_engine):
def test_extractor_uses_fast_path_when_requested(docling_engine):
from application.parser.file.bulk import get_default_file_extractor
from application.parser.file.pdfium_parser import PdfiumTextParser
from docsgpt.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.pdfium_parser import PdfiumTextParser
pdf_parser = get_default_file_extractor(pdf_text_fast_path=True)[".pdf"]
@@ -192,8 +192,8 @@ def test_extractor_uses_fast_path_when_requested(docling_engine):
def test_anydoc_engine_ignores_fast_path(monkeypatch):
"""anydoc already reads the text layer in milliseconds and keeps structure."""
pytest.importorskip("anydoc")
from application.core.settings import settings
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.core.settings import settings
from docsgpt.parser.file.bulk import get_default_file_extractor
monkeypatch.setattr(settings, "DOC_PARSER_ENGINE", "anydoc")
@@ -203,7 +203,7 @@ def test_anydoc_engine_ignores_fast_path(monkeypatch):
def test_fast_path_does_not_change_non_pdf_parsers(docling_engine):
from application.parser.file.bulk import get_default_file_extractor
from docsgpt.parser.file.bulk import get_default_file_extractor
plain = get_default_file_extractor()
fast = get_default_file_extractor(pdf_text_fast_path=True)
tests/parser/file/test_pptx_parser.py
+1 -1
View File
@@ -2,7 +2,7 @@ import pytest
from pathlib import Path
from unittest.mock import patch
from application.parser.file.pptx_parser import PPTXParser
from docsgpt.parser.file.pptx_parser import PPTXParser
def test_pptx_init_parser():
tests/parser/file/test_rst_parser.py
+1 -1
View File
@@ -2,7 +2,7 @@ import pytest
from pathlib import Path
from unittest.mock import patch, mock_open
from application.parser.file.rst_parser import RstParser
from docsgpt.parser.file.rst_parser import RstParser
@pytest.fixture
tests/parser/file/test_tableize.py
+3 -3
View File
@@ -1,5 +1,5 @@
"""Tests for the dot-leader/whitespace table reconstruction (``ANYDOC_TABLEIZE``)."""
from application.parser.file.tableize import tableize
from docsgpt.parser.file.tableize import tableize
DOT_LEADER = """Revenues
Insurance premiums ............ 83,431 77,731
@@ -76,7 +76,7 @@ def test_non_table_text_passes_through_verbatim():
def test_single_trailing_number_lines_are_not_a_table():
"""Headings, footnotes and version lists look like 'word number' rows; leave them alone."""
from application.parser.file.tableize import tableize
from docsgpt.parser.file.tableize import tableize
for block in (
"Chapter 1\nChapter 2\nChapter 3",
@@ -88,7 +88,7 @@ def test_single_trailing_number_lines_are_not_a_table():
def test_leader_rows_with_one_value_still_convert():
from application.parser.file.tableize import tableize
from docsgpt.parser.file.tableize import tableize
block = "Revenue ...... 1,234\nCosts ...... 567\nProfit ...... 667"
assert "| --- |" in tableize(block)
tests/parser/file/test_tabular_parser.py
+2 -2
View File
@@ -2,7 +2,7 @@ import pytest
from pathlib import Path
from unittest.mock import patch, MagicMock, mock_open
from application.parser.file.tabular_parser import CSVParser, PandasCSVParser, ExcelParser
from docsgpt.parser.file.tabular_parser import CSVParser, PandasCSVParser, ExcelParser
@pytest.fixture
@@ -220,7 +220,7 @@ def test_excel_numeric_headers_do_not_crash(tmp_path):
to this parser, so it must handle numeric headers."""
from openpyxl import Workbook
from application.parser.file.tabular_parser import ExcelParser
from docsgpt.parser.file.tabular_parser import ExcelParser
wb = Workbook()
ws = wb.active
tests/parser/file/test_tabular_parser_real_fixtures.py
+1 -1
View File
@@ -16,7 +16,7 @@ from pathlib import Path
import pytest
from application.parser.file.tabular_parser import (
from docsgpt.parser.file.tabular_parser import (
ExcelParser,
PandasCSVParser,
cell_to_text,
tests/parser/remote/test_crawler_loader.py
+16 -16
View File
@@ -2,8 +2,8 @@ from unittest.mock import MagicMock, patch
import pytest
from application.parser.remote.crawler_loader import CrawlerLoader
from application.parser.schema.base import Document
from docsgpt.parser.remote.crawler_loader import CrawlerLoader
from docsgpt.parser.schema.base import Document
class DummyResponse:
@@ -22,8 +22,8 @@ def _mock_validate_url(url):
return url
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.crawler_loader.pinned_request")
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
def test_load_data_crawls_same_domain_links(mock_pinned_request, mock_validate_url):
responses = {
"http://example.com": DummyResponse(
@@ -65,8 +65,8 @@ def test_load_data_crawls_same_domain_links(mock_pinned_request, mock_validate_u
assert mock_pinned_request.call_count == 2
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.crawler_loader.pinned_request")
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
def test_load_data_accepts_list_input_and_adds_scheme(mock_pinned_request, mock_validate_url):
mock_pinned_request.return_value = DummyResponse("<html><body>No links here</body></html>")
crawler = CrawlerLoader()
@@ -83,8 +83,8 @@ def test_load_data_accepts_list_input_and_adds_scheme(mock_pinned_request, mock_
}
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.crawler_loader.pinned_request")
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
def test_load_data_respects_limit(mock_pinned_request, mock_validate_url):
responses = {
"http://example.com": DummyResponse(
@@ -110,9 +110,9 @@ def test_load_data_respects_limit(mock_pinned_request, mock_validate_url):
assert mock_pinned_request.call_count == 1
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.crawler_loader.logging")
@patch("application.parser.remote.crawler_loader.pinned_request")
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.crawler_loader.logging")
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
def test_load_data_logs_and_skips_on_request_error(mock_pinned_request, mock_logging, mock_validate_url):
mock_pinned_request.side_effect = Exception("load failure")
crawler = CrawlerLoader()
@@ -128,10 +128,10 @@ def test_load_data_logs_and_skips_on_request_error(mock_pinned_request, mock_log
assert mock_logging.error.call_args.kwargs.get("exc_info") is True
@patch("application.parser.remote.crawler_loader.validate_url")
@patch("docsgpt.parser.remote.crawler_loader.validate_url")
def test_load_data_returns_empty_on_ssrf_validation_failure(mock_validate_url):
"""Test that SSRF validation failure returns empty list."""
from application.core.url_validation import SSRFError
from docsgpt.core.url_validation import SSRFError
mock_validate_url.side_effect = SSRFError("Access to private IP not allowed")
crawler = CrawlerLoader()
@@ -165,15 +165,15 @@ def test_url_to_virtual_path_variants():
class TestCrawlerLoaderGaps:
def test_pinned_fetch_builds_document_without_webbase_loader(self):
"""The crawler should index the response body it already fetched."""
from application.parser.remote.crawler_loader import CrawlerLoader
from docsgpt.parser.remote.crawler_loader import CrawlerLoader
loader = CrawlerLoader(limit=5)
with patch(
"application.parser.remote.crawler_loader.validate_url",
"docsgpt.parser.remote.crawler_loader.validate_url",
return_value="https://example.com",
):
with patch(
"application.parser.remote.crawler_loader.pinned_request"
"docsgpt.parser.remote.crawler_loader.pinned_request"
) as mock_pinned_request:
mock_response = MagicMock()
mock_response.status_code = 200
tests/parser/remote/test_crawler_markdown.py
+8 -8
View File
@@ -5,8 +5,8 @@ from urllib.parse import urlparse
import pytest
import requests
from application.parser.remote.crawler_markdown import CrawlerLoader
from application.parser.schema.base import Document
from docsgpt.parser.remote.crawler_markdown import CrawlerLoader
from docsgpt.parser.schema.base import Document
class DummyResponse:
@@ -40,7 +40,7 @@ def _mock_validate_url(url):
@pytest.fixture(autouse=True)
def _patch_validate_url(monkeypatch):
monkeypatch.setattr(
"application.parser.remote.crawler_markdown.validate_url",
"docsgpt.parser.remote.crawler_markdown.validate_url",
_mock_validate_url,
)
@@ -48,7 +48,7 @@ def _patch_validate_url(monkeypatch):
@pytest.fixture(autouse=True)
def _patch_tldextract(monkeypatch):
monkeypatch.setattr(
"application.parser.remote.crawler_markdown._extract",
"docsgpt.parser.remote.crawler_markdown._extract",
_fake_extract,
)
@@ -61,7 +61,7 @@ def _patch_markdownify(monkeypatch):
return outputs.get(html, html)
monkeypatch.setattr(
"application.parser.remote.crawler_markdown.markdownify",
"docsgpt.parser.remote.crawler_markdown.markdownify",
fake_markdownify,
)
return outputs
@@ -73,7 +73,7 @@ def _patch_pinned_request(monkeypatch, side_effect):
return side_effect(url)
monkeypatch.setattr(
"application.parser.remote.crawler_markdown.pinned_request",
"docsgpt.parser.remote.crawler_markdown.pinned_request",
fake_pinned_request,
)
@@ -164,13 +164,13 @@ def test_load_data_handles_fetch_errors(monkeypatch, _patch_markdownify, _patch_
def test_load_data_returns_empty_on_ssrf_validation_failure(monkeypatch):
"""Test that SSRF validation failure returns empty list."""
from application.core.url_validation import SSRFError
from docsgpt.core.url_validation import SSRFError
def raise_ssrf_error(url):
raise SSRFError("Access to private IP not allowed")
monkeypatch.setattr(
"application.parser.remote.crawler_markdown.validate_url",
"docsgpt.parser.remote.crawler_markdown.validate_url",
raise_ssrf_error,
)
tests/parser/remote/test_github_loader.py
+31 -31
View File
@@ -3,7 +3,7 @@ import pytest
from unittest.mock import patch, MagicMock
import requests
from application.parser.remote.github_loader import GitHubLoader
from docsgpt.parser.remote.github_loader import GitHubLoader
def make_response(json_data=None, status_code=200, raise_error=None):
@@ -18,7 +18,7 @@ def make_response(json_data=None, status_code=200, raise_error=None):
class TestGitHubLoaderFetchFileContent:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_text_file_base64_decoded(self, mock_get):
loader = GitHubLoader()
content_str = "Hello from README"
@@ -34,7 +34,7 @@ class TestGitHubLoaderFetchFileContent:
timeout=100,
)
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_binary_file_skipped(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response({"encoding": "base64", "content": "AAAA"})
@@ -43,7 +43,7 @@ class TestGitHubLoaderFetchFileContent:
assert result is None
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_non_base64_plain_content(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response({"encoding": "", "content": "Plain text"})
@@ -52,7 +52,7 @@ class TestGitHubLoaderFetchFileContent:
assert result == "Plain text"
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_http_error_raises(self, mock_get):
loader = GitHubLoader()
http_err = requests.HTTPError("Not found")
@@ -63,7 +63,7 @@ class TestGitHubLoaderFetchFileContent:
class TestGitHubLoaderFetchRepoFiles:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_recurses_directories(self, mock_get):
loader = GitHubLoader()
@@ -133,7 +133,7 @@ class TestGitHubLoaderIsTextFile:
loader = GitHubLoader()
assert loader.is_text_file("image.png") is False
@patch("application.parser.remote.github_loader.mimetypes.guess_type")
@patch("docsgpt.parser.remote.github_loader.mimetypes.guess_type")
def test_mime_fallback_text(self, mock_mime):
mock_mime.return_value = ("text/plain", None)
loader = GitHubLoader()
@@ -141,15 +141,15 @@ class TestGitHubLoaderIsTextFile:
class TestGitHubLoaderMakeRequest:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_success(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response({"ok": True}, 200)
resp = loader._make_request("http://example.com")
assert resp.status_code == 200
@patch("application.parser.remote.github_loader.time.sleep")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.time.sleep")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_rate_limit_retry(self, mock_get, mock_sleep):
loader = GitHubLoader()
rate_resp = MagicMock()
@@ -166,7 +166,7 @@ class TestGitHubLoaderMakeRequest:
assert resp.status_code == 200
mock_sleep.assert_called_once()
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_rate_limit_exhausted(self, mock_get):
loader = GitHubLoader()
rate_resp = MagicMock()
@@ -181,7 +181,7 @@ class TestGitHubLoaderMakeRequest:
with pytest.raises(Exception, match="rate limit exceeded"):
loader._make_request("http://example.com", max_retries=1)
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_403_non_rate_limit(self, mock_get):
loader = GitHubLoader()
resp = MagicMock()
@@ -193,7 +193,7 @@ class TestGitHubLoaderMakeRequest:
with pytest.raises(Exception, match="GitHub API error"):
loader._make_request("http://example.com", max_retries=1)
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_other_error_raises(self, mock_get):
loader = GitHubLoader()
resp = make_response(
@@ -207,7 +207,7 @@ class TestGitHubLoaderMakeRequest:
class TestGitHubLoaderFetchRepoFilesErrors:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_api_error_message_in_dict(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response(
@@ -217,7 +217,7 @@ class TestGitHubLoaderFetchRepoFilesErrors:
with pytest.raises(Exception, match="GitHub API error"):
loader.fetch_repo_files("owner/repo")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_non_list_response(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response("not a list", 200)
@@ -227,7 +227,7 @@ class TestGitHubLoaderFetchRepoFilesErrors:
class TestGitHubLoaderFetchFileContentEdgeCases:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_empty_base64_text_returns_none(self, mock_get):
loader = GitHubLoader()
b64 = base64.b64encode(b"").decode("utf-8")
@@ -237,7 +237,7 @@ class TestGitHubLoaderFetchFileContentEdgeCases:
result = loader.fetch_file_content("owner/repo", "empty.py")
assert result is None
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_empty_non_base64_returns_none(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response(
@@ -246,7 +246,7 @@ class TestGitHubLoaderFetchFileContentEdgeCases:
result = loader.fetch_file_content("owner/repo", "empty.txt")
assert result is None
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_decode_failure_returns_none(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response(
@@ -275,7 +275,7 @@ class TestGitHubLoaderLoadDataSkipsNone:
class TestGitHubLoaderRobustness:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_fetch_repo_files_non_json_raises(self, mock_get):
resp = MagicMock()
resp.json.side_effect = ValueError("No JSON")
@@ -283,14 +283,14 @@ class TestGitHubLoaderRobustness:
with pytest.raises(ValueError):
GitHubLoader().fetch_repo_files("owner/repo")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_fetch_repo_files_unexpected_shape_missing_type_raises(self, mock_get):
# Missing 'type' in items should raise KeyError when accessed
mock_get.return_value = make_response([{"path": "README.md"}])
with pytest.raises(KeyError):
GitHubLoader().fetch_repo_files("owner/repo")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_fetch_file_content_non_json_raises(self, mock_get):
resp = MagicMock()
resp.status_code = 200
@@ -299,7 +299,7 @@ class TestGitHubLoaderRobustness:
with pytest.raises(ValueError):
GitHubLoader().fetch_file_content("owner/repo", "README.md")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_fetch_file_content_unexpected_shape_missing_content_returns_none(self, mock_get):
# encoding indicates base64 text, but 'content' key is missing
# With the new code, the exception is caught and returns None (treated as binary/skipped)
@@ -308,8 +308,8 @@ class TestGitHubLoaderRobustness:
result = GitHubLoader().fetch_file_content("owner/repo", "file.txt")
assert result is None
@patch("application.parser.remote.github_loader.base64.b64decode")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.base64.b64decode")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_large_binary_skip_does_not_decode(self, mock_get, mock_b64decode):
# Ensure we don't attempt to decode large binary content for non-text files
mock_b64decode.side_effect = AssertionError("b64decode should not be called for binary files")
@@ -384,7 +384,7 @@ class TestGitHubLoaderSelectFiles:
def test_applies_size_cap(self, monkeypatch):
loader = GitHubLoader()
monkeypatch.setattr(
"application.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
"docsgpt.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
100, raising=False,
)
entries = [("small.py", 50), ("huge.py", 5000), ("ok.md", 99)]
@@ -393,7 +393,7 @@ class TestGitHubLoaderSelectFiles:
def test_zero_cap_disables_limit(self, monkeypatch):
loader = GitHubLoader()
monkeypatch.setattr(
"application.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
"docsgpt.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
0, raising=False,
)
assert loader.select_files([("huge.py", 10**9)]) == ["huge.py"]
@@ -407,7 +407,7 @@ class TestGitHubLoaderSelectFiles:
class TestGitHubLoaderTree:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_single_request_lists_all_blobs(self, mock_get):
mock_get.return_value = make_response({
"tree": [
@@ -424,7 +424,7 @@ class TestGitHubLoaderTree:
# One call for the whole repo, versus one per directory before.
assert mock_get.call_count == 1
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_truncated_tree_falls_back_to_walk(self, mock_get, monkeypatch):
loader = GitHubLoader()
mock_get.return_value = make_response({"tree": [], "truncated": True})
@@ -435,12 +435,12 @@ class TestGitHubLoaderTree:
class TestGitHubLoaderDefaultBranch:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_uses_repo_default_branch(self, mock_get):
mock_get.return_value = make_response({"default_branch": "master"})
assert GitHubLoader().get_default_branch("owner/repo") == "master"
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_falls_back_to_main(self, mock_get):
mock_get.side_effect = requests.ConnectionError("boom")
assert GitHubLoader().get_default_branch("owner/repo") == "main"
@@ -496,7 +496,7 @@ class TestGitHubLoaderParallelFetch:
class TestGitHubLoaderStaleTokenFallback:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_401_retries_unauthenticated(self, mock_get):
"""An expired PAT 401s even public repos; fall back to anonymous
rather than failing the ingest outright."""
tests/parser/remote/test_reddit_loader.py
+1 -1
View File
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
import pytest
from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
from docsgpt.parser.remote.reddit_loader import RedditPostsLoaderRemote
def _post(post_id, title, body):
tests/parser/remote/test_remote_creator.py
+16 -16
View File
@@ -1,4 +1,4 @@
"""Tests for application.parser.remote.remote_creator."""
"""Tests for docsgpt.parser.remote.remote_creator."""
import json
@@ -10,7 +10,7 @@ from unittest.mock import MagicMock
class TestRemoteCreator:
def test_create_loader_valid_type(self):
"""Cover line 34: returns loader instance for valid type."""
from application.parser.remote.remote_creator import RemoteCreator
from docsgpt.parser.remote.remote_creator import RemoteCreator
mock_loader_cls = MagicMock()
original_loaders = RemoteCreator.loaders.copy()
@@ -23,14 +23,14 @@ class TestRemoteCreator:
def test_create_loader_invalid_type_raises(self):
"""Cover lines 32-33: raises ValueError for unknown type."""
from application.parser.remote.remote_creator import RemoteCreator
from docsgpt.parser.remote.remote_creator import RemoteCreator
with pytest.raises(ValueError, match="No loader class found"):
RemoteCreator.create_loader("nonexistent_xyz")
def test_create_loader_case_insensitive(self):
"""Cover line 31: type.lower() normalization."""
from application.parser.remote.remote_creator import RemoteCreator
from docsgpt.parser.remote.remote_creator import RemoteCreator
mock_loader_cls = MagicMock()
original_loaders = RemoteCreator.loaders.copy()
@@ -48,12 +48,12 @@ class TestNormalizeRemoteData:
back to the ``source_data`` shape each loader expects."""
def test_none_passes_through(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
assert normalize_remote_data("crawler", None) is None
def test_crawler_dict_with_url_key(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data(
"crawler", {"url": "https://example.com", "provider": "crawler"}
@@ -61,20 +61,20 @@ class TestNormalizeRemoteData:
assert result == "https://example.com"
def test_url_dict_with_url_key(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data("url", {"url": "https://example.com"})
assert result == "https://example.com"
def test_url_legacy_raw_key(self):
"""Legacy rows wrapped a bare URL string as ``{"raw": ...}``."""
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data("crawler", {"raw": "https://legacy.example.com"})
assert result == "https://legacy.example.com"
def test_url_dict_with_urls_list(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data(
"url", {"urls": ["https://a.example.com", "https://b.example.com"]}
@@ -82,7 +82,7 @@ class TestNormalizeRemoteData:
assert result == ["https://a.example.com", "https://b.example.com"]
def test_github_repo_url_key(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data(
"github", {"repo_url": "https://github.com/arc53/DocsGPT"}
@@ -90,25 +90,25 @@ class TestNormalizeRemoteData:
assert result == "https://github.com/arc53/DocsGPT"
def test_sitemap_dict_with_url_key(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data("sitemap", {"url": "https://example.com/sitemap.xml"})
assert result == "https://example.com/sitemap.xml"
def test_plain_string_url_passes_through(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
assert normalize_remote_data("crawler", "https://example.com") == "https://example.com"
def test_url_dict_without_url_key_returns_none(self):
"""A URL-type loader must never receive a dict, even a malformed one."""
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
assert normalize_remote_data("crawler", {"provider": "crawler"}) is None
def test_reddit_dict_serialized_to_json_string(self):
"""reddit's loader runs json.loads() — it needs a JSON string."""
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data(
"reddit", {"client_id": "x", "search_queries": ["y"]}
@@ -118,14 +118,14 @@ class TestNormalizeRemoteData:
def test_s3_dict_passes_through(self):
"""S3Loader.load_data() accepts a dict, so it is left untouched."""
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
data = {"bucket": "b", "prefix": "k"}
assert normalize_remote_data("s3", data) == data
def test_json_string_remote_data_is_parsed(self):
"""Legacy rows that stored the JSON itself as a string still resolve."""
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data("crawler", '{"url": "https://example.com"}')
assert result == "https://example.com"
tests/parser/remote/test_s3_loader.py
+15 -15
View File
@@ -11,14 +11,14 @@ from botocore.exceptions import ClientError, NoCredentialsError
def mock_boto3():
"""Mock boto3 module."""
with patch.dict("sys.modules", {"boto3": MagicMock()}):
with patch("application.parser.remote.s3_loader.boto3") as mock:
with patch("docsgpt.parser.remote.s3_loader.boto3") as mock:
yield mock
@pytest.fixture
def s3_loader(mock_boto3):
"""Create S3Loader instance with mocked boto3."""
from application.parser.remote.s3_loader import S3Loader
from docsgpt.parser.remote.s3_loader import S3Loader
loader = S3Loader()
return loader
@@ -29,15 +29,15 @@ class TestS3LoaderInit:
def test_init_raises_import_error_when_boto3_missing(self):
"""Should raise ImportError when boto3 is not installed."""
with patch("application.parser.remote.s3_loader.boto3", None):
from application.parser.remote.s3_loader import S3Loader
with patch("docsgpt.parser.remote.s3_loader.boto3", None):
from docsgpt.parser.remote.s3_loader import S3Loader
with pytest.raises(ImportError, match="boto3 is required"):
S3Loader()
def test_init_sets_client_to_none(self, mock_boto3):
"""Should initialize with s3_client as None."""
from application.parser.remote.s3_loader import S3Loader
from docsgpt.parser.remote.s3_loader import S3Loader
loader = S3Loader()
assert loader.s3_client is None
@@ -127,7 +127,7 @@ class TestInitClient:
def test_init_client_with_custom_endpoint(self, s3_loader, mock_boto3):
"""Should configure path-style addressing for custom endpoints."""
with patch(
"application.parser.remote.s3_loader.validate_url",
"docsgpt.parser.remote.s3_loader.validate_url",
side_effect=lambda u: u,
):
s3_loader._init_client(
@@ -145,7 +145,7 @@ class TestInitClient:
def test_init_client_normalizes_do_endpoint(self, s3_loader, mock_boto3):
"""Should normalize DigitalOcean Spaces bucket-prefixed URLs."""
with patch(
"application.parser.remote.s3_loader.validate_url",
"docsgpt.parser.remote.s3_loader.validate_url",
side_effect=lambda u: u,
):
corrected_bucket = s3_loader._init_client(
@@ -564,7 +564,7 @@ class TestLoadData:
}
with patch(
"application.parser.remote.s3_loader.validate_url",
"docsgpt.parser.remote.s3_loader.validate_url",
side_effect=lambda u: u,
):
s3_loader.load_data(input_data)
@@ -644,7 +644,7 @@ class TestLoadData:
}
with patch(
"application.parser.remote.s3_loader.validate_url",
"docsgpt.parser.remote.s3_loader.validate_url",
side_effect=lambda u: u,
):
docs = s3_loader.load_data(input_data)
@@ -663,7 +663,7 @@ class TestProcessDocument:
mock_doc.text = "Extracted document text"
with patch(
"application.parser.file.bulk.SimpleDirectoryReader"
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
) as mock_reader_class:
mock_reader = MagicMock()
mock_reader.load_data.return_value = [mock_doc]
@@ -687,7 +687,7 @@ class TestProcessDocument:
def test_process_document_returns_none_on_error(self, s3_loader):
"""Should return None when document processing fails."""
with patch(
"application.parser.file.bulk.SimpleDirectoryReader"
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
) as mock_reader_class:
mock_reader_class.side_effect = Exception("Parse error")
@@ -709,7 +709,7 @@ class TestProcessDocument:
def test_process_document_cleans_up_temp_file(self, s3_loader):
"""Should clean up temporary file after processing."""
with patch(
"application.parser.file.bulk.SimpleDirectoryReader"
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
) as mock_reader_class:
mock_reader = MagicMock()
mock_reader.load_data.return_value = []
@@ -831,7 +831,7 @@ class TestProcessDocumentAdditional:
def test_process_document_empty_documents_returns_none(self, s3_loader):
"""Cover line 347-348: no documents extracted returns None."""
with patch(
"application.parser.file.bulk.SimpleDirectoryReader"
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
) as mock_reader_class:
mock_reader = MagicMock()
mock_reader.load_data.return_value = []
@@ -906,8 +906,8 @@ class TestSSRFValidation:
def test_is_supported_document_follows_the_upload_whitelist():
from application.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
from application.parser.remote.s3_loader import S3Loader
from docsgpt.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
from docsgpt.parser.remote.s3_loader import S3Loader
for suffix in SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS:
key = f"bucket/file{suffix}"
tests/parser/remote/test_share_point_loader.py
+17 -17
View File
@@ -2,7 +2,7 @@
from unittest.mock import patch, MagicMock
from application.parser.connectors.share_point.loader import SharePointLoader
from docsgpt.parser.connectors.share_point.loader import SharePointLoader
def make_response(json_data=None, status_code=200, raise_error=None):
@@ -65,10 +65,10 @@ class TestSharePointLoaderProcessFile:
class TestSharePointLoaderLoadFileById:
"""Test _load_file_by_id method."""
@patch("application.parser.connectors.share_point.loader.requests.get")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
def test_load_file_by_id_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
"""Should include size field in $select parameter."""
mock_get_token.return_value = {
@@ -96,10 +96,10 @@ class TestSharePointLoaderLoadFileById:
params = call_args[1]["params"]
assert "size" in params["$select"]
@patch("application.parser.connectors.share_point.loader.requests.get")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
def test_load_file_by_id_returns_document_with_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
"""Should return document with size from API response."""
mock_get_token.return_value = {
@@ -133,10 +133,10 @@ class TestSharePointLoaderLoadFileById:
class TestSharePointLoaderListItems:
"""Test _list_items_in_parent method."""
@patch("application.parser.connectors.share_point.loader.requests.get")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
def test_list_items_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
"""Should include size field in $select parameter when listing items."""
mock_get_token.return_value = {
@@ -168,10 +168,10 @@ class TestSharePointLoaderListItems:
params = call_args[1]["params"]
assert "size" in params["$select"]
@patch("application.parser.connectors.share_point.loader.requests.get")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
def test_list_items_folders_include_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
"""Should include size for folders as well."""
mock_get_token.return_value = {
tests/parser/remote/test_sitemap_loader.py
+22 -22
View File
@@ -1,4 +1,4 @@
"""Comprehensive tests for application/parser/remote/sitemap_loader.py
"""Comprehensive tests for docsgpt/parser/remote/sitemap_loader.py
Covers: SitemapLoader (init, load_data, _extract_urls, _is_sitemap,
_parse_sitemap, URL validation, error handling).
@@ -9,8 +9,8 @@ from unittest.mock import MagicMock, patch
import pytest
import requests
from application.parser.remote.sitemap_loader import SitemapLoader
from application.parser.schema.base import Document
from docsgpt.parser.remote.sitemap_loader import SitemapLoader
from docsgpt.parser.schema.base import Document
# =====================================================================
@@ -159,7 +159,7 @@ class TestParseSitemap:
@pytest.mark.unit
class TestExtractUrls:
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_extract_urls_from_sitemap(self, mock_pinned_request):
loader = SitemapLoader()
@@ -177,7 +177,7 @@ class TestExtractUrls:
urls = loader._extract_urls("https://example.com/sitemap.xml")
assert "https://example.com/p" in urls
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_extract_urls_not_sitemap(self, mock_pinned_request):
loader = SitemapLoader()
@@ -191,7 +191,7 @@ class TestExtractUrls:
urls = loader._extract_urls("https://example.com/page")
assert urls == ["https://example.com/page"]
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_extract_urls_http_error(self, mock_pinned_request):
loader = SitemapLoader()
mock_pinned_request.side_effect = requests.exceptions.HTTPError("404")
@@ -199,7 +199,7 @@ class TestExtractUrls:
urls = loader._extract_urls("https://example.com/missing")
assert urls == []
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_extract_urls_connection_error(self, mock_pinned_request):
loader = SitemapLoader()
mock_pinned_request.side_effect = requests.exceptions.ConnectionError()
@@ -208,12 +208,12 @@ class TestExtractUrls:
assert urls == []
def test_extract_urls_ssrf_blocked(self):
from application.security.safe_url import UnsafeUserUrlError
from docsgpt.security.safe_url import UnsafeUserUrlError
loader = SitemapLoader()
with patch(
"application.parser.remote.sitemap_loader.pinned_request",
"docsgpt.parser.remote.sitemap_loader.pinned_request",
side_effect=UnsafeUserUrlError("blocked"),
):
urls = loader._extract_urls("http://169.254.169.254/")
@@ -228,8 +228,8 @@ class TestExtractUrls:
@pytest.mark.unit
class TestSitemapLoaderLoadData:
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_load_data_success(self, mock_pinned_request, mock_validate):
loader = SitemapLoader(limit=10)
mock_validate.side_effect = lambda url: url
@@ -248,7 +248,7 @@ class TestSitemapLoaderLoadData:
assert docs[0].text == "Page body"
assert docs[0].extra_info == {"source": "https://example.com/page1"}
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
def test_load_data_no_urls(self, mock_validate):
loader = SitemapLoader()
@@ -256,8 +256,8 @@ class TestSitemapLoaderLoadData:
docs = loader.load_data("https://example.com/empty-sitemap.xml")
assert docs == []
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_load_data_list_input(self, mock_pinned_request, mock_validate):
loader = SitemapLoader()
mock_validate.side_effect = lambda url: url
@@ -274,8 +274,8 @@ class TestSitemapLoaderLoadData:
assert len(docs) == 1
assert docs[0].text == "List body"
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_load_data_respects_limit(self, mock_pinned_request, mock_validate):
loader = SitemapLoader(limit=2)
mock_validate.side_effect = lambda url: url
@@ -290,8 +290,8 @@ class TestSitemapLoaderLoadData:
assert len(docs) == 2
assert mock_pinned_request.call_count == 2
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_load_data_handles_url_error(self, mock_pinned_request, mock_validate):
loader = SitemapLoader()
mock_validate.side_effect = lambda url: url
@@ -305,19 +305,19 @@ class TestSitemapLoaderLoadData:
assert docs == []
def test_load_data_ssrf_blocked(self):
from application.core.url_validation import SSRFError
from docsgpt.core.url_validation import SSRFError
loader = SitemapLoader()
with patch(
"application.parser.remote.sitemap_loader.validate_url",
"docsgpt.parser.remote.sitemap_loader.validate_url",
side_effect=SSRFError("blocked"),
):
docs = loader.load_data("http://169.254.169.254/")
assert docs == []
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_load_data_no_limit(self, mock_pinned_request, mock_validate):
loader = SitemapLoader(limit=None)
mock_validate.side_effect = lambda url: url
tests/parser/remote/test_web_loader.py
+33 -33
View File
@@ -2,10 +2,10 @@ import pytest
from unittest.mock import patch, MagicMock
from urllib.parse import urlparse
from application.core.url_validation import SSRFError
from application.parser.remote.web_loader import WebLoader, headers
from application.parser.schema.base import Document
from application.vectorstore.document_class import Document as LCDocument
from docsgpt.core.url_validation import SSRFError
from docsgpt.parser.remote.web_loader import WebLoader, headers
from docsgpt.parser.schema.base import Document
from docsgpt.vectorstore.document_class import Document as LCDocument
def _mock_validate_url(url):
@@ -60,8 +60,8 @@ class TestWebLoaderHeaders:
class TestWebLoaderLoadData:
"""Test WebLoader load_data method."""
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_single_url_string(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response(
"<html lang='en'><head><title>Test Page</title></head>"
@@ -82,8 +82,8 @@ class TestWebLoaderLoadData:
"GET", "https://example.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_multiple_urls_list(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.side_effect = [
_fake_response("<html><body>Content from site 1</body></html>"),
@@ -108,8 +108,8 @@ class TestWebLoaderLoadData:
"GET", "https://site2.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_url_without_scheme(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response(
"<html><body>Schemeless</body></html>"
@@ -122,8 +122,8 @@ class TestWebLoaderLoadData:
"GET", "http://example.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_url_with_scheme(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response(
"<html><body>Schemed</body></html>"
@@ -136,8 +136,8 @@ class TestWebLoaderLoadData:
"GET", "https://example.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_skips_pages_without_title(self, mock_pinned_request, mock_validate, web_loader):
"""A page without <title> or <html lang=...> still loads, just with bare metadata."""
mock_pinned_request.return_value = _fake_response("<p>Bare body</p>")
@@ -148,8 +148,8 @@ class TestWebLoaderLoadData:
assert result[0].text == "Bare body"
assert result[0].extra_info == {"source": "https://example.com"}
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_extracts_title_with_nested_markup(self, mock_pinned_request, mock_validate, web_loader):
"""<title> with nested-looking content must still produce a non-empty title.
@@ -175,9 +175,9 @@ class TestWebLoaderLoadData:
class TestWebLoaderErrorHandling:
"""Test WebLoader error handling."""
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("application.parser.remote.web_loader.logging")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.logging")
def test_load_data_single_url_error(self, mock_logging, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.side_effect = Exception("Network error")
@@ -189,9 +189,9 @@ class TestWebLoaderErrorHandling:
assert "Error processing URL https://invalid-url.com" in error_call[0][0]
assert error_call[1]["exc_info"] is True
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("application.parser.remote.web_loader.logging")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.logging")
def test_load_data_partial_failure(self, mock_logging, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.side_effect = [
_fake_response("<html><body>Success content</body></html>"),
@@ -213,8 +213,8 @@ class TestWebLoaderErrorHandling:
class TestWebLoaderSSRF:
"""Test WebLoader SSRF protection."""
@patch("application.parser.remote.web_loader.validate_url")
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url")
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_skips_url_failing_ssrf_validation(self, mock_pinned_request, mock_validate, web_loader):
"""A URL that fails SSRF validation must be skipped, never reaching the fetcher."""
mock_validate.side_effect = SSRFError("Access to private/internal IP addresses is not allowed.")
@@ -225,8 +225,8 @@ class TestWebLoaderSSRF:
mock_validate.assert_called_once_with("http://169.254.169.254/latest/meta-data/")
mock_pinned_request.assert_not_called()
@patch("application.parser.remote.web_loader.validate_url")
@patch("application.parser.remote.web_loader.logging")
@patch("docsgpt.parser.remote.web_loader.validate_url")
@patch("docsgpt.parser.remote.web_loader.logging")
def test_load_data_logs_warning_on_ssrf_failure(self, mock_logging, mock_validate, web_loader):
mock_validate.side_effect = SSRFError("blocked")
@@ -237,8 +237,8 @@ class TestWebLoaderSSRF:
assert "SSRF" in warning_msg
assert "127.0.0.1" in warning_msg
@patch("application.parser.remote.web_loader.validate_url")
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url")
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_mixed_safe_and_unsafe_urls(self, mock_pinned_request, mock_validate, web_loader):
def validate(url):
if "169.254.169.254" in url:
@@ -267,8 +267,8 @@ class TestWebLoaderEdgeCases:
result = web_loader.load_data([])
assert result == []
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_empty_response_body(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response("")
@@ -292,15 +292,15 @@ class TestWebLoaderIntegration:
"""Test WebLoader integration with base class."""
def test_inherits_from_base_remote(self, web_loader):
from application.parser.remote.base import BaseRemote
from docsgpt.parser.remote.base import BaseRemote
assert isinstance(web_loader, BaseRemote)
def test_implements_load_data_method(self, web_loader):
assert hasattr(web_loader, "load_data")
assert callable(web_loader.load_data)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_vector_documents_method(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response(
"<html><head><title>Test Page</title></head>"
tests/parser/test_chunking.py
+3 -3
View File
@@ -1,4 +1,4 @@
"""Comprehensive tests for application/parser/chunking.py
"""Comprehensive tests for docsgpt/parser/chunking.py
Covers: Chunker (init, separate_header_and_body, split_document,
classic_chunk, chunk), edge cases, token counting.
@@ -6,8 +6,8 @@ classic_chunk, chunk), edge cases, token counting.
import pytest
from application.parser.chunking import Chunker
from application.parser.schema.base import Document
from docsgpt.parser.chunking import Chunker
from docsgpt.parser.schema.base import Document
# =====================================================================
tests/parser/test_chunking_creator.py
+3 -3
View File
@@ -4,9 +4,9 @@ from __future__ import annotations
import pytest
from application.parser.chunking import Chunker
from application.parser.chunking_creator import ChunkerCreator
from application.parser.schema.base import Document
from docsgpt.parser.chunking import Chunker
from docsgpt.parser.chunking_creator import ChunkerCreator
from docsgpt.parser.schema.base import Document
def _docs():
tests/parser/test_chunking_strategies.py
+8 -8
View File
@@ -6,16 +6,16 @@ from unittest.mock import patch
import pytest
from application.parser.chunking import Chunker
from application.parser.chunking_creator import ChunkerCreator
from application.parser.chunking_strategies import (
from docsgpt.parser.chunking import Chunker
from docsgpt.parser.chunking_creator import ChunkerCreator
from docsgpt.parser.chunking_strategies import (
MarkdownChunker,
ParentChildChunker,
RecursiveChunker,
SemanticChunker,
)
from application.parser.schema.base import Document
from application.parser.tokenization import get_token_counter
from docsgpt.parser.schema.base import Document
from docsgpt.parser.tokenization import get_token_counter
def _tok(text: str) -> int:
@@ -129,7 +129,7 @@ class TestParentChild:
assert all("parent_text" in c.extra_info for c in out)
_EMB_TARGET = "application.vectorstore.base.EmbeddingsSingleton.get_instance"
_EMB_TARGET = "docsgpt.vectorstore.base.EmbeddingsSingleton.get_instance"
class _FakeEmbeddings:
@@ -145,7 +145,7 @@ class TestSemantic:
@pytest.fixture(autouse=True)
def _no_remote_embeddings(self, monkeypatch):
"""The resolver short-circuits to the remote API when this is set."""
from application.core.settings import settings
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "EMBEDDINGS_BASE_URL", None)
@@ -274,7 +274,7 @@ class TestSemanticEmbeddingsResolution:
chunker = SemanticChunker(max_tokens=2000, min_tokens=0)
with patch(
"application.vectorstore.base.get_embeddings",
"docsgpt.vectorstore.base.get_embeddings",
return_value=_FakeEmbeddings(vectors),
) as mock_resolver:
out = chunker.chunk([Document(text=text, doc_id="d")])
tests/parser/test_document_reader.py
+8 -8
View File
@@ -11,8 +11,8 @@ from typing import Any, Dict, List
import pytest
import application.parser.document_reader as dr
from application.parser.document_reader import (
import docsgpt.parser.document_reader as dr
from docsgpt.parser.document_reader import (
bound_parse_payload,
parse_document_bytes,
truncate_text_head_tail,
@@ -229,7 +229,7 @@ def test_default_markdown_with_tables_converts_docling_once(monkeypatch):
# engine=auto resolves to the server default; this test is about the docling path.
monkeypatch.setattr(dr.settings, "DOC_PARSER_ENGINE", "docling")
from application.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.docling_parser import DoclingParser
counter = {"instances": 0, "converts": 0}
@@ -546,7 +546,7 @@ def test_vanilla_converter_applies_inference_settings(monkeypatch, tmp_path):
called = []
monkeypatch.setattr(
"application.parser.file.docling_parser._apply_inference_settings",
"docsgpt.parser.file.docling_parser._apply_inference_settings",
lambda: called.append(True),
)
@@ -618,7 +618,7 @@ def test_structured_tables_render_blank_cells_as_strings(monkeypatch, tmp_path):
return types.SimpleNamespace(document=_Doc())
monkeypatch.setattr(
"application.parser.file.docling_parser._apply_inference_settings",
"docsgpt.parser.file.docling_parser._apply_inference_settings",
lambda: None,
)
fake_docling = types.ModuleType("docling")
@@ -753,7 +753,7 @@ def test_tables_are_not_collected_when_the_pdf_parser_is_not_docling(monkeypatch
"""Under OCR_BACKEND=native the docling engine hands PDFs to the native OCR
parser; a vanilla DocumentConverter table pass would OCR the scan again."""
from application.parser.file.ocr_parser import NativeOcrPdfParser
from docsgpt.parser.file.ocr_parser import NativeOcrPdfParser
native = NativeOcrPdfParser()
native._parser_config = {}
@@ -777,8 +777,8 @@ def test_tables_survive_native_ocr_delegation_for_text_only_pdfs(monkeypatch):
to that parser, so tables must ride that single conversion instead of being
dropped by the scan exclusion."""
from application.parser.file.docling_parser import DoclingParser
from application.parser.file.ocr_parser import NativeOcrPdfParser
from docsgpt.parser.file.docling_parser import DoclingParser
from docsgpt.parser.file.ocr_parser import NativeOcrPdfParser
class _Docling(DoclingParser):
def __init__(self):
tests/parser/test_schema.py
+8 -8
View File
@@ -1,6 +1,6 @@
import pytest
from application.parser.schema.schema import BaseDocument
from docsgpt.parser.schema.schema import BaseDocument
class ConcreteDoc(BaseDocument):
@@ -59,7 +59,7 @@ class TestBaseDocument:
# =====================================================================
# Coverage gap tests for application/parser/schema/base.py (lines 19, 27, 34)
# Coverage gap tests for docsgpt/parser/schema/base.py (lines 19, 27, 34)
# =====================================================================
@@ -68,14 +68,14 @@ class TestDocumentBase:
def test_document_post_init_raises_on_none_text(self):
"""Cover line 19: Document.__post_init__ raises ValueError for None text."""
from application.parser.schema.base import Document
from docsgpt.parser.schema.base import Document
with pytest.raises(ValueError, match="text field not set"):
Document(text=None)
def test_document_to_vector_format(self):
"""Cover line 27: Document.to_vector_format converts correctly."""
from application.parser.schema.base import Document
from docsgpt.parser.schema.base import Document
doc = Document(text="hello world", extra_info={"source": "test"})
lc_doc = doc.to_vector_format()
@@ -84,7 +84,7 @@ class TestDocumentBase:
def test_document_to_vector_format_no_extra_info(self):
"""Cover: to_vector_format with no extra_info uses empty dict."""
from application.parser.schema.base import Document
from docsgpt.parser.schema.base import Document
doc = Document(text="hello")
lc_doc = doc.to_vector_format()
@@ -92,8 +92,8 @@ class TestDocumentBase:
def test_document_from_vector_format(self):
"""Cover line 34: Document.from_vector_format creates Document."""
from application.parser.schema.base import Document
from application.vectorstore.document_class import Document as LCDocument
from docsgpt.parser.schema.base import Document
from docsgpt.vectorstore.document_class import Document as LCDocument
lc_doc = LCDocument(page_content="test content", metadata={"key": "val"})
doc = Document.from_vector_format(lc_doc)
@@ -102,6 +102,6 @@ class TestDocumentBase:
def test_document_get_type(self):
"""Cover line 24: Document.get_type returns 'Document'."""
from application.parser.schema.base import Document
from docsgpt.parser.schema.base import Document
assert Document.get_type() == "Document"
tests/parser/test_tokenization.py
+2 -2
View File
@@ -6,8 +6,8 @@ import types
import pytest
from application.parser import tokenization
from application.parser.tokenization import (
from docsgpt.parser import tokenization
from docsgpt.parser.tokenization import (
HuggingFaceCounter,
TiktokenCounter,
get_token_counter,