refactor: rename the application package to docsgpt

The backend import package is now docsgpt, the name it will carry on PyPI;
application was far too generic to install into anyone's site-packages.
git mv plus a mechanical rewrite of every import, dotted string and path
reference: 734 Python files, the compose files, Dockerfile, workflows, docs,
setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage
config, .gitignore. Behaviour is unchanged.

Kept for one release:
- A top-level application package whose meta-path finder resolves
  application.x.y to the already-imported docsgpt.x.y object, so old imports
  and entry points (celery -A application.app.celery,
  uvicorn application.asgi:asgi_app) keep working with a FutureWarning.
- Celery registers every application.* task name as an alias of its
  docsgpt.* task on start-up, so messages queued by the previous release still
  run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries
  the previous release wrote are left unread instead of firing twice.

The backend image builds from the repository root (docker build -f
docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore
allow-lists docsgpt/ and application/ and keeps caches, local data, .env
files, the sample index files and the Dockerfile out. Compose and the image
workflows point at the new context.
This commit is contained in:
Alex committed 2026-09-07 10:20:43 +01:00
1 parent 3e7f1f91b4
commit 574f96341e
984 files changed
+9847 -9682

No files matched your search

+16 -16
View File
@@ -2,8 +2,8 @@ from unittest.mock import MagicMock, patch
import pytest
from application.parser.remote.crawler_loader import CrawlerLoader
from application.parser.schema.base import Document
from docsgpt.parser.remote.crawler_loader import CrawlerLoader
from docsgpt.parser.schema.base import Document
class DummyResponse:
@@ -22,8 +22,8 @@ def _mock_validate_url(url):
return url
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.crawler_loader.pinned_request")
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
def test_load_data_crawls_same_domain_links(mock_pinned_request, mock_validate_url):
responses = {
"http://example.com": DummyResponse(
@@ -65,8 +65,8 @@ def test_load_data_crawls_same_domain_links(mock_pinned_request, mock_validate_u
assert mock_pinned_request.call_count == 2
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.crawler_loader.pinned_request")
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
def test_load_data_accepts_list_input_and_adds_scheme(mock_pinned_request, mock_validate_url):
mock_pinned_request.return_value = DummyResponse("<html><body>No links here</body></html>")
crawler = CrawlerLoader()
@@ -83,8 +83,8 @@ def test_load_data_accepts_list_input_and_adds_scheme(mock_pinned_request, mock_
}
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.crawler_loader.pinned_request")
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
def test_load_data_respects_limit(mock_pinned_request, mock_validate_url):
responses = {
"http://example.com": DummyResponse(
@@ -110,9 +110,9 @@ def test_load_data_respects_limit(mock_pinned_request, mock_validate_url):
assert mock_pinned_request.call_count == 1
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.crawler_loader.logging")
@patch("application.parser.remote.crawler_loader.pinned_request")
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.crawler_loader.logging")
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
def test_load_data_logs_and_skips_on_request_error(mock_pinned_request, mock_logging, mock_validate_url):
mock_pinned_request.side_effect = Exception("load failure")
crawler = CrawlerLoader()
@@ -128,10 +128,10 @@ def test_load_data_logs_and_skips_on_request_error(mock_pinned_request, mock_log
assert mock_logging.error.call_args.kwargs.get("exc_info") is True
@patch("application.parser.remote.crawler_loader.validate_url")
@patch("docsgpt.parser.remote.crawler_loader.validate_url")
def test_load_data_returns_empty_on_ssrf_validation_failure(mock_validate_url):
"""Test that SSRF validation failure returns empty list."""
from application.core.url_validation import SSRFError
from docsgpt.core.url_validation import SSRFError
mock_validate_url.side_effect = SSRFError("Access to private IP not allowed")
crawler = CrawlerLoader()
@@ -165,15 +165,15 @@ def test_url_to_virtual_path_variants():
class TestCrawlerLoaderGaps:
def test_pinned_fetch_builds_document_without_webbase_loader(self):
"""The crawler should index the response body it already fetched."""
from application.parser.remote.crawler_loader import CrawlerLoader
from docsgpt.parser.remote.crawler_loader import CrawlerLoader
loader = CrawlerLoader(limit=5)
with patch(
"application.parser.remote.crawler_loader.validate_url",
"docsgpt.parser.remote.crawler_loader.validate_url",
return_value="https://example.com",
):
with patch(
"application.parser.remote.crawler_loader.pinned_request"
"docsgpt.parser.remote.crawler_loader.pinned_request"
) as mock_pinned_request:
mock_response = MagicMock()
mock_response.status_code = 200
+8 -8
View File
@@ -5,8 +5,8 @@ from urllib.parse import urlparse
import pytest
import requests
from application.parser.remote.crawler_markdown import CrawlerLoader
from application.parser.schema.base import Document
from docsgpt.parser.remote.crawler_markdown import CrawlerLoader
from docsgpt.parser.schema.base import Document
class DummyResponse:
@@ -40,7 +40,7 @@ def _mock_validate_url(url):
@pytest.fixture(autouse=True)
def _patch_validate_url(monkeypatch):
monkeypatch.setattr(
"application.parser.remote.crawler_markdown.validate_url",
"docsgpt.parser.remote.crawler_markdown.validate_url",
_mock_validate_url,
)
@@ -48,7 +48,7 @@ def _patch_validate_url(monkeypatch):
@pytest.fixture(autouse=True)
def _patch_tldextract(monkeypatch):
monkeypatch.setattr(
"application.parser.remote.crawler_markdown._extract",
"docsgpt.parser.remote.crawler_markdown._extract",
_fake_extract,
)
@@ -61,7 +61,7 @@ def _patch_markdownify(monkeypatch):
return outputs.get(html, html)
monkeypatch.setattr(
"application.parser.remote.crawler_markdown.markdownify",
"docsgpt.parser.remote.crawler_markdown.markdownify",
fake_markdownify,
)
return outputs
@@ -73,7 +73,7 @@ def _patch_pinned_request(monkeypatch, side_effect):
return side_effect(url)
monkeypatch.setattr(
"application.parser.remote.crawler_markdown.pinned_request",
"docsgpt.parser.remote.crawler_markdown.pinned_request",
fake_pinned_request,
)
@@ -164,13 +164,13 @@ def test_load_data_handles_fetch_errors(monkeypatch, _patch_markdownify, _patch_
def test_load_data_returns_empty_on_ssrf_validation_failure(monkeypatch):
"""Test that SSRF validation failure returns empty list."""
from application.core.url_validation import SSRFError
from docsgpt.core.url_validation import SSRFError
def raise_ssrf_error(url):
raise SSRFError("Access to private IP not allowed")
monkeypatch.setattr(
"application.parser.remote.crawler_markdown.validate_url",
"docsgpt.parser.remote.crawler_markdown.validate_url",
raise_ssrf_error,
)
+31 -31
View File
@@ -3,7 +3,7 @@ import pytest
from unittest.mock import patch, MagicMock
import requests
from application.parser.remote.github_loader import GitHubLoader
from docsgpt.parser.remote.github_loader import GitHubLoader
def make_response(json_data=None, status_code=200, raise_error=None):
@@ -18,7 +18,7 @@ def make_response(json_data=None, status_code=200, raise_error=None):
class TestGitHubLoaderFetchFileContent:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_text_file_base64_decoded(self, mock_get):
loader = GitHubLoader()
content_str = "Hello from README"
@@ -34,7 +34,7 @@ class TestGitHubLoaderFetchFileContent:
timeout=100,
)
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_binary_file_skipped(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response({"encoding": "base64", "content": "AAAA"})
@@ -43,7 +43,7 @@ class TestGitHubLoaderFetchFileContent:
assert result is None
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_non_base64_plain_content(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response({"encoding": "", "content": "Plain text"})
@@ -52,7 +52,7 @@ class TestGitHubLoaderFetchFileContent:
assert result == "Plain text"
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_http_error_raises(self, mock_get):
loader = GitHubLoader()
http_err = requests.HTTPError("Not found")
@@ -63,7 +63,7 @@ class TestGitHubLoaderFetchFileContent:
class TestGitHubLoaderFetchRepoFiles:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_recurses_directories(self, mock_get):
loader = GitHubLoader()
@@ -133,7 +133,7 @@ class TestGitHubLoaderIsTextFile:
loader = GitHubLoader()
assert loader.is_text_file("image.png") is False
@patch("application.parser.remote.github_loader.mimetypes.guess_type")
@patch("docsgpt.parser.remote.github_loader.mimetypes.guess_type")
def test_mime_fallback_text(self, mock_mime):
mock_mime.return_value = ("text/plain", None)
loader = GitHubLoader()
@@ -141,15 +141,15 @@ class TestGitHubLoaderIsTextFile:
class TestGitHubLoaderMakeRequest:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_success(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response({"ok": True}, 200)
resp = loader._make_request("http://example.com")
assert resp.status_code == 200
@patch("application.parser.remote.github_loader.time.sleep")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.time.sleep")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_rate_limit_retry(self, mock_get, mock_sleep):
loader = GitHubLoader()
rate_resp = MagicMock()
@@ -166,7 +166,7 @@ class TestGitHubLoaderMakeRequest:
assert resp.status_code == 200
mock_sleep.assert_called_once()
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_rate_limit_exhausted(self, mock_get):
loader = GitHubLoader()
rate_resp = MagicMock()
@@ -181,7 +181,7 @@ class TestGitHubLoaderMakeRequest:
with pytest.raises(Exception, match="rate limit exceeded"):
loader._make_request("http://example.com", max_retries=1)
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_403_non_rate_limit(self, mock_get):
loader = GitHubLoader()
resp = MagicMock()
@@ -193,7 +193,7 @@ class TestGitHubLoaderMakeRequest:
with pytest.raises(Exception, match="GitHub API error"):
loader._make_request("http://example.com", max_retries=1)
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_other_error_raises(self, mock_get):
loader = GitHubLoader()
resp = make_response(
@@ -207,7 +207,7 @@ class TestGitHubLoaderMakeRequest:
class TestGitHubLoaderFetchRepoFilesErrors:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_api_error_message_in_dict(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response(
@@ -217,7 +217,7 @@ class TestGitHubLoaderFetchRepoFilesErrors:
with pytest.raises(Exception, match="GitHub API error"):
loader.fetch_repo_files("owner/repo")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_non_list_response(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response("not a list", 200)
@@ -227,7 +227,7 @@ class TestGitHubLoaderFetchRepoFilesErrors:
class TestGitHubLoaderFetchFileContentEdgeCases:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_empty_base64_text_returns_none(self, mock_get):
loader = GitHubLoader()
b64 = base64.b64encode(b"").decode("utf-8")
@@ -237,7 +237,7 @@ class TestGitHubLoaderFetchFileContentEdgeCases:
result = loader.fetch_file_content("owner/repo", "empty.py")
assert result is None
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_empty_non_base64_returns_none(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response(
@@ -246,7 +246,7 @@ class TestGitHubLoaderFetchFileContentEdgeCases:
result = loader.fetch_file_content("owner/repo", "empty.txt")
assert result is None
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_decode_failure_returns_none(self, mock_get):
loader = GitHubLoader()
mock_get.return_value = make_response(
@@ -275,7 +275,7 @@ class TestGitHubLoaderLoadDataSkipsNone:
class TestGitHubLoaderRobustness:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_fetch_repo_files_non_json_raises(self, mock_get):
resp = MagicMock()
resp.json.side_effect = ValueError("No JSON")
@@ -283,14 +283,14 @@ class TestGitHubLoaderRobustness:
with pytest.raises(ValueError):
GitHubLoader().fetch_repo_files("owner/repo")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_fetch_repo_files_unexpected_shape_missing_type_raises(self, mock_get):
# Missing 'type' in items should raise KeyError when accessed
mock_get.return_value = make_response([{"path": "README.md"}])
with pytest.raises(KeyError):
GitHubLoader().fetch_repo_files("owner/repo")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_fetch_file_content_non_json_raises(self, mock_get):
resp = MagicMock()
resp.status_code = 200
@@ -299,7 +299,7 @@ class TestGitHubLoaderRobustness:
with pytest.raises(ValueError):
GitHubLoader().fetch_file_content("owner/repo", "README.md")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_fetch_file_content_unexpected_shape_missing_content_returns_none(self, mock_get):
# encoding indicates base64 text, but 'content' key is missing
# With the new code, the exception is caught and returns None (treated as binary/skipped)
@@ -308,8 +308,8 @@ class TestGitHubLoaderRobustness:
result = GitHubLoader().fetch_file_content("owner/repo", "file.txt")
assert result is None
@patch("application.parser.remote.github_loader.base64.b64decode")
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.base64.b64decode")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_large_binary_skip_does_not_decode(self, mock_get, mock_b64decode):
# Ensure we don't attempt to decode large binary content for non-text files
mock_b64decode.side_effect = AssertionError("b64decode should not be called for binary files")
@@ -384,7 +384,7 @@ class TestGitHubLoaderSelectFiles:
def test_applies_size_cap(self, monkeypatch):
loader = GitHubLoader()
monkeypatch.setattr(
"application.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
"docsgpt.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
100, raising=False,
)
entries = [("small.py", 50), ("huge.py", 5000), ("ok.md", 99)]
@@ -393,7 +393,7 @@ class TestGitHubLoaderSelectFiles:
def test_zero_cap_disables_limit(self, monkeypatch):
loader = GitHubLoader()
monkeypatch.setattr(
"application.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
"docsgpt.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
0, raising=False,
)
assert loader.select_files([("huge.py", 10**9)]) == ["huge.py"]
@@ -407,7 +407,7 @@ class TestGitHubLoaderSelectFiles:
class TestGitHubLoaderTree:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_single_request_lists_all_blobs(self, mock_get):
mock_get.return_value = make_response({
"tree": [
@@ -424,7 +424,7 @@ class TestGitHubLoaderTree:
# One call for the whole repo, versus one per directory before.
assert mock_get.call_count == 1
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_truncated_tree_falls_back_to_walk(self, mock_get, monkeypatch):
loader = GitHubLoader()
mock_get.return_value = make_response({"tree": [], "truncated": True})
@@ -435,12 +435,12 @@ class TestGitHubLoaderTree:
class TestGitHubLoaderDefaultBranch:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_uses_repo_default_branch(self, mock_get):
mock_get.return_value = make_response({"default_branch": "master"})
assert GitHubLoader().get_default_branch("owner/repo") == "master"
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_falls_back_to_main(self, mock_get):
mock_get.side_effect = requests.ConnectionError("boom")
assert GitHubLoader().get_default_branch("owner/repo") == "main"
@@ -496,7 +496,7 @@ class TestGitHubLoaderParallelFetch:
class TestGitHubLoaderStaleTokenFallback:
@patch("application.parser.remote.github_loader.requests.get")
@patch("docsgpt.parser.remote.github_loader.requests.get")
def test_401_retries_unauthenticated(self, mock_get):
"""An expired PAT 401s even public repos; fall back to anonymous
rather than failing the ingest outright."""
+1 -1
View File
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
import pytest
from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
from docsgpt.parser.remote.reddit_loader import RedditPostsLoaderRemote
def _post(post_id, title, body):
+16 -16
View File
@@ -1,4 +1,4 @@
"""Tests for application.parser.remote.remote_creator."""
"""Tests for docsgpt.parser.remote.remote_creator."""
import json
@@ -10,7 +10,7 @@ from unittest.mock import MagicMock
class TestRemoteCreator:
def test_create_loader_valid_type(self):
"""Cover line 34: returns loader instance for valid type."""
from application.parser.remote.remote_creator import RemoteCreator
from docsgpt.parser.remote.remote_creator import RemoteCreator
mock_loader_cls = MagicMock()
original_loaders = RemoteCreator.loaders.copy()
@@ -23,14 +23,14 @@ class TestRemoteCreator:
def test_create_loader_invalid_type_raises(self):
"""Cover lines 32-33: raises ValueError for unknown type."""
from application.parser.remote.remote_creator import RemoteCreator
from docsgpt.parser.remote.remote_creator import RemoteCreator
with pytest.raises(ValueError, match="No loader class found"):
RemoteCreator.create_loader("nonexistent_xyz")
def test_create_loader_case_insensitive(self):
"""Cover line 31: type.lower() normalization."""
from application.parser.remote.remote_creator import RemoteCreator
from docsgpt.parser.remote.remote_creator import RemoteCreator
mock_loader_cls = MagicMock()
original_loaders = RemoteCreator.loaders.copy()
@@ -48,12 +48,12 @@ class TestNormalizeRemoteData:
back to the ``source_data`` shape each loader expects."""
def test_none_passes_through(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
assert normalize_remote_data("crawler", None) is None
def test_crawler_dict_with_url_key(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data(
"crawler", {"url": "https://example.com", "provider": "crawler"}
@@ -61,20 +61,20 @@ class TestNormalizeRemoteData:
assert result == "https://example.com"
def test_url_dict_with_url_key(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data("url", {"url": "https://example.com"})
assert result == "https://example.com"
def test_url_legacy_raw_key(self):
"""Legacy rows wrapped a bare URL string as ``{"raw": ...}``."""
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data("crawler", {"raw": "https://legacy.example.com"})
assert result == "https://legacy.example.com"
def test_url_dict_with_urls_list(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data(
"url", {"urls": ["https://a.example.com", "https://b.example.com"]}
@@ -82,7 +82,7 @@ class TestNormalizeRemoteData:
assert result == ["https://a.example.com", "https://b.example.com"]
def test_github_repo_url_key(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data(
"github", {"repo_url": "https://github.com/arc53/DocsGPT"}
@@ -90,25 +90,25 @@ class TestNormalizeRemoteData:
assert result == "https://github.com/arc53/DocsGPT"
def test_sitemap_dict_with_url_key(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data("sitemap", {"url": "https://example.com/sitemap.xml"})
assert result == "https://example.com/sitemap.xml"
def test_plain_string_url_passes_through(self):
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
assert normalize_remote_data("crawler", "https://example.com") == "https://example.com"
def test_url_dict_without_url_key_returns_none(self):
"""A URL-type loader must never receive a dict, even a malformed one."""
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
assert normalize_remote_data("crawler", {"provider": "crawler"}) is None
def test_reddit_dict_serialized_to_json_string(self):
"""reddit's loader runs json.loads() — it needs a JSON string."""
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data(
"reddit", {"client_id": "x", "search_queries": ["y"]}
@@ -118,14 +118,14 @@ class TestNormalizeRemoteData:
def test_s3_dict_passes_through(self):
"""S3Loader.load_data() accepts a dict, so it is left untouched."""
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
data = {"bucket": "b", "prefix": "k"}
assert normalize_remote_data("s3", data) == data
def test_json_string_remote_data_is_parsed(self):
"""Legacy rows that stored the JSON itself as a string still resolve."""
from application.parser.remote.remote_creator import normalize_remote_data
from docsgpt.parser.remote.remote_creator import normalize_remote_data
result = normalize_remote_data("crawler", '{"url": "https://example.com"}')
assert result == "https://example.com"
+15 -15
View File
@@ -11,14 +11,14 @@ from botocore.exceptions import ClientError, NoCredentialsError
def mock_boto3():
"""Mock boto3 module."""
with patch.dict("sys.modules", {"boto3": MagicMock()}):
with patch("application.parser.remote.s3_loader.boto3") as mock:
with patch("docsgpt.parser.remote.s3_loader.boto3") as mock:
yield mock
@pytest.fixture
def s3_loader(mock_boto3):
"""Create S3Loader instance with mocked boto3."""
from application.parser.remote.s3_loader import S3Loader
from docsgpt.parser.remote.s3_loader import S3Loader
loader = S3Loader()
return loader
@@ -29,15 +29,15 @@ class TestS3LoaderInit:
def test_init_raises_import_error_when_boto3_missing(self):
"""Should raise ImportError when boto3 is not installed."""
with patch("application.parser.remote.s3_loader.boto3", None):
from application.parser.remote.s3_loader import S3Loader
with patch("docsgpt.parser.remote.s3_loader.boto3", None):
from docsgpt.parser.remote.s3_loader import S3Loader
with pytest.raises(ImportError, match="boto3 is required"):
S3Loader()
def test_init_sets_client_to_none(self, mock_boto3):
"""Should initialize with s3_client as None."""
from application.parser.remote.s3_loader import S3Loader
from docsgpt.parser.remote.s3_loader import S3Loader
loader = S3Loader()
assert loader.s3_client is None
@@ -127,7 +127,7 @@ class TestInitClient:
def test_init_client_with_custom_endpoint(self, s3_loader, mock_boto3):
"""Should configure path-style addressing for custom endpoints."""
with patch(
"application.parser.remote.s3_loader.validate_url",
"docsgpt.parser.remote.s3_loader.validate_url",
side_effect=lambda u: u,
):
s3_loader._init_client(
@@ -145,7 +145,7 @@ class TestInitClient:
def test_init_client_normalizes_do_endpoint(self, s3_loader, mock_boto3):
"""Should normalize DigitalOcean Spaces bucket-prefixed URLs."""
with patch(
"application.parser.remote.s3_loader.validate_url",
"docsgpt.parser.remote.s3_loader.validate_url",
side_effect=lambda u: u,
):
corrected_bucket = s3_loader._init_client(
@@ -564,7 +564,7 @@ class TestLoadData:
}
with patch(
"application.parser.remote.s3_loader.validate_url",
"docsgpt.parser.remote.s3_loader.validate_url",
side_effect=lambda u: u,
):
s3_loader.load_data(input_data)
@@ -644,7 +644,7 @@ class TestLoadData:
}
with patch(
"application.parser.remote.s3_loader.validate_url",
"docsgpt.parser.remote.s3_loader.validate_url",
side_effect=lambda u: u,
):
docs = s3_loader.load_data(input_data)
@@ -663,7 +663,7 @@ class TestProcessDocument:
mock_doc.text = "Extracted document text"
with patch(
"application.parser.file.bulk.SimpleDirectoryReader"
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
) as mock_reader_class:
mock_reader = MagicMock()
mock_reader.load_data.return_value = [mock_doc]
@@ -687,7 +687,7 @@ class TestProcessDocument:
def test_process_document_returns_none_on_error(self, s3_loader):
"""Should return None when document processing fails."""
with patch(
"application.parser.file.bulk.SimpleDirectoryReader"
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
) as mock_reader_class:
mock_reader_class.side_effect = Exception("Parse error")
@@ -709,7 +709,7 @@ class TestProcessDocument:
def test_process_document_cleans_up_temp_file(self, s3_loader):
"""Should clean up temporary file after processing."""
with patch(
"application.parser.file.bulk.SimpleDirectoryReader"
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
) as mock_reader_class:
mock_reader = MagicMock()
mock_reader.load_data.return_value = []
@@ -831,7 +831,7 @@ class TestProcessDocumentAdditional:
def test_process_document_empty_documents_returns_none(self, s3_loader):
"""Cover line 347-348: no documents extracted returns None."""
with patch(
"application.parser.file.bulk.SimpleDirectoryReader"
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
) as mock_reader_class:
mock_reader = MagicMock()
mock_reader.load_data.return_value = []
@@ -906,8 +906,8 @@ class TestSSRFValidation:
def test_is_supported_document_follows_the_upload_whitelist():
from application.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
from application.parser.remote.s3_loader import S3Loader
from docsgpt.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
from docsgpt.parser.remote.s3_loader import S3Loader
for suffix in SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS:
key = f"bucket/file{suffix}"
+17 -17
View File
@@ -2,7 +2,7 @@
from unittest.mock import patch, MagicMock
from application.parser.connectors.share_point.loader import SharePointLoader
from docsgpt.parser.connectors.share_point.loader import SharePointLoader
def make_response(json_data=None, status_code=200, raise_error=None):
@@ -65,10 +65,10 @@ class TestSharePointLoaderProcessFile:
class TestSharePointLoaderLoadFileById:
"""Test _load_file_by_id method."""
@patch("application.parser.connectors.share_point.loader.requests.get")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
def test_load_file_by_id_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
"""Should include size field in $select parameter."""
mock_get_token.return_value = {
@@ -96,10 +96,10 @@ class TestSharePointLoaderLoadFileById:
params = call_args[1]["params"]
assert "size" in params["$select"]
@patch("application.parser.connectors.share_point.loader.requests.get")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
def test_load_file_by_id_returns_document_with_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
"""Should return document with size from API response."""
mock_get_token.return_value = {
@@ -133,10 +133,10 @@ class TestSharePointLoaderLoadFileById:
class TestSharePointLoaderListItems:
"""Test _list_items_in_parent method."""
@patch("application.parser.connectors.share_point.loader.requests.get")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
def test_list_items_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
"""Should include size field in $select parameter when listing items."""
mock_get_token.return_value = {
@@ -168,10 +168,10 @@ class TestSharePointLoaderListItems:
params = call_args[1]["params"]
assert "size" in params["$select"]
@patch("application.parser.connectors.share_point.loader.requests.get")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
def test_list_items_folders_include_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
"""Should include size for folders as well."""
mock_get_token.return_value = {
+22 -22
View File
@@ -1,4 +1,4 @@
"""Comprehensive tests for application/parser/remote/sitemap_loader.py
"""Comprehensive tests for docsgpt/parser/remote/sitemap_loader.py
Covers: SitemapLoader (init, load_data, _extract_urls, _is_sitemap,
_parse_sitemap, URL validation, error handling).
@@ -9,8 +9,8 @@ from unittest.mock import MagicMock, patch
import pytest
import requests
from application.parser.remote.sitemap_loader import SitemapLoader
from application.parser.schema.base import Document
from docsgpt.parser.remote.sitemap_loader import SitemapLoader
from docsgpt.parser.schema.base import Document
# =====================================================================
@@ -159,7 +159,7 @@ class TestParseSitemap:
@pytest.mark.unit
class TestExtractUrls:
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_extract_urls_from_sitemap(self, mock_pinned_request):
loader = SitemapLoader()
@@ -177,7 +177,7 @@ class TestExtractUrls:
urls = loader._extract_urls("https://example.com/sitemap.xml")
assert "https://example.com/p" in urls
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_extract_urls_not_sitemap(self, mock_pinned_request):
loader = SitemapLoader()
@@ -191,7 +191,7 @@ class TestExtractUrls:
urls = loader._extract_urls("https://example.com/page")
assert urls == ["https://example.com/page"]
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_extract_urls_http_error(self, mock_pinned_request):
loader = SitemapLoader()
mock_pinned_request.side_effect = requests.exceptions.HTTPError("404")
@@ -199,7 +199,7 @@ class TestExtractUrls:
urls = loader._extract_urls("https://example.com/missing")
assert urls == []
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_extract_urls_connection_error(self, mock_pinned_request):
loader = SitemapLoader()
mock_pinned_request.side_effect = requests.exceptions.ConnectionError()
@@ -208,12 +208,12 @@ class TestExtractUrls:
assert urls == []
def test_extract_urls_ssrf_blocked(self):
from application.security.safe_url import UnsafeUserUrlError
from docsgpt.security.safe_url import UnsafeUserUrlError
loader = SitemapLoader()
with patch(
"application.parser.remote.sitemap_loader.pinned_request",
"docsgpt.parser.remote.sitemap_loader.pinned_request",
side_effect=UnsafeUserUrlError("blocked"),
):
urls = loader._extract_urls("http://169.254.169.254/")
@@ -228,8 +228,8 @@ class TestExtractUrls:
@pytest.mark.unit
class TestSitemapLoaderLoadData:
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_load_data_success(self, mock_pinned_request, mock_validate):
loader = SitemapLoader(limit=10)
mock_validate.side_effect = lambda url: url
@@ -248,7 +248,7 @@ class TestSitemapLoaderLoadData:
assert docs[0].text == "Page body"
assert docs[0].extra_info == {"source": "https://example.com/page1"}
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
def test_load_data_no_urls(self, mock_validate):
loader = SitemapLoader()
@@ -256,8 +256,8 @@ class TestSitemapLoaderLoadData:
docs = loader.load_data("https://example.com/empty-sitemap.xml")
assert docs == []
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_load_data_list_input(self, mock_pinned_request, mock_validate):
loader = SitemapLoader()
mock_validate.side_effect = lambda url: url
@@ -274,8 +274,8 @@ class TestSitemapLoaderLoadData:
assert len(docs) == 1
assert docs[0].text == "List body"
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_load_data_respects_limit(self, mock_pinned_request, mock_validate):
loader = SitemapLoader(limit=2)
mock_validate.side_effect = lambda url: url
@@ -290,8 +290,8 @@ class TestSitemapLoaderLoadData:
assert len(docs) == 2
assert mock_pinned_request.call_count == 2
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_load_data_handles_url_error(self, mock_pinned_request, mock_validate):
loader = SitemapLoader()
mock_validate.side_effect = lambda url: url
@@ -305,19 +305,19 @@ class TestSitemapLoaderLoadData:
assert docs == []
def test_load_data_ssrf_blocked(self):
from application.core.url_validation import SSRFError
from docsgpt.core.url_validation import SSRFError
loader = SitemapLoader()
with patch(
"application.parser.remote.sitemap_loader.validate_url",
"docsgpt.parser.remote.sitemap_loader.validate_url",
side_effect=SSRFError("blocked"),
):
docs = loader.load_data("http://169.254.169.254/")
assert docs == []
@patch("application.parser.remote.sitemap_loader.validate_url")
@patch("application.parser.remote.sitemap_loader.pinned_request")
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
def test_load_data_no_limit(self, mock_pinned_request, mock_validate):
loader = SitemapLoader(limit=None)
mock_validate.side_effect = lambda url: url
+33 -33
View File
@@ -2,10 +2,10 @@ import pytest
from unittest.mock import patch, MagicMock
from urllib.parse import urlparse
from application.core.url_validation import SSRFError
from application.parser.remote.web_loader import WebLoader, headers
from application.parser.schema.base import Document
from application.vectorstore.document_class import Document as LCDocument
from docsgpt.core.url_validation import SSRFError
from docsgpt.parser.remote.web_loader import WebLoader, headers
from docsgpt.parser.schema.base import Document
from docsgpt.vectorstore.document_class import Document as LCDocument
def _mock_validate_url(url):
@@ -60,8 +60,8 @@ class TestWebLoaderHeaders:
class TestWebLoaderLoadData:
"""Test WebLoader load_data method."""
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_single_url_string(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response(
"<html lang='en'><head><title>Test Page</title></head>"
@@ -82,8 +82,8 @@ class TestWebLoaderLoadData:
"GET", "https://example.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_multiple_urls_list(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.side_effect = [
_fake_response("<html><body>Content from site 1</body></html>"),
@@ -108,8 +108,8 @@ class TestWebLoaderLoadData:
"GET", "https://site2.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_url_without_scheme(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response(
"<html><body>Schemeless</body></html>"
@@ -122,8 +122,8 @@ class TestWebLoaderLoadData:
"GET", "http://example.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_url_with_scheme(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response(
"<html><body>Schemed</body></html>"
@@ -136,8 +136,8 @@ class TestWebLoaderLoadData:
"GET", "https://example.com", headers=headers, timeout=30
)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_skips_pages_without_title(self, mock_pinned_request, mock_validate, web_loader):
"""A page without <title> or <html lang=...> still loads, just with bare metadata."""
mock_pinned_request.return_value = _fake_response("<p>Bare body</p>")
@@ -148,8 +148,8 @@ class TestWebLoaderLoadData:
assert result[0].text == "Bare body"
assert result[0].extra_info == {"source": "https://example.com"}
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_extracts_title_with_nested_markup(self, mock_pinned_request, mock_validate, web_loader):
"""<title> with nested-looking content must still produce a non-empty title.
@@ -175,9 +175,9 @@ class TestWebLoaderLoadData:
class TestWebLoaderErrorHandling:
"""Test WebLoader error handling."""
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("application.parser.remote.web_loader.logging")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.logging")
def test_load_data_single_url_error(self, mock_logging, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.side_effect = Exception("Network error")
@@ -189,9 +189,9 @@ class TestWebLoaderErrorHandling:
assert "Error processing URL https://invalid-url.com" in error_call[0][0]
assert error_call[1]["exc_info"] is True
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("application.parser.remote.web_loader.logging")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.logging")
def test_load_data_partial_failure(self, mock_logging, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.side_effect = [
_fake_response("<html><body>Success content</body></html>"),
@@ -213,8 +213,8 @@ class TestWebLoaderErrorHandling:
class TestWebLoaderSSRF:
"""Test WebLoader SSRF protection."""
@patch("application.parser.remote.web_loader.validate_url")
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url")
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_skips_url_failing_ssrf_validation(self, mock_pinned_request, mock_validate, web_loader):
"""A URL that fails SSRF validation must be skipped, never reaching the fetcher."""
mock_validate.side_effect = SSRFError("Access to private/internal IP addresses is not allowed.")
@@ -225,8 +225,8 @@ class TestWebLoaderSSRF:
mock_validate.assert_called_once_with("http://169.254.169.254/latest/meta-data/")
mock_pinned_request.assert_not_called()
@patch("application.parser.remote.web_loader.validate_url")
@patch("application.parser.remote.web_loader.logging")
@patch("docsgpt.parser.remote.web_loader.validate_url")
@patch("docsgpt.parser.remote.web_loader.logging")
def test_load_data_logs_warning_on_ssrf_failure(self, mock_logging, mock_validate, web_loader):
mock_validate.side_effect = SSRFError("blocked")
@@ -237,8 +237,8 @@ class TestWebLoaderSSRF:
assert "SSRF" in warning_msg
assert "127.0.0.1" in warning_msg
@patch("application.parser.remote.web_loader.validate_url")
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url")
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_mixed_safe_and_unsafe_urls(self, mock_pinned_request, mock_validate, web_loader):
def validate(url):
if "169.254.169.254" in url:
@@ -267,8 +267,8 @@ class TestWebLoaderEdgeCases:
result = web_loader.load_data([])
assert result == []
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_data_empty_response_body(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response("")
@@ -292,15 +292,15 @@ class TestWebLoaderIntegration:
"""Test WebLoader integration with base class."""
def test_inherits_from_base_remote(self, web_loader):
from application.parser.remote.base import BaseRemote
from docsgpt.parser.remote.base import BaseRemote
assert isinstance(web_loader, BaseRemote)
def test_implements_load_data_method(self, web_loader):
assert hasattr(web_loader, "load_data")
assert callable(web_loader.load_data)
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("application.parser.remote.web_loader.pinned_request")
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
@patch("docsgpt.parser.remote.web_loader.pinned_request")
def test_load_vector_documents_method(self, mock_pinned_request, mock_validate, web_loader):
mock_pinned_request.return_value = _fake_response(
"<html><head><title>Test Page</title></head>"