mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 20:16:48 +00:00
refactor: rename the application package to docsgpt
The backend import package is now docsgpt, the name it will carry on PyPI; application was far too generic to install into anyone's site-packages. git mv plus a mechanical rewrite of every import, dotted string and path reference: 734 Python files, the compose files, Dockerfile, workflows, docs, setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage config, .gitignore. Behaviour is unchanged. Kept for one release: - A top-level application package whose meta-path finder resolves application.x.y to the already-imported docsgpt.x.y object, so old imports and entry points (celery -A application.app.celery, uvicorn application.asgi:asgi_app) keep working with a FutureWarning. - Celery registers every application.* task name as an alias of its docsgpt.* task on start-up, so messages queued by the previous release still run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries the previous release wrote are left unread instead of firing twice. The backend image builds from the repository root (docker build -f docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore allow-lists docsgpt/ and application/ and keeps caches, local data, .env files, the sample index files and the Dockerfile out. Compose and the image workflows point at the new context.
This commit is contained in:
1 parent
3e7f1f91b4
commit
574f96341e
984 files changed
+9847
-9682
No files matched your search
@@ -2,8 +2,8 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from application.parser.remote.crawler_loader import CrawlerLoader
|
||||
from application.parser.schema.base import Document
|
||||
from docsgpt.parser.remote.crawler_loader import CrawlerLoader
|
||||
from docsgpt.parser.schema.base import Document
|
||||
|
||||
|
||||
class DummyResponse:
|
||||
@@ -22,8 +22,8 @@ def _mock_validate_url(url):
|
||||
return url
|
||||
|
||||
|
||||
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.crawler_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
|
||||
def test_load_data_crawls_same_domain_links(mock_pinned_request, mock_validate_url):
|
||||
responses = {
|
||||
"http://example.com": DummyResponse(
|
||||
@@ -65,8 +65,8 @@ def test_load_data_crawls_same_domain_links(mock_pinned_request, mock_validate_u
|
||||
assert mock_pinned_request.call_count == 2
|
||||
|
||||
|
||||
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.crawler_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
|
||||
def test_load_data_accepts_list_input_and_adds_scheme(mock_pinned_request, mock_validate_url):
|
||||
mock_pinned_request.return_value = DummyResponse("<html><body>No links here</body></html>")
|
||||
crawler = CrawlerLoader()
|
||||
@@ -83,8 +83,8 @@ def test_load_data_accepts_list_input_and_adds_scheme(mock_pinned_request, mock_
|
||||
}
|
||||
|
||||
|
||||
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.crawler_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
|
||||
def test_load_data_respects_limit(mock_pinned_request, mock_validate_url):
|
||||
responses = {
|
||||
"http://example.com": DummyResponse(
|
||||
@@ -110,9 +110,9 @@ def test_load_data_respects_limit(mock_pinned_request, mock_validate_url):
|
||||
assert mock_pinned_request.call_count == 1
|
||||
|
||||
|
||||
@patch("application.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.crawler_loader.logging")
|
||||
@patch("application.parser.remote.crawler_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.crawler_loader.logging")
|
||||
@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
|
||||
def test_load_data_logs_and_skips_on_request_error(mock_pinned_request, mock_logging, mock_validate_url):
|
||||
mock_pinned_request.side_effect = Exception("load failure")
|
||||
crawler = CrawlerLoader()
|
||||
@@ -128,10 +128,10 @@ def test_load_data_logs_and_skips_on_request_error(mock_pinned_request, mock_log
|
||||
assert mock_logging.error.call_args.kwargs.get("exc_info") is True
|
||||
|
||||
|
||||
@patch("application.parser.remote.crawler_loader.validate_url")
|
||||
@patch("docsgpt.parser.remote.crawler_loader.validate_url")
|
||||
def test_load_data_returns_empty_on_ssrf_validation_failure(mock_validate_url):
|
||||
"""Test that SSRF validation failure returns empty list."""
|
||||
from application.core.url_validation import SSRFError
|
||||
from docsgpt.core.url_validation import SSRFError
|
||||
mock_validate_url.side_effect = SSRFError("Access to private IP not allowed")
|
||||
|
||||
crawler = CrawlerLoader()
|
||||
@@ -165,15 +165,15 @@ def test_url_to_virtual_path_variants():
|
||||
class TestCrawlerLoaderGaps:
|
||||
def test_pinned_fetch_builds_document_without_webbase_loader(self):
|
||||
"""The crawler should index the response body it already fetched."""
|
||||
from application.parser.remote.crawler_loader import CrawlerLoader
|
||||
from docsgpt.parser.remote.crawler_loader import CrawlerLoader
|
||||
|
||||
loader = CrawlerLoader(limit=5)
|
||||
with patch(
|
||||
"application.parser.remote.crawler_loader.validate_url",
|
||||
"docsgpt.parser.remote.crawler_loader.validate_url",
|
||||
return_value="https://example.com",
|
||||
):
|
||||
with patch(
|
||||
"application.parser.remote.crawler_loader.pinned_request"
|
||||
"docsgpt.parser.remote.crawler_loader.pinned_request"
|
||||
) as mock_pinned_request:
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
|
||||
@@ -5,8 +5,8 @@ from urllib.parse import urlparse
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from application.parser.remote.crawler_markdown import CrawlerLoader
|
||||
from application.parser.schema.base import Document
|
||||
from docsgpt.parser.remote.crawler_markdown import CrawlerLoader
|
||||
from docsgpt.parser.schema.base import Document
|
||||
|
||||
|
||||
class DummyResponse:
|
||||
@@ -40,7 +40,7 @@ def _mock_validate_url(url):
|
||||
@pytest.fixture(autouse=True)
|
||||
def _patch_validate_url(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"application.parser.remote.crawler_markdown.validate_url",
|
||||
"docsgpt.parser.remote.crawler_markdown.validate_url",
|
||||
_mock_validate_url,
|
||||
)
|
||||
|
||||
@@ -48,7 +48,7 @@ def _patch_validate_url(monkeypatch):
|
||||
@pytest.fixture(autouse=True)
|
||||
def _patch_tldextract(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"application.parser.remote.crawler_markdown._extract",
|
||||
"docsgpt.parser.remote.crawler_markdown._extract",
|
||||
_fake_extract,
|
||||
)
|
||||
|
||||
@@ -61,7 +61,7 @@ def _patch_markdownify(monkeypatch):
|
||||
return outputs.get(html, html)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"application.parser.remote.crawler_markdown.markdownify",
|
||||
"docsgpt.parser.remote.crawler_markdown.markdownify",
|
||||
fake_markdownify,
|
||||
)
|
||||
return outputs
|
||||
@@ -73,7 +73,7 @@ def _patch_pinned_request(monkeypatch, side_effect):
|
||||
return side_effect(url)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"application.parser.remote.crawler_markdown.pinned_request",
|
||||
"docsgpt.parser.remote.crawler_markdown.pinned_request",
|
||||
fake_pinned_request,
|
||||
)
|
||||
|
||||
@@ -164,13 +164,13 @@ def test_load_data_handles_fetch_errors(monkeypatch, _patch_markdownify, _patch_
|
||||
|
||||
def test_load_data_returns_empty_on_ssrf_validation_failure(monkeypatch):
|
||||
"""Test that SSRF validation failure returns empty list."""
|
||||
from application.core.url_validation import SSRFError
|
||||
from docsgpt.core.url_validation import SSRFError
|
||||
|
||||
def raise_ssrf_error(url):
|
||||
raise SSRFError("Access to private IP not allowed")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"application.parser.remote.crawler_markdown.validate_url",
|
||||
"docsgpt.parser.remote.crawler_markdown.validate_url",
|
||||
raise_ssrf_error,
|
||||
)
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
import requests
|
||||
|
||||
from application.parser.remote.github_loader import GitHubLoader
|
||||
from docsgpt.parser.remote.github_loader import GitHubLoader
|
||||
|
||||
|
||||
def make_response(json_data=None, status_code=200, raise_error=None):
|
||||
@@ -18,7 +18,7 @@ def make_response(json_data=None, status_code=200, raise_error=None):
|
||||
|
||||
|
||||
class TestGitHubLoaderFetchFileContent:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_text_file_base64_decoded(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
content_str = "Hello from README"
|
||||
@@ -34,7 +34,7 @@ class TestGitHubLoaderFetchFileContent:
|
||||
timeout=100,
|
||||
)
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_binary_file_skipped(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response({"encoding": "base64", "content": "AAAA"})
|
||||
@@ -43,7 +43,7 @@ class TestGitHubLoaderFetchFileContent:
|
||||
|
||||
assert result is None
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_non_base64_plain_content(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response({"encoding": "", "content": "Plain text"})
|
||||
@@ -52,7 +52,7 @@ class TestGitHubLoaderFetchFileContent:
|
||||
|
||||
assert result == "Plain text"
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_http_error_raises(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
http_err = requests.HTTPError("Not found")
|
||||
@@ -63,7 +63,7 @@ class TestGitHubLoaderFetchFileContent:
|
||||
|
||||
|
||||
class TestGitHubLoaderFetchRepoFiles:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_recurses_directories(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
|
||||
@@ -133,7 +133,7 @@ class TestGitHubLoaderIsTextFile:
|
||||
loader = GitHubLoader()
|
||||
assert loader.is_text_file("image.png") is False
|
||||
|
||||
@patch("application.parser.remote.github_loader.mimetypes.guess_type")
|
||||
@patch("docsgpt.parser.remote.github_loader.mimetypes.guess_type")
|
||||
def test_mime_fallback_text(self, mock_mime):
|
||||
mock_mime.return_value = ("text/plain", None)
|
||||
loader = GitHubLoader()
|
||||
@@ -141,15 +141,15 @@ class TestGitHubLoaderIsTextFile:
|
||||
|
||||
|
||||
class TestGitHubLoaderMakeRequest:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_success(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response({"ok": True}, 200)
|
||||
resp = loader._make_request("http://example.com")
|
||||
assert resp.status_code == 200
|
||||
|
||||
@patch("application.parser.remote.github_loader.time.sleep")
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.time.sleep")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_rate_limit_retry(self, mock_get, mock_sleep):
|
||||
loader = GitHubLoader()
|
||||
rate_resp = MagicMock()
|
||||
@@ -166,7 +166,7 @@ class TestGitHubLoaderMakeRequest:
|
||||
assert resp.status_code == 200
|
||||
mock_sleep.assert_called_once()
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_rate_limit_exhausted(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
rate_resp = MagicMock()
|
||||
@@ -181,7 +181,7 @@ class TestGitHubLoaderMakeRequest:
|
||||
with pytest.raises(Exception, match="rate limit exceeded"):
|
||||
loader._make_request("http://example.com", max_retries=1)
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_403_non_rate_limit(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
resp = MagicMock()
|
||||
@@ -193,7 +193,7 @@ class TestGitHubLoaderMakeRequest:
|
||||
with pytest.raises(Exception, match="GitHub API error"):
|
||||
loader._make_request("http://example.com", max_retries=1)
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_other_error_raises(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
resp = make_response(
|
||||
@@ -207,7 +207,7 @@ class TestGitHubLoaderMakeRequest:
|
||||
|
||||
|
||||
class TestGitHubLoaderFetchRepoFilesErrors:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_api_error_message_in_dict(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response(
|
||||
@@ -217,7 +217,7 @@ class TestGitHubLoaderFetchRepoFilesErrors:
|
||||
with pytest.raises(Exception, match="GitHub API error"):
|
||||
loader.fetch_repo_files("owner/repo")
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_non_list_response(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response("not a list", 200)
|
||||
@@ -227,7 +227,7 @@ class TestGitHubLoaderFetchRepoFilesErrors:
|
||||
|
||||
|
||||
class TestGitHubLoaderFetchFileContentEdgeCases:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_empty_base64_text_returns_none(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
b64 = base64.b64encode(b"").decode("utf-8")
|
||||
@@ -237,7 +237,7 @@ class TestGitHubLoaderFetchFileContentEdgeCases:
|
||||
result = loader.fetch_file_content("owner/repo", "empty.py")
|
||||
assert result is None
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_empty_non_base64_returns_none(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response(
|
||||
@@ -246,7 +246,7 @@ class TestGitHubLoaderFetchFileContentEdgeCases:
|
||||
result = loader.fetch_file_content("owner/repo", "empty.txt")
|
||||
assert result is None
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_decode_failure_returns_none(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response(
|
||||
@@ -275,7 +275,7 @@ class TestGitHubLoaderLoadDataSkipsNone:
|
||||
|
||||
|
||||
class TestGitHubLoaderRobustness:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_fetch_repo_files_non_json_raises(self, mock_get):
|
||||
resp = MagicMock()
|
||||
resp.json.side_effect = ValueError("No JSON")
|
||||
@@ -283,14 +283,14 @@ class TestGitHubLoaderRobustness:
|
||||
with pytest.raises(ValueError):
|
||||
GitHubLoader().fetch_repo_files("owner/repo")
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_fetch_repo_files_unexpected_shape_missing_type_raises(self, mock_get):
|
||||
# Missing 'type' in items should raise KeyError when accessed
|
||||
mock_get.return_value = make_response([{"path": "README.md"}])
|
||||
with pytest.raises(KeyError):
|
||||
GitHubLoader().fetch_repo_files("owner/repo")
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_fetch_file_content_non_json_raises(self, mock_get):
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
@@ -299,7 +299,7 @@ class TestGitHubLoaderRobustness:
|
||||
with pytest.raises(ValueError):
|
||||
GitHubLoader().fetch_file_content("owner/repo", "README.md")
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_fetch_file_content_unexpected_shape_missing_content_returns_none(self, mock_get):
|
||||
# encoding indicates base64 text, but 'content' key is missing
|
||||
# With the new code, the exception is caught and returns None (treated as binary/skipped)
|
||||
@@ -308,8 +308,8 @@ class TestGitHubLoaderRobustness:
|
||||
result = GitHubLoader().fetch_file_content("owner/repo", "file.txt")
|
||||
assert result is None
|
||||
|
||||
@patch("application.parser.remote.github_loader.base64.b64decode")
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.base64.b64decode")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_large_binary_skip_does_not_decode(self, mock_get, mock_b64decode):
|
||||
# Ensure we don't attempt to decode large binary content for non-text files
|
||||
mock_b64decode.side_effect = AssertionError("b64decode should not be called for binary files")
|
||||
@@ -384,7 +384,7 @@ class TestGitHubLoaderSelectFiles:
|
||||
def test_applies_size_cap(self, monkeypatch):
|
||||
loader = GitHubLoader()
|
||||
monkeypatch.setattr(
|
||||
"application.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
|
||||
"docsgpt.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
|
||||
100, raising=False,
|
||||
)
|
||||
entries = [("small.py", 50), ("huge.py", 5000), ("ok.md", 99)]
|
||||
@@ -393,7 +393,7 @@ class TestGitHubLoaderSelectFiles:
|
||||
def test_zero_cap_disables_limit(self, monkeypatch):
|
||||
loader = GitHubLoader()
|
||||
monkeypatch.setattr(
|
||||
"application.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
|
||||
"docsgpt.parser.remote.github_loader.settings.GITHUB_INGEST_MAX_FILE_BYTES",
|
||||
0, raising=False,
|
||||
)
|
||||
assert loader.select_files([("huge.py", 10**9)]) == ["huge.py"]
|
||||
@@ -407,7 +407,7 @@ class TestGitHubLoaderSelectFiles:
|
||||
|
||||
|
||||
class TestGitHubLoaderTree:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_single_request_lists_all_blobs(self, mock_get):
|
||||
mock_get.return_value = make_response({
|
||||
"tree": [
|
||||
@@ -424,7 +424,7 @@ class TestGitHubLoaderTree:
|
||||
# One call for the whole repo, versus one per directory before.
|
||||
assert mock_get.call_count == 1
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_truncated_tree_falls_back_to_walk(self, mock_get, monkeypatch):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response({"tree": [], "truncated": True})
|
||||
@@ -435,12 +435,12 @@ class TestGitHubLoaderTree:
|
||||
|
||||
|
||||
class TestGitHubLoaderDefaultBranch:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_uses_repo_default_branch(self, mock_get):
|
||||
mock_get.return_value = make_response({"default_branch": "master"})
|
||||
assert GitHubLoader().get_default_branch("owner/repo") == "master"
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_falls_back_to_main(self, mock_get):
|
||||
mock_get.side_effect = requests.ConnectionError("boom")
|
||||
assert GitHubLoader().get_default_branch("owner/repo") == "main"
|
||||
@@ -496,7 +496,7 @@ class TestGitHubLoaderParallelFetch:
|
||||
|
||||
|
||||
class TestGitHubLoaderStaleTokenFallback:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
@patch("docsgpt.parser.remote.github_loader.requests.get")
|
||||
def test_401_retries_unauthenticated(self, mock_get):
|
||||
"""An expired PAT 401s even public repos; fall back to anonymous
|
||||
rather than failing the ingest outright."""
|
||||
|
||||
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
|
||||
from docsgpt.parser.remote.reddit_loader import RedditPostsLoaderRemote
|
||||
|
||||
|
||||
def _post(post_id, title, body):
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Tests for application.parser.remote.remote_creator."""
|
||||
"""Tests for docsgpt.parser.remote.remote_creator."""
|
||||
|
||||
import json
|
||||
|
||||
@@ -10,7 +10,7 @@ from unittest.mock import MagicMock
|
||||
class TestRemoteCreator:
|
||||
def test_create_loader_valid_type(self):
|
||||
"""Cover line 34: returns loader instance for valid type."""
|
||||
from application.parser.remote.remote_creator import RemoteCreator
|
||||
from docsgpt.parser.remote.remote_creator import RemoteCreator
|
||||
|
||||
mock_loader_cls = MagicMock()
|
||||
original_loaders = RemoteCreator.loaders.copy()
|
||||
@@ -23,14 +23,14 @@ class TestRemoteCreator:
|
||||
|
||||
def test_create_loader_invalid_type_raises(self):
|
||||
"""Cover lines 32-33: raises ValueError for unknown type."""
|
||||
from application.parser.remote.remote_creator import RemoteCreator
|
||||
from docsgpt.parser.remote.remote_creator import RemoteCreator
|
||||
|
||||
with pytest.raises(ValueError, match="No loader class found"):
|
||||
RemoteCreator.create_loader("nonexistent_xyz")
|
||||
|
||||
def test_create_loader_case_insensitive(self):
|
||||
"""Cover line 31: type.lower() normalization."""
|
||||
from application.parser.remote.remote_creator import RemoteCreator
|
||||
from docsgpt.parser.remote.remote_creator import RemoteCreator
|
||||
|
||||
mock_loader_cls = MagicMock()
|
||||
original_loaders = RemoteCreator.loaders.copy()
|
||||
@@ -48,12 +48,12 @@ class TestNormalizeRemoteData:
|
||||
back to the ``source_data`` shape each loader expects."""
|
||||
|
||||
def test_none_passes_through(self):
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
assert normalize_remote_data("crawler", None) is None
|
||||
|
||||
def test_crawler_dict_with_url_key(self):
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
result = normalize_remote_data(
|
||||
"crawler", {"url": "https://example.com", "provider": "crawler"}
|
||||
@@ -61,20 +61,20 @@ class TestNormalizeRemoteData:
|
||||
assert result == "https://example.com"
|
||||
|
||||
def test_url_dict_with_url_key(self):
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
result = normalize_remote_data("url", {"url": "https://example.com"})
|
||||
assert result == "https://example.com"
|
||||
|
||||
def test_url_legacy_raw_key(self):
|
||||
"""Legacy rows wrapped a bare URL string as ``{"raw": ...}``."""
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
result = normalize_remote_data("crawler", {"raw": "https://legacy.example.com"})
|
||||
assert result == "https://legacy.example.com"
|
||||
|
||||
def test_url_dict_with_urls_list(self):
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
result = normalize_remote_data(
|
||||
"url", {"urls": ["https://a.example.com", "https://b.example.com"]}
|
||||
@@ -82,7 +82,7 @@ class TestNormalizeRemoteData:
|
||||
assert result == ["https://a.example.com", "https://b.example.com"]
|
||||
|
||||
def test_github_repo_url_key(self):
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
result = normalize_remote_data(
|
||||
"github", {"repo_url": "https://github.com/arc53/DocsGPT"}
|
||||
@@ -90,25 +90,25 @@ class TestNormalizeRemoteData:
|
||||
assert result == "https://github.com/arc53/DocsGPT"
|
||||
|
||||
def test_sitemap_dict_with_url_key(self):
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
result = normalize_remote_data("sitemap", {"url": "https://example.com/sitemap.xml"})
|
||||
assert result == "https://example.com/sitemap.xml"
|
||||
|
||||
def test_plain_string_url_passes_through(self):
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
assert normalize_remote_data("crawler", "https://example.com") == "https://example.com"
|
||||
|
||||
def test_url_dict_without_url_key_returns_none(self):
|
||||
"""A URL-type loader must never receive a dict, even a malformed one."""
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
assert normalize_remote_data("crawler", {"provider": "crawler"}) is None
|
||||
|
||||
def test_reddit_dict_serialized_to_json_string(self):
|
||||
"""reddit's loader runs json.loads() — it needs a JSON string."""
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
result = normalize_remote_data(
|
||||
"reddit", {"client_id": "x", "search_queries": ["y"]}
|
||||
@@ -118,14 +118,14 @@ class TestNormalizeRemoteData:
|
||||
|
||||
def test_s3_dict_passes_through(self):
|
||||
"""S3Loader.load_data() accepts a dict, so it is left untouched."""
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
data = {"bucket": "b", "prefix": "k"}
|
||||
assert normalize_remote_data("s3", data) == data
|
||||
|
||||
def test_json_string_remote_data_is_parsed(self):
|
||||
"""Legacy rows that stored the JSON itself as a string still resolve."""
|
||||
from application.parser.remote.remote_creator import normalize_remote_data
|
||||
from docsgpt.parser.remote.remote_creator import normalize_remote_data
|
||||
|
||||
result = normalize_remote_data("crawler", '{"url": "https://example.com"}')
|
||||
assert result == "https://example.com"
|
||||
@@ -11,14 +11,14 @@ from botocore.exceptions import ClientError, NoCredentialsError
|
||||
def mock_boto3():
|
||||
"""Mock boto3 module."""
|
||||
with patch.dict("sys.modules", {"boto3": MagicMock()}):
|
||||
with patch("application.parser.remote.s3_loader.boto3") as mock:
|
||||
with patch("docsgpt.parser.remote.s3_loader.boto3") as mock:
|
||||
yield mock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def s3_loader(mock_boto3):
|
||||
"""Create S3Loader instance with mocked boto3."""
|
||||
from application.parser.remote.s3_loader import S3Loader
|
||||
from docsgpt.parser.remote.s3_loader import S3Loader
|
||||
|
||||
loader = S3Loader()
|
||||
return loader
|
||||
@@ -29,15 +29,15 @@ class TestS3LoaderInit:
|
||||
|
||||
def test_init_raises_import_error_when_boto3_missing(self):
|
||||
"""Should raise ImportError when boto3 is not installed."""
|
||||
with patch("application.parser.remote.s3_loader.boto3", None):
|
||||
from application.parser.remote.s3_loader import S3Loader
|
||||
with patch("docsgpt.parser.remote.s3_loader.boto3", None):
|
||||
from docsgpt.parser.remote.s3_loader import S3Loader
|
||||
|
||||
with pytest.raises(ImportError, match="boto3 is required"):
|
||||
S3Loader()
|
||||
|
||||
def test_init_sets_client_to_none(self, mock_boto3):
|
||||
"""Should initialize with s3_client as None."""
|
||||
from application.parser.remote.s3_loader import S3Loader
|
||||
from docsgpt.parser.remote.s3_loader import S3Loader
|
||||
|
||||
loader = S3Loader()
|
||||
assert loader.s3_client is None
|
||||
@@ -127,7 +127,7 @@ class TestInitClient:
|
||||
def test_init_client_with_custom_endpoint(self, s3_loader, mock_boto3):
|
||||
"""Should configure path-style addressing for custom endpoints."""
|
||||
with patch(
|
||||
"application.parser.remote.s3_loader.validate_url",
|
||||
"docsgpt.parser.remote.s3_loader.validate_url",
|
||||
side_effect=lambda u: u,
|
||||
):
|
||||
s3_loader._init_client(
|
||||
@@ -145,7 +145,7 @@ class TestInitClient:
|
||||
def test_init_client_normalizes_do_endpoint(self, s3_loader, mock_boto3):
|
||||
"""Should normalize DigitalOcean Spaces bucket-prefixed URLs."""
|
||||
with patch(
|
||||
"application.parser.remote.s3_loader.validate_url",
|
||||
"docsgpt.parser.remote.s3_loader.validate_url",
|
||||
side_effect=lambda u: u,
|
||||
):
|
||||
corrected_bucket = s3_loader._init_client(
|
||||
@@ -564,7 +564,7 @@ class TestLoadData:
|
||||
}
|
||||
|
||||
with patch(
|
||||
"application.parser.remote.s3_loader.validate_url",
|
||||
"docsgpt.parser.remote.s3_loader.validate_url",
|
||||
side_effect=lambda u: u,
|
||||
):
|
||||
s3_loader.load_data(input_data)
|
||||
@@ -644,7 +644,7 @@ class TestLoadData:
|
||||
}
|
||||
|
||||
with patch(
|
||||
"application.parser.remote.s3_loader.validate_url",
|
||||
"docsgpt.parser.remote.s3_loader.validate_url",
|
||||
side_effect=lambda u: u,
|
||||
):
|
||||
docs = s3_loader.load_data(input_data)
|
||||
@@ -663,7 +663,7 @@ class TestProcessDocument:
|
||||
mock_doc.text = "Extracted document text"
|
||||
|
||||
with patch(
|
||||
"application.parser.file.bulk.SimpleDirectoryReader"
|
||||
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
|
||||
) as mock_reader_class:
|
||||
mock_reader = MagicMock()
|
||||
mock_reader.load_data.return_value = [mock_doc]
|
||||
@@ -687,7 +687,7 @@ class TestProcessDocument:
|
||||
def test_process_document_returns_none_on_error(self, s3_loader):
|
||||
"""Should return None when document processing fails."""
|
||||
with patch(
|
||||
"application.parser.file.bulk.SimpleDirectoryReader"
|
||||
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
|
||||
) as mock_reader_class:
|
||||
mock_reader_class.side_effect = Exception("Parse error")
|
||||
|
||||
@@ -709,7 +709,7 @@ class TestProcessDocument:
|
||||
def test_process_document_cleans_up_temp_file(self, s3_loader):
|
||||
"""Should clean up temporary file after processing."""
|
||||
with patch(
|
||||
"application.parser.file.bulk.SimpleDirectoryReader"
|
||||
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
|
||||
) as mock_reader_class:
|
||||
mock_reader = MagicMock()
|
||||
mock_reader.load_data.return_value = []
|
||||
@@ -831,7 +831,7 @@ class TestProcessDocumentAdditional:
|
||||
def test_process_document_empty_documents_returns_none(self, s3_loader):
|
||||
"""Cover line 347-348: no documents extracted returns None."""
|
||||
with patch(
|
||||
"application.parser.file.bulk.SimpleDirectoryReader"
|
||||
"docsgpt.parser.file.bulk.SimpleDirectoryReader"
|
||||
) as mock_reader_class:
|
||||
mock_reader = MagicMock()
|
||||
mock_reader.load_data.return_value = []
|
||||
@@ -906,8 +906,8 @@ class TestSSRFValidation:
|
||||
|
||||
|
||||
def test_is_supported_document_follows_the_upload_whitelist():
|
||||
from application.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
|
||||
from application.parser.remote.s3_loader import S3Loader
|
||||
from docsgpt.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
|
||||
from docsgpt.parser.remote.s3_loader import S3Loader
|
||||
|
||||
for suffix in SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS:
|
||||
key = f"bucket/file{suffix}"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from application.parser.connectors.share_point.loader import SharePointLoader
|
||||
from docsgpt.parser.connectors.share_point.loader import SharePointLoader
|
||||
|
||||
|
||||
def make_response(json_data=None, status_code=200, raise_error=None):
|
||||
@@ -65,10 +65,10 @@ class TestSharePointLoaderProcessFile:
|
||||
class TestSharePointLoaderLoadFileById:
|
||||
"""Test _load_file_by_id method."""
|
||||
|
||||
@patch("application.parser.connectors.share_point.loader.requests.get")
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
||||
def test_load_file_by_id_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
|
||||
"""Should include size field in $select parameter."""
|
||||
mock_get_token.return_value = {
|
||||
@@ -96,10 +96,10 @@ class TestSharePointLoaderLoadFileById:
|
||||
params = call_args[1]["params"]
|
||||
assert "size" in params["$select"]
|
||||
|
||||
@patch("application.parser.connectors.share_point.loader.requests.get")
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
||||
def test_load_file_by_id_returns_document_with_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
|
||||
"""Should return document with size from API response."""
|
||||
mock_get_token.return_value = {
|
||||
@@ -133,10 +133,10 @@ class TestSharePointLoaderLoadFileById:
|
||||
class TestSharePointLoaderListItems:
|
||||
"""Test _list_items_in_parent method."""
|
||||
|
||||
@patch("application.parser.connectors.share_point.loader.requests.get")
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
||||
def test_list_items_includes_size_in_select(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
|
||||
"""Should include size field in $select parameter when listing items."""
|
||||
mock_get_token.return_value = {
|
||||
@@ -168,10 +168,10 @@ class TestSharePointLoaderListItems:
|
||||
params = call_args[1]["params"]
|
||||
assert "size" in params["$select"]
|
||||
|
||||
@patch("application.parser.connectors.share_point.loader.requests.get")
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
||||
@patch("application.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.requests.get")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.get_token_info_from_session")
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointAuth.__init__", return_value=None)
|
||||
@patch("docsgpt.parser.connectors.share_point.loader.SharePointLoader._ensure_valid_token")
|
||||
def test_list_items_folders_include_size(self, mock_ensure_token, mock_auth_init, mock_get_token, mock_get):
|
||||
"""Should include size for folders as well."""
|
||||
mock_get_token.return_value = {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Comprehensive tests for application/parser/remote/sitemap_loader.py
|
||||
"""Comprehensive tests for docsgpt/parser/remote/sitemap_loader.py
|
||||
|
||||
Covers: SitemapLoader (init, load_data, _extract_urls, _is_sitemap,
|
||||
_parse_sitemap, URL validation, error handling).
|
||||
@@ -9,8 +9,8 @@ from unittest.mock import MagicMock, patch
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from application.parser.remote.sitemap_loader import SitemapLoader
|
||||
from application.parser.schema.base import Document
|
||||
from docsgpt.parser.remote.sitemap_loader import SitemapLoader
|
||||
from docsgpt.parser.schema.base import Document
|
||||
|
||||
|
||||
# =====================================================================
|
||||
@@ -159,7 +159,7 @@ class TestParseSitemap:
|
||||
@pytest.mark.unit
|
||||
class TestExtractUrls:
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
|
||||
def test_extract_urls_from_sitemap(self, mock_pinned_request):
|
||||
loader = SitemapLoader()
|
||||
|
||||
@@ -177,7 +177,7 @@ class TestExtractUrls:
|
||||
urls = loader._extract_urls("https://example.com/sitemap.xml")
|
||||
assert "https://example.com/p" in urls
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
|
||||
def test_extract_urls_not_sitemap(self, mock_pinned_request):
|
||||
loader = SitemapLoader()
|
||||
|
||||
@@ -191,7 +191,7 @@ class TestExtractUrls:
|
||||
urls = loader._extract_urls("https://example.com/page")
|
||||
assert urls == ["https://example.com/page"]
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
|
||||
def test_extract_urls_http_error(self, mock_pinned_request):
|
||||
loader = SitemapLoader()
|
||||
mock_pinned_request.side_effect = requests.exceptions.HTTPError("404")
|
||||
@@ -199,7 +199,7 @@ class TestExtractUrls:
|
||||
urls = loader._extract_urls("https://example.com/missing")
|
||||
assert urls == []
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
|
||||
def test_extract_urls_connection_error(self, mock_pinned_request):
|
||||
loader = SitemapLoader()
|
||||
mock_pinned_request.side_effect = requests.exceptions.ConnectionError()
|
||||
@@ -208,12 +208,12 @@ class TestExtractUrls:
|
||||
assert urls == []
|
||||
|
||||
def test_extract_urls_ssrf_blocked(self):
|
||||
from application.security.safe_url import UnsafeUserUrlError
|
||||
from docsgpt.security.safe_url import UnsafeUserUrlError
|
||||
|
||||
loader = SitemapLoader()
|
||||
|
||||
with patch(
|
||||
"application.parser.remote.sitemap_loader.pinned_request",
|
||||
"docsgpt.parser.remote.sitemap_loader.pinned_request",
|
||||
side_effect=UnsafeUserUrlError("blocked"),
|
||||
):
|
||||
urls = loader._extract_urls("http://169.254.169.254/")
|
||||
@@ -228,8 +228,8 @@ class TestExtractUrls:
|
||||
@pytest.mark.unit
|
||||
class TestSitemapLoaderLoadData:
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("application.parser.remote.sitemap_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
|
||||
def test_load_data_success(self, mock_pinned_request, mock_validate):
|
||||
loader = SitemapLoader(limit=10)
|
||||
mock_validate.side_effect = lambda url: url
|
||||
@@ -248,7 +248,7 @@ class TestSitemapLoaderLoadData:
|
||||
assert docs[0].text == "Page body"
|
||||
assert docs[0].extra_info == {"source": "https://example.com/page1"}
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
|
||||
def test_load_data_no_urls(self, mock_validate):
|
||||
loader = SitemapLoader()
|
||||
|
||||
@@ -256,8 +256,8 @@ class TestSitemapLoaderLoadData:
|
||||
docs = loader.load_data("https://example.com/empty-sitemap.xml")
|
||||
assert docs == []
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("application.parser.remote.sitemap_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
|
||||
def test_load_data_list_input(self, mock_pinned_request, mock_validate):
|
||||
loader = SitemapLoader()
|
||||
mock_validate.side_effect = lambda url: url
|
||||
@@ -274,8 +274,8 @@ class TestSitemapLoaderLoadData:
|
||||
assert len(docs) == 1
|
||||
assert docs[0].text == "List body"
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("application.parser.remote.sitemap_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
|
||||
def test_load_data_respects_limit(self, mock_pinned_request, mock_validate):
|
||||
loader = SitemapLoader(limit=2)
|
||||
mock_validate.side_effect = lambda url: url
|
||||
@@ -290,8 +290,8 @@ class TestSitemapLoaderLoadData:
|
||||
assert len(docs) == 2
|
||||
assert mock_pinned_request.call_count == 2
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("application.parser.remote.sitemap_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
|
||||
def test_load_data_handles_url_error(self, mock_pinned_request, mock_validate):
|
||||
loader = SitemapLoader()
|
||||
mock_validate.side_effect = lambda url: url
|
||||
@@ -305,19 +305,19 @@ class TestSitemapLoaderLoadData:
|
||||
assert docs == []
|
||||
|
||||
def test_load_data_ssrf_blocked(self):
|
||||
from application.core.url_validation import SSRFError
|
||||
from docsgpt.core.url_validation import SSRFError
|
||||
|
||||
loader = SitemapLoader()
|
||||
|
||||
with patch(
|
||||
"application.parser.remote.sitemap_loader.validate_url",
|
||||
"docsgpt.parser.remote.sitemap_loader.validate_url",
|
||||
side_effect=SSRFError("blocked"),
|
||||
):
|
||||
docs = loader.load_data("http://169.254.169.254/")
|
||||
assert docs == []
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("application.parser.remote.sitemap_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
|
||||
def test_load_data_no_limit(self, mock_pinned_request, mock_validate):
|
||||
loader = SitemapLoader(limit=None)
|
||||
mock_validate.side_effect = lambda url: url
|
||||
|
||||
@@ -2,10 +2,10 @@ import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from application.core.url_validation import SSRFError
|
||||
from application.parser.remote.web_loader import WebLoader, headers
|
||||
from application.parser.schema.base import Document
|
||||
from application.vectorstore.document_class import Document as LCDocument
|
||||
from docsgpt.core.url_validation import SSRFError
|
||||
from docsgpt.parser.remote.web_loader import WebLoader, headers
|
||||
from docsgpt.parser.schema.base import Document
|
||||
from docsgpt.vectorstore.document_class import Document as LCDocument
|
||||
|
||||
|
||||
def _mock_validate_url(url):
|
||||
@@ -60,8 +60,8 @@ class TestWebLoaderHeaders:
|
||||
class TestWebLoaderLoadData:
|
||||
"""Test WebLoader load_data method."""
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
def test_load_data_single_url_string(self, mock_pinned_request, mock_validate, web_loader):
|
||||
mock_pinned_request.return_value = _fake_response(
|
||||
"<html lang='en'><head><title>Test Page</title></head>"
|
||||
@@ -82,8 +82,8 @@ class TestWebLoaderLoadData:
|
||||
"GET", "https://example.com", headers=headers, timeout=30
|
||||
)
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
def test_load_data_multiple_urls_list(self, mock_pinned_request, mock_validate, web_loader):
|
||||
mock_pinned_request.side_effect = [
|
||||
_fake_response("<html><body>Content from site 1</body></html>"),
|
||||
@@ -108,8 +108,8 @@ class TestWebLoaderLoadData:
|
||||
"GET", "https://site2.com", headers=headers, timeout=30
|
||||
)
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
def test_load_data_url_without_scheme(self, mock_pinned_request, mock_validate, web_loader):
|
||||
mock_pinned_request.return_value = _fake_response(
|
||||
"<html><body>Schemeless</body></html>"
|
||||
@@ -122,8 +122,8 @@ class TestWebLoaderLoadData:
|
||||
"GET", "http://example.com", headers=headers, timeout=30
|
||||
)
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
def test_load_data_url_with_scheme(self, mock_pinned_request, mock_validate, web_loader):
|
||||
mock_pinned_request.return_value = _fake_response(
|
||||
"<html><body>Schemed</body></html>"
|
||||
@@ -136,8 +136,8 @@ class TestWebLoaderLoadData:
|
||||
"GET", "https://example.com", headers=headers, timeout=30
|
||||
)
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
def test_load_data_skips_pages_without_title(self, mock_pinned_request, mock_validate, web_loader):
|
||||
"""A page without <title> or <html lang=...> still loads, just with bare metadata."""
|
||||
mock_pinned_request.return_value = _fake_response("<p>Bare body</p>")
|
||||
@@ -148,8 +148,8 @@ class TestWebLoaderLoadData:
|
||||
assert result[0].text == "Bare body"
|
||||
assert result[0].extra_info == {"source": "https://example.com"}
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
def test_load_data_extracts_title_with_nested_markup(self, mock_pinned_request, mock_validate, web_loader):
|
||||
"""<title> with nested-looking content must still produce a non-empty title.
|
||||
|
||||
@@ -175,9 +175,9 @@ class TestWebLoaderLoadData:
|
||||
class TestWebLoaderErrorHandling:
|
||||
"""Test WebLoader error handling."""
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("application.parser.remote.web_loader.logging")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.logging")
|
||||
def test_load_data_single_url_error(self, mock_logging, mock_pinned_request, mock_validate, web_loader):
|
||||
mock_pinned_request.side_effect = Exception("Network error")
|
||||
|
||||
@@ -189,9 +189,9 @@ class TestWebLoaderErrorHandling:
|
||||
assert "Error processing URL https://invalid-url.com" in error_call[0][0]
|
||||
assert error_call[1]["exc_info"] is True
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("application.parser.remote.web_loader.logging")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.logging")
|
||||
def test_load_data_partial_failure(self, mock_logging, mock_pinned_request, mock_validate, web_loader):
|
||||
mock_pinned_request.side_effect = [
|
||||
_fake_response("<html><body>Success content</body></html>"),
|
||||
@@ -213,8 +213,8 @@ class TestWebLoaderErrorHandling:
|
||||
class TestWebLoaderSSRF:
|
||||
"""Test WebLoader SSRF protection."""
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url")
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url")
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
def test_load_data_skips_url_failing_ssrf_validation(self, mock_pinned_request, mock_validate, web_loader):
|
||||
"""A URL that fails SSRF validation must be skipped, never reaching the fetcher."""
|
||||
mock_validate.side_effect = SSRFError("Access to private/internal IP addresses is not allowed.")
|
||||
@@ -225,8 +225,8 @@ class TestWebLoaderSSRF:
|
||||
mock_validate.assert_called_once_with("http://169.254.169.254/latest/meta-data/")
|
||||
mock_pinned_request.assert_not_called()
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url")
|
||||
@patch("application.parser.remote.web_loader.logging")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url")
|
||||
@patch("docsgpt.parser.remote.web_loader.logging")
|
||||
def test_load_data_logs_warning_on_ssrf_failure(self, mock_logging, mock_validate, web_loader):
|
||||
mock_validate.side_effect = SSRFError("blocked")
|
||||
|
||||
@@ -237,8 +237,8 @@ class TestWebLoaderSSRF:
|
||||
assert "SSRF" in warning_msg
|
||||
assert "127.0.0.1" in warning_msg
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url")
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url")
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
def test_load_data_mixed_safe_and_unsafe_urls(self, mock_pinned_request, mock_validate, web_loader):
|
||||
def validate(url):
|
||||
if "169.254.169.254" in url:
|
||||
@@ -267,8 +267,8 @@ class TestWebLoaderEdgeCases:
|
||||
result = web_loader.load_data([])
|
||||
assert result == []
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
def test_load_data_empty_response_body(self, mock_pinned_request, mock_validate, web_loader):
|
||||
mock_pinned_request.return_value = _fake_response("")
|
||||
|
||||
@@ -292,15 +292,15 @@ class TestWebLoaderIntegration:
|
||||
"""Test WebLoader integration with base class."""
|
||||
|
||||
def test_inherits_from_base_remote(self, web_loader):
|
||||
from application.parser.remote.base import BaseRemote
|
||||
from docsgpt.parser.remote.base import BaseRemote
|
||||
assert isinstance(web_loader, BaseRemote)
|
||||
|
||||
def test_implements_load_data_method(self, web_loader):
|
||||
assert hasattr(web_loader, "load_data")
|
||||
assert callable(web_loader.load_data)
|
||||
|
||||
@patch("application.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("application.parser.remote.web_loader.pinned_request")
|
||||
@patch("docsgpt.parser.remote.web_loader.validate_url", side_effect=_mock_validate_url)
|
||||
@patch("docsgpt.parser.remote.web_loader.pinned_request")
|
||||
def test_load_vector_documents_method(self, mock_pinned_request, mock_validate, web_loader):
|
||||
mock_pinned_request.return_value = _fake_response(
|
||||
"<html><head><title>Test Page</title></head>"
|
||||
|
||||
Reference in new issue
Block a user