mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 06:17:23 +00:00
chore: more tests
This commit is contained in:
Alex committed
1 parent
dc6db847ca
commit
d5c0322e2a
64 files changed
+24436
-140
No files matched your search
Whitespace-only changes.
@@ -116,6 +116,159 @@ class TestGitHubLoaderLoadData:
|
||||
|
||||
|
||||
|
||||
class TestGitHubLoaderIsTextFile:
|
||||
def test_known_extension(self):
|
||||
loader = GitHubLoader()
|
||||
assert loader.is_text_file("app.py") is True
|
||||
assert loader.is_text_file("data.json") is True
|
||||
|
||||
def test_unknown_extension_with_text_mime(self):
|
||||
loader = GitHubLoader()
|
||||
assert loader.is_text_file("file.xml") is True
|
||||
|
||||
def test_binary_file(self):
|
||||
loader = GitHubLoader()
|
||||
assert loader.is_text_file("image.png") is False
|
||||
|
||||
@patch("application.parser.remote.github_loader.mimetypes.guess_type")
|
||||
def test_mime_fallback_text(self, mock_mime):
|
||||
mock_mime.return_value = ("text/plain", None)
|
||||
loader = GitHubLoader()
|
||||
assert loader.is_text_file("unknownfile.xyz") is True
|
||||
|
||||
|
||||
class TestGitHubLoaderMakeRequest:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
def test_success(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response({"ok": True}, 200)
|
||||
resp = loader._make_request("http://example.com")
|
||||
assert resp.status_code == 200
|
||||
|
||||
@patch("application.parser.remote.github_loader.time.sleep")
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
def test_rate_limit_retry(self, mock_get, mock_sleep):
|
||||
loader = GitHubLoader()
|
||||
rate_resp = MagicMock()
|
||||
rate_resp.status_code = 403
|
||||
rate_resp.json.return_value = {"message": "API rate limit exceeded"}
|
||||
rate_resp.headers = {
|
||||
"X-RateLimit-Remaining": "0",
|
||||
"X-RateLimit-Reset": "9999999",
|
||||
}
|
||||
ok_resp = make_response({"ok": True}, 200)
|
||||
mock_get.side_effect = [rate_resp, ok_resp]
|
||||
|
||||
resp = loader._make_request("http://example.com", max_retries=2)
|
||||
assert resp.status_code == 200
|
||||
mock_sleep.assert_called_once()
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
def test_rate_limit_exhausted(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
rate_resp = MagicMock()
|
||||
rate_resp.status_code = 403
|
||||
rate_resp.json.return_value = {"message": "API rate limit exceeded"}
|
||||
rate_resp.headers = {
|
||||
"X-RateLimit-Remaining": "0",
|
||||
"X-RateLimit-Reset": "9999",
|
||||
}
|
||||
mock_get.return_value = rate_resp
|
||||
|
||||
with pytest.raises(Exception, match="rate limit exceeded"):
|
||||
loader._make_request("http://example.com", max_retries=1)
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
def test_403_non_rate_limit(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
resp = MagicMock()
|
||||
resp.status_code = 403
|
||||
resp.json.return_value = {"message": "Forbidden - need auth"}
|
||||
resp.headers = {"X-RateLimit-Remaining": "50", "X-RateLimit-Reset": "9999"}
|
||||
mock_get.return_value = resp
|
||||
|
||||
with pytest.raises(Exception, match="GitHub API error"):
|
||||
loader._make_request("http://example.com", max_retries=1)
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
def test_other_error_raises(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
resp = make_response(
|
||||
status_code=500,
|
||||
raise_error=requests.HTTPError("Server Error"),
|
||||
)
|
||||
mock_get.return_value = resp
|
||||
|
||||
with pytest.raises(requests.HTTPError):
|
||||
loader._make_request("http://example.com", max_retries=1)
|
||||
|
||||
|
||||
class TestGitHubLoaderFetchRepoFilesErrors:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
def test_api_error_message_in_dict(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response(
|
||||
{"message": "Not Found"}, 200
|
||||
)
|
||||
|
||||
with pytest.raises(Exception, match="GitHub API error"):
|
||||
loader.fetch_repo_files("owner/repo")
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
def test_non_list_response(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response("not a list", 200)
|
||||
|
||||
with pytest.raises(TypeError, match="Expected list"):
|
||||
loader.fetch_repo_files("owner/repo")
|
||||
|
||||
|
||||
class TestGitHubLoaderFetchFileContentEdgeCases:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
def test_empty_base64_text_returns_none(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
b64 = base64.b64encode(b"").decode("utf-8")
|
||||
mock_get.return_value = make_response(
|
||||
{"encoding": "base64", "content": b64}
|
||||
)
|
||||
result = loader.fetch_file_content("owner/repo", "empty.py")
|
||||
assert result is None
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
def test_empty_non_base64_returns_none(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response(
|
||||
{"encoding": "none", "content": " "}
|
||||
)
|
||||
result = loader.fetch_file_content("owner/repo", "empty.txt")
|
||||
assert result is None
|
||||
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
def test_decode_failure_returns_none(self, mock_get):
|
||||
loader = GitHubLoader()
|
||||
mock_get.return_value = make_response(
|
||||
{"encoding": "base64", "content": "invalid!!base64"}
|
||||
)
|
||||
result = loader.fetch_file_content("owner/repo", "broken.py")
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestGitHubLoaderLoadDataSkipsNone:
|
||||
def test_skips_binary_files(self, monkeypatch):
|
||||
loader = GitHubLoader()
|
||||
monkeypatch.setattr(
|
||||
loader, "fetch_repo_files", lambda repo, path="": ["a.py", "b.png"]
|
||||
)
|
||||
|
||||
def fake_content(repo, fp):
|
||||
return "code" if fp == "a.py" else None
|
||||
|
||||
monkeypatch.setattr(loader, "fetch_file_content", fake_content)
|
||||
docs = loader.load_data("https://github.com/o/r")
|
||||
assert len(docs) == 1
|
||||
assert docs[0].doc_id == "a.py"
|
||||
|
||||
|
||||
class TestGitHubLoaderRobustness:
|
||||
@patch("application.parser.remote.github_loader.requests.get")
|
||||
def test_fetch_repo_files_non_json_raises(self, mock_get):
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
"""Comprehensive tests for application/parser/remote/sitemap_loader.py
|
||||
|
||||
Covers: SitemapLoader (init, load_data, _extract_urls, _is_sitemap,
|
||||
_parse_sitemap, URL validation, error handling).
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from application.parser.remote.sitemap_loader import SitemapLoader
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# SitemapLoader - Init
|
||||
# =====================================================================
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSitemapLoaderInit:
|
||||
|
||||
def test_default_limit(self):
|
||||
loader = SitemapLoader()
|
||||
assert loader.limit == 20
|
||||
|
||||
def test_custom_limit(self):
|
||||
loader = SitemapLoader(limit=5)
|
||||
assert loader.limit == 5
|
||||
|
||||
def test_has_loader_class(self):
|
||||
loader = SitemapLoader()
|
||||
assert loader.loader is not None
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# _is_sitemap
|
||||
# =====================================================================
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestIsSitemap:
|
||||
|
||||
def test_xml_content_type(self):
|
||||
loader = SitemapLoader()
|
||||
response = MagicMock()
|
||||
response.headers = {"Content-Type": "application/xml"}
|
||||
response.url = "https://example.com/sitemap.xml"
|
||||
response.text = ""
|
||||
assert loader._is_sitemap(response) is True
|
||||
|
||||
def test_xml_url_extension(self):
|
||||
loader = SitemapLoader()
|
||||
response = MagicMock()
|
||||
response.headers = {"Content-Type": "text/html"}
|
||||
response.url = "https://example.com/sitemap.xml"
|
||||
response.text = ""
|
||||
assert loader._is_sitemap(response) is True
|
||||
|
||||
def test_sitemapindex_in_body(self):
|
||||
loader = SitemapLoader()
|
||||
response = MagicMock()
|
||||
response.headers = {"Content-Type": "text/html"}
|
||||
response.url = "https://example.com/sitemap"
|
||||
response.text = "<sitemapindex><sitemap></sitemap></sitemapindex>"
|
||||
assert loader._is_sitemap(response) is True
|
||||
|
||||
def test_urlset_in_body(self):
|
||||
loader = SitemapLoader()
|
||||
response = MagicMock()
|
||||
response.headers = {"Content-Type": "text/html"}
|
||||
response.url = "https://example.com/page"
|
||||
response.text = "<urlset><url></url></urlset>"
|
||||
assert loader._is_sitemap(response) is True
|
||||
|
||||
def test_regular_page(self):
|
||||
loader = SitemapLoader()
|
||||
response = MagicMock()
|
||||
response.headers = {"Content-Type": "text/html"}
|
||||
response.url = "https://example.com/about"
|
||||
response.text = "<html><body>About us</body></html>"
|
||||
assert loader._is_sitemap(response) is False
|
||||
|
||||
def test_text_xml_content_type(self):
|
||||
loader = SitemapLoader()
|
||||
response = MagicMock()
|
||||
response.headers = {"Content-Type": "text/xml; charset=utf-8"}
|
||||
response.url = "https://example.com/feed"
|
||||
response.text = ""
|
||||
assert loader._is_sitemap(response) is True
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# _parse_sitemap
|
||||
# =====================================================================
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestParseSitemap:
|
||||
|
||||
def test_parse_basic_sitemap(self):
|
||||
loader = SitemapLoader()
|
||||
sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
<url><loc>https://example.com/page1</loc></url>
|
||||
<url><loc>https://example.com/page2</loc></url>
|
||||
</urlset>"""
|
||||
|
||||
urls = loader._parse_sitemap(sitemap_xml)
|
||||
assert "https://example.com/page1" in urls
|
||||
assert "https://example.com/page2" in urls
|
||||
|
||||
def test_parse_nested_sitemap(self):
|
||||
loader = SitemapLoader()
|
||||
|
||||
parent_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
<sitemap>
|
||||
<loc>https://example.com/sitemap-child.xml</loc>
|
||||
</sitemap>
|
||||
</sitemapindex>"""
|
||||
|
||||
with patch.object(
|
||||
loader, "_extract_urls",
|
||||
return_value=["https://example.com/page1"]
|
||||
):
|
||||
urls = loader._parse_sitemap(parent_xml)
|
||||
assert "https://example.com/page1" in urls
|
||||
|
||||
def test_parse_empty_sitemap(self):
|
||||
loader = SitemapLoader()
|
||||
sitemap_xml = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
</urlset>"""
|
||||
|
||||
urls = loader._parse_sitemap(sitemap_xml)
|
||||
assert urls == []
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# _extract_urls
|
||||
# =====================================================================
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestExtractUrls:
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("application.parser.remote.sitemap_loader.requests.get")
|
||||
def test_extract_urls_from_sitemap(self, mock_get, mock_validate):
|
||||
loader = SitemapLoader()
|
||||
|
||||
response = MagicMock()
|
||||
response.headers = {"Content-Type": "application/xml"}
|
||||
response.url = "https://example.com/sitemap.xml"
|
||||
response.text = "<urlset><url><loc>https://example.com/p</loc></url></urlset>"
|
||||
response.content = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||
<url><loc>https://example.com/p</loc></url>
|
||||
</urlset>"""
|
||||
mock_get.return_value = response
|
||||
|
||||
urls = loader._extract_urls("https://example.com/sitemap.xml")
|
||||
assert "https://example.com/p" in urls
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("application.parser.remote.sitemap_loader.requests.get")
|
||||
def test_extract_urls_not_sitemap(self, mock_get, mock_validate):
|
||||
loader = SitemapLoader()
|
||||
|
||||
response = MagicMock()
|
||||
response.headers = {"Content-Type": "text/html"}
|
||||
response.url = "https://example.com/page"
|
||||
response.text = "<html>Normal page</html>"
|
||||
mock_get.return_value = response
|
||||
|
||||
urls = loader._extract_urls("https://example.com/page")
|
||||
assert urls == ["https://example.com/page"]
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("application.parser.remote.sitemap_loader.requests.get")
|
||||
def test_extract_urls_http_error(self, mock_get, mock_validate):
|
||||
loader = SitemapLoader()
|
||||
mock_get.side_effect = requests.exceptions.HTTPError("404")
|
||||
|
||||
urls = loader._extract_urls("https://example.com/missing")
|
||||
assert urls == []
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
@patch("application.parser.remote.sitemap_loader.requests.get")
|
||||
def test_extract_urls_connection_error(self, mock_get, mock_validate):
|
||||
loader = SitemapLoader()
|
||||
mock_get.side_effect = requests.exceptions.ConnectionError()
|
||||
|
||||
urls = loader._extract_urls("https://example.com/bad")
|
||||
assert urls == []
|
||||
|
||||
def test_extract_urls_ssrf_blocked(self):
|
||||
from application.core.url_validation import SSRFError
|
||||
|
||||
loader = SitemapLoader()
|
||||
|
||||
with patch(
|
||||
"application.parser.remote.sitemap_loader.validate_url",
|
||||
side_effect=SSRFError("blocked"),
|
||||
):
|
||||
urls = loader._extract_urls("http://169.254.169.254/")
|
||||
assert urls == []
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# load_data
|
||||
# =====================================================================
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSitemapLoaderLoadData:
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
def test_load_data_success(self, mock_validate):
|
||||
loader = SitemapLoader(limit=10)
|
||||
|
||||
mock_doc = MagicMock()
|
||||
mock_loader_instance = MagicMock()
|
||||
mock_loader_instance.load.return_value = [mock_doc]
|
||||
loader.loader = MagicMock(return_value=mock_loader_instance)
|
||||
|
||||
with patch.object(
|
||||
loader, "_extract_urls",
|
||||
return_value=["https://example.com/page1"]
|
||||
):
|
||||
docs = loader.load_data("https://example.com/sitemap.xml")
|
||||
assert len(docs) == 1
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
def test_load_data_no_urls(self, mock_validate):
|
||||
loader = SitemapLoader()
|
||||
|
||||
with patch.object(loader, "_extract_urls", return_value=[]):
|
||||
docs = loader.load_data("https://example.com/empty-sitemap.xml")
|
||||
assert docs == []
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
def test_load_data_list_input(self, mock_validate):
|
||||
loader = SitemapLoader()
|
||||
|
||||
mock_loader_instance = MagicMock()
|
||||
mock_loader_instance.load.return_value = [MagicMock()]
|
||||
loader.loader = MagicMock(return_value=mock_loader_instance)
|
||||
|
||||
with patch.object(
|
||||
loader, "_extract_urls",
|
||||
return_value=["https://example.com/page1"]
|
||||
):
|
||||
docs = loader.load_data(["https://example.com/sitemap.xml"])
|
||||
assert len(docs) == 1
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
def test_load_data_respects_limit(self, mock_validate):
|
||||
loader = SitemapLoader(limit=2)
|
||||
|
||||
mock_loader_instance = MagicMock()
|
||||
mock_loader_instance.load.return_value = [MagicMock()]
|
||||
loader.loader = MagicMock(return_value=mock_loader_instance)
|
||||
|
||||
urls = [f"https://example.com/page{i}" for i in range(10)]
|
||||
with patch.object(loader, "_extract_urls", return_value=urls):
|
||||
docs = loader.load_data("https://example.com/sitemap.xml")
|
||||
assert len(docs) == 2
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
def test_load_data_handles_url_error(self, mock_validate):
|
||||
loader = SitemapLoader()
|
||||
loader.loader = MagicMock(side_effect=Exception("Load failed"))
|
||||
|
||||
with patch.object(
|
||||
loader, "_extract_urls",
|
||||
return_value=["https://example.com/broken"]
|
||||
):
|
||||
docs = loader.load_data("https://example.com/sitemap.xml")
|
||||
assert docs == []
|
||||
|
||||
def test_load_data_ssrf_blocked(self):
|
||||
from application.core.url_validation import SSRFError
|
||||
|
||||
loader = SitemapLoader()
|
||||
|
||||
with patch(
|
||||
"application.parser.remote.sitemap_loader.validate_url",
|
||||
side_effect=SSRFError("blocked"),
|
||||
):
|
||||
docs = loader.load_data("http://169.254.169.254/")
|
||||
assert docs == []
|
||||
|
||||
@patch("application.parser.remote.sitemap_loader.validate_url")
|
||||
def test_load_data_no_limit(self, mock_validate):
|
||||
loader = SitemapLoader(limit=None)
|
||||
|
||||
mock_loader_instance = MagicMock()
|
||||
mock_loader_instance.load.return_value = [MagicMock()]
|
||||
loader.loader = MagicMock(return_value=mock_loader_instance)
|
||||
|
||||
urls = [f"https://example.com/page{i}" for i in range(5)]
|
||||
with patch.object(loader, "_extract_urls", return_value=urls):
|
||||
docs = loader.load_data("https://example.com/sitemap.xml")
|
||||
assert len(docs) == 5
|
||||
Reference in new issue
Block a user