rabbit fixes

tests/parser/remote/test_base.py

@@ -5,6 +5,7 @@ import pytest
 from docsgpt.parser.remote.base import (
     MAX_QUERY_SEGMENT_LENGTH,
     dedupe_virtual_paths,
+    normalize_page_url,
     url_to_virtual_path,
 )
 from docsgpt.parser.schema.base import Document
@@ -131,3 +132,29 @@ class TestDedupeVirtualPaths:
 
         assert dedupe_virtual_paths([doc]) == [doc]
         assert "file_path" not in doc.extra_info
+
+
+@pytest.mark.unit
+class TestNormalizePageUrl:
+    def test_query_order_does_not_matter(self):
+        assert normalize_page_url("https://x.io/p?a=1&b=2") == normalize_page_url("https://x.io/p?b=2&a=1")
+
+    def test_fragment_is_dropped(self):
+        assert normalize_page_url("https://x.io/p?a=1#top") == normalize_page_url("https://x.io/p?a=1")
+
+    def test_distinct_queries_stay_distinct(self):
+        assert normalize_page_url("https://x.io/p?page=1") != normalize_page_url("https://x.io/p?page=2")
+
+    def test_blank_values_are_kept(self):
+        assert normalize_page_url("https://x.io/p?flag") != normalize_page_url("https://x.io/p")
+
+    def test_url_without_query_or_fragment_is_unchanged(self):
+        assert normalize_page_url("https://x.io/guides/setup") == "https://x.io/guides/setup"
+
+
+@pytest.mark.unit
+class TestDedupeReorderedQuery:
+    def test_reordered_query_is_one_page(self):
+        docs = [_doc("https://x.com/p?a=1&b=2"), _doc("https://x.com/p?b=2&a=1")]
+
+        assert _paths(dedupe_virtual_paths(docs)) == ["p__a=1&b=2.md", "p__a=1&b=2.md"]

tests/parser/remote/test_crawler_loader.py

@@ -208,3 +208,43 @@ def test_colliding_pages_get_distinct_file_paths(mock_pinned_request, mock_valid
         "http://example.com/a.html": "a-2.md",
         "http://example.com/l?page=2": "l__page=2.md",
     }
+
+
+@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
+@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
+def test_reordered_query_and_fragment_variants_are_fetched_once(mock_pinned_request, mock_validate_url):
+    responses = {
+        "http://example.com": DummyResponse(
+            "<html><body><a href='/p?a=1&b=2'>P</a><a href='/p?b=2&a=1'>P2</a>"
+            "<a href='/p?a=1&b=2#top'>P3</a></body></html>"
+        ),
+        "http://example.com/p?a=1&b=2": DummyResponse("<html><body>p</body></html>"),
+        "http://example.com/p?b=2&a=1": DummyResponse("<html><body>p</body></html>"),
+        "http://example.com/p?a=1&b=2#top": DummyResponse("<html><body>p</body></html>"),
+    }
+    mock_pinned_request.side_effect = lambda _m, url, timeout=30: responses[url]
+
+    result = CrawlerLoader(limit=10).load_data("http://example.com")
+
+    fetched = [c.args[1] for c in mock_pinned_request.call_args_list]
+    assert fetched[0] == "http://example.com"
+    assert len(fetched) == 2
+    # The page is fetched as a link wrote it, not as its normalized key.
+    assert fetched[1] in responses
+    assert len(result) == 2
+
+
+@patch("docsgpt.parser.remote.crawler_loader.validate_url", side_effect=_mock_validate_url)
+@patch("docsgpt.parser.remote.crawler_loader.pinned_request")
+def test_malformed_link_does_not_abort_the_crawl(mock_pinned_request, mock_validate_url):
+    responses = {
+        "http://example.com": DummyResponse(
+            "<html><body><a href='http://[bad'>Bad</a><a href='/good'>Good</a></body></html>"
+        ),
+        "http://example.com/good": DummyResponse("<html><body>good</body></html>"),
+    }
+    mock_pinned_request.side_effect = lambda _m, url, timeout=30: responses[url]
+
+    result = CrawlerLoader(limit=10).load_data("http://example.com")
+
+    assert {d.extra_info["source"] for d in result} == {"http://example.com", "http://example.com/good"}

tests/parser/remote/test_crawler_markdown.py

@@ -204,3 +204,49 @@ def test_colliding_pages_get_distinct_file_paths(monkeypatch, _patch_markdownify
         "http://example.com/a.html": "a-2.md",
         "http://example.com/l?page=2": "l__page=2.md",
     }
+
+
+def test_reordered_query_and_fragment_variants_are_fetched_once(monkeypatch, _patch_markdownify):
+    root_html = (
+        "<html><head><title>Home</title></head><body>"
+        "<a href='/p?a=1&b=2'>P</a><a href='/p?b=2&a=1'>P2</a><a href='/p?a=1&b=2#top'>P3</a>"
+        "</body></html>"
+    )
+    page_html = "<html><body>p</body></html>"
+    responses = {
+        "http://example.com": DummyResponse(root_html),
+        "http://example.com/p?a=1&b=2": DummyResponse(page_html),
+        "http://example.com/p?b=2&a=1": DummyResponse(page_html),
+        "http://example.com/p?a=1&b=2#top": DummyResponse(page_html),
+    }
+    fetched = []
+
+    def side_effect(url):
+        fetched.append(url)
+        return responses[url]
+
+    _patch_pinned_request(monkeypatch, side_effect)
+
+    docs = CrawlerLoader(limit=10).load_data("http://example.com")
+
+    assert len(fetched) == 2
+    assert fetched[0] == "http://example.com"
+    # The page is fetched as a link wrote it, not as its normalized key.
+    assert fetched[1] in responses
+    assert len(docs) == 2
+
+
+def test_malformed_link_does_not_abort_the_crawl(monkeypatch, _patch_markdownify):
+    root_html = (
+        "<html><head><title>Home</title></head><body>"
+        "<a href='http://[bad'>Bad</a><a href='/good'>Good</a></body></html>"
+    )
+    responses = {
+        "http://example.com": DummyResponse(root_html),
+        "http://example.com/good": DummyResponse("<html><body>good</body></html>"),
+    }
+    _patch_pinned_request(monkeypatch, lambda url: responses[url])
+
+    docs = CrawlerLoader(limit=10).load_data("http://example.com")
+
+    assert {d.extra_info["source"] for d in docs} == {"http://example.com", "http://example.com/good"}

tests/parser/remote/test_sitemap_loader.py

@@ -351,3 +351,27 @@ class TestSitemapLoaderDistinctPaths:
             docs = loader.load_data("https://x.io/sitemap.xml")
 
         assert [d.extra_info["file_path"] for d in docs] == ["p__page=2.md", "p.md", "p-2.md"]
+
+
+@pytest.mark.unit
+class TestSitemapLoaderMalformedEntry:
+
+    @patch("docsgpt.core.url_validation.resolve_hostname", return_value="93.184.216.34")
+    @patch("docsgpt.parser.remote.sitemap_loader.pinned_request")
+    def test_malformed_entry_is_skipped_and_the_rest_ingest(self, mock_pinned_request, _mock_resolve):
+        loader = SitemapLoader()
+        response = MagicMock()
+        response.text = "Good body"
+        response.raise_for_status.return_value = None
+        mock_pinned_request.return_value = response
+
+        with patch.object(
+            loader, "_extract_urls",
+            return_value=["http://[bad", "https://example.com/good"],
+        ):
+            docs = loader.load_data("https://example.com/sitemap.xml")
+
+        assert [d.extra_info for d in docs] == [
+            {"source": "https://example.com/good", "file_path": "good.md"}
+        ]
+        mock_pinned_request.assert_called_once()

tests/parser/remote/test_web_loader.py

@@ -351,3 +351,16 @@ class TestWebLoaderDistinctPaths:
         assert [d.extra_info["file_path"] for d in result] == [
             "list__page=1.md", "list__page=2.md", "a-2.md", "a.md"
         ]
+
+
+@pytest.mark.unit
+class TestWebLoaderMalformedUrl:
+    @patch("docsgpt.core.url_validation.resolve_hostname", return_value="93.184.216.34")
+    @patch("docsgpt.parser.remote.web_loader.pinned_request")
+    def test_malformed_url_is_skipped_and_the_rest_ingest(self, mock_pinned_request, _mock_resolve, web_loader):
+        mock_pinned_request.side_effect = lambda *a, **k: _fake_response("<html><body>good</body></html>")
+
+        result = web_loader.load_data(["http://[bad", "https://x.io/good"])
+
+        assert [d.extra_info["source"] for d in result] == ["https://x.io/good"]
+        assert result[0].extra_info["file_path"] == "good.md"