Big source revamp

docsgpt/parser/remote/crawler_loader.py

@@ -1,9 +1,8 @@
 import logging
-import os
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin, urlparse
 
-from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.remote.base import BaseRemote, dedupe_virtual_paths, url_to_virtual_path
 from docsgpt.parser.schema.base import Document
 from docsgpt.core.url_validation import validate_url, SSRFError
 from docsgpt.security.safe_url import pinned_request
@@ -73,30 +72,8 @@ class CrawlerLoader(BaseRemote):
             if self.limit is not None and len(visited_urls) >= self.limit:
                 break
 
-        return loaded_content
+        return dedupe_virtual_paths(loaded_content)
 
     def _url_to_virtual_path(self, url):
-        """
-        Convert a URL to a virtual file path ending with .md.
-
-        Examples:
-            https://docs.docsgpt.cloud/ -> index.md
-            https://docs.docsgpt.cloud/guides/setup -> guides/setup.md
-            https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md
-            https://example.com/page.html -> page.md
-        """
-        parsed = urlparse(url)
-        path = parsed.path.strip("/")
-
-        if not path:
-            return "index.md"
-
-        # Remove common file extensions and add .md
-        base, ext = os.path.splitext(path)
-        if ext.lower() in [".html", ".htm", ".php", ".asp", ".aspx", ".jsp"]:
-            path = base
-
-        if not path.endswith(".md"):
-            path = f"{path}.md"
-
-        return path
+        """Convert a URL to a virtual ``.md`` path; see ``url_to_virtual_path``."""
+        return url_to_virtual_path(url)