rabbit fixes

docsgpt/parser/remote/crawler_loader.py

@@ -2,7 +2,7 @@ import logging
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin, urlparse
 
-from docsgpt.parser.remote.base import BaseRemote, dedupe_virtual_paths, url_to_virtual_path
+from docsgpt.parser.remote.base import BaseRemote, dedupe_virtual_paths, normalize_page_url, url_to_virtual_path
 from docsgpt.parser.schema.base import Document
 from docsgpt.core.url_validation import validate_url, SSRFError
 from docsgpt.security.safe_url import pinned_request
@@ -24,6 +24,8 @@ class CrawlerLoader(BaseRemote):
             logging.error(f"URL validation failed: {e}")
             return []
 
+        # Keyed by normalize_page_url, so fragment and query-order variants of
+        # one page are fetched once.
         visited_urls = set()
         base_url = urlparse(url).scheme + "://" + urlparse(url).hostname
         urls_to_visit = [url]
@@ -31,7 +33,10 @@ class CrawlerLoader(BaseRemote):
 
         while urls_to_visit:
             current_url = urls_to_visit.pop(0)
-            visited_urls.add(current_url)
+            page_key = normalize_page_url(current_url)
+            if page_key in visited_urls:
+                continue
+            visited_urls.add(page_key)
 
             try:
                 response = pinned_request("GET", current_url, timeout=30)
@@ -57,15 +62,21 @@ class CrawlerLoader(BaseRemote):
                 logging.error(f"Error processing URL {current_url}: {e}", exc_info=True)
                 continue
 
-            # Parse the HTML content to extract all links
-            all_links = [
-                urljoin(current_url, a['href'])
-                for a in soup.find_all('a', href=True)
-                if base_url in urljoin(current_url, a['href'])
-            ]
+            # Parse the HTML content to extract all links. A malformed href
+            # (``http://[bad``) makes urljoin or the key parse raise; skip it
+            # rather than abort the crawl.
+            all_links = []
+            for a in soup.find_all('a', href=True):
+                try:
+                    link = urljoin(current_url, a['href'])
+                    link_key = normalize_page_url(link)
+                except ValueError:
+                    continue
+                if base_url in link and link_key not in visited_urls:
+                    all_links.append(link)
 
             # Add new links to the list of URLs to visit if they haven't been visited yet
-            urls_to_visit.extend([link for link in all_links if link not in visited_urls])
+            urls_to_visit.extend(all_links)
             urls_to_visit = list(set(urls_to_visit))
 
             # Stop crawling if the limit of pages to scrape is reached