Merge remote-tracking branch 'origin/main' into connectors

# Conflicts:
#	frontend/DESIGN.md
#	frontend/src/locale/de.json
#	frontend/src/locale/en.json
#	frontend/src/locale/es.json
#	frontend/src/locale/jp.json
#	frontend/src/locale/ru.json
#	frontend/src/locale/zh-TW.json
#	frontend/src/locale/zh.json
#	frontend/src/upload/Upload.tsx

docsgpt/parser/remote/base.py

@@ -1,10 +1,191 @@
 """Base reader class."""
+import hashlib
+import os
+import re
 from abc import abstractmethod
-from typing import Any, List
+from typing import Any, Dict, List
+from urllib.parse import parse_qsl, urlencode, urlparse, urlsplit, urlunsplit
 
 from docsgpt.parser.schema.base import Document
 from docsgpt.vectorstore.document_class import Document as VectorDocument
 
+_PAGE_EXTENSIONS = (".html", ".htm", ".php", ".asp", ".aspx", ".jsp")
+
+# Joins a page's path to its encoded query string in the virtual file name.
+_QUERY_SEPARATOR = "__"
+# Longest encoded query kept verbatim; longer ones keep a prefix plus a hash.
+MAX_QUERY_SEGMENT_LENGTH = 64
+_QUERY_HASH_LENGTH = 10
+# Anything but letters, digits, ``_``, ``.`` and ``-`` in a query key or value.
+_UNSAFE_QUERY_CHARS = re.compile(r"[^\w.-]+")
+
+
+def _query_segment(query: str) -> str:
+    """Encode a URL query string as a deterministic, tree-safe name segment.
+
+    Parameters are sorted so the same page maps to the same name across
+    re-syncs whatever order its links list them in. Keys and values are
+    percent-decoded and every run of characters outside ``[\\w.-]`` becomes
+    ``-``, so the segment carries no ``/``, ``?``, ``#`` or ``%``. A segment
+    longer than ``MAX_QUERY_SEGMENT_LENGTH`` keeps a prefix and a hash of the
+    full query, which keeps it bounded and still distinct.
+
+    Args:
+        query: The raw query string, without the leading ``?``.
+
+    Returns:
+        The encoded segment, or ``""`` when the query has no parameters.
+    """
+    pairs = sorted(parse_qsl(query, keep_blank_values=True))
+    if not pairs:
+        return ""
+    parts = []
+    for key, value in pairs:
+        key = _UNSAFE_QUERY_CHARS.sub("-", key)
+        value = _UNSAFE_QUERY_CHARS.sub("-", value)
+        parts.append(f"{key}={value}" if value else key)
+    segment = "&".join(parts)
+    if len(segment) <= MAX_QUERY_SEGMENT_LENGTH:
+        return segment
+    digest = hashlib.sha1(urlencode(pairs).encode("utf-8")).hexdigest()
+    keep = MAX_QUERY_SEGMENT_LENGTH - _QUERY_HASH_LENGTH - 1
+    return f"{segment[:keep]}-{digest[:_QUERY_HASH_LENGTH]}"
+
+
+def url_to_virtual_path(url: str, include_host: bool = False) -> str:
+    """Convert a page URL to the virtual ``.md`` path used as its tree key.
+
+    Web ingests store this as the chunk's ``file_path``: the worker keys the
+    source's file tree by it and the chunks view filters by it, so the two
+    agree on which chunks belong to which file.
+
+    The fragment is dropped (it names a spot on the same page), but the query
+    string is kept, encoded into the file name (see ``_query_segment``), so
+    ``/p?page=1`` and ``/p?page=2`` stay two files. A URL without a query maps
+    exactly as before. Paths that still collide, such as ``/a`` and
+    ``/a.html``, are told apart by ``dedupe_virtual_paths``.
+
+    Args:
+        url: Page URL, e.g. ``"https://docs.docsgpt.cloud/guides/setup"``.
+        include_host: Prefix the host, for ingests spanning several hosts
+            whose paths would otherwise collide (every root is ``index.md``).
+
+    Returns:
+        A relative path such as ``"index.md"``, ``"guides/setup.md"``,
+        ``"list__page=2.md"`` or, with ``include_host``,
+        ``"docs.docsgpt.cloud/guides/setup.md"``.
+    """
+    parsed = urlparse(url)
+    path = parsed.path.strip("/")
+
+    if not path:
+        path = "index.md"
+    else:
+        base, ext = os.path.splitext(path)
+        if ext.lower() in _PAGE_EXTENSIONS:
+            path = base
+        if not path.endswith(".md"):
+            path = f"{path}.md"
+
+    query = _query_segment(parsed.query)
+    if query:
+        path = f"{path[:-len('.md')]}{_QUERY_SEPARATOR}{query}.md"
+
+    if include_host and parsed.netloc:
+        return f"{parsed.netloc}/{path}"
+    return path
+
+
+def spans_multiple_hosts(urls: List[str]) -> bool:
+    """Return whether ``urls`` point at more than one host.
+
+    A URL ``urlparse`` rejects (an unclosed IPv6 bracket, say) is left out:
+    it fails validation and is never fetched, so it names no host here.
+
+    Args:
+        urls: Page URLs about to be ingested together.
+
+    Returns:
+        True when paths alone could collide and need a host prefix.
+    """
+    hosts = set()
+    for url in urls:
+        try:
+            hosts.add(urlparse(url).netloc)
+        except ValueError:
+            continue
+    return len(hosts) > 1
+
+
+def normalize_page_url(url: str) -> str:
+    """Return the identity of the page ``url`` names, for deduplication.
+
+    The fragment is dropped (it names a spot on the same page) and the query
+    parameters are sorted, as ``_query_segment`` sorts them, so
+    ``/p?a=1&b=2`` and ``/p?b=2&a=1#top`` are one page. The result is a key
+    only; fetch the URL as it was written.
+
+    Args:
+        url: Page URL.
+
+    Returns:
+        The normalized URL.
+
+    Raises:
+        ValueError: If ``urlsplit`` cannot parse ``url``.
+    """
+    parts = urlsplit(url)
+    query = urlencode(sorted(parse_qsl(parts.query, keep_blank_values=True)))
+    return urlunsplit(parts._replace(query=query, fragment=""))
+
+
+def dedupe_virtual_paths(documents: List[Document]) -> List[Document]:
+    """Give distinct pages that share a virtual ``file_path`` distinct ones.
+
+    ``url_to_virtual_path`` folds some different URLs onto one path (``/a`` and
+    ``/a.html`` are both ``a.md``), and the worker would then merge those pages
+    into one tree entry. Within a path, the pages are ordered by URL (fragment
+    dropped, query parameters sorted; see ``normalize_page_url``): the first
+    keeps the path and each later one gets ``-2``, ``-3`` and so on before
+    ``.md``, skipping any path another page already has.
+    Ordering by URL rather than by fetch order keeps the names stable across
+    re-syncs of the same pages. Documents for the same URL (one page reached
+    via two fragments, or with its query parameters in another order) are one
+    page and keep one path.
+
+    Args:
+        documents: Loaded documents; each ``extra_info`` carries ``source``
+            (the page URL) and ``file_path``. Others are left untouched.
+
+    Returns:
+        ``documents``, with colliding ``file_path`` values rewritten in place.
+    """
+    pages_by_path: Dict[str, Dict[str, List[Document]]] = {}
+    for doc in documents:
+        info = doc.extra_info or {}
+        path = info.get("file_path")
+        if not path:
+            continue
+        page = normalize_page_url(str(info.get("source") or path))
+        pages_by_path.setdefault(path, {}).setdefault(page, []).append(doc)
+
+    taken = set(pages_by_path)
+    for path in sorted(pages_by_path):
+        pages = pages_by_path[path]
+        if len(pages) < 2:
+            continue
+        stem = path[: -len(".md")] if path.endswith(".md") else path
+        suffix = 1
+        for page in sorted(pages)[1:]:
+            suffix += 1
+            while f"{stem}-{suffix}.md" in taken:
+                suffix += 1
+            new_path = f"{stem}-{suffix}.md"
+            taken.add(new_path)
+            for doc in pages[page]:
+                doc.extra_info["file_path"] = new_path
+    return documents
+
 
 class BaseRemote:
     """Utilities for loading data from a directory."""

docsgpt/parser/remote/crawler_loader.py

@@ -1,9 +1,8 @@
 import logging
-import os
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin, urlparse
 
-from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.remote.base import BaseRemote, dedupe_virtual_paths, normalize_page_url, url_to_virtual_path
 from docsgpt.parser.schema.base import Document
 from docsgpt.core.url_validation import validate_url, SSRFError
 from docsgpt.security.safe_url import pinned_request
@@ -25,6 +24,8 @@ class CrawlerLoader(BaseRemote):
             logging.error(f"URL validation failed: {e}")
             return []
 
+        # Keyed by normalize_page_url, so fragment and query-order variants of
+        # one page are fetched once.
         visited_urls = set()
         base_url = urlparse(url).scheme + "://" + urlparse(url).hostname
         urls_to_visit = [url]
@@ -32,7 +33,10 @@ class CrawlerLoader(BaseRemote):
 
         while urls_to_visit:
             current_url = urls_to_visit.pop(0)
-            visited_urls.add(current_url)
+            page_key = normalize_page_url(current_url)
+            if page_key in visited_urls:
+                continue
+            visited_urls.add(page_key)
 
             try:
                 response = pinned_request("GET", current_url, timeout=30)
@@ -58,45 +62,29 @@ class CrawlerLoader(BaseRemote):
                 logging.error(f"Error processing URL {current_url}: {e}", exc_info=True)
                 continue
 
-            # Parse the HTML content to extract all links
-            all_links = [
-                urljoin(current_url, a['href'])
-                for a in soup.find_all('a', href=True)
-                if base_url in urljoin(current_url, a['href'])
-            ]
+            # Parse the HTML content to extract all links. A malformed href
+            # (``http://[bad``) makes urljoin or the key parse raise; skip it
+            # rather than abort the crawl.
+            all_links = []
+            for a in soup.find_all('a', href=True):
+                try:
+                    link = urljoin(current_url, a['href'])
+                    link_key = normalize_page_url(link)
+                except ValueError:
+                    continue
+                if base_url in link and link_key not in visited_urls:
+                    all_links.append(link)
 
             # Add new links to the list of URLs to visit if they haven't been visited yet
-            urls_to_visit.extend([link for link in all_links if link not in visited_urls])
+            urls_to_visit.extend(all_links)
             urls_to_visit = list(set(urls_to_visit))
 
             # Stop crawling if the limit of pages to scrape is reached
             if self.limit is not None and len(visited_urls) >= self.limit:
                 break
 
-        return loaded_content
+        return dedupe_virtual_paths(loaded_content)
 
     def _url_to_virtual_path(self, url):
-        """
-        Convert a URL to a virtual file path ending with .md.
-
-        Examples:
-            https://docs.docsgpt.cloud/ -> index.md
-            https://docs.docsgpt.cloud/guides/setup -> guides/setup.md
-            https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md
-            https://example.com/page.html -> page.md
-        """
-        parsed = urlparse(url)
-        path = parsed.path.strip("/")
-
-        if not path:
-            return "index.md"
-
-        # Remove common file extensions and add .md
-        base, ext = os.path.splitext(path)
-        if ext.lower() in [".html", ".htm", ".php", ".asp", ".aspx", ".jsp"]:
-            path = base
-
-        if not path.endswith(".md"):
-            path = f"{path}.md"
-
-        return path
+        """Convert a URL to a virtual ``.md`` path; see ``url_to_virtual_path``."""
+        return url_to_virtual_path(url)

docsgpt/parser/remote/crawler_markdown.py

@@ -1,13 +1,12 @@
 from urllib.parse import urlparse, urljoin
 from bs4 import BeautifulSoup
-from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.remote.base import BaseRemote, dedupe_virtual_paths, normalize_page_url, url_to_virtual_path
 from docsgpt.core.url_validation import validate_url, SSRFError
 from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
 import re
 from markdownify import markdownify
 from docsgpt.parser.schema.base import Document
 import tldextract
-import os
 
 # The bundled public-suffix snapshot is enough for domain matching; the
 # default extractor would fetch the live list on first use and cache it on
@@ -38,7 +37,8 @@ class CrawlerLoader(BaseRemote):
             print(f"URL validation failed: {e}")
             return []
 
-        # Keep track of visited URLs to avoid revisiting the same page
+        # Keep track of visited pages to avoid revisiting the same page. Keyed
+        # by normalize_page_url, so fragment and query-order variants are one.
         visited_urls = set()
 
         # Determine the base domain for link filtering using tldextract
@@ -50,9 +50,10 @@ class CrawlerLoader(BaseRemote):
             current_url = urls_to_visit.pop()
 
             # Skip if already visited
-            if current_url in visited_urls:
+            page_key = normalize_page_url(current_url)
+            if page_key in visited_urls:
                 continue
-            visited_urls.add(current_url)
+            visited_urls.add(page_key)
 
             # Fetch the page content
             html_content = self._fetch_page(current_url)
@@ -85,13 +86,15 @@ class CrawlerLoader(BaseRemote):
             filtered_links = self._filter_links(new_links, base_domain)
 
             # Add any new, not-yet-visited links to the queue
-            urls_to_visit.update(link for link in filtered_links if link not in visited_urls)
+            urls_to_visit.update(
+                link for link in filtered_links if normalize_page_url(link) not in visited_urls
+            )
 
             # If we've reached the limit, stop crawling
             if self.limit is not None and len(visited_urls) >= self.limit:
                 break
 
-        return documents
+        return dedupe_virtual_paths(documents)
 
     def _fetch_page(self, url):
         try:
@@ -124,7 +127,13 @@ class CrawlerLoader(BaseRemote):
         soup = BeautifulSoup(html_content, 'html.parser')
         links = []
         for a in soup.find_all('a', href=True):
-            full_url = urljoin(current_url, a['href'])
+            # A malformed href (``http://[bad``) makes urljoin or the later
+            # parse raise; skip that link rather than abort the crawl.
+            try:
+                full_url = urljoin(current_url, a['href'])
+                normalize_page_url(full_url)
+            except ValueError:
+                continue
             links.append((full_url, a.text.strip()))
         return links
 
@@ -160,28 +169,5 @@ class CrawlerLoader(BaseRemote):
         return filtered
 
     def _url_to_virtual_path(self, url):
-        """
-        Convert a URL to a virtual file path ending with .md.
-
-        Examples:
-            https://docs.docsgpt.cloud/ -> index.md
-            https://docs.docsgpt.cloud/guides/setup -> guides/setup.md
-            https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md
-            https://example.com/page.html -> page.md
-        """
-        parsed = urlparse(url)
-        path = parsed.path.strip("/")
-
-        if not path:
-            return "index.md"
-
-        # Remove common file extensions and add .md
-        base, ext = os.path.splitext(path)
-        if ext.lower() in [".html", ".htm", ".php", ".asp", ".aspx", ".jsp"]:
-            path = base
-
-        # Ensure path ends with .md
-        if not path.endswith(".md"):
-            path = path + ".md"
-
-        return path
+        """Convert a URL to a virtual ``.md`` path; see ``url_to_virtual_path``."""
+        return url_to_virtual_path(url)

docsgpt/parser/remote/sitemap_loader.py

@@ -4,7 +4,12 @@ import re
 import defusedxml.ElementTree as ET
 from bs4 import BeautifulSoup
 
-from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.remote.base import (
+    BaseRemote,
+    dedupe_virtual_paths,
+    spans_multiple_hosts,
+    url_to_virtual_path,
+)
 from docsgpt.parser.schema.base import Document
 from docsgpt.core.url_validation import validate_url, SSRFError
 from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
@@ -32,6 +37,7 @@ class SitemapLoader(BaseRemote):
             return []
 
         # Load content of extracted URLs
+        include_host = spans_multiple_hosts(urls)
         documents = []
         processed_urls = 0  # Counter for processed URLs
         for url in urls:
@@ -50,7 +56,13 @@ class SitemapLoader(BaseRemote):
                 documents.append(
                     Document(
                         soup.get_text(separator="\n", strip=True),
-                        extra_info={"source": url},
+                        # Without file_path the worker had no tree key (no
+                        # title, key or doc_id), so sitemap pages never
+                        # appeared in the file tree.
+                        extra_info={
+                            "source": url,
+                            "file_path": url_to_virtual_path(url, include_host),
+                        },
                     )
                 )
                 processed_urls += 1  # Increment the counter after processing each URL
@@ -58,7 +70,7 @@ class SitemapLoader(BaseRemote):
                 logging.error(f"Error processing URL {url}: {e}", exc_info=True)
                 continue
 
-        return documents
+        return dedupe_virtual_paths(documents)
 
     def _extract_urls(self, sitemap_url):
         try:

docsgpt/parser/remote/web_loader.py

@@ -3,7 +3,12 @@ import logging
 from bs4 import BeautifulSoup
 
 from docsgpt.core.url_validation import SSRFError, validate_url
-from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.remote.base import (
+    BaseRemote,
+    dedupe_virtual_paths,
+    spans_multiple_hosts,
+    url_to_virtual_path,
+)
 from docsgpt.parser.schema.base import Document
 from docsgpt.security.safe_url import pinned_request
 
@@ -24,15 +29,17 @@ class WebLoader(BaseRemote):
         urls = inputs
         if isinstance(urls, str):
             urls = [urls]
-        documents = []
+        valid_urls = []
         for url in urls:
             try:
-                url = validate_url(url)
+                valid_urls.append(validate_url(url))
             except SSRFError as e:
                 logging.warning(
                     f"Skipping URL due to SSRF validation failure: {url} - {e}"
                 )
-                continue
+        include_host = spans_multiple_hosts(valid_urls)
+        documents = []
+        for url in valid_urls:
             try:
                 response = pinned_request("GET", url, headers=headers, timeout=30)
                 response.raise_for_status()
@@ -45,6 +52,9 @@ class WebLoader(BaseRemote):
                 html_tag = soup.find("html")
                 if html_tag and html_tag.get("lang"):
                     metadata["language"] = html_tag.get("lang")
+                # The worker keys the file tree by file_path; without it the
+                # tree fell back to the title, which the chunks view can't match.
+                metadata["file_path"] = url_to_virtual_path(url, include_host)
                 documents.append(
                     Document(
                         soup.get_text(separator="\n", strip=True),
@@ -54,4 +64,4 @@ class WebLoader(BaseRemote):
             except Exception as e:
                 logging.error(f"Error processing URL {url}: {e}", exc_info=True)
                 continue
-        return documents
+        return dedupe_virtual_paths(documents)