Big source revamp

docsgpt/parser/remote/base.py

@@ -1,10 +1,158 @@
 """Base reader class."""
+import hashlib
+import os
+import re
 from abc import abstractmethod
-from typing import Any, List
+from typing import Any, Dict, List
+from urllib.parse import parse_qsl, urldefrag, urlencode, urlparse
 
 from docsgpt.parser.schema.base import Document
 from docsgpt.vectorstore.document_class import Document as VectorDocument
 
+_PAGE_EXTENSIONS = (".html", ".htm", ".php", ".asp", ".aspx", ".jsp")
+
+# Joins a page's path to its encoded query string in the virtual file name.
+_QUERY_SEPARATOR = "__"
+# Longest encoded query kept verbatim; longer ones keep a prefix plus a hash.
+MAX_QUERY_SEGMENT_LENGTH = 64
+_QUERY_HASH_LENGTH = 10
+# Anything but letters, digits, ``_``, ``.`` and ``-`` in a query key or value.
+_UNSAFE_QUERY_CHARS = re.compile(r"[^\w.-]+")
+
+
+def _query_segment(query: str) -> str:
+    """Encode a URL query string as a deterministic, tree-safe name segment.
+
+    Parameters are sorted so the same page maps to the same name across
+    re-syncs whatever order its links list them in. Keys and values are
+    percent-decoded and every run of characters outside ``[\\w.-]`` becomes
+    ``-``, so the segment carries no ``/``, ``?``, ``#`` or ``%``. A segment
+    longer than ``MAX_QUERY_SEGMENT_LENGTH`` keeps a prefix and a hash of the
+    full query, which keeps it bounded and still distinct.
+
+    Args:
+        query: The raw query string, without the leading ``?``.
+
+    Returns:
+        The encoded segment, or ``""`` when the query has no parameters.
+    """
+    pairs = sorted(parse_qsl(query, keep_blank_values=True))
+    if not pairs:
+        return ""
+    parts = []
+    for key, value in pairs:
+        key = _UNSAFE_QUERY_CHARS.sub("-", key)
+        value = _UNSAFE_QUERY_CHARS.sub("-", value)
+        parts.append(f"{key}={value}" if value else key)
+    segment = "&".join(parts)
+    if len(segment) <= MAX_QUERY_SEGMENT_LENGTH:
+        return segment
+    digest = hashlib.sha1(urlencode(pairs).encode("utf-8")).hexdigest()
+    keep = MAX_QUERY_SEGMENT_LENGTH - _QUERY_HASH_LENGTH - 1
+    return f"{segment[:keep]}-{digest[:_QUERY_HASH_LENGTH]}"
+
+
+def url_to_virtual_path(url: str, include_host: bool = False) -> str:
+    """Convert a page URL to the virtual ``.md`` path used as its tree key.
+
+    Web ingests store this as the chunk's ``file_path``: the worker keys the
+    source's file tree by it and the chunks view filters by it, so the two
+    agree on which chunks belong to which file.
+
+    The fragment is dropped (it names a spot on the same page), but the query
+    string is kept, encoded into the file name (see ``_query_segment``), so
+    ``/p?page=1`` and ``/p?page=2`` stay two files. A URL without a query maps
+    exactly as before. Paths that still collide, such as ``/a`` and
+    ``/a.html``, are told apart by ``dedupe_virtual_paths``.
+
+    Args:
+        url: Page URL, e.g. ``"https://docs.docsgpt.cloud/guides/setup"``.
+        include_host: Prefix the host, for ingests spanning several hosts
+            whose paths would otherwise collide (every root is ``index.md``).
+
+    Returns:
+        A relative path such as ``"index.md"``, ``"guides/setup.md"``,
+        ``"list__page=2.md"`` or, with ``include_host``,
+        ``"docs.docsgpt.cloud/guides/setup.md"``.
+    """
+    parsed = urlparse(url)
+    path = parsed.path.strip("/")
+
+    if not path:
+        path = "index.md"
+    else:
+        base, ext = os.path.splitext(path)
+        if ext.lower() in _PAGE_EXTENSIONS:
+            path = base
+        if not path.endswith(".md"):
+            path = f"{path}.md"
+
+    query = _query_segment(parsed.query)
+    if query:
+        path = f"{path[:-len('.md')]}{_QUERY_SEPARATOR}{query}.md"
+
+    if include_host and parsed.netloc:
+        return f"{parsed.netloc}/{path}"
+    return path
+
+
+def spans_multiple_hosts(urls: List[str]) -> bool:
+    """Return whether ``urls`` point at more than one host.
+
+    Args:
+        urls: Page URLs about to be ingested together.
+
+    Returns:
+        True when paths alone could collide and need a host prefix.
+    """
+    return len({urlparse(u).netloc for u in urls}) > 1
+
+
+def dedupe_virtual_paths(documents: List[Document]) -> List[Document]:
+    """Give distinct pages that share a virtual ``file_path`` distinct ones.
+
+    ``url_to_virtual_path`` folds some different URLs onto one path (``/a`` and
+    ``/a.html`` are both ``a.md``), and the worker would then merge those pages
+    into one tree entry. Within a path, the pages are ordered by URL (fragment
+    dropped): the first keeps the path and each later one gets ``-2``, ``-3``
+    and so on before ``.md``, skipping any path another page already has.
+    Ordering by URL rather than by fetch order keeps the names stable across
+    re-syncs of the same pages. Documents for the same URL (one page reached
+    via two fragments) are one page and keep one path.
+
+    Args:
+        documents: Loaded documents; each ``extra_info`` carries ``source``
+            (the page URL) and ``file_path``. Others are left untouched.
+
+    Returns:
+        ``documents``, with colliding ``file_path`` values rewritten in place.
+    """
+    pages_by_path: Dict[str, Dict[str, List[Document]]] = {}
+    for doc in documents:
+        info = doc.extra_info or {}
+        path = info.get("file_path")
+        if not path:
+            continue
+        page = urldefrag(str(info.get("source") or path))[0]
+        pages_by_path.setdefault(path, {}).setdefault(page, []).append(doc)
+
+    taken = set(pages_by_path)
+    for path in sorted(pages_by_path):
+        pages = pages_by_path[path]
+        if len(pages) < 2:
+            continue
+        stem = path[: -len(".md")] if path.endswith(".md") else path
+        suffix = 1
+        for page in sorted(pages)[1:]:
+            suffix += 1
+            while f"{stem}-{suffix}.md" in taken:
+                suffix += 1
+            new_path = f"{stem}-{suffix}.md"
+            taken.add(new_path)
+            for doc in pages[page]:
+                doc.extra_info["file_path"] = new_path
+    return documents
+
 
 class BaseRemote:
     """Utilities for loading data from a directory."""

docsgpt/parser/remote/crawler_loader.py

@@ -1,9 +1,8 @@
 import logging
-import os
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin, urlparse
 
-from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.remote.base import BaseRemote, dedupe_virtual_paths, url_to_virtual_path
 from docsgpt.parser.schema.base import Document
 from docsgpt.core.url_validation import validate_url, SSRFError
 from docsgpt.security.safe_url import pinned_request
@@ -73,30 +72,8 @@ class CrawlerLoader(BaseRemote):
             if self.limit is not None and len(visited_urls) >= self.limit:
                 break
 
-        return loaded_content
+        return dedupe_virtual_paths(loaded_content)
 
     def _url_to_virtual_path(self, url):
-        """
-        Convert a URL to a virtual file path ending with .md.
-
-        Examples:
-            https://docs.docsgpt.cloud/ -> index.md
-            https://docs.docsgpt.cloud/guides/setup -> guides/setup.md
-            https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md
-            https://example.com/page.html -> page.md
-        """
-        parsed = urlparse(url)
-        path = parsed.path.strip("/")
-
-        if not path:
-            return "index.md"
-
-        # Remove common file extensions and add .md
-        base, ext = os.path.splitext(path)
-        if ext.lower() in [".html", ".htm", ".php", ".asp", ".aspx", ".jsp"]:
-            path = base
-
-        if not path.endswith(".md"):
-            path = f"{path}.md"
-
-        return path
+        """Convert a URL to a virtual ``.md`` path; see ``url_to_virtual_path``."""
+        return url_to_virtual_path(url)

docsgpt/parser/remote/crawler_markdown.py

@@ -1,13 +1,12 @@
 from urllib.parse import urlparse, urljoin
 from bs4 import BeautifulSoup
-from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.remote.base import BaseRemote, dedupe_virtual_paths, url_to_virtual_path
 from docsgpt.core.url_validation import validate_url, SSRFError
 from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
 import re
 from markdownify import markdownify
 from docsgpt.parser.schema.base import Document
 import tldextract
-import os
 
 # The bundled public-suffix snapshot is enough for domain matching; the
 # default extractor would fetch the live list on first use and cache it on
@@ -91,7 +90,7 @@ class CrawlerLoader(BaseRemote):
             if self.limit is not None and len(visited_urls) >= self.limit:
                 break
 
-        return documents
+        return dedupe_virtual_paths(documents)
 
     def _fetch_page(self, url):
         try:
@@ -160,28 +159,5 @@ class CrawlerLoader(BaseRemote):
         return filtered
 
     def _url_to_virtual_path(self, url):
-        """
-        Convert a URL to a virtual file path ending with .md.
-
-        Examples:
-            https://docs.docsgpt.cloud/ -> index.md
-            https://docs.docsgpt.cloud/guides/setup -> guides/setup.md
-            https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md
-            https://example.com/page.html -> page.md
-        """
-        parsed = urlparse(url)
-        path = parsed.path.strip("/")
-
-        if not path:
-            return "index.md"
-
-        # Remove common file extensions and add .md
-        base, ext = os.path.splitext(path)
-        if ext.lower() in [".html", ".htm", ".php", ".asp", ".aspx", ".jsp"]:
-            path = base
-
-        # Ensure path ends with .md
-        if not path.endswith(".md"):
-            path = path + ".md"
-
-        return path
+        """Convert a URL to a virtual ``.md`` path; see ``url_to_virtual_path``."""
+        return url_to_virtual_path(url)

docsgpt/parser/remote/sitemap_loader.py

@@ -4,7 +4,12 @@ import re
 import defusedxml.ElementTree as ET
 from bs4 import BeautifulSoup
 
-from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.remote.base import (
+    BaseRemote,
+    dedupe_virtual_paths,
+    spans_multiple_hosts,
+    url_to_virtual_path,
+)
 from docsgpt.parser.schema.base import Document
 from docsgpt.core.url_validation import validate_url, SSRFError
 from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
@@ -32,6 +37,7 @@ class SitemapLoader(BaseRemote):
             return []
 
         # Load content of extracted URLs
+        include_host = spans_multiple_hosts(urls)
         documents = []
         processed_urls = 0  # Counter for processed URLs
         for url in urls:
@@ -50,7 +56,13 @@ class SitemapLoader(BaseRemote):
                 documents.append(
                     Document(
                         soup.get_text(separator="\n", strip=True),
-                        extra_info={"source": url},
+                        # Without file_path the worker had no tree key (no
+                        # title, key or doc_id), so sitemap pages never
+                        # appeared in the file tree.
+                        extra_info={
+                            "source": url,
+                            "file_path": url_to_virtual_path(url, include_host),
+                        },
                     )
                 )
                 processed_urls += 1  # Increment the counter after processing each URL
@@ -58,7 +70,7 @@ class SitemapLoader(BaseRemote):
                 logging.error(f"Error processing URL {url}: {e}", exc_info=True)
                 continue
 
-        return documents
+        return dedupe_virtual_paths(documents)
 
     def _extract_urls(self, sitemap_url):
         try:

docsgpt/parser/remote/web_loader.py

@@ -3,7 +3,12 @@ import logging
 from bs4 import BeautifulSoup
 
 from docsgpt.core.url_validation import SSRFError, validate_url
-from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.remote.base import (
+    BaseRemote,
+    dedupe_virtual_paths,
+    spans_multiple_hosts,
+    url_to_virtual_path,
+)
 from docsgpt.parser.schema.base import Document
 from docsgpt.security.safe_url import pinned_request
 
@@ -24,15 +29,17 @@ class WebLoader(BaseRemote):
         urls = inputs
         if isinstance(urls, str):
             urls = [urls]
-        documents = []
+        valid_urls = []
         for url in urls:
             try:
-                url = validate_url(url)
+                valid_urls.append(validate_url(url))
             except SSRFError as e:
                 logging.warning(
                     f"Skipping URL due to SSRF validation failure: {url} - {e}"
                 )
-                continue
+        include_host = spans_multiple_hosts(valid_urls)
+        documents = []
+        for url in valid_urls:
             try:
                 response = pinned_request("GET", url, headers=headers, timeout=30)
                 response.raise_for_status()
@@ -45,6 +52,9 @@ class WebLoader(BaseRemote):
                 html_tag = soup.find("html")
                 if html_tag and html_tag.get("lang"):
                     metadata["language"] = html_tag.get("lang")
+                # The worker keys the file tree by file_path; without it the
+                # tree fell back to the title, which the chunks view can't match.
+                metadata["file_path"] = url_to_virtual_path(url, include_host)
                 documents.append(
                     Document(
                         soup.get_text(separator="\n", strip=True),
@@ -54,4 +64,4 @@ class WebLoader(BaseRemote):
             except Exception as e:
                 logging.error(f"Error processing URL {url}: {e}", exc_info=True)
                 continue
-        return documents
+        return dedupe_virtual_paths(documents)