refactor: rename the application package to docsgpt

The backend import package is now docsgpt, the name it will carry on PyPI;
application was far too generic to install into anyone's site-packages.
git mv plus a mechanical rewrite of every import, dotted string and path
reference: 734 Python files, the compose files, Dockerfile, workflows, docs,
setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage
config, .gitignore. Behaviour is unchanged.

Kept for one release:
- A top-level application package whose meta-path finder resolves
  application.x.y to the already-imported docsgpt.x.y object, so old imports
  and entry points (celery -A application.app.celery,
  uvicorn application.asgi:asgi_app) keep working with a FutureWarning.
- Celery registers every application.* task name as an alias of its
  docsgpt.* task on start-up, so messages queued by the previous release still
  run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries
  the previous release wrote are left unread instead of firing twice.

The backend image builds from the repository root (docker build -f
docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore
allow-lists docsgpt/ and application/ and keeps caches, local data, .env
files, the sample index files and the Dockerfile out. Compose and the image
workflows point at the new context.

docsgpt/parser/remote/base.py

@@ -0,0 +1,19 @@
+"""Base reader class."""
+from abc import abstractmethod
+from typing import Any, List
+
+from docsgpt.parser.schema.base import Document
+from docsgpt.vectorstore.document_class import Document as VectorDocument
+
+
+class BaseRemote:
+    """Utilities for loading data from a directory."""
+
+    @abstractmethod
+    def load_data(self, *args: Any, **load_kwargs: Any) -> List[Document]:
+        """Load data from the input directory."""
+
+    def load_vector_documents(self, **load_kwargs: Any) -> List[VectorDocument]:
+        """Load data in the vector-store document format."""
+        docs = self.load_data(**load_kwargs)
+        return [d.to_vector_format() for d in docs]

docsgpt/parser/remote/crawler_loader.py

@@ -0,0 +1,102 @@
+import logging
+import os
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+from docsgpt.core.url_validation import validate_url, SSRFError
+from docsgpt.security.safe_url import pinned_request
+
+
+class CrawlerLoader(BaseRemote):
+    def __init__(self, limit=10):
+        self.limit = limit  # Set the limit for the number of pages to scrape
+
+    def load_data(self, inputs):
+        url = inputs
+        if isinstance(url, list) and url:
+            url = url[0]
+
+        # Validate URL to prevent SSRF attacks
+        try:
+            url = validate_url(url)
+        except SSRFError as e:
+            logging.error(f"URL validation failed: {e}")
+            return []
+
+        visited_urls = set()
+        base_url = urlparse(url).scheme + "://" + urlparse(url).hostname
+        urls_to_visit = [url]
+        loaded_content = []
+
+        while urls_to_visit:
+            current_url = urls_to_visit.pop(0)
+            visited_urls.add(current_url)
+
+            try:
+                response = pinned_request("GET", current_url, timeout=30)
+                response.raise_for_status()
+                soup = BeautifulSoup(response.text, "html.parser")
+                extra_info = {
+                    "source": current_url,
+                    "file_path": self._url_to_virtual_path(current_url),
+                }
+                # Mirror WebLoader: without a title, citations fall back to the
+                # chunk body and render as a wall of text.
+                if soup.title:
+                    title = soup.title.get_text(strip=True)
+                    if title:
+                        extra_info["title"] = title
+                loaded_content.append(
+                    Document(
+                        soup.get_text(separator="\n", strip=True),
+                        extra_info=extra_info,
+                    )
+                )
+            except Exception as e:
+                logging.error(f"Error processing URL {current_url}: {e}", exc_info=True)
+                continue
+
+            # Parse the HTML content to extract all links
+            all_links = [
+                urljoin(current_url, a['href'])
+                for a in soup.find_all('a', href=True)
+                if base_url in urljoin(current_url, a['href'])
+            ]
+
+            # Add new links to the list of URLs to visit if they haven't been visited yet
+            urls_to_visit.extend([link for link in all_links if link not in visited_urls])
+            urls_to_visit = list(set(urls_to_visit))
+
+            # Stop crawling if the limit of pages to scrape is reached
+            if self.limit is not None and len(visited_urls) >= self.limit:
+                break
+
+        return loaded_content
+
+    def _url_to_virtual_path(self, url):
+        """
+        Convert a URL to a virtual file path ending with .md.
+
+        Examples:
+            https://docs.docsgpt.cloud/ -> index.md
+            https://docs.docsgpt.cloud/guides/setup -> guides/setup.md
+            https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md
+            https://example.com/page.html -> page.md
+        """
+        parsed = urlparse(url)
+        path = parsed.path.strip("/")
+
+        if not path:
+            return "index.md"
+
+        # Remove common file extensions and add .md
+        base, ext = os.path.splitext(path)
+        if ext.lower() in [".html", ".htm", ".php", ".asp", ".aspx", ".jsp"]:
+            path = base
+
+        if not path.endswith(".md"):
+            path = f"{path}.md"
+
+        return path

docsgpt/parser/remote/crawler_markdown.py

@@ -0,0 +1,187 @@
+from urllib.parse import urlparse, urljoin
+from bs4 import BeautifulSoup
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.core.url_validation import validate_url, SSRFError
+from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
+import re
+from markdownify import markdownify
+from docsgpt.parser.schema.base import Document
+import tldextract
+import os
+
+# The bundled public-suffix snapshot is enough for domain matching; the
+# default extractor would fetch the live list on first use and cache it on
+# disk, which is a network round trip the ingest worker should not depend on.
+_extract = tldextract.TLDExtract(suffix_list_urls=(), cache_dir=None)
+
+class CrawlerLoader(BaseRemote):
+    def __init__(self, limit=10, allow_subdomains=False):
+        """
+        Given a URL crawl web pages up to `self.limit`,
+        convert HTML content to Markdown, and returning a list of Document objects.
+
+        :param limit: The maximum number of pages to crawl.
+        :param allow_subdomains: If True, crawl pages on subdomains of the base domain.
+        """
+        self.limit = limit
+        self.allow_subdomains = allow_subdomains
+
+    def load_data(self, inputs):
+        url = inputs
+        if isinstance(url, list) and url:
+            url = url[0]
+
+        # Validate URL to prevent SSRF attacks
+        try:
+            url = validate_url(url)
+        except SSRFError as e:
+            print(f"URL validation failed: {e}")
+            return []
+
+        # Keep track of visited URLs to avoid revisiting the same page
+        visited_urls = set()
+
+        # Determine the base domain for link filtering using tldextract
+        base_domain = self._get_base_domain(url)
+        urls_to_visit = {url}
+        documents = []
+
+        while urls_to_visit:
+            current_url = urls_to_visit.pop()
+
+            # Skip if already visited
+            if current_url in visited_urls:
+                continue
+            visited_urls.add(current_url)
+
+            # Fetch the page content
+            html_content = self._fetch_page(current_url)
+            if html_content is None:
+                continue
+
+            # Convert the HTML to Markdown for cleaner text formatting
+            title, language, processed_markdown = self._process_html_to_markdown(html_content, current_url)
+            if processed_markdown:
+                # Generate virtual file path from URL for consistent file-like matching
+                virtual_path = self._url_to_virtual_path(current_url)
+                
+                # Create a Document for each visited page
+                documents.append(
+                    Document(
+                        processed_markdown,  # content
+                        None,  # doc_id
+                        None,  # embedding
+                        {
+                            "source": current_url,
+                            "title": title,
+                            "language": language,
+                            "file_path": virtual_path,
+                        },  # extra_info
+                    )
+                )
+
+            # Extract links and filter them according to domain rules
+            new_links = self._extract_links(html_content, current_url)
+            filtered_links = self._filter_links(new_links, base_domain)
+
+            # Add any new, not-yet-visited links to the queue
+            urls_to_visit.update(link for link in filtered_links if link not in visited_urls)
+
+            # If we've reached the limit, stop crawling
+            if self.limit is not None and len(visited_urls) >= self.limit:
+                break
+
+        return documents
+
+    def _fetch_page(self, url):
+        try:
+            response = pinned_request("GET", url, timeout=10)
+            response.raise_for_status()
+            return response.text
+        except UnsafeUserUrlError as e:
+            print(f"URL validation failed for {url}: {e}")
+            return None
+        except Exception as e:
+            print(f"Error fetching URL {url}: {e}")
+            return None
+
+    def _process_html_to_markdown(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        title_tag = soup.find('title')
+        title = title_tag.text.strip() if title_tag else "No Title"
+
+        # Extract language
+        language_tag = soup.find('html')
+        language = language_tag.get('lang', 'en') if language_tag else "en"
+
+        markdownified = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH")
+        # Collapse runs of blank lines to a single one — the same shape
+        # ``html_to_markdown`` gives uploaded HTML files.
+        markdownified = re.sub(r'\n{3,}', '\n\n', markdownified)
+        return title, language, markdownified
+
+    def _extract_links(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        links = []
+        for a in soup.find_all('a', href=True):
+            full_url = urljoin(current_url, a['href'])
+            links.append((full_url, a.text.strip()))
+        return links
+
+    def _get_base_domain(self, url):
+        extracted = _extract(url)
+        # Reconstruct the domain as domain.suffix
+        base_domain = f"{extracted.domain}.{extracted.suffix}"
+        return base_domain
+
+    def _filter_links(self, links, base_domain):
+        """
+        Filter the extracted links to only include those that match the crawling criteria:
+        - If allow_subdomains is True, allow any link whose domain ends with the base_domain.
+        - If allow_subdomains is False, only allow exact matches of the base_domain.
+        """
+        filtered = []
+        for link, _ in links:
+            parsed_link = urlparse(link)
+            if not parsed_link.netloc:
+                continue
+
+            extracted = _extract(parsed_link.netloc)
+            link_base = f"{extracted.domain}.{extracted.suffix}"
+
+            if self.allow_subdomains:
+                # For subdomains: sub.example.com ends with example.com
+                if link_base == base_domain or link_base.endswith("." + base_domain):
+                    filtered.append(link)
+            else:
+                # Exact domain match
+                if link_base == base_domain:
+                    filtered.append(link)
+        return filtered
+
+    def _url_to_virtual_path(self, url):
+        """
+        Convert a URL to a virtual file path ending with .md.
+
+        Examples:
+            https://docs.docsgpt.cloud/ -> index.md
+            https://docs.docsgpt.cloud/guides/setup -> guides/setup.md
+            https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md
+            https://example.com/page.html -> page.md
+        """
+        parsed = urlparse(url)
+        path = parsed.path.strip("/")
+
+        if not path:
+            return "index.md"
+
+        # Remove common file extensions and add .md
+        base, ext = os.path.splitext(path)
+        if ext.lower() in [".html", ".htm", ".php", ".asp", ".aspx", ".jsp"]:
+            path = base
+
+        # Ensure path ends with .md
+        if not path.endswith(".md"):
+            path = path + ".md"
+
+        return path

docsgpt/parser/remote/github_loader.py

@@ -0,0 +1,454 @@
+import base64
+import logging
+import mimetypes
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Dict, List, Optional, Tuple
+
+import requests
+
+from docsgpt.core.settings import settings
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+
+logger = logging.getLogger(__name__)
+
+# Directory names that hold vendored or generated output. Anything under one
+# of these is build product, not source: it bloats the index, costs an API
+# call each, and answers no question a user would ask of the repo.
+SKIP_DIRECTORIES = {
+    ".git", ".github/workflows/generated", ".idea", ".vscode",
+    "__pycache__", ".mypy_cache", ".pytest_cache", ".tox", ".venv", "venv",
+    "node_modules", "bower_components", "vendor", "third_party", "external",
+    "dist", "build", "out", "target", "bin", "obj",
+    "site-packages", "coverage", "htmlcov", ".next", ".nuxt", ".svelte-kit",
+}
+
+# Exact filenames that are machine-generated and semantically empty.
+SKIP_FILENAMES = {
+    "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lockb",
+    "composer.lock", "gemfile.lock", "cargo.lock", "poetry.lock",
+    "pipfile.lock", "go.sum", "mix.lock", "podfile.lock",
+}
+
+# Suffixes that mark minified or map output even under a source directory.
+SKIP_SUFFIXES = (
+    ".min.js", ".min.css", ".map", ".lock",
+    ".pyc", ".pyo", ".class", ".jar", ".war", ".o", ".so", ".dylib", ".dll",
+    ".exe", ".wasm", ".bin", ".pdb",
+)
+
+
+class GitHubLoader(BaseRemote):
+    """Load a GitHub repository's text files as ``Document`` objects.
+
+    Uses the git *tree* API to enumerate the repository in a single request
+    rather than walking ``/contents/`` directory by directory, filters out
+    binaries, vendored output and oversized blobs *before* fetching them, and
+    downloads the survivors in parallel.
+    """
+
+    def __init__(self):
+        self.access_token = settings.GITHUB_ACCESS_TOKEN
+        self.headers = {
+            "Authorization": f"token {self.access_token}",
+            "Accept": "application/vnd.github.v3+json"
+        } if self.access_token else {
+            "Accept": "application/vnd.github.v3+json"
+        }
+        return
+
+    def is_text_file(self, file_path: str) -> bool:
+        """Determine if a file is a text file based on extension."""
+        # Common text file extensions
+        text_extensions = {
+            '.txt', '.md', '.markdown', '.rst', '.adoc', '.org',
+            '.json', '.jsonc', '.xml', '.yaml', '.yml', '.toml',
+            '.py', '.pyi', '.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx',
+            '.vue', '.svelte', '.astro',
+            '.java', '.c', '.cc', '.cpp', '.h', '.hpp', '.hxx',
+            '.cs', '.go', '.rs', '.rb', '.php', '.swift', '.kt', '.kts',
+            '.scala', '.sc', '.clj', '.cljs', '.edn', '.ex', '.exs', '.erl',
+            '.hs', '.ml', '.mli', '.fs', '.fsx', '.dart', '.lua', '.jl',
+            '.zig', '.nim', '.v', '.pl', '.pm', '.groovy', '.gradle',
+            '.html', '.htm', '.css', '.scss', '.sass', '.less',
+            '.sh', '.bash', '.zsh', '.fish', '.ps1', '.bat',
+            '.sql', '.graphql', '.gql', '.proto', '.thrift',
+            '.tf', '.tfvars', '.hcl', '.dockerfile', '.cmake', '.mk',
+            '.ini', '.cfg', '.conf', '.config', '.properties',
+            '.gitignore', '.dockerignore', '.editorconfig', '.gitattributes',
+            '.csv', '.tsv',
+        }
+
+        # Get file extension
+        file_lower = file_path.lower()
+        for ext in text_extensions:
+            if file_lower.endswith(ext):
+                return True
+
+        # Extension-less files that are conventionally text.
+        basename = file_lower.rsplit("/", 1)[-1]
+        if basename in {
+            "dockerfile", "makefile", "rakefile", "gemfile", "procfile",
+            "license", "licence", "notice", "authors", "codeowners", "readme",
+        }:
+            return True
+
+        # Also check MIME type
+        mime_type, _ = mimetypes.guess_type(file_path)
+        if mime_type and (mime_type.startswith("text") or mime_type in ["application/json", "application/xml"]):
+            return True
+
+        return False
+
+    def should_skip_path(self, file_path: str) -> bool:
+        """Return ``True`` for vendored, generated or binary-by-name paths.
+
+        Applied to the tree listing before any content is fetched, so a
+        skipped file costs nothing. Complements :meth:`is_text_file`, which
+        only knows about extensions.
+
+        Args:
+            file_path: Repo-relative path, e.g. ``"web/dist/app.min.js"``.
+
+        Returns:
+            ``True`` when the path should not be ingested.
+        """
+        lowered = file_path.lower()
+        parts = lowered.split("/")
+        if any(part in SKIP_DIRECTORIES for part in parts[:-1]):
+            return True
+        if parts[-1] in SKIP_FILENAMES:
+            return True
+        if lowered.endswith(SKIP_SUFFIXES):
+            return True
+        # Dot-directories (.git, .cache, ...) never carry documentation.
+        if any(part.startswith(".") and part not in {".github"} for part in parts[:-1]):
+            return True
+        return False
+
+    def _max_file_bytes(self) -> int:
+        """Resolve the per-blob size cap; ``0`` disables it."""
+        raw = getattr(settings, "GITHUB_INGEST_MAX_FILE_BYTES", None)
+        if isinstance(raw, bool) or not isinstance(raw, (int, str)):
+            return 1048576
+        try:
+            return max(0, int(raw))
+        except (TypeError, ValueError):
+            return 1048576
+
+    def _max_workers(self) -> int:
+        """Resolve the parallel-fetch width, clamped to a sane range."""
+        raw = getattr(settings, "GITHUB_INGEST_MAX_WORKERS", None)
+        if isinstance(raw, bool) or not isinstance(raw, (int, str)):
+            return 8
+        try:
+            return max(1, min(32, int(raw)))
+        except (TypeError, ValueError):
+            return 8
+
+    @staticmethod
+    def normalize_repo(repo_url: str) -> str:
+        """Reduce a user-pasted repo URL to ``owner/name``.
+
+        Strips the scheme/host, a trailing ``.git`` and any trailing slash —
+        the three shapes that previously 404'd the contents API.
+
+        Args:
+            repo_url: Anything from ``owner/name`` to
+                ``https://github.com/owner/name.git/``.
+
+        Returns:
+            The ``owner/name`` segment.
+        """
+        repo = (repo_url or "").strip()
+        if not repo:
+            return ""
+
+        if repo.startswith("git@"):
+            # git@github.com:owner/name.git
+            host, _, path = repo[len("git@"):].partition(":")
+            if host.lower() != "github.com":
+                return ""
+            repo = path
+        elif "://" in repo:
+            remainder = repo.split("://", 1)[1]
+            host, _, path = remainder.partition("/")
+            # Anything not hosted on github.com is not a repo. Previously such
+            # a URL was pasted straight into the contents path and 404-looped.
+            if host.lower() not in {"github.com", "www.github.com"}:
+                return ""
+            repo = path
+        elif repo.lower().startswith(("github.com/", "www.github.com/")):
+            repo = repo.split("/", 1)[1]
+
+        repo = repo.strip("/")
+        if repo.lower().endswith(".git"):
+            repo = repo[: -len(".git")]
+        parts = [p for p in repo.split("/") if p]
+        if len(parts) < 2:
+            return ""
+        owner, name = parts[0], parts[1]
+        if name.lower().endswith(".git"):
+            name = name[: -len(".git")]
+        # A bare host slipping through ("suat-handbook.netlify.app") has no
+        # owner segment, so require both halves to look like path segments.
+        if not owner or not name or ":" in owner or ":" in name:
+            return ""
+        return f"{owner}/{name}"
+
+    def get_default_branch(self, repo_name: str) -> str:
+        """Return the repo's default branch, falling back to ``main``.
+
+        The blob URL used for citations was hard-coded to ``main``, which
+        produced dead source links for every ``master``-default repo.
+        """
+        try:
+            response = self._make_request(f"https://api.github.com/repos/{repo_name}")
+            branch = response.json().get("default_branch")
+            if branch:
+                return str(branch)
+        except Exception as e:
+            logger.warning(
+                "Could not resolve default branch for %s (%s); assuming 'main'",
+                repo_name, e,
+            )
+        return "main"
+
+    def fetch_repo_tree(
+        self, repo_name: str, branch: str
+    ) -> Tuple[List[Tuple[str, int]], bool]:
+        """List every blob in the repo with one recursive tree request.
+
+        Replaces the per-directory ``/contents/`` walk, which cost one API
+        call per directory (720 for a mid-size repo) before a single file
+        was read.
+
+        Args:
+            repo_name: ``owner/name``.
+            branch: Branch or ref to enumerate.
+
+        Returns:
+            ``(entries, truncated)`` where ``entries`` is a list of
+            ``(path, size_bytes)`` and ``truncated`` flags a repo too large
+            for the tree endpoint (caller should fall back to the walk).
+        """
+        url = (
+            f"https://api.github.com/repos/{repo_name}/git/trees/"
+            f"{branch}?recursive=1"
+        )
+        response = self._make_request(url)
+        payload = response.json()
+        if isinstance(payload, dict) and "tree" not in payload and "message" in payload:
+            raise Exception(f"GitHub API error: {payload.get('message')}")
+        entries = [
+            (item.get("path", ""), int(item.get("size") or 0))
+            for item in payload.get("tree", [])
+            if item.get("type") == "blob"
+        ]
+        return entries, bool(payload.get("truncated"))
+
+    def select_files(self, entries: List[Tuple[str, int]]) -> List[str]:
+        """Filter tree entries down to the paths worth fetching.
+
+        Args:
+            entries: ``(path, size_bytes)`` pairs from :meth:`fetch_repo_tree`.
+
+        Returns:
+            Repo-relative paths that pass the skip-list, the text-extension
+            check and the size cap, in tree order.
+        """
+        max_bytes = self._max_file_bytes()
+        selected: List[str] = []
+        skipped_big = 0
+        for path, size in entries:
+            if not path or self.should_skip_path(path) or not self.is_text_file(path):
+                continue
+            if max_bytes and size > max_bytes:
+                skipped_big += 1
+                continue
+            selected.append(path)
+        if skipped_big:
+            logger.info(
+                "Skipped %d file(s) over the %d-byte cap", skipped_big, max_bytes
+            )
+        return selected
+
+    def fetch_file_content(self, repo_url: str, file_path: str) -> Optional[str]:
+        """Fetch file content. Returns None if file should be skipped (binary files or empty files)."""
+        url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}"
+        response = self._make_request(url)
+
+        content = response.json()
+
+        if content.get("encoding") == "base64":
+            if self.is_text_file(file_path):  # Handle only text files
+                try:
+                    decoded_content = base64.b64decode(content["content"]).decode("utf-8").strip()
+                    # Skip empty files
+                    if not decoded_content:
+                        return None
+                    return decoded_content
+                except Exception:
+                    # If decoding fails, it's probably a binary file
+                    return None
+            else:
+                # Skip binary files by returning None
+                return None
+        else:
+            file_content = content['content'].strip()
+            # Skip empty files
+            if not file_content:
+                return None
+            return file_content
+
+    def _make_request(self, url: str, max_retries: int = 3) -> requests.Response:
+        """Make a request with retry logic for rate limiting"""
+        for attempt in range(max_retries):
+            response = requests.get(url, headers=self.headers, timeout=100)
+
+            if response.status_code == 200:
+                return response
+            elif response.status_code == 403:
+                # Check if it's a rate limit issue
+                try:
+                    error_data = response.json()
+                    error_msg = error_data.get("message", "")
+
+                    # Check rate limit headers
+                    remaining = response.headers.get("X-RateLimit-Remaining", "unknown")
+                    reset_time = response.headers.get("X-RateLimit-Reset", "unknown")
+
+                    logger.warning("GitHub API 403 Error: %s", error_msg)
+                    logger.warning(
+                        "Rate limit remaining: %s, Reset time: %s", remaining, reset_time
+                    )
+
+                    if "rate limit" in error_msg.lower():
+                        if attempt < max_retries - 1:
+                            wait_time = 2 ** attempt  # Exponential backoff
+                            logger.warning(
+                                "Rate limit hit, waiting %s seconds before retry...",
+                                wait_time,
+                            )
+                            time.sleep(wait_time)
+                            continue
+
+                    # Provide helpful error message
+                    if remaining == "0":
+                        raise Exception(f"GitHub API rate limit exceeded. Please set GITHUB_ACCESS_TOKEN environment variable. Reset time: {reset_time}")
+                    else:
+                        raise Exception(f"GitHub API error: {error_msg}. This may require authentication - set GITHUB_ACCESS_TOKEN environment variable.")
+                except Exception as e:
+                    if isinstance(e, Exception) and "GitHub API" in str(e):
+                        raise
+                    # If we can't parse the response, raise the original error
+                    response.raise_for_status()
+            elif response.status_code == 401 and self.access_token:
+                # An expired or revoked PAT makes even public repos 401, which
+                # is strictly worse than not sending one. Retry unauthenticated
+                # so a stale credential degrades instead of failing the ingest.
+                logger.warning(
+                    "GitHub rejected the configured token (401); "
+                    "retrying %s unauthenticated", url,
+                )
+                anon = {"Accept": "application/vnd.github.v3+json"}
+                anon_response = requests.get(url, headers=anon, timeout=100)
+                if anon_response.status_code == 200:
+                    return anon_response
+                anon_response.raise_for_status()
+                return anon_response
+            else:
+                response.raise_for_status()
+
+        return response
+
+    def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]:
+        """Walk ``/contents/`` recursively (fallback for truncated trees)."""
+        url = f"https://api.github.com/repos/{repo_url}/contents/{path}"
+        response = self._make_request(url)
+
+        contents = response.json()
+
+        # Handle error responses from GitHub API
+        if isinstance(contents, dict) and "message" in contents:
+            raise Exception(f"GitHub API error: {contents.get('message')}")
+
+        # Ensure contents is a list
+        if not isinstance(contents, list):
+            raise TypeError(f"Expected list from GitHub API, got {type(contents).__name__}: {contents}")
+
+        files = []
+        for item in contents:
+            if item["type"] == "file":
+                files.append(item["path"])
+            elif item["type"] == "dir":
+                files.extend(self.fetch_repo_files(repo_url, item["path"]))
+        return files
+
+    def _list_candidate_files(self, repo_name: str, branch: str) -> List[str]:
+        """Enumerate ingestable paths, preferring the single tree request."""
+        try:
+            entries, truncated = self.fetch_repo_tree(repo_name, branch)
+            if not truncated:
+                return self.select_files(entries)
+            logger.warning(
+                "Tree for %s is truncated; falling back to the directory walk",
+                repo_name,
+            )
+        except Exception as e:
+            logger.warning(
+                "Tree listing failed for %s (%s); falling back to the "
+                "directory walk", repo_name, e,
+            )
+        paths = self.fetch_repo_files(repo_name)
+        return self.select_files([(p, 0) for p in paths])
+
+    def load_data(self, repo_url: str) -> List[Document]:
+        """Load every ingestable text file in ``repo_url`` as a Document."""
+        repo_name = self.normalize_repo(repo_url)
+        if not repo_name or "/" not in repo_name:
+            raise ValueError(
+                f"Not a valid GitHub repository: {repo_url!r}. "
+                "Expected a github.com URL like https://github.com/owner/name."
+            )
+        branch = self.get_default_branch(repo_name)
+        files = self._list_candidate_files(repo_name, branch)
+        logger.info(
+            "Fetching %d file(s) from %s@%s", len(files), repo_name, branch
+        )
+
+        # Fetch in parallel: this phase is pure network latency, and it was
+        # previously one blocking round-trip per file.
+        contents: Dict[str, Optional[str]] = {}
+
+        def _fetch(file_path: str) -> Tuple[str, Optional[str]]:
+            try:
+                return file_path, self.fetch_file_content(repo_name, file_path)
+            except Exception as e:
+                # One unreadable file must not sink the whole repo ingest.
+                logger.warning("Skipping %s: %s", file_path, e)
+                return file_path, None
+
+        max_workers = min(self._max_workers(), len(files)) or 1
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            for file_path, content in pool.map(_fetch, files):
+                contents[file_path] = content
+
+        documents = []
+        for file_path in files:
+            content = contents.get(file_path)
+            # Skip binary files (content is None)
+            if not content:
+                continue
+            documents.append(Document(
+                text=content,
+                doc_id=file_path,
+                extra_info={
+                    "title": file_path,
+                    "source": (
+                        f"https://github.com/{repo_name}/blob/{branch}/{file_path}"
+                    ),
+                }
+            ))
+        return documents

docsgpt/parser/remote/reddit_loader.py

@@ -0,0 +1,89 @@
+import json
+from typing import Any, Iterable, List
+
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+
+REQUIRED_FIELDS = ["client_id", "client_secret", "user_agent", "search_queries"]
+
+
+class RedditPostsLoaderRemote(BaseRemote):
+    """Load Reddit posts through praw.
+
+    Create an app at https://www.reddit.com/prefs/apps/ to obtain the
+    ``client_id`` and ``client_secret``.
+    """
+
+    def load_data(self, inputs) -> List[Document]:
+        """Load posts for every configured subreddit or user.
+
+        Args:
+            inputs: JSON string holding the praw credentials plus
+                ``search_queries``, and optionally ``mode``
+                (``subreddit``/``username``), ``categories`` and
+                ``number_posts``.
+
+        Returns:
+            One document per post, carrying the post body as text.
+        """
+        try:
+            data = json.loads(inputs)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON input: {e}")
+
+        missing_fields = [field for field in REQUIRED_FIELDS if field not in data]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
+
+        mode = data.get("mode", "subreddit")
+        if mode not in ("subreddit", "username"):
+            raise ValueError(
+                "mode not correct, please enter 'username' or 'subreddit' as mode"
+            )
+        categories = data.get("categories", ["new", "hot"])
+        search_queries = data.get("search_queries")
+        number_posts = data.get("number_posts", 10)
+
+        import praw
+
+        reddit = praw.Reddit(
+            client_id=data.get("client_id"),
+            client_secret=data.get("client_secret"),
+            user_agent=data.get("user_agent"),
+        )
+
+        documents: List[Document] = []
+        for search_query in search_queries:
+            for category in categories:
+                if mode == "subreddit":
+                    listing = reddit.subreddit(search_query)
+                else:
+                    listing = reddit.redditor(search_query).submissions
+                documents.extend(
+                    self._posts_to_documents(
+                        getattr(listing, category)(limit=number_posts), category
+                    )
+                )
+        return documents
+
+    @staticmethod
+    def _posts_to_documents(posts: Iterable[Any], category: str) -> List[Document]:
+        """Convert praw submissions into documents."""
+        documents = []
+        for post in posts:
+            documents.append(
+                Document(
+                    text=post.selftext,
+                    doc_id=post.id,
+                    extra_info={
+                        "post_subreddit": post.subreddit_name_prefixed,
+                        "post_category": category,
+                        "title": post.title,
+                        "post_score": post.score,
+                        "post_id": post.id,
+                        "source": post.url,
+                        "post_author": str(post.author),
+                    },
+                )
+            )
+        return documents

docsgpt/parser/remote/remote_creator.py

@@ -0,0 +1,92 @@
+import json
+
+from docsgpt.parser.remote.sitemap_loader import SitemapLoader
+from docsgpt.parser.remote.crawler_loader import CrawlerLoader
+from docsgpt.parser.remote.web_loader import WebLoader
+from docsgpt.parser.remote.reddit_loader import RedditPostsLoaderRemote
+from docsgpt.parser.remote.github_loader import GitHubLoader
+from docsgpt.parser.remote.s3_loader import S3Loader
+
+
+class RemoteCreator:
+    """
+    Factory class for creating remote content loaders.
+
+    These loaders fetch content from remote web sources like URLs,
+    sitemaps, web crawlers, social media platforms, etc.
+
+    For external knowledge base connectors (like Google Drive),
+    use ConnectorCreator instead.
+    """
+
+    loaders = {
+        "url": WebLoader,
+        "sitemap": SitemapLoader,
+        "crawler": CrawlerLoader,
+        "reddit": RedditPostsLoaderRemote,
+        "github": GitHubLoader,
+        "s3": S3Loader,
+    }
+
+    @classmethod
+    def create_loader(cls, type, *args, **kwargs):
+        loader_class = cls.loaders.get(type.lower())
+        if not loader_class:
+            raise ValueError(f"No loader class found for type {type}")
+        return loader_class(*args, **kwargs)
+
+
+# Loader types whose load_data expects a URL string, not a config dict.
+_URL_LOADER_TYPES = {"url", "crawler", "sitemap", "github"}
+
+# Keys a remote_data dict may hold the URL under (``raw`` is the legacy shape).
+_URL_DATA_KEYS = ("url", "urls", "repo_url", "raw")
+
+
+def normalize_remote_data(source_type, remote_data):
+    """Convert a stored ``sources.remote_data`` JSONB value into the
+    ``source_data`` shape the matching loader expects.
+
+    Args:
+        source_type: The ``sources.type`` value (the loader name).
+        remote_data: The stored ``remote_data`` (dict, list, str, or None).
+
+    Returns:
+        Loader input: a URL string or list for url/crawler/sitemap/github,
+        a JSON string for reddit, a dict for s3; ``None`` when the row has
+        nothing syncable.
+    """
+    if remote_data is None:
+        return None
+
+    # Some legacy rows stored the JSON itself as a string.
+    if isinstance(remote_data, str):
+        stripped = remote_data.strip()
+        if stripped[:1] in ("{", "["):
+            try:
+                remote_data = json.loads(stripped)
+            except json.JSONDecodeError:
+                # Not actually JSON — leave remote_data as the original
+                # string; the per-loader branches below handle a string.
+                pass
+
+    loader = (source_type or "").lower()
+
+    if loader in _URL_LOADER_TYPES:
+        if isinstance(remote_data, dict):
+            for key in _URL_DATA_KEYS:
+                value = remote_data.get(key)
+                if value:
+                    return value
+            # No URL key — None keeps the loader off the dict-crash path.
+            return None
+        return remote_data
+
+    if loader == "reddit":
+        # reddit's loader runs json.loads() on its input — needs a string.
+        if isinstance(remote_data, (dict, list)):
+            return json.dumps(remote_data)
+        return remote_data
+
+    # s3's loader accepts a dict or JSON string; pass it through unchanged.
+    return remote_data

docsgpt/parser/remote/s3_loader.py

@@ -0,0 +1,440 @@
+import json
+import logging
+import os
+import tempfile
+import mimetypes
+from typing import List, Optional
+from docsgpt.core.url_validation import SSRFError, validate_url
+from docsgpt.parser.file.constants import SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+
+try:
+    import boto3
+    from botocore.exceptions import ClientError, NoCredentialsError
+except ImportError:
+    boto3 = None
+
+logger = logging.getLogger(__name__)
+
+
+# Plain-text suffixes the loader reads directly (no document parser).
+_TEXT_EXTENSIONS = {
+        ".txt",
+        ".md",
+        ".markdown",
+        ".rst",
+        ".json",
+        ".xml",
+        ".yaml",
+        ".yml",
+        ".py",
+        ".js",
+        ".ts",
+        ".jsx",
+        ".tsx",
+        ".java",
+        ".c",
+        ".cpp",
+        ".h",
+        ".hpp",
+        ".cs",
+        ".go",
+        ".rs",
+        ".rb",
+        ".php",
+        ".swift",
+        ".kt",
+        ".scala",
+        ".html",
+        ".css",
+        ".scss",
+        ".sass",
+        ".less",
+        ".sh",
+        ".bash",
+        ".zsh",
+        ".fish",
+        ".sql",
+        ".r",
+        ".m",
+        ".mat",
+        ".ini",
+        ".cfg",
+        ".conf",
+        ".config",
+        ".env",
+        ".gitignore",
+        ".dockerignore",
+        ".editorconfig",
+        ".log",
+        ".csv",
+        ".tsv",
+    }
+
+
+class S3Loader(BaseRemote):
+    """Load documents from an AWS S3 bucket."""
+
+    def __init__(self):
+        if boto3 is None:
+            raise ImportError(
+                "boto3 is required for S3Loader. Install it with: pip install boto3"
+            )
+        self.s3_client = None
+
+    def _normalize_endpoint_url(self, endpoint_url: str, bucket: str) -> tuple[str, str]:
+        """
+        Normalize endpoint URL for S3-compatible services.
+
+        Detects common mistakes like using bucket-prefixed URLs and extracts
+        the correct endpoint and bucket name.
+
+        Args:
+            endpoint_url: The provided endpoint URL
+            bucket: The provided bucket name
+
+        Returns:
+            Tuple of (normalized_endpoint_url, bucket_name)
+        """
+        import re
+        from urllib.parse import urlparse
+
+        if not endpoint_url:
+            return endpoint_url, bucket
+
+        parsed = urlparse(endpoint_url)
+        host = parsed.netloc or parsed.path
+
+        # Check for DigitalOcean Spaces bucket-prefixed URL pattern
+        # e.g., https://mybucket.nyc3.digitaloceanspaces.com
+        do_match = re.match(r"^([^.]+)\.([a-z0-9]+)\.digitaloceanspaces\.com$", host)
+        if do_match:
+            extracted_bucket = do_match.group(1)
+            region = do_match.group(2)
+            correct_endpoint = f"https://{region}.digitaloceanspaces.com"
+            logger.warning(
+                f"Detected bucket-prefixed DigitalOcean Spaces URL. "
+                f"Extracted bucket '{extracted_bucket}' from endpoint. "
+                f"Using endpoint: {correct_endpoint}"
+            )
+            # If bucket wasn't provided or differs, use extracted one
+            if not bucket or bucket != extracted_bucket:
+                logger.info(f"Using extracted bucket name: '{extracted_bucket}' (was: '{bucket}')")
+                bucket = extracted_bucket
+            return correct_endpoint, bucket
+
+        # Check for just "digitaloceanspaces.com" without region
+        if host == "digitaloceanspaces.com":
+            logger.error(
+                "Invalid DigitalOcean Spaces endpoint: missing region. "
+                "Use format: https://<region>.digitaloceanspaces.com (e.g., https://lon1.digitaloceanspaces.com)"
+            )
+
+        return endpoint_url, bucket
+
+    def _init_client(
+        self,
+        aws_access_key_id: str,
+        aws_secret_access_key: str,
+        region_name: str = "us-east-1",
+        endpoint_url: Optional[str] = None,
+        bucket: Optional[str] = None,
+    ) -> Optional[str]:
+        """
+        Initialize the S3 client with credentials.
+
+        Returns:
+            The potentially corrected bucket name if endpoint URL was normalized
+        """
+        from botocore.config import Config
+
+        client_kwargs = {
+            "aws_access_key_id": aws_access_key_id,
+            "aws_secret_access_key": aws_secret_access_key,
+            "region_name": region_name,
+        }
+
+        logger.info(f"Initializing S3 client with region: {region_name}")
+
+        corrected_bucket = bucket
+        if endpoint_url:
+            # Normalize the endpoint URL and potentially extract bucket name
+            normalized_endpoint, corrected_bucket = self._normalize_endpoint_url(endpoint_url, bucket)
+            logger.info(f"Original endpoint URL: {endpoint_url}")
+            logger.info(f"Normalized endpoint URL: {normalized_endpoint}")
+            logger.info(f"Bucket name: '{corrected_bucket}'")
+
+            try:
+                normalized_endpoint = validate_url(normalized_endpoint)
+            except SSRFError as e:
+                raise ValueError(f"Invalid S3 endpoint_url: {e}") from e
+
+            client_kwargs["endpoint_url"] = normalized_endpoint
+            # Use path-style addressing for S3-compatible services
+            # (DigitalOcean Spaces, MinIO, etc.)
+            client_kwargs["config"] = Config(s3={"addressing_style": "path"})
+        else:
+            logger.info("Using default AWS S3 endpoint")
+
+        self.s3_client = boto3.client("s3", **client_kwargs)
+        logger.info("S3 client initialized successfully")
+
+        return corrected_bucket
+
+    def is_text_file(self, file_path: str) -> bool:
+        """Determine if a file is a text file based on extension."""
+        text_extensions = _TEXT_EXTENSIONS
+
+        file_lower = file_path.lower()
+        for ext in text_extensions:
+            if file_lower.endswith(ext):
+                return True
+
+        mime_type, _ = mimetypes.guess_type(file_path)
+        if mime_type and (
+            mime_type.startswith("text")
+            or mime_type in ["application/json", "application/xml"]
+        ):
+            return True
+
+        return False
+
+    def is_supported_document(self, file_path: str) -> bool:
+        """Check if file is a supported document type for parsing."""
+        # The upload whitelist plus the historical S3 list, so an object the
+        # upload API accepts is never silently skipped here.
+        document_extensions = (set(SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS) - _TEXT_EXTENSIONS) | {
+            ".pdf",
+            ".docx",
+            ".doc",
+            ".xlsx",
+            ".xls",
+            ".pptx",
+            ".ppt",
+            ".epub",
+            ".odt",
+            ".rtf",
+        }
+
+        file_lower = file_path.lower()
+        for ext in document_extensions:
+            if file_lower.endswith(ext):
+                return True
+
+        return False
+
+    def list_objects(self, bucket: str, prefix: str = "") -> List[str]:
+        """
+        List all objects in the bucket with the given prefix.
+
+        Args:
+            bucket: S3 bucket name
+            prefix: Optional path prefix to filter objects
+
+        Returns:
+            List of object keys
+        """
+        objects = []
+        paginator = self.s3_client.get_paginator("list_objects_v2")
+
+        logger.info(f"Listing objects in bucket: '{bucket}' with prefix: '{prefix}'")
+        logger.debug(f"S3 client endpoint: {self.s3_client.meta.endpoint_url}")
+
+        try:
+            page_count = 0
+            for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+                page_count += 1
+                logger.debug(f"Processing page {page_count}, keys in response: {list(page.keys())}")
+                if "Contents" in page:
+                    for obj in page["Contents"]:
+                        key = obj["Key"]
+                        if not key.endswith("/"):
+                            objects.append(key)
+                            logger.debug(f"Found object: {key}")
+                else:
+                    logger.info(f"Page {page_count} has no 'Contents' key - bucket may be empty or prefix not found")
+
+            logger.info(f"Found {len(objects)} objects in bucket '{bucket}'")
+
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "")
+            error_message = e.response.get("Error", {}).get("Message", "")
+            logger.error(f"ClientError listing objects - Code: {error_code}, Message: {error_message}")
+            logger.error(f"Full error response: {e.response}")
+            logger.error(f"Bucket: '{bucket}', Prefix: '{prefix}', Endpoint: {self.s3_client.meta.endpoint_url}")
+
+            if error_code == "NoSuchBucket":
+                raise Exception(f"S3 bucket '{bucket}' does not exist")
+            elif error_code == "AccessDenied":
+                raise Exception(
+                    f"Access denied to S3 bucket '{bucket}'. Check your credentials and permissions."
+                )
+            elif error_code == "NoSuchKey":
+                # This is unusual for ListObjectsV2 - may indicate endpoint/bucket configuration issue
+                logger.error(
+                    "NoSuchKey error on ListObjectsV2 - this may indicate the bucket name "
+                    "is incorrect or the endpoint URL format is wrong. "
+                    "For DigitalOcean Spaces, the endpoint should be like: "
+                    "https://<region>.digitaloceanspaces.com and bucket should be just the space name."
+                )
+                raise Exception(
+                    f"S3 error: {e}. For S3-compatible services, verify: "
+                    f"1) Endpoint URL format (e.g., https://nyc3.digitaloceanspaces.com), "
+                    f"2) Bucket name is just the space/bucket name without region prefix"
+                )
+            else:
+                raise Exception(f"S3 error: {e}")
+        except NoCredentialsError:
+            raise Exception(
+                "AWS credentials not found. Please provide valid credentials."
+            )
+
+        return objects
+
+    def get_object_content(self, bucket: str, key: str) -> Optional[str]:
+        """
+        Get the content of an S3 object as text.
+
+        Args:
+            bucket: S3 bucket name
+            key: Object key
+
+        Returns:
+            File content as string, or None if file should be skipped
+        """
+        if not self.is_text_file(key) and not self.is_supported_document(key):
+            return None
+
+        try:
+            response = self.s3_client.get_object(Bucket=bucket, Key=key)
+            content = response["Body"].read()
+
+            if self.is_text_file(key):
+                try:
+                    decoded_content = content.decode("utf-8").strip()
+                    if not decoded_content:
+                        return None
+                    return decoded_content
+                except UnicodeDecodeError:
+                    return None
+            elif self.is_supported_document(key):
+                return self._process_document(content, key)
+
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "")
+            if error_code == "NoSuchKey":
+                return None
+            elif error_code == "AccessDenied":
+                print(f"Access denied to object: {key}")
+                return None
+            else:
+                print(f"Error fetching object {key}: {e}")
+                return None
+
+        return None
+
+    def _process_document(self, content: bytes, key: str) -> Optional[str]:
+        """
+        Process a document file (PDF, DOCX, etc.) and extract text.
+
+        Args:
+            content: File content as bytes
+            key: Object key (filename)
+
+        Returns:
+            Extracted text content
+        """
+        ext = os.path.splitext(key)[1].lower()
+
+        with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp_file:
+            tmp_file.write(content)
+            tmp_path = tmp_file.name
+
+        try:
+            from docsgpt.parser.file.bulk import SimpleDirectoryReader
+
+            reader = SimpleDirectoryReader(input_files=[tmp_path])
+            documents = reader.load_data()
+            if documents:
+                return "\n\n".join(doc.text for doc in documents if doc.text)
+            return None
+        except Exception as e:
+            print(f"Error processing document {key}: {e}")
+            return None
+        finally:
+            if os.path.exists(tmp_path):
+                os.unlink(tmp_path)
+
+    def load_data(self, inputs) -> List[Document]:
+        """
+        Load documents from an S3 bucket.
+
+        Args:
+            inputs: JSON string or dict containing:
+                - aws_access_key_id: AWS access key ID
+                - aws_secret_access_key: AWS secret access key
+                - bucket: S3 bucket name
+                - prefix: Optional path prefix to filter objects
+                - region: AWS region (default: us-east-1)
+                - endpoint_url: Custom S3 endpoint URL (for MinIO, R2, etc.)
+
+        Returns:
+            List of Document objects
+        """
+        if isinstance(inputs, str):
+            try:
+                data = json.loads(inputs)
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Invalid JSON input: {e}")
+        else:
+            data = inputs
+
+        required_fields = ["aws_access_key_id", "aws_secret_access_key", "bucket"]
+        missing_fields = [field for field in required_fields if not data.get(field)]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
+
+        aws_access_key_id = data["aws_access_key_id"]
+        aws_secret_access_key = data["aws_secret_access_key"]
+        bucket = data["bucket"]
+        prefix = data.get("prefix", "")
+        region = data.get("region", "us-east-1")
+        endpoint_url = data.get("endpoint_url", "")
+
+        logger.info(f"Loading data from S3 - Bucket: '{bucket}', Prefix: '{prefix}', Region: '{region}'")
+        if endpoint_url:
+            logger.info(f"Custom endpoint URL provided: '{endpoint_url}'")
+
+        corrected_bucket = self._init_client(
+            aws_access_key_id, aws_secret_access_key, region, endpoint_url or None, bucket
+        )
+
+        # Use the corrected bucket name if endpoint URL normalization extracted one
+        if corrected_bucket and corrected_bucket != bucket:
+            logger.info(f"Using corrected bucket name: '{corrected_bucket}' (original: '{bucket}')")
+            bucket = corrected_bucket
+
+        objects = self.list_objects(bucket, prefix)
+        documents = []
+
+        for key in objects:
+            content = self.get_object_content(bucket, key)
+            if content is None:
+                continue
+
+            documents.append(
+                Document(
+                    text=content,
+                    doc_id=key,
+                    extra_info={
+                        "title": os.path.basename(key),
+                        "source": f"s3://{bucket}/{key}",
+                        "bucket": bucket,
+                        "key": key,
+                    },
+                )
+            )
+
+        logger.info(f"Loaded {len(documents)} documents from S3 bucket '{bucket}'")
+        return documents

docsgpt/parser/remote/sitemap_loader.py

@@ -0,0 +1,111 @@
+import logging
+import re
+
+import defusedxml.ElementTree as ET
+from bs4 import BeautifulSoup
+
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+from docsgpt.core.url_validation import validate_url, SSRFError
+from docsgpt.security.safe_url import UnsafeUserUrlError, pinned_request
+
+class SitemapLoader(BaseRemote):
+    def __init__(self, limit=20):
+        self.limit = limit  # Adding limit to control the number of URLs to process
+
+    def load_data(self, inputs):
+        sitemap_url= inputs
+        # Check if the input is a list and if it is, use the first element
+        if isinstance(sitemap_url, list) and sitemap_url:
+            sitemap_url = sitemap_url[0]
+
+        # Validate URL to prevent SSRF attacks
+        try:
+            sitemap_url = validate_url(sitemap_url)
+        except SSRFError as e:
+            logging.error(f"URL validation failed: {e}")
+            return []
+
+        urls = self._extract_urls(sitemap_url)
+        if not urls:
+            print(f"No URLs found in the sitemap: {sitemap_url}")
+            return []
+
+        # Load content of extracted URLs
+        documents = []
+        processed_urls = 0  # Counter for processed URLs
+        for url in urls:
+            if self.limit is not None and processed_urls >= self.limit:
+                break  # Stop processing if the limit is reached
+
+            try:
+                url = validate_url(url)
+            except SSRFError as e:
+                logging.error(f"URL validation failed for sitemap entry {url}: {e}")
+                continue
+            try:
+                response = pinned_request("GET", url, timeout=30)
+                response.raise_for_status()
+                soup = BeautifulSoup(response.text, "html.parser")
+                documents.append(
+                    Document(
+                        soup.get_text(separator="\n", strip=True),
+                        extra_info={"source": url},
+                    )
+                )
+                processed_urls += 1  # Increment the counter after processing each URL
+            except Exception as e:
+                logging.error(f"Error processing URL {url}: {e}", exc_info=True)
+                continue
+
+        return documents
+
+    def _extract_urls(self, sitemap_url):
+        try:
+            response = pinned_request("GET", sitemap_url, timeout=30)
+            response.raise_for_status()
+        except UnsafeUserUrlError as e:
+            print(f"URL validation failed for sitemap: {sitemap_url}. Error: {e}")
+            return []
+        except Exception as e:
+            print(f"Failed to fetch sitemap: {sitemap_url}. Error: {e}")
+            return []
+
+        # Determine if this is a sitemap or a URL
+        if self._is_sitemap(response):
+            # It's a sitemap, so parse it and extract URLs
+            return self._parse_sitemap(response.content)
+        else:
+            # It's not a sitemap, return the URL itself
+            return [sitemap_url]
+
+    def _is_sitemap(self, response):
+        content_type = response.headers.get('Content-Type', '')
+        if 'xml' in content_type or response.url.endswith('.xml'):
+            return True
+
+        if '<sitemapindex' in response.text or '<urlset' in response.text:
+            return True
+
+        return False
+
+    def _parse_sitemap(self, sitemap_content):
+        # Remove namespaces
+        sitemap_content = re.sub(' xmlns="[^"]+"', '', sitemap_content.decode('utf-8'), count=1)
+
+        root = ET.fromstring(sitemap_content)
+
+        urls = []
+        for loc in root.findall('.//url/loc'):
+            if not loc.text:
+                continue
+            urls.append(loc.text)
+
+        # Check for nested sitemaps
+        for sitemap in root.findall('.//sitemap/loc'):
+            nested_sitemap_url = sitemap.text
+            if not nested_sitemap_url:
+                continue
+            urls.extend(self._extract_urls(nested_sitemap_url))
+
+        return urls

docsgpt/parser/remote/web_loader.py

@@ -0,0 +1,57 @@
+import logging
+
+from bs4 import BeautifulSoup
+
+from docsgpt.core.url_validation import SSRFError, validate_url
+from docsgpt.parser.remote.base import BaseRemote
+from docsgpt.parser.schema.base import Document
+from docsgpt.security.safe_url import pinned_request
+
+headers = {
+    "User-Agent": "Mozilla/5.0",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*"
+    ";q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Referer": "https://www.google.com/",
+    "DNT": "1",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+}
+
+
+class WebLoader(BaseRemote):
+    def load_data(self, inputs):
+        urls = inputs
+        if isinstance(urls, str):
+            urls = [urls]
+        documents = []
+        for url in urls:
+            try:
+                url = validate_url(url)
+            except SSRFError as e:
+                logging.warning(
+                    f"Skipping URL due to SSRF validation failure: {url} - {e}"
+                )
+                continue
+            try:
+                response = pinned_request("GET", url, headers=headers, timeout=30)
+                response.raise_for_status()
+                soup = BeautifulSoup(response.text, "html.parser")
+                metadata = {"source": url}
+                if soup.title:
+                    title = soup.title.get_text(strip=True)
+                    if title:
+                        metadata["title"] = title
+                html_tag = soup.find("html")
+                if html_tag and html_tag.get("lang"):
+                    metadata["language"] = html_tag.get("lang")
+                documents.append(
+                    Document(
+                        soup.get_text(separator="\n", strip=True),
+                        extra_info=metadata,
+                    )
+                )
+            except Exception as e:
+                logging.error(f"Error processing URL {url}: {e}", exc_info=True)
+                continue
+        return documents