refactor: rename the application package to docsgpt

The backend import package is now docsgpt, the name it will carry on PyPI;
application was far too generic to install into anyone's site-packages.
git mv plus a mechanical rewrite of every import, dotted string and path
reference: 734 Python files, the compose files, Dockerfile, workflows, docs,
setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage
config, .gitignore. Behaviour is unchanged.

Kept for one release:
- A top-level application package whose meta-path finder resolves
  application.x.y to the already-imported docsgpt.x.y object, so old imports
  and entry points (celery -A application.app.celery,
  uvicorn application.asgi:asgi_app) keep working with a FutureWarning.
- Celery registers every application.* task name as an alias of its
  docsgpt.* task on start-up, so messages queued by the previous release still
  run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries
  the previous release wrote are left unread instead of firing twice.

The backend image builds from the repository root (docker build -f
docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore
allow-lists docsgpt/ and application/ and keeps caches, local data, .env
files, the sample index files and the Dockerfile out. Compose and the image
workflows point at the new context.

docsgpt/security/encryption.py

@@ -0,0 +1,88 @@
+import base64
+import json
+import logging
+import os
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.ciphers import algorithms, Cipher, modes
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+
+from docsgpt.core.settings import settings
+
+logger = logging.getLogger(__name__)
+
+
+def _derive_key(user_id: str, salt: bytes) -> bytes:
+    app_secret = settings.ENCRYPTION_SECRET_KEY
+
+    password = f"{app_secret}#{user_id}".encode()
+
+    kdf = PBKDF2HMAC(
+        algorithm=hashes.SHA256(),
+        length=32,
+        salt=salt,
+        iterations=100000,
+        backend=default_backend(),
+    )
+
+    return kdf.derive(password)
+
+
+def encrypt_credentials(credentials: dict, user_id: str) -> str:
+    if not credentials:
+        return ""
+    try:
+        salt = os.urandom(16)
+        iv = os.urandom(16)
+        key = _derive_key(user_id, salt)
+
+        json_str = json.dumps(credentials)
+
+        cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
+        encryptor = cipher.encryptor()
+
+        padded_data = _pad_data(json_str.encode())
+        encrypted_data = encryptor.update(padded_data) + encryptor.finalize()
+
+        result = salt + iv + encrypted_data
+        return base64.b64encode(result).decode()
+    except Exception as e:
+        logger.warning(f"Failed to encrypt credentials: {e}")
+        return ""
+
+
+def decrypt_credentials(encrypted_data: str, user_id: str) -> dict:
+    if not encrypted_data:
+        return {}
+    try:
+        data = base64.b64decode(encrypted_data.encode())
+
+        salt = data[:16]
+        iv = data[16:32]
+        encrypted_content = data[32:]
+
+        key = _derive_key(user_id, salt)
+
+        cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
+        decryptor = cipher.decryptor()
+
+        decrypted_padded = decryptor.update(encrypted_content) + decryptor.finalize()
+        decrypted_data = _unpad_data(decrypted_padded)
+
+        return json.loads(decrypted_data.decode())
+    except Exception as e:
+        logger.warning(f"Failed to decrypt credentials: {e}")
+        return {}
+
+
+def _pad_data(data: bytes) -> bytes:
+    block_size = 16
+    padding_len = block_size - (len(data) % block_size)
+    padding = bytes([padding_len]) * padding_len
+    return data + padding
+
+
+def _unpad_data(data: bytes) -> bytes:
+    padding_len = data[-1]
+    return data[:-padding_len]

docsgpt/security/safe_url.py

@@ -0,0 +1,593 @@
+"""SSRF protection for user-supplied OpenAI-compatible base URLs.
+
+This module is the single chokepoint for validating any URL that a user
+provides as an OpenAI-compatible ``base_url`` ("Bring Your Own Model").
+The backend will later issue outbound HTTP requests to that URL on the
+user's behalf, so we must reject anything that could be used to reach
+internal-network resources (cloud metadata services, RFC 1918 ranges,
+loopback, link-local, etc.).
+
+Three entry points:
+
+* :func:`validate_user_base_url` — called at create/update time on REST
+  routes that persist the URL, to give the user immediate feedback.
+* :func:`pinned_post` — called at dispatch time when the caller drives
+  ``requests`` directly (e.g. the ``/api/models/test`` endpoint).
+  Resolves once, dials the IP literal, preserves the original hostname
+  in the ``Host`` header and via SNI / cert verification for HTTPS.
+* :func:`pinned_httpx_client` — called at dispatch time when the caller
+  hands an ``httpx.Client`` to a third-party SDK (e.g. the OpenAI
+  Python SDK via ``OpenAI(http_client=...)``). Same DNS-rebinding
+  closure on the httpx transport layer.
+
+Why all three: the OpenAI / httpx ecosystem performs its own DNS lookup
+inside ``socket.getaddrinfo`` when a connection opens, so a hostile DNS
+server can hand a public IP to the validator and a loopback / link-local
+address to the HTTP client. Validate-then-construct-SDK is unsafe; the
+pinned variants close that TOCTOU window by resolving exactly once and
+dialing the chosen IP literal directly.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import socket
+from typing import Any, Iterable
+from urllib.parse import urlsplit, urlunsplit
+
+import httpx
+import requests
+from requests.adapters import HTTPAdapter
+
+# Allowed URL schemes. Anything else (file, gopher, ftp, data, ...) is
+# rejected outright because it either bypasses HTTP entirely or enables
+# protocol smuggling against the proxy stack.
+_ALLOWED_SCHEMES: frozenset[str] = frozenset({"http", "https"})
+
+# Hostnames that resolve to a loopback / metadata / unspecified address
+# but which we want to reject *by name* as well, so the rejection
+# message is unambiguous and so we never accidentally call DNS on them.
+_BLOCKED_HOSTNAMES: frozenset[str] = frozenset(
+    {
+        "localhost",
+        "localhost.localdomain",
+        "0.0.0.0",
+        "::",
+        "::1",
+        "ip6-localhost",
+        "ip6-loopback",
+        # GCP metadata service. AWS/Azure use 169.254.169.254 which the
+        # IP-range check below already covers via the link-local range,
+        # but Google's hostname does not always resolve to a link-local
+        # IP from every VPC, so we hard-deny the string too.
+        "metadata.google.internal",
+    }
+)
+
+# Carrier-grade NAT (RFC 6598). Python's ``ipaddress`` module does NOT
+# classify this range as ``is_private``, so we must check it explicitly.
+_CGNAT_NETWORK_V4: ipaddress.IPv4Network = ipaddress.IPv4Network("100.64.0.0/10")
+
+
+class UnsafeUserUrlError(ValueError):
+    """Raised when a user-supplied URL fails SSRF validation.
+
+    Subclasses :class:`ValueError` so call sites that already treat
+    invalid input as a 400-class error continue to work. The string
+    message names the specific reason (scheme, hostname, resolved IP,
+    DNS failure, ...) so that it can be surfaced to the user verbatim.
+    """
+
+
+class ResponseTooLargeError(Exception):
+    """Raised by :func:`pinned_fetch_bytes` when a response body exceeds
+    the caller's byte ceiling (declared via ``Content-Length`` or found
+    during the bounded read)."""
+
+
+def _strip_ipv6_brackets(host: str) -> str:
+    """Return ``host`` with surrounding ``[`` / ``]`` removed if present."""
+
+    if host.startswith("[") and host.endswith("]"):
+        return host[1:-1]
+    return host
+
+
+def _is_blocked_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
+    """Return ``True`` if ``ip`` falls in any range we refuse to dial.
+
+    This is the single source of truth for the IP-range policy:
+
+    * loopback (``127.0.0.0/8``, ``::1``)
+    * private (RFC 1918, ULA ``fc00::/7``)
+    * link-local (``169.254.0.0/16``, ``fe80::/10``)
+    * multicast (``224.0.0.0/4``, ``ff00::/8``)
+    * unspecified (``0.0.0.0``, ``::``)
+    * reserved (``240.0.0.0/4``, etc.)
+    * carrier-grade NAT (``100.64.0.0/10``) — not covered by ``is_private``
+    """
+
+    if (
+        ip.is_loopback
+        or ip.is_private
+        or ip.is_link_local
+        or ip.is_multicast
+        or ip.is_unspecified
+        or ip.is_reserved
+    ):
+        return True
+    if isinstance(ip, ipaddress.IPv4Address) and ip in _CGNAT_NETWORK_V4:
+        return True
+    return False
+
+
+def _resolve(host: str) -> Iterable[ipaddress.IPv4Address | ipaddress.IPv6Address]:
+    """Resolve ``host`` to every A/AAAA record returned by the system.
+
+    Returning *all* addresses (rather than the first one) is critical:
+    a hostile DNS server can return a public IP first followed by a
+    private IP, and the underlying HTTP client may fail over to the
+    private one on connect. We treat the set as unsafe if any element
+    is unsafe.
+    """
+
+    try:
+        results = socket.getaddrinfo(host, None)
+    except socket.gaierror as exc:  # noqa: PERF203 — re-raise as our own type
+        raise UnsafeUserUrlError(f"could not resolve hostname {host!r}: {exc}") from exc
+
+    addresses: list[ipaddress.IPv4Address | ipaddress.IPv6Address] = []
+    for entry in results:
+        sockaddr = entry[4]
+        # IPv4 sockaddr: (host, port). IPv6 sockaddr: (host, port, flowinfo, scope_id).
+        ip_str = sockaddr[0]
+        # Strip IPv6 zone-id ("fe80::1%lo0") before parsing.
+        if "%" in ip_str:
+            ip_str = ip_str.split("%", 1)[0]
+        try:
+            addresses.append(ipaddress.ip_address(ip_str))
+        except ValueError:
+            # An entry we can't parse is itself suspicious; treat as unsafe.
+            raise UnsafeUserUrlError(
+                f"hostname {host!r} resolved to unparseable address {ip_str!r}"
+            ) from None
+    return addresses
+
+
+def _validate_and_pick_ip(
+    url: str,
+) -> tuple[str, ipaddress.IPv4Address | ipaddress.IPv6Address, "urlsplit"]:
+    """Run the SSRF guard and return the data needed to dial safely.
+
+    Performs every check :func:`validate_user_base_url` performs, but
+    additionally returns ``(hostname, ip, parts)`` where ``ip`` is one
+    of the validated addresses (the first record returned by the
+    resolver, or the literal itself if the URL already used an IP) and
+    ``parts`` is the :func:`urllib.parse.urlsplit` result so callers do
+    not have to re-parse the URL.
+
+    Raises :class:`UnsafeUserUrlError` on the same conditions as
+    :func:`validate_user_base_url`.
+    """
+
+    if not isinstance(url, str) or not url.strip():
+        raise UnsafeUserUrlError("url must be a non-empty string")
+
+    try:
+        parts = urlsplit(url)
+    except ValueError as exc:
+        raise UnsafeUserUrlError(f"could not parse url {url!r}: {exc}") from exc
+
+    scheme = parts.scheme.lower()
+    if scheme not in _ALLOWED_SCHEMES:
+        raise UnsafeUserUrlError(
+            f"scheme {scheme!r} is not allowed; only http and https are permitted"
+        )
+
+    # ``urlsplit`` returns the bracketed form for IPv6 in ``netloc`` but
+    # the bare form in ``hostname``. Normalize via lower() because
+    # hostnames are case-insensitive and we compare against a lowercase
+    # blocklist.
+    raw_host = parts.hostname
+    if not raw_host:
+        raise UnsafeUserUrlError(f"url {url!r} has no hostname")
+
+    host = raw_host.lower()
+
+    # Check the literal-string blocklist first. urlsplit().hostname strips
+    # IPv6 brackets, so we also test the bracketed form for completeness
+    # (matches the public-spec note about ``[::]``).
+    bracketed = f"[{host}]"
+    if host in _BLOCKED_HOSTNAMES or bracketed in _BLOCKED_HOSTNAMES:
+        raise UnsafeUserUrlError(
+            f"hostname {raw_host!r} is not allowed (matches internal-only name)"
+        )
+
+    # If the host is already an IP literal (with or without IPv6 brackets),
+    # check it directly without going to DNS — DNS for an IP literal is a
+    # no-op but it's clearer to short-circuit and gives a better message.
+    candidate = _strip_ipv6_brackets(host)
+    try:
+        literal = ipaddress.ip_address(candidate)
+    except ValueError:
+        literal = None
+
+    if literal is not None:
+        if _is_blocked_ip(literal):
+            raise UnsafeUserUrlError(
+                f"hostname {raw_host!r} resolves to blocked address {literal} "
+                f"(loopback/private/link-local/multicast/reserved/CGNAT)"
+            )
+        return host, literal, parts
+
+    # Hostname (not an IP literal) — resolve and validate every record.
+    addresses = list(_resolve(host))
+    for ip in addresses:
+        if _is_blocked_ip(ip):
+            raise UnsafeUserUrlError(
+                f"hostname {raw_host!r} resolves to blocked address {ip} "
+                f"(loopback/private/link-local/multicast/reserved/CGNAT)"
+            )
+    if not addresses:
+        # ``getaddrinfo`` would normally raise instead of returning an
+        # empty list, but treat the degenerate case as unsafe too — we
+        # have nothing to bind a connection to.
+        raise UnsafeUserUrlError(
+            f"hostname {raw_host!r} returned no addresses from DNS"
+        )
+    return host, addresses[0], parts
+
+
+def validate_user_base_url(url: str) -> None:
+    """Validate that ``url`` is safe to use as an outbound base URL.
+
+    Resolve the URL's hostname to one or more IPs and reject if any
+    resolved IP is private/loopback/link-local/multicast/reserved, or if
+    the URL uses a non-http(s) scheme, or if the hostname is one of the
+    known dangerous strings (``localhost``, ``0.0.0.0``, ``[::]``).
+
+    Raises :class:`UnsafeUserUrlError` on rejection. Returns ``None`` on
+    success.
+
+    This function is the create/update-time check. At dispatch time use
+    :func:`pinned_post` instead, which performs the same validation
+    *and* pins the outbound connection to the validated IP so a DNS
+    rebinder cannot flip the resolution between check and connect.
+
+    Args:
+        url: The user-supplied URL to validate. Expected to be an
+            absolute URL with an ``http`` or ``https`` scheme.
+
+    Raises:
+        UnsafeUserUrlError: If the URL fails to parse, uses a forbidden
+            scheme, has an empty/blocklisted hostname, fails DNS
+            resolution, or resolves to any IP in a blocked range.
+    """
+
+    _validate_and_pick_ip(url)
+
+
+class _PinnedHostAdapter(HTTPAdapter):
+    """HTTPS adapter that performs SNI and cert verification against a
+    fixed hostname even when the URL connects to an IP literal.
+
+    Used by :func:`pinned_post` so that resolving the user-supplied
+    hostname once and dialing the resolved IP doesn't break TLS.
+    Without this, ``urllib3`` would default ``server_hostname`` /
+    ``assert_hostname`` to the connect host (the IP) and either send the
+    wrong SNI or fail cert verification — the cert is for the original
+    hostname, not the IP literal.
+    """
+
+    def __init__(self, server_hostname: str, *args: Any, **kwargs: Any) -> None:
+        self._server_hostname = server_hostname
+        super().__init__(*args, **kwargs)
+
+    def init_poolmanager(self, *args: Any, **kwargs: Any) -> None:
+        kwargs["server_hostname"] = self._server_hostname
+        kwargs["assert_hostname"] = self._server_hostname
+        super().init_poolmanager(*args, **kwargs)
+
+
+def _ip_to_url_host(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> str:
+    """Return ``ip`` formatted for use in a URL netloc (brackets for v6)."""
+
+    if isinstance(ip, ipaddress.IPv6Address):
+        return f"[{ip}]"
+    return str(ip)
+
+
+def _url_userinfo_prefix(netloc: str) -> str:
+    """Return the exact ``userinfo@`` prefix from a URL netloc, if present."""
+
+    if "@" not in netloc:
+        return ""
+    return f"{netloc.rsplit('@', 1)[0]}@"
+
+
+def _pinned_session(
+    url: str, headers: dict[str, str] | None
+) -> tuple[requests.Session, str, dict[str, str]]:
+    """Run the SSRF guard once and build a session pinned to the chosen IP.
+
+    Single home for the pinning preamble (validate/pick IP, netloc rewrite,
+    ``Host`` header, HTTPS adapter mount) shared by :func:`pinned_request`
+    and :func:`pinned_fetch_bytes`, so the guard cannot drift between them.
+
+    Returns:
+        Tuple of the session (caller must close it), the IP-literal URL,
+        and the request headers carrying the original ``Host``.
+
+    Raises:
+        UnsafeUserUrlError: If the URL fails the SSRF guard.
+    """
+
+    host, ip, parts = _validate_and_pick_ip(url)
+
+    netloc = f"{_url_userinfo_prefix(parts.netloc)}{_ip_to_url_host(ip)}"
+    if parts.port is not None:
+        netloc = f"{netloc}:{parts.port}"
+    pinned_url = urlunsplit(
+        (parts.scheme, netloc, parts.path, parts.query, parts.fragment)
+    )
+
+    request_headers = dict(headers or {})
+    host_header = host if parts.port is None else f"{host}:{parts.port}"
+    request_headers["Host"] = host_header
+
+    session = requests.Session()
+    if parts.scheme == "https":
+        session.mount("https://", _PinnedHostAdapter(host))
+    return session, pinned_url, request_headers
+
+
+def pinned_request(
+    method: str,
+    url: str,
+    *,
+    data: Any = None,
+    json: Any = None,
+    headers: dict[str, str] | None = None,
+    timeout: float = 90.0,
+    allow_redirects: bool = False,
+) -> requests.Response:
+    """Send an HTTP request with the connection pinned to a validated IP,
+    closing the DNS-rebinding TOCTOU window left by the naive
+    validate-then-``requests`` pattern.
+
+    Raises:
+        UnsafeUserUrlError: If the URL fails the SSRF guard.
+        requests.RequestException: For network-level failures.
+    """
+
+    session, pinned_url, request_headers = _pinned_session(url, headers)
+    try:
+        return session.request(
+            method=method.upper(),
+            url=pinned_url,
+            data=data,
+            json=json,
+            headers=request_headers,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+        )
+    finally:
+        session.close()
+
+
+def pinned_fetch_bytes(
+    url: str,
+    *,
+    max_bytes: int,
+    headers: dict[str, str] | None = None,
+    timeout: float = 10.0,
+    allow_redirects: bool = False,
+) -> tuple[bytes, requests.Response]:
+    """GET ``url`` with SSRF pinning, streaming at most ``max_bytes`` of body.
+
+    The size limit is enforced twice: a declared ``Content-Length`` above
+    the ceiling is rejected before any body is read, and the streamed
+    read itself stops the moment the ceiling is crossed — a server that
+    lies about (or omits) ``Content-Length`` cannot bypass the cap.
+
+    Returns:
+        Tuple of the body bytes and the (already-closed) ``Response``,
+        kept for its status code and headers.
+
+    Raises:
+        UnsafeUserUrlError: If the URL fails the SSRF guard.
+        ResponseTooLargeError: If the body exceeds ``max_bytes``.
+        requests.RequestException: For network-level failures.
+    """
+
+    session, pinned_url, request_headers = _pinned_session(url, headers)
+    try:
+        response = session.request(
+            method="GET",
+            url=pinned_url,
+            headers=request_headers,
+            timeout=timeout,
+            allow_redirects=allow_redirects,
+            stream=True,
+        )
+        try:
+            # An error response's body is never used (callers
+            # raise_for_status) — skip the transfer and let the caller
+            # surface the more informative HTTP-status error.
+            if response.status_code >= 400:
+                return b"", response
+            declared = response.headers.get("Content-Length")
+            if declared and declared.isdigit() and int(declared) > max_bytes:
+                raise ResponseTooLargeError(
+                    f"response declares {declared} bytes, over the "
+                    f"{max_bytes}-byte limit"
+                )
+            chunks: list[bytes] = []
+            received = 0
+            for chunk in response.iter_content(chunk_size=65536):
+                received += len(chunk)
+                if received > max_bytes:
+                    raise ResponseTooLargeError(
+                        f"response body exceeds the {max_bytes}-byte limit"
+                    )
+                chunks.append(chunk)
+            return b"".join(chunks), response
+        finally:
+            response.close()
+    finally:
+        session.close()
+
+
+def pinned_post(
+    url: str,
+    *,
+    json: Any = None,
+    headers: dict[str, str] | None = None,
+    timeout: float = 5.0,
+    allow_redirects: bool = False,
+) -> requests.Response:
+    """POST to ``url`` with the outbound connection pinned to a single
+    validated IP, closing the DNS-rebinding TOCTOU window left by the
+    naive validate-then-``requests.post`` pattern.
+
+    The URL's hostname is resolved exactly once. Every returned address
+    must pass the same SSRF guard as :func:`validate_user_base_url`. The
+    outbound request is issued against the chosen IP literal (so
+    ``urllib3`` cannot ask the resolver again and receive a different
+    answer); the original hostname is preserved in the ``Host`` header
+    and, for HTTPS, via :class:`_PinnedHostAdapter` for SNI and cert
+    verification.
+
+    Args:
+        url: Absolute http(s) URL to POST to.
+        json: JSON-serializable payload — passed through to ``requests``.
+        headers: Caller-supplied headers. Any caller-supplied ``Host``
+            entry is overwritten so the in-flight request matches what
+            was validated.
+        timeout: Per-request timeout (seconds).
+        allow_redirects: Forwarded to ``requests``. Defaults to
+            ``False`` because the SSRF guard only inspects the supplied
+            URL — following redirects would let a hostile upstream
+            bounce the request to an internal address.
+
+    Raises:
+        UnsafeUserUrlError: If the URL fails the SSRF guard.
+        requests.RequestException: For network-level failures.
+    """
+
+    return pinned_request(
+        "POST",
+        url,
+        json=json,
+        headers=headers,
+        timeout=timeout,
+        allow_redirects=allow_redirects,
+    )
+
+
+class _PinnedHTTPSTransport(httpx.HTTPTransport):
+    """``httpx`` transport pinned to a single validated IP literal.
+
+    Closes the DNS-rebinding TOCTOU window that
+    :func:`validate_user_base_url` cannot close on its own. The OpenAI
+    Python SDK (and any other SDK that uses ``httpx``) re-resolves the
+    hostname inside ``socket.getaddrinfo`` at request time, so a
+    hostile DNS server can return a public IP at validation time and a
+    private IP at request time. This transport rewrites every outgoing
+    request's URL host to the validated IP literal so ``httpcore``
+    dials that IP without a fresh lookup.
+
+    The original hostname is preserved in two places:
+
+    1. ``Host`` header — ``httpx.Request._prepare`` set it from the URL
+       netloc *before* this transport runs, so it carries the hostname
+       not the IP literal. We deliberately do not touch headers here.
+    2. TLS SNI / cert verification — set via the
+       ``request.extensions["sni_hostname"]`` extension which
+       ``httpcore`` feeds into ``start_tls``'s ``server_hostname``
+       parameter. Without this, ``urllib3``-equivalent code would use
+       the IP literal as SNI and cert verification would fail (the
+       cert is for the original hostname, not the IP).
+    """
+
+    def __init__(
+        self,
+        validated_host: str,
+        validated_ip: ipaddress.IPv4Address | ipaddress.IPv6Address,
+        **kwargs: Any,
+    ) -> None:
+        # http2=False (the httpx default) — defense in depth against
+        # HTTP/2 connection coalescing (RFC 7540 §9.1.1), where a
+        # client may reuse a TCP connection for any host whose cert
+        # covers it. Per-IP pinning never shares connections across
+        # hosts, but explicit is safer than relying on the default.
+        kwargs.setdefault("http2", False)
+        super().__init__(**kwargs)
+        self._host = validated_host
+        self._ip_netloc = _ip_to_url_host(validated_ip)
+
+    def handle_request(self, request: httpx.Request) -> httpx.Response:
+        # Defense in depth: refuse if the request URL's host doesn't
+        # match what we validated. Catches any future SDK regression
+        # that rewrites the URL between Request construction and dial,
+        # and any rare case where the SDK reuses our pinned client for
+        # a different host (which it shouldn't, but assert it anyway).
+        if request.url.host != self._host:
+            raise UnsafeUserUrlError(
+                f"pinned transport bound to {self._host!r}, refused "
+                f"request for {request.url.host!r}"
+            )
+        # SNI/server_hostname for TLS verification. httpcore reads this
+        # extension at _sync/connection.py and feeds it into
+        # start_tls's server_hostname argument. Set before the URL host
+        # is rewritten so cert validation continues to use the original
+        # hostname even though TCP dials the IP literal.
+        request.extensions = {
+            **request.extensions,
+            "sni_hostname": self._host.encode("ascii"),
+        }
+        request.url = request.url.copy_with(host=self._ip_netloc)
+        return super().handle_request(request)
+
+
+def pinned_httpx_client(
+    base_url: str,
+    *,
+    timeout: float = 600.0,
+) -> httpx.Client:
+    """Return an :class:`httpx.Client` whose connections are pinned to
+    one validated IP, closing the DNS-rebinding TOCTOU window the naive
+    ``OpenAI(base_url=...)`` flow leaves open.
+
+    The hostname in ``base_url`` is resolved exactly once. Every
+    returned address must pass :func:`_validate_and_pick_ip`'s SSRF
+    guard (loopback, RFC 1918, link-local, multicast, reserved, CGNAT,
+    cloud metadata names). The chosen IP becomes the URL host on every
+    outgoing request so ``httpcore`` cannot ask the resolver again.
+
+    Pass via ``OpenAI(http_client=pinned_httpx_client(base_url))`` (or
+    any other SDK that accepts an ``httpx.Client``) to make BYOM
+    dispatch immune to DNS-rebinding TOCTOU.
+
+    Args:
+        base_url: User-supplied http(s) URL. Validated through the same
+            SSRF guard as :func:`validate_user_base_url`.
+        timeout: Per-request timeout (seconds). Defaults to 600 to
+            match the OpenAI SDK's default; callers should override
+            for non-LLM workloads.
+
+    Raises:
+        UnsafeUserUrlError: If ``base_url`` fails the SSRF guard.
+    """
+
+    host, ip, _parts = _validate_and_pick_ip(base_url)
+    transport = _PinnedHTTPSTransport(host, ip)
+    # follow_redirects=False — the SSRF guard only inspects the
+    # supplied URL; following 3xx would let a hostile upstream bounce
+    # the in-network request to an internal address (cloud metadata,
+    # RFC1918, loopback) carrying whatever credentials the SDK adds.
+    return httpx.Client(
+        transport=transport,
+        timeout=timeout,
+        follow_redirects=False,
+    )

docsgpt/security/zip_archive.py

@@ -0,0 +1,401 @@
+"""Safe, bounded extraction for untrusted ZIP archives."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import stat
+import tempfile
+import zipfile
+from contextlib import suppress
+from dataclasses import dataclass
+from pathlib import PurePosixPath
+
+
+_COPY_CHUNK_BYTES = 64 * 1024
+_MAX_MEMBER_NAME_BYTES = 4096
+_MAX_MEMBER_DEPTH = 64
+
+
+class ZipExtractionError(ValueError):
+    """Raised when a ZIP is invalid or exceeds an extraction safety limit."""
+
+
+@dataclass(frozen=True)
+class ZipExtractionLimits:
+    """Resource ceilings for one extraction tree."""
+
+    max_uncompressed_bytes: int
+    max_files: int
+    max_compression_ratio: int
+    max_member_bytes: int | None = None
+    max_depth: int = 3
+
+
+@dataclass
+class ZipExtractionBudget:
+    """Cumulative resources reserved across nested archives."""
+
+    uncompressed_bytes: int = 0
+    files: int = 0
+
+
+@dataclass(frozen=True)
+class _ValidatedMember:
+    info: zipfile.ZipInfo
+    relative_path: str
+
+
+def _collision_safe_member_path(
+    relative_path: str,
+    assignments: dict[tuple[tuple[str, ...], str], str],
+    occupied_names: dict[tuple[str, ...], set[str]],
+) -> str:
+    """Map case-colliding member components to stable, portable names."""
+    original_parent: tuple[str, ...] = ()
+    assigned_parent: tuple[str, ...] = ()
+    assigned_parts: list[str] = []
+    for component in relative_path.split("/"):
+        assignment_key = (original_parent, component)
+        assigned_component = assignments.get(assignment_key)
+        if assigned_component is None:
+            used = occupied_names.setdefault(assigned_parent, set())
+            assigned_component = component
+            suffix = 2
+            stem, extension = os.path.splitext(component)
+            while assigned_component.casefold() in used:
+                assigned_component = f"{stem} ({suffix}){extension}"
+                suffix += 1
+            assignments[assignment_key] = assigned_component
+            used.add(assigned_component.casefold())
+        assigned_parts.append(assigned_component)
+        original_parent += (component,)
+        assigned_parent += (assigned_component,)
+    return "/".join(assigned_parts)
+
+
+def _safe_member_path(name: str) -> str:
+    """Normalize a ZIP member name and reject paths escaping extraction."""
+    if not name or "\x00" in name:
+        raise ZipExtractionError("ZIP contains an invalid empty member name")
+    if len(name.encode("utf-8", errors="surrogatepass")) > _MAX_MEMBER_NAME_BYTES:
+        raise ZipExtractionError("ZIP member name exceeds the safety limit")
+
+    normalized = name.replace("\\", "/")
+    path = PurePosixPath(normalized)
+    parts = path.parts
+    if (
+        path.is_absolute()
+        or normalized.startswith("//")
+        or any(part in {"", ".", ".."} for part in parts)
+        or (parts and parts[0].endswith(":"))
+    ):
+        raise ZipExtractionError(f"ZIP contains a path traversal attempt: {name}")
+    if len(parts) > _MAX_MEMBER_DEPTH:
+        raise ZipExtractionError("ZIP member path is nested too deeply")
+    return "/".join(parts)
+
+
+def _reject_special_member(info: zipfile.ZipInfo) -> None:
+    """Reject symlinks and device-like Unix entries."""
+    mode = info.external_attr >> 16
+    file_type = stat.S_IFMT(mode)
+    if file_type not in {0, stat.S_IFREG, stat.S_IFDIR}:
+        raise ZipExtractionError(f"ZIP contains a special file: {info.filename}")
+    if info.flag_bits & 0x1:
+        raise ZipExtractionError("Encrypted ZIP members are not supported")
+
+
+def _validate_members(
+    archive: zipfile.ZipFile,
+    limits: ZipExtractionLimits,
+    budget: ZipExtractionBudget,
+) -> tuple[list[_ValidatedMember], int, int]:
+    members: list[_ValidatedMember] = []
+    seen_paths: dict[str, bool] = {}
+    component_assignments: dict[tuple[tuple[str, ...], str], str] = {}
+    occupied_names: dict[tuple[str, ...], set[str]] = {}
+    added_bytes = 0
+    added_files = 0
+    added_compressed_bytes = 0
+
+    for info in archive.infolist():
+        relative_path = _safe_member_path(info.filename)
+        is_directory = info.is_dir()
+        prior_is_directory = seen_paths.get(relative_path)
+        if prior_is_directory is not None and not (
+            prior_is_directory and is_directory
+        ):
+            raise ZipExtractionError(f"ZIP contains a duplicate path: {info.filename}")
+        seen_paths[relative_path] = is_directory
+        _reject_special_member(info)
+        extraction_path = _collision_safe_member_path(
+            relative_path, component_assignments, occupied_names
+        )
+
+        if is_directory:
+            members.append(_ValidatedMember(info=info, relative_path=extraction_path))
+            continue
+
+        added_files += 1
+        added_bytes += int(info.file_size)
+        added_compressed_bytes += int(info.compress_size)
+        if limits.max_member_bytes and info.file_size > limits.max_member_bytes:
+            raise ZipExtractionError(
+                "ZIP member exceeds the "
+                f"{limits.max_member_bytes}-byte per-file limit"
+            )
+        if budget.files + added_files > limits.max_files:
+            raise ZipExtractionError(
+                f"ZIP contains too many files (>{limits.max_files})"
+            )
+        if budget.uncompressed_bytes + added_bytes > limits.max_uncompressed_bytes:
+            raise ZipExtractionError(
+                "ZIP extraction exceeds the "
+                f"{limits.max_uncompressed_bytes}-byte expansion limit"
+            )
+
+        members.append(_ValidatedMember(info=info, relative_path=extraction_path))
+
+    if added_bytes:
+        if added_compressed_bytes <= 0:
+            raise ZipExtractionError("ZIP has an invalid compression ratio")
+        ratio = added_bytes / added_compressed_bytes
+        if ratio > limits.max_compression_ratio:
+            raise ZipExtractionError(
+                "ZIP exceeds the "
+                f"{limits.max_compression_ratio}:1 compression ratio limit"
+            )
+
+    return members, added_bytes, added_files
+
+
+def validate_zip_archive(
+    zip_path: str | os.PathLike[str],
+    limits: ZipExtractionLimits,
+    budget: ZipExtractionBudget | None = None,
+) -> None:
+    """Validate central-directory metadata without extracting any content."""
+    active_budget = budget or ZipExtractionBudget()
+    try:
+        with zipfile.ZipFile(zip_path, "r") as archive:
+            _validate_members(archive, limits, active_budget)
+    except (OSError, zipfile.BadZipFile) as exc:
+        raise ZipExtractionError(f"Invalid or corrupted ZIP file: {exc}") from exc
+
+
+def _extract_archive_into(
+    zip_path: str | os.PathLike[str],
+    destination_root: str,
+    limits: ZipExtractionLimits,
+    budget: ZipExtractionBudget,
+) -> list[str]:
+    """Extract one validated archive into a private staging directory."""
+    extracted_files: list[str] = []
+    with zipfile.ZipFile(zip_path, "r") as archive:
+        members, added_bytes, added_files = _validate_members(
+            archive, limits, budget
+        )
+
+        # Reserve declared resources before decompression so nested callers
+        # cannot reset the allowance between layers.
+        budget.uncompressed_bytes += added_bytes
+        budget.files += added_files
+
+        for member in members:
+            target = os.path.realpath(
+                os.path.join(destination_root, *member.relative_path.split("/"))
+            )
+            try:
+                inside_destination = (
+                    os.path.commonpath((destination_root, target))
+                    == destination_root
+                )
+            except ValueError as exc:
+                raise ZipExtractionError(
+                    f"ZIP contains a path traversal attempt: {member.info.filename}"
+                ) from exc
+            if not inside_destination:
+                raise ZipExtractionError(
+                    "ZIP contains a path traversal attempt: "
+                    f"{member.info.filename}"
+                )
+            if member.info.is_dir():
+                os.makedirs(target, exist_ok=True)
+                continue
+            if os.path.lexists(target):
+                raise ZipExtractionError(
+                    f"ZIP member would overwrite an existing path: {member.info.filename}"
+                )
+
+            os.makedirs(os.path.dirname(target), exist_ok=True)
+            written = 0
+            with archive.open(member.info, "r") as source, open(target, "xb") as output:
+                while True:
+                    chunk = source.read(_COPY_CHUNK_BYTES)
+                    if not chunk:
+                        break
+                    written += len(chunk)
+                    if written > member.info.file_size:
+                        raise ZipExtractionError(
+                            "ZIP member expanded past its declared size: "
+                            f"{member.info.filename}"
+                        )
+                    output.write(chunk)
+            if written != member.info.file_size:
+                raise ZipExtractionError(
+                    f"ZIP member size did not match metadata: {member.info.filename}"
+                )
+            extracted_files.append(target)
+    return extracted_files
+
+
+def _extract_tree_into(
+    zip_path: str | os.PathLike[str],
+    destination_root: str,
+    limits: ZipExtractionLimits,
+    budget: ZipExtractionBudget,
+    depth: int,
+) -> None:
+    """Extract an archive and recursively expand nested ``.zip`` members."""
+    extracted_files = _extract_archive_into(
+        zip_path, destination_root, limits, budget
+    )
+    for extracted_path in extracted_files:
+        if not extracted_path.lower().endswith(".zip"):
+            continue
+        if not zipfile.is_zipfile(extracted_path):
+            continue
+        if depth >= limits.max_depth:
+            raise ZipExtractionError(
+                f"ZIP nesting exceeds the {limits.max_depth}-level limit"
+            )
+        parent = os.path.dirname(extracted_path)
+        stem = os.path.splitext(os.path.basename(extracted_path))[0] or "archive"
+        occupied_names = {name.casefold() for name in os.listdir(parent)}
+        nested_name = stem
+        suffix = 2
+        while nested_name.casefold() in occupied_names:
+            nested_name = f"{stem} ({suffix})"
+            suffix += 1
+        nested_root = os.path.join(parent, nested_name)
+        os.makedirs(nested_root)
+        _extract_tree_into(
+            extracted_path,
+            nested_root,
+            limits,
+            budget,
+            depth + 1,
+        )
+        os.remove(extracted_path)
+
+
+def _commit_staging(staging_root: str, destination_root: str) -> None:
+    """Commit a complete staging tree, rolling back an interrupted merge."""
+    if not os.path.lexists(destination_root):
+        os.replace(staging_root, destination_root)
+        return
+    if not os.path.isdir(destination_root):
+        raise ZipExtractionError("ZIP extraction destination is not a directory")
+
+    staged_directories: list[tuple[str, str]] = []
+    staged_files: list[tuple[str, str]] = []
+    for root, directories, files in os.walk(staging_root):
+        relative_root = os.path.relpath(root, staging_root)
+        destination_directory = (
+            destination_root
+            if relative_root == "."
+            else os.path.join(destination_root, relative_root)
+        )
+        for directory in directories:
+            source = os.path.join(root, directory)
+            target = os.path.join(destination_directory, directory)
+            if os.path.lexists(target) and not os.path.isdir(target):
+                raise ZipExtractionError(
+                    "ZIP member would overwrite an existing path"
+                )
+            staged_directories.append((source, target))
+        for filename in files:
+            source = os.path.join(root, filename)
+            target = os.path.join(destination_directory, filename)
+            if os.path.lexists(target):
+                raise ZipExtractionError(
+                    "ZIP member would overwrite an existing path"
+                )
+            staged_files.append((source, target))
+
+    created_directories: list[str] = []
+    moved_files: list[str] = []
+    try:
+        for _, target in staged_directories:
+            if not os.path.isdir(target):
+                os.makedirs(target)
+                created_directories.append(target)
+        for source, target in staged_files:
+            os.makedirs(os.path.dirname(target), exist_ok=True)
+            os.replace(source, target)
+            moved_files.append(target)
+    except Exception:
+        # Rollback is best effort and must not hide the original commit failure.
+        for target in reversed(moved_files):
+            with suppress(OSError):
+                os.remove(target)
+        for target in reversed(created_directories):
+            with suppress(OSError):
+                os.rmdir(target)
+        raise
+
+
+def extract_zip_safely(
+    zip_path: str | os.PathLike[str],
+    extract_to: str | os.PathLike[str],
+    limits: ZipExtractionLimits,
+    budget: ZipExtractionBudget | None = None,
+) -> ZipExtractionBudget:
+    """Transactionally extract a ZIP tree under cumulative resource limits.
+
+    Nested ``.zip`` members are expanded inside a private staging directory.
+    The destination is changed only after every layer has passed validation
+    and extraction, so a corrupt member cannot leave partial content behind.
+    """
+    active_budget = budget or ZipExtractionBudget()
+    destination_root = os.path.realpath(extract_to)
+    destination_parent = os.path.dirname(destination_root)
+    staging_root: str | None = None
+
+    try:
+        os.makedirs(destination_parent, exist_ok=True)
+        staging_root = tempfile.mkdtemp(
+            prefix=".zip-extract-", dir=destination_parent
+        )
+        _extract_tree_into(zip_path, staging_root, limits, active_budget, depth=0)
+        _commit_staging(staging_root, destination_root)
+    except ZipExtractionError:
+        raise
+    except (
+        OSError,
+        RuntimeError,
+        ValueError,
+        zipfile.BadZipFile,
+        zipfile.LargeZipFile,
+    ) as exc:
+        raise ZipExtractionError(f"Unable to extract ZIP safely: {exc}") from exc
+    finally:
+        if staging_root is not None:
+            shutil.rmtree(staging_root, ignore_errors=True)
+
+    return active_budget
+
+
+def safe_zip_error_message(error: BaseException | str, max_chars: int = 512) -> str:
+    """Return a bounded, single-line representation safe for logs and JSON."""
+    raw = str(error)
+    escaped = "".join(
+        f"\\x{ord(character):02x}"
+        if ord(character) < 32 or 127 <= ord(character) <= 159
+        else character
+        for character in raw
+    )
+    if len(escaped) > max_chars:
+        return f"{escaped[: max_chars - 3]}..."
+    return escaped