refactor: rename the application package to docsgpt

The backend import package is now docsgpt, the name it will carry on PyPI;
application was far too generic to install into anyone's site-packages.
git mv plus a mechanical rewrite of every import, dotted string and path
reference: 734 Python files, the compose files, Dockerfile, workflows, docs,
setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage
config, .gitignore. Behaviour is unchanged.

Kept for one release:
- A top-level application package whose meta-path finder resolves
  application.x.y to the already-imported docsgpt.x.y object, so old imports
  and entry points (celery -A application.app.celery,
  uvicorn application.asgi:asgi_app) keep working with a FutureWarning.
- Celery registers every application.* task name as an alias of its
  docsgpt.* task on start-up, so messages queued by the previous release still
  run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries
  the previous release wrote are left unread instead of firing twice.

The backend image builds from the repository root (docker build -f
docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore
allow-lists docsgpt/ and application/ and keeps caches, local data, .env
files, the sample index files and the Dockerfile out. Compose and the image
workflows point at the new context.

docsgpt/agents/tools/read_document.py

@@ -0,0 +1,479 @@
+"""Read Document tool: parse an input artifact to text/markdown/structured/chunks via the backend parser.
+
+The ``read_document`` action resolves a parent-scoped input artifact, enqueues a
+``parse_document`` task on the dedicated ``parsing`` Celery queue, and awaits the
+result with a timeout (inside a worker it parses inline under the same bound — see
+``_dispatch``). The run-scoped authz gate is enforced TWICE — here before
+enqueue (reject cross-tenant) and again in the worker (re-resolve, never trusting a
+raw path). When a ``json_schema`` is supplied the structured payload is validated
+through the existing jsonschema path; the full result may also be persisted as a
+``data`` artifact by reference (handled in the worker).
+"""
+
+from __future__ import annotations
+
+import logging
+import signal
+import threading
+from typing import Any, Callable, Dict, List, Optional
+
+from celery import current_task
+
+from docsgpt.agents.tools.artifact_ref import resolve_artifact_id
+from docsgpt.agents.tools.attachment_bridge import (
+    AttachmentBridgeError,
+    bridge_attachment,
+    match_attachment,
+)
+from docsgpt.agents.tools.base import Tool
+from docsgpt.core.json_schema_utils import (
+    JsonSchemaValidationError,
+    normalize_json_schema_payload,
+)
+from docsgpt.core.settings import settings
+from docsgpt.storage.db.repositories.artifacts import ArtifactsRepository
+from docsgpt.storage.db.session import db_readonly
+
+logger = logging.getLogger(__name__)
+
+try:
+    import jsonschema
+except Exception:  # pragma: no cover - jsonschema is a declared dependency
+    jsonschema = None  # type: ignore[assignment]
+
+
+class _InlineParseTimeout(BaseException):
+    """Raised when an inline (in-worker) parse outlives its window.
+
+    Derives from ``BaseException`` on purpose: the SIGALRM interrupt lands deep inside the
+    parser, whose blanket ``except Exception`` would otherwise swallow it and report a
+    generic parse failure instead of the shared timed-out result.
+    """
+
+
+class ReadDocumentTool(Tool):
+    """Read Document
+    Parse a document (PDF, Word, PowerPoint, ...) to text, markdown, or structured data.
+    """
+
+    # Hidden from the Add-Tool catalog; surfaced (workflow-only) via the
+    # BUILTIN_AGENT_TOOLS synthetic-id path. Does not gate tool_manager loading
+    # nor synthetic-id execution.
+    internal: bool = True
+
+    def __init__(self, tool_config: Optional[Dict[str, Any]] = None, user_id: Optional[str] = None) -> None:
+        """Bind the tool to the invoker and its conversation/run scope."""
+        self.config: Dict[str, Any] = tool_config or {}
+        self.user_id: Optional[str] = user_id
+        self.tool_id: Optional[str] = self.config.get("tool_id")
+        self.conversation_id: Optional[str] = self.config.get("conversation_id")
+        self.workflow_run_id: Optional[str] = self.config.get("workflow_run_id")
+        self.message_id: Optional[str] = self.config.get("message_id")
+        self._last_artifact_id: Optional[str] = None
+        # Byte size of the resolved input, used to scale the awaited parse window.
+        self._input_size: Optional[int] = None
+
+    # ------------------------------------------------------------------
+    # Tool ABC
+    # ------------------------------------------------------------------
+    def get_actions_metadata(self) -> List[Dict[str, Any]]:
+        """Return JSON metadata describing the ``read_document`` action for tool schemas."""
+        return [
+            {
+                "name": "read_document",
+                "description": (
+                    "Read a document artifact (pdf/docx/pptx/...) and return its parsed content as "
+                    "markdown, plain text, structured JSON (with tables), or chunks. Optionally "
+                    "validate the structured result against a json_schema and persist it as a "
+                    "downloadable data artifact."
+                ),
+                "active": True,
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "input": {
+                            "type": "string",
+                            "description": "Document to read; accepts the short ref like `A1` returned by a "
+                            "previous artifact action, a full artifact id, or the name/id of a file the user "
+                            "attached to this conversation.",
+                        },
+                        "output": {
+                            "type": "string",
+                            "enum": ["markdown", "text", "structured", "chunks"],
+                            "description": "Shape of the parsed result (default: markdown). Note: "
+                            "`structured` always uses the Docling engine regardless of `engine` "
+                            "(requires the optional docling install; the other engines are "
+                            "markdown/text only).",
+                        },
+                        "ocr": {
+                            "type": "string",
+                            "enum": ["auto", "on", "off"],
+                            "description": "OCR mode for scanned pages/images (default: auto, follows server config).",
+                        },
+                        "pages": {
+                            "type": "string",
+                            "description": "Optional page range to read, e.g. `1-3` or `2` (best-effort).",
+                        },
+                        "engine": {
+                            "type": "string",
+                            "enum": ["auto", "docling", "anydoc", "fast"],
+                            "description": "Parser engine (default: auto = the server's configured "
+                            "engine). `anydoc` is the fast Markdown converter, `docling` the "
+                            "layout-model engine (optional install, needed for tables/OCR), "
+                            "`fast` a plain-text legacy path. Ignored when `output='structured'`, "
+                            "which always uses Docling.",
+                        },
+                        "max_chars": {
+                            "type": "integer",
+                            "description": "Optional cap on returned characters.",
+                        },
+                        "include_tables": {
+                            "type": "boolean",
+                            "description": "Include extracted tables in the result (default: true).",
+                        },
+                        "json_schema": {
+                            "type": "object",
+                            "description": "Optional JSON schema the structured payload must satisfy.",
+                        },
+                        "persist": {
+                            "type": "boolean",
+                            "description": "Persist the parsed result as a downloadable data artifact (default true).",
+                        },
+                    },
+                    "required": ["input"],
+                },
+            }
+        ]
+
+    def get_config_requirements(self) -> Dict[str, Any]:
+        """Return configuration requirements (none beyond a running parsing worker)."""
+        return {}
+
+    def get_artifact_id(self, action_name: str, **kwargs: Any) -> Optional[str]:
+        """Return the persisted parse artifact id so the UI artifact rail lights up."""
+        return self._last_artifact_id
+
+    # ------------------------------------------------------------------
+    # Dispatch
+    # ------------------------------------------------------------------
+    def execute_action(self, action_name: str, **kwargs: Any) -> Dict[str, Any]:
+        """Dispatch a tool action; only ``read_document`` is supported."""
+        self._last_artifact_id = None
+        if action_name != "read_document":
+            return {"status": "error", "error": f"unknown action: {action_name}"}
+        if not self.user_id:
+            return {"status": "error", "error": "read_document requires a valid user_id."}
+        if self.conversation_id is None and self.workflow_run_id is None:
+            return {"status": "error", "error": "read_document requires a conversation_id or workflow_run_id."}
+        return self._read(**kwargs)
+
+    # ------------------------------------------------------------------
+    # Read
+    # ------------------------------------------------------------------
+    def _read(self, **kwargs: Any) -> Dict[str, Any]:
+        """Resolve the input run-scoped (reject cross-tenant before enqueue), enqueue+await, validate."""
+        input_id = kwargs.get("input")
+        json_schema = kwargs.get("json_schema")
+        self._input_size = None
+        if not isinstance(input_id, str) or not input_id.strip():
+            return {"status": "error", "error": "input artifact id is required."}
+        if json_schema is not None:
+            schema_err = self._check_schema(json_schema)
+            if schema_err is not None:
+                return schema_err
+
+        artifact_id = self._resolve_input(input_id.strip())
+        if isinstance(artifact_id, dict):
+            return artifact_id  # error payload
+
+        options = {
+            "output": kwargs.get("output", "markdown"),
+            "ocr": kwargs.get("ocr", "auto"),
+            "pages": kwargs.get("pages"),
+            "engine": kwargs.get("engine", "auto"),
+            "max_chars": kwargs.get("max_chars"),
+            "include_tables": kwargs.get("include_tables", True),
+            "persist": kwargs.get("persist", True),
+            "tool_id": self.tool_id,
+        }
+        result = self._dispatch(artifact_id, options)
+        if result.get("status") == "error":
+            return result
+        if json_schema is not None:
+            valid = self._validate(json_schema, result.get("structured"))
+            if valid is not None:
+                return valid
+        artifact = result.get("artifact")
+        if isinstance(artifact, dict) and artifact.get("artifact_id"):
+            self._last_artifact_id = artifact["artifact_id"]
+        return result
+
+    def _dispatch(self, artifact_id: str, options: Dict[str, Any]) -> Dict[str, Any]:
+        """Parse INLINE inside a Celery worker, else dispatch to the parsing queue and await.
+
+        This tool runs in the WEB process (/stream) OR inside a Celery worker
+        (headless/scheduled/workflow agents). When it already runs inside a worker that also
+        serves the ``parsing`` queue (the shipped default ``-Q docsgpt,parsing``), dispatching
+        and blocking on ``get()`` would self-deadlock: concurrent agent tasks each hold a pool
+        slot blocked in ``get()`` so ``parse_document`` never gets a free slot. So parse INLINE
+        in-process inside a worker; only dispatch+await (degrading on timeout/failure) from web.
+
+        Both branches are bounded by the SAME size-scaled window: the inline parse carries no
+        Celery time limit of its own, so an unbounded one would pin the agent's worker slot
+        until the OUTER task's limit (webhook runs have none) kills the whole agent run.
+        """
+        parent = self._parent()
+        from docsgpt.api.user.tasks import parse_timeout_for_size
+
+        # OCR cost scales with pages, so the parse window grows with the document's size
+        # (floored at DOCUMENT_PARSE_TIMEOUT).
+        timeout = parse_timeout_for_size(self._input_size)
+
+        # ``current_task`` is a Celery proxy: truthy only while this runs inside a worker task,
+        # falsy in the web process (the bare proxy is NOT identity-None, so test truthiness).
+        if current_task:
+            from docsgpt.worker import run_parse_document
+
+            try:
+                result = self._run_inline_bounded(
+                    lambda: run_parse_document(artifact_id, parent, self.user_id, options), timeout
+                )
+            except _InlineParseTimeout:
+                logger.warning("read_document: inline parse timed out after %ss; abandoning it", int(timeout))
+                return {"status": "error", "error": f"document parsing timed out after {int(timeout)}s."}
+            except Exception as exc:
+                logger.exception("read_document: inline parse failed")
+                return {"status": "error", "error": f"document parsing failed: {type(exc).__name__}: {exc}"}
+            if not isinstance(result, dict):
+                return {"status": "error", "error": "document parsing produced an unexpected result."}
+            return result
+
+        from celery.exceptions import TimeoutError as CeleryTimeoutError
+
+        from docsgpt.api.user.tasks import parse_document, parse_task_time_limits
+
+        # The task's per-call time limits are raised to match the awaited window: bound to
+        # the base timeout at import, the worker would otherwise self-terminate a large
+        # parse long before this await gives up.
+        queue = getattr(settings, "DOCUMENT_PARSE_QUEUE", "parsing")
+        try:
+            async_result = parse_document.apply_async(
+                args=[artifact_id, parent, self.user_id, options],
+                queue=queue,
+                **parse_task_time_limits(timeout),
+            )
+            # The web process (not a worker) awaits here; ``disable_sync_subtasks=False`` keeps
+            # the call correct if invoked from a non-prefork (eventlet/gevent) worker where the
+            # inline branch above still ran but the blanket guard would otherwise raise.
+            result = async_result.get(timeout=timeout, disable_sync_subtasks=False)
+        except (CeleryTimeoutError, TimeoutError):
+            return {"status": "error", "error": f"document parsing timed out after {int(timeout)}s."}
+        except Exception as exc:
+            logger.exception("read_document: parse task failed")
+            return {"status": "error", "error": f"document parsing failed: {type(exc).__name__}: {exc}"}
+        if not isinstance(result, dict):
+            return {"status": "error", "error": "document parsing produced an unexpected result."}
+        return result
+
+    # ------------------------------------------------------------------
+    # Inline parse bound
+    # ------------------------------------------------------------------
+    @classmethod
+    def _run_inline_bounded(cls, fn: Callable[[], Any], timeout: Optional[float]) -> Any:
+        """Run ``fn`` under a wall-clock bound, raising ``_InlineParseTimeout`` on expiry.
+
+        Prefers a SIGALRM interval timer (POSIX, main thread — prefork/solo Celery pools run
+        the task on the child's main thread), which actually interrupts the parse between
+        bytecodes and frees the slot. Otherwise falls back to a helper thread, which can only
+        stop WAITING: the orphaned parse keeps running until the parser finishes.
+
+        Args:
+            fn: Zero-argument callable performing the parse.
+            timeout: Window in seconds; ``None`` or non-positive runs unbounded.
+
+        Returns:
+            Whatever ``fn`` returns.
+        """
+        if not timeout or timeout <= 0:
+            return fn()
+        if cls._sigalrm_usable():
+            return cls._run_with_sigalrm(fn, timeout)
+        return cls._run_in_thread(fn, timeout)
+
+    @staticmethod
+    def _sigalrm_usable() -> bool:
+        """True when this thread may arm SIGALRM without clobbering another user of the timer."""
+        if not hasattr(signal, "SIGALRM") or not hasattr(signal, "setitimer"):
+            return False
+        if threading.current_thread() is not threading.main_thread():
+            return False
+        try:
+            previous = signal.getsignal(signal.SIGALRM)
+            armed = signal.getitimer(signal.ITIMER_REAL)
+        except (ValueError, OSError):  # pragma: no cover - non-main thread / no timer
+            return False
+        if previous not in (None, signal.SIG_DFL):
+            return False
+        return not (armed[0] or armed[1])
+
+    @staticmethod
+    def _run_with_sigalrm(fn: Callable[[], Any], timeout: float) -> Any:
+        """Run ``fn`` with a SIGALRM deadline, always restoring the handler and cancelling the timer."""
+        completed = False
+
+        def _on_alarm(signum: int, frame: Any) -> None:
+            # A straggler racing the disarm below: ``fn`` already returned, and
+            # raising here would escape the ``finally`` and replace its value
+            # with a timeout for a document that parsed fine.
+            if completed:
+                return
+            raise _InlineParseTimeout()
+
+        previous = signal.signal(signal.SIGALRM, _on_alarm)
+        signal.setitimer(signal.ITIMER_REAL, timeout)
+        try:
+            result = fn()
+            completed = True
+            return result
+        finally:
+            # Nested so the handler is restored even if the alarm lands in this very
+            # window (fn returned just as the timer expired).
+            try:
+                signal.setitimer(signal.ITIMER_REAL, 0)
+            finally:
+                signal.signal(signal.SIGALRM, previous)
+
+    @staticmethod
+    def _run_in_thread(fn: Callable[[], Any], timeout: float) -> Any:
+        """Run ``fn`` in a daemon helper thread, giving up at ``timeout`` (the parse is NOT killed)."""
+        # A raw daemon thread rather than a ThreadPoolExecutor: executor threads
+        # are non-daemon and registered with ``concurrent.futures``' atexit hook,
+        # which joins them -- so an abandoned parse would hold up worker
+        # shutdown for the rest of its (size-scaled) window. Same reasoning as
+        # docsgpt/guardrails/engine.py. ``shutdown(cancel_futures=True)``
+        # is not an alternative: it only drops queued work items, never the one
+        # already running.
+        slot: Dict[str, Any] = {}
+
+        def _fill() -> None:
+            try:
+                slot["value"] = fn()
+            except BaseException as exc:  # noqa: BLE001 - re-raised on the caller's thread
+                slot["error"] = exc
+
+        thread = threading.Thread(target=_fill, daemon=True, name="read-document-inline")
+        thread.start()
+        thread.join(timeout)
+        if thread.is_alive():
+            logger.warning(
+                "read_document: inline parse exceeded %.0fs off the main thread; the parse thread "
+                "keeps running until the parser finishes (it cannot be interrupted)",
+                timeout,
+            )
+            raise _InlineParseTimeout()
+        # ``fn`` raising TimeoutError is a parse failure, not our deadline; with
+        # join()+is_alive() the two are distinguishable without a done() probe.
+        if "error" in slot:
+            raise slot["error"]
+        return slot.get("value")
+
+    # ------------------------------------------------------------------
+    # Input resolution (run-scoped gate, before enqueue)
+    # ------------------------------------------------------------------
+    def _resolve_input(self, raw_id: str) -> Any:
+        """Resolve a short ref/uuid to a parent-scoped artifact id; an error dict on miss/cross-tenant."""
+        try:
+            with db_readonly() as conn:
+                repo = ArtifactsRepository(conn)
+                artifact_id = resolve_artifact_id(
+                    repo,
+                    raw_id,
+                    conversation_id=self.conversation_id,
+                    workflow_run_id=self.workflow_run_id,
+                )
+                artifact = (
+                    repo.get_artifact_in_parent(
+                        artifact_id,
+                        conversation_id=self.conversation_id,
+                        workflow_run_id=self.workflow_run_id,
+                    )
+                    if artifact_id is not None
+                    else None
+                )
+                if artifact is not None:
+                    self._input_size = self._version_size(repo, artifact_id, artifact)
+        except Exception:
+            logger.exception("read_document: failed to resolve input artifact")
+            return {"status": "error", "error": f"failed to resolve input artifact {raw_id}."}
+        if artifact is None:
+            # Conversation scope only: a raw ref that is not an artifact may name a
+            # chat attachment; bridge it on demand. Workflows bridge up front.
+            bridged_id = self._bridge_chat_attachment(raw_id)
+            if isinstance(bridged_id, dict):
+                return bridged_id
+            if bridged_id is not None:
+                return bridged_id
+            return {"status": "error", "error": f"input artifact {raw_id} not found in this conversation/run."}
+        return str(artifact_id)
+
+    @staticmethod
+    def _version_size(repo: Any, artifact_id: Any, artifact: Dict[str, Any]) -> Optional[int]:
+        """Best-effort byte size of the artifact's current version (scales the parse window)."""
+        try:
+            version = repo.get_version(artifact_id, artifact.get("current_version"))
+        except Exception:
+            logger.debug("read_document: could not read input size; using the base parse window")
+            return None
+        size = (version or {}).get("size")
+        return int(size) if isinstance(size, (int, float)) else None
+
+    def _bridge_chat_attachment(self, raw_id: str) -> Any:
+        """Bridge a referenced chat attachment to a conversation artifact id; None on miss, error dict on failure."""
+        if not self.conversation_id or not self.user_id:
+            return None
+        attachment = match_attachment(self.config.get("attachments"), raw_id, self.user_id)
+        if attachment is None:
+            return None
+        # The attachment row carries the authoritative byte size; the bridged artifact
+        # is a copy of the same bytes, so it sizes the parse window just as well.
+        size = attachment.get("size")
+        self._input_size = int(size) if isinstance(size, (int, float)) else None
+        try:
+            return bridge_attachment(attachment, user_id=self.user_id, conversation_id=self.conversation_id)
+        except AttachmentBridgeError as exc:
+            return {"status": "error", "error": f"failed to attach {raw_id}: {exc}"}
+
+    def _parent(self) -> Dict[str, Any]:
+        """Build the run-scoped parent dict passed to the worker for its independent re-resolve."""
+        if self.conversation_id is not None:
+            parent: Dict[str, Any] = {"conversation_id": self.conversation_id}
+            if self.message_id:
+                parent["message_id"] = self.message_id
+            return parent
+        return {"workflow_run_id": self.workflow_run_id}
+
+    # ------------------------------------------------------------------
+    # Schema validation
+    # ------------------------------------------------------------------
+    @staticmethod
+    def _check_schema(json_schema: Any) -> Optional[Dict[str, Any]]:
+        """Return an error payload when ``json_schema`` itself is malformed, else None."""
+        try:
+            normalize_json_schema_payload(json_schema)
+        except JsonSchemaValidationError as exc:
+            return {"status": "error", "error": f"invalid json_schema: {exc}"}
+        return None
+
+    @staticmethod
+    def _validate(json_schema: Any, instance: Any) -> Optional[Dict[str, Any]]:
+        """Validate ``instance`` against the (already-normalized) json_schema; error payload on mismatch."""
+        if jsonschema is None:
+            return {"status": "error", "error": "jsonschema is required for json_schema validation."}
+        if instance is None:
+            return {"status": "error", "error": "json_schema validation requires output='structured'."}
+        schema = normalize_json_schema_payload(json_schema)
+        try:
+            jsonschema.validate(instance=instance, schema=schema)
+        except jsonschema.exceptions.ValidationError as exc:
+            return {"status": "error", "error": f"parsed structure did not match json_schema: {exc.message}"}
+        return None