From f23a32d9c5a3e27939a45038d2653ff4878fc2ab Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 16 Sep 2026 21:51:43 +0100 Subject: [PATCH] feat: docsgpt up --native Run DocsGPT without Docker: the API and the worker each become a service on the machine itself, a launchd agent on macOS and a systemd user unit on Linux, pointed at a PostgreSQL and a Redis that already run. `docsgpt up --native --postgres-uri ... --redis-url ...` writes the same .env a Docker install uses, applies the migrations and starts both services. status, logs, down and uninstall work on a native install the same way they do on a Docker one, and never touch the database or Redis: they were the user's to begin with. One Redis URL covers the broker, the result backend and the cache on three consecutive databases, starting at the one the URL names, so a Redis that already holds something else can be shared. Windows has neither service manager, so native mode refuses it and says what to do instead. --- .../config/vocabularies/DocsGPT/accept.txt | 6 +- docs/content/Deploying/Pip-Install.mdx | 29 ++ docs/content/changelog.mdx | 8 + docsgpt/cli.py | 4 + docsgpt/deploy/commands.py | 180 ++++++++++- docsgpt/deploy/native.py | 218 ++++++++++++++ tests/deploy/test_native.py | 284 ++++++++++++++++++ 7 files changed, 722 insertions(+), 7 deletions(-) create mode 100644 docsgpt/deploy/native.py create mode 100644 tests/deploy/test_native.py diff --git a/.github/styles/config/vocabularies/DocsGPT/accept.txt b/.github/styles/config/vocabularies/DocsGPT/accept.txt index 7605df49..48520858 100644 --- a/.github/styles/config/vocabularies/DocsGPT/accept.txt +++ b/.github/styles/config/vocabularies/DocsGPT/accept.txt @@ -3,9 +3,9 @@ Anthropic's api APIs Atlassian -automations autoescaping Autoescaping +automations backfill backfills bool @@ -21,9 +21,9 @@ diarization Docling docsgpt docstrings +enqueues Entra env -enqueues EOL ESLint feedbacks @@ -35,6 +35,7 @@ hardcoding Idempotency JSONPath kubectl +launchd Lightsail llama_cpp llm @@ -70,6 +71,7 @@ SGLang Shareability Signup Supabase +systemd UIs uncomment URl diff --git a/docs/content/Deploying/Pip-Install.mdx b/docs/content/Deploying/Pip-Install.mdx index 1b946687..04574490 100644 --- a/docs/content/Deploying/Pip-Install.mdx +++ b/docs/content/Deploying/Pip-Install.mdx @@ -99,6 +99,35 @@ Other commands: - `docsgpt verify-offline`: check that a prepared install starts with networking off. - `docsgpt reembed`: re-embed every index after changing `EMBEDDINGS_NAME` (see [Upgrading](/upgrading)). +## Run it as services, without Docker + +`docsgpt up --native` runs the API and the worker as services on the machine itself: launchd agents on macOS, systemd user units on Linux. It does not start PostgreSQL or Redis; point it at ones you already run. + +```bash +docsgpt up --native \ + --postgres-uri postgresql://docsgpt:@localhost:5432/docsgpt \ + --redis-url redis://localhost:6379 +``` + +Without a terminal both flags are required; with one, it asks for them and for the model provider. It writes the same `.env` a Docker install uses (minus the image settings), generates `INTERNAL_KEY` and `JWT_SECRET_KEY` on the first run, applies the migrations, then starts `docsgpt-api` and `docsgpt-worker` and waits for the API to answer. + +One Redis URL covers all three uses: the Celery broker, its result backend and the cache go on databases 0, 1 and 2 of it. Name a database in the URL and the three start there instead, so `redis://localhost:6379/5` puts them on 5, 6 and 7 — that is how you share a Redis that already holds something else. + +The same commands manage it: + +| Command | In native mode | +| --- | --- | +| `docsgpt status` | Which services run, the address, and whether the API answers | +| `docsgpt logs [api\|worker]` | The service log files under `/logs` | +| `docsgpt down` | Stops both services; settings stay | +| `docsgpt uninstall [--purge]` | Removes the services; `--purge` also deletes the stack directory | + +`uninstall` never touches the database or Redis: they were yours to begin with. + + + Windows has neither launchd nor systemd, so native mode is macOS and Linux only. On Windows, run DocsGPT on Docker with `docsgpt up`, or start `docsgpt api` and `docsgpt worker` yourself. + + ## Upgrade ```bash diff --git a/docs/content/changelog.mdx b/docs/content/changelog.mdx index 2c5f9501..107685e7 100644 --- a/docs/content/changelog.mdx +++ b/docs/content/changelog.mdx @@ -13,6 +13,14 @@ request, and [Upgrading](/upgrading) covers the steps an existing deployment has ## Unreleased +### Run DocsGPT without Docker + +`docsgpt up --native` runs the API and the worker as services on the machine itself, launchd on +macOS and systemd user units on Linux, against a PostgreSQL and Redis you already have +(`--postgres-uri`, `--redis-url`). `status`, `logs`, `down` and `uninstall` work on such an install +the same way they do on a Docker one, and never touch the database or Redis. See +[Run it as services, without Docker](/Deploying/Pip-Install#run-it-as-services-without-docker). + ### Back up and restore an install `docsgpt backup` writes a dump of the database and a tar of each data volume into one archive, and diff --git a/docsgpt/cli.py b/docsgpt/cli.py index e5a5af2a..1ca2a187 100644 --- a/docsgpt/cli.py +++ b/docsgpt/cli.py @@ -206,6 +206,10 @@ def _add_deploy_commands(commands) -> None: help="go back to the default image") up.set_defaults(docling=None) up.add_argument("--image-tag", help="image tag to run instead of this package's version, e.g. develop") + up.add_argument("--native", action="store_true", + help="run the API and worker as services on this machine instead of on Docker") + up.add_argument("--postgres-uri", help="native mode: the PostgreSQL DocsGPT should use") + up.add_argument("--redis-url", help="native mode: the Redis for the queue and the cache (default: localhost:6379)") up.add_argument("-y", "--yes", action="store_true", help="ask nothing: use the flags, then the defaults") up.add_argument("--reconfigure", action="store_true", help="ask the setup questions again") up.add_argument("--adopt", action="store_true", help="take over a DocsGPT stack started from another folder") diff --git a/docsgpt/deploy/commands.py b/docsgpt/deploy/commands.py index fb9c5e25..26c287fc 100644 --- a/docsgpt/deploy/commands.py +++ b/docsgpt/deploy/commands.py @@ -6,6 +6,7 @@ import getpass import json import os import re +import secrets import shutil import subprocess import sys @@ -18,7 +19,7 @@ from pathlib import Path from typing import Any, Optional from docsgpt.deploy import backup as backup_format -from docsgpt.deploy import envfile, stack +from docsgpt.deploy import envfile, native, stack from docsgpt.deploy.docker import DeployError, Docker, lan_ip, wait_healthy PROJECT = "docsgpt" @@ -78,9 +79,10 @@ def detect_installer() -> str: return "pip" -def _run_command(args: list[str]) -> int: +def _run_command(args: list[str], env: Optional[Mapping[str, str]] = None) -> int: + """Run ``args``; ``env`` adds to this process's environment rather than replacing it.""" try: - return subprocess.call(args) + return subprocess.call(args, env={**os.environ, **env} if env else None) except FileNotFoundError as exc: raise DeployError(f"{args[0]} is not on PATH") from exc @@ -106,8 +108,9 @@ class Context: wait: Callable[[str, float], bool] = wait_healthy open_browser: Callable[[str], Any] = webbrowser.open installer: Callable[[], str] = detect_installer - run: Callable[[list[str]], int] = _run_command + run: Callable[..., int] = _run_command exec_up: Callable[[list[str]], int] = _exec_command + services: Any = None @classmethod def default(cls, args) -> "Context": @@ -116,10 +119,33 @@ class Context: interactive = sys.stdin.isatty() and not getattr(args, "yes", False) return cls(docker=Docker(), prompter=Prompter(), interactive=interactive, version=__version__) + def service_manager(self): + """The launchd or systemd wrapper, made on first use so Docker installs never touch it.""" + if self.services is None: + self.services = native.services_for_platform() + return self.services + + +def _record(directory: Path) -> dict: + """What ``install.json`` says about this install (empty when there is none).""" + path = directory / stack.RECORD_FILE + if not path.is_file(): + return {} + try: + return json.loads(path.read_text(encoding="utf-8")) + except ValueError: + return {} + + +def _mode(directory: Path) -> str: + """``native`` or ``docker``, from the install record.""" + return "native" if _record(directory).get("mode") == "native" else "docker" + def _installed(directory: Path) -> Optional[dict[str, str]]: """The stack's settings, or None (with a message) when there is no install in ``directory``.""" - if not (directory / stack.COMPOSE_FILE).is_file(): + installed = (directory / stack.COMPOSE_FILE).is_file() or _mode(directory) == "native" + if not installed: print(f"No DocsGPT install in {directory}. Run `docsgpt up` first, or pass --dir.", file=sys.stderr) return None return envfile.read(directory / ".env") @@ -174,10 +200,110 @@ def _choose_provider(args, context: Context, existing: Mapping[str, str], ask: b raise DeployError(str(exc)) from exc +def _redis_urls(base: str) -> dict[str, str]: + """Celery's broker and result backend, and the cache, on three consecutive databases of one Redis. + + They start at database 0, or at the one the URL names: ``redis://host:6379/5`` puts them on 5, 6 + and 7, which is how one Redis is shared with something that already uses the first databases. + """ + trimmed = base.rstrip("/") + first = 0 + head, _, tail = trimmed.rpartition("/") + if tail.isdigit(): + trimmed, first = head, int(tail) + return { + "CELERY_BROKER_URL": f"{trimmed}/{first}", + "CELERY_RESULT_BACKEND": f"{trimmed}/{first + 1}", + "CACHE_REDIS_URL": f"{trimmed}/{first + 2}", + } + + +def _native_up(args, context: Context, directory: Path) -> int: + """Run the API and the worker as services on this machine, against an existing Postgres and Redis.""" + services = context.service_manager() + env_path = directory / ".env" + existing = envfile.read(env_path) + record = _record(directory) + configured = bool(record) + + postgres = args.postgres_uri or existing.get("POSTGRES_URI") + redis = args.redis_url or "" + ask = context.interactive and (not configured or args.reconfigure) + if not postgres and ask: + postgres = context.prompter.text("PostgreSQL URL (postgresql://user:password@host:5432/docsgpt)") + if not redis and ask: + redis = context.prompter.text("Redis URL", default="redis://localhost:6379") + if not postgres: + raise DeployError( + "native mode needs a database: pass --postgres-uri postgresql://user:password@host:5432/docsgpt " + "(Redis defaults to redis://localhost:6379)." + ) + + # An install keeps the port it was given: a later `up` with no --port must not move it back to the default. + port = int(args.port or existing.get("DOCSGPT_PORT") or stack.DEFAULT_PORT) + updates: dict[str, Optional[str]] = { + "POSTGRES_URI": postgres, + "API_URL": f"http://127.0.0.1:{port}", + "DOCSGPT_PORT": str(port), + } + if redis or "CELERY_BROKER_URL" not in existing: + updates.update(_redis_urls(redis or "redis://localhost:6379")) + for key in ("INTERNAL_KEY", "JWT_SECRET_KEY"): + if not existing.get(key): + updates[key] = secrets.token_hex(32) + if "VITE_API_STREAMING" not in existing: + updates["VITE_API_STREAMING"] = "true" + provider = _choose_provider(args, context, existing, ask) + if provider: + updates.update(provider) + elif "LLM_PROVIDER" not in existing: + updates.update(stack.provider_settings("docsgpt")) + + directory.mkdir(parents=True, exist_ok=True) + (directory / "logs").mkdir(exist_ok=True) + envfile.update(env_path, updates) + + executable = shutil.which("docsgpt") or sys.argv[0] + print("Applying database migrations ...") + # The child reads the stack's settings, not a .env in whatever directory this was run from. + stack_env = {"DOCSGPT_HOME": str(directory), "DOCSGPT_ENV_FILE": str(env_path)} + if context.run([executable, "migrate"], stack_env) != 0: + raise DeployError("`docsgpt migrate` failed; check the database URL and that the server is reachable.") + + for unit in native.units_for(directory, executable, port, directory): + services.install(unit) + for name in native.SERVICES: + print(f"Starting {name} ...") + services.start(name) + + now = datetime.now(timezone.utc).isoformat(timespec="seconds") + record = record or {"installed_at": now} + record.update(version=context.version, mode="native", updated_at=now) + (directory / stack.RECORD_FILE).write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + + health = f"http://127.0.0.1:{port}/api/health" + print("Waiting for DocsGPT to answer ...") + if not context.wait(health, args.timeout): + print( + f"DocsGPT did not answer at {health} within {args.timeout} seconds. " + "See what happened with `docsgpt logs`.", + file=sys.stderr, + ) + return 1 + + print(f"\nDocsGPT is running at http://localhost:{port}") + print(f"Services: {', '.join(native.SERVICES)} under {services.name}") + print(f"Settings: {env_path}") + print("Manage it with: docsgpt status | logs | down | uninstall") + return 0 + + def up(args, context: Optional[Context] = None) -> int: """Install or update the stack in its directory and start it.""" context = context or Context.default(args) directory = stack.stack_dir(args.dir) + if getattr(args, "native", False) or _mode(directory) == "native": + return _native_up(args, context, directory) env_path = directory / ".env" record_path = directory / stack.RECORD_FILE existing = envfile.read(env_path) @@ -269,6 +395,12 @@ def down(args, context: Optional[Context] = None) -> int: directory = stack.stack_dir(args.dir) if _installed(directory) is None: return 1 + if _mode(directory) == "native": + services = context.service_manager() + # The worker goes first: it talks to the API, not the other way round. + for name in reversed(native.SERVICES): + services.stop(name) + return 0 context.docker.compose(directory, *_EVERY_PROFILE, "down") return 0 @@ -280,6 +412,16 @@ def status(args, context: Optional[Context] = None) -> int: env = _installed(directory) if env is None: return 1 + if _mode(directory) == "native": + services = context.service_manager() + print(f"DocsGPT {_record(directory).get('version', 'unknown')} in {directory} (native, {services.name})") + print(f"Address: {stack.url(env, context.lan_ip())}") + for name in native.SERVICES: + print(f" {name}: {'running' if services.is_running(name) else 'stopped'}") + healthy = context.wait(stack.health_url(env), 0) + print("API: answering" if healthy else "API: not answering (see `docsgpt logs`)") + return 0 if healthy else 1 + tag = env.get("DOCSGPT_IMAGE_TAG", "unknown") + env.get("DOCSGPT_IMAGE_VARIANT", "") print(f"DocsGPT {tag} in {directory}") print(f"Address: {stack.url(env, context.lan_ip())} ({stack.exposure(env)})") @@ -295,6 +437,20 @@ def logs(args, context: Optional[Context] = None) -> int: directory = stack.stack_dir(args.dir) if _installed(directory) is None: return 1 + if _mode(directory) == "native": + logs_dir = directory / "logs" + wanted = args.services or [name.removeprefix("docsgpt-") for name in native.SERVICES] + for service in wanted: + path = logs_dir / f"{service}.log" + print(f"=== {path}") + if path.is_file(): + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + print("\n".join(lines[-args.tail:] if args.tail else lines)) + else: + print("(nothing logged yet)") + if args.follow: + print("Following is not supported in native mode; use `tail -f` on the files above.", file=sys.stderr) + return 0 options = (["--follow"] if args.follow else []) + (["--tail", str(args.tail)] if args.tail else []) return context.docker.compose(directory, "logs", *options, *args.services, check=False).returncode @@ -499,6 +655,20 @@ def uninstall(args, context: Optional[Context] = None) -> int: if not context.prompter.confirm(f"Remove the DocsGPT {what} in {directory}?", default=False): print("Nothing removed.") return 1 + if _mode(directory) == "native": + services = context.service_manager() + for name in reversed(native.SERVICES): + services.remove(name) + print(f"Removed the {', '.join(native.SERVICES)} services.") + if args.purge: + shutil.rmtree(directory) + print(f"Removed {directory}. The database and Redis it used are untouched.") + else: + (directory / stack.RECORD_FILE).unlink(missing_ok=True) + print(f"Settings stay in {directory / '.env'}; the database and Redis are untouched.") + print(f"To remove the docsgpt command too: {_uninstall_hint(context.installer())}") + return 0 + context.docker.compose(directory, *_EVERY_PROFILE, "down", "--remove-orphans", *(["-v"] if args.purge else [])) if args.purge: shutil.rmtree(directory) diff --git a/docsgpt/deploy/native.py b/docsgpt/deploy/native.py new file mode 100644 index 00000000..0ef0cf19 --- /dev/null +++ b/docsgpt/deploy/native.py @@ -0,0 +1,218 @@ +"""Running DocsGPT without Docker, supervised by the system's own service manager. + +The API and the worker each become one service: a launchd agent on macOS, a +systemd user unit on Linux. Both run the ``docsgpt`` command of this +installation with the stack directory as their data home, so a native install +keeps its settings in the same ``.env`` a Docker install would. + +Postgres and Redis are not started here; a native install points at ones that +already run (``--postgres-uri``, ``--redis-url``). +""" + +from __future__ import annotations + +import os +import plistlib +import shlex +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from docsgpt.deploy.docker import DeployError + +API_SERVICE = "docsgpt-api" +WORKER_SERVICE = "docsgpt-worker" +SERVICES = (API_SERVICE, WORKER_SERVICE) +LABEL_PREFIX = "cloud.docsgpt" + + +@dataclass +class Unit: + """One supervised process, in the terms every service manager needs.""" + + name: str + arguments: list[str] + environment: dict[str, str] + working_directory: str + log_file: str + + +def label_for(name: str) -> str: + """The launchd label for a service name (``docsgpt-api`` -> ``cloud.docsgpt.api``).""" + return f"{LABEL_PREFIX}.{name.removeprefix('docsgpt-')}" + + +def launchd_plist(unit: Unit, label: Optional[str] = None) -> str: + """The launchd agent for ``unit``: kept alive, with its output in the stack's log file.""" + body = { + "Label": label or label_for(unit.name), + "ProgramArguments": list(unit.arguments), + "EnvironmentVariables": dict(unit.environment), + "WorkingDirectory": unit.working_directory, + "StandardOutPath": unit.log_file, + "StandardErrorPath": unit.log_file, + "KeepAlive": True, + "RunAtLoad": True, + "ProcessType": "Background", + } + return plistlib.dumps(body).decode("utf-8") + + +def systemd_unit(unit: Unit) -> str: + """The systemd user unit for ``unit``; arguments are quoted, so a path with spaces survives.""" + environment = "\n".join(f'Environment="{key}={value}"' for key, value in sorted(unit.environment.items())) + command = " ".join(shlex.quote(argument) for argument in unit.arguments) + return f"""[Unit] +Description=DocsGPT ({unit.name}) +After=network-online.target + +[Service] +Type=simple +ExecStart={command} +WorkingDirectory={unit.working_directory} +{environment} +Restart=always +RestartSec=5 +StandardOutput=append:{unit.log_file} +StandardError=append:{unit.log_file} + +[Install] +WantedBy=default.target +""" + + +class LaunchdServices: + """launchd user agents in ~/Library/LaunchAgents (macOS).""" + + name = "launchd" + poll_interval = 0.2 + unload_timeout = 15.0 + + def __init__(self, runner=subprocess.run, home: Optional[Path] = None) -> None: + self._run = runner + self.directory = (home or Path.home()) / "Library" / "LaunchAgents" + + def _plist(self, name: str) -> Path: + return self.directory / f"{label_for(name)}.plist" + + def _target(self, name: str) -> str: + return f"gui/{os.getuid()}/{label_for(name)}" + + def _print(self, name: str): + return self._run(["launchctl", "print", self._target(name)], capture_output=True, text=True, check=False) + + def _bootout(self, name: str) -> None: + """Unload the job and wait for it to go. + + ``bootout`` returns before launchd has finished unloading, and bootstrapping a label that is + still on its way out fails with "Bootstrap failed: 5: Input/output error" — which is what a + restart used to hit. Waiting for the label to stop resolving makes stop and start ordered. + """ + self._run(["launchctl", "bootout", self._target(name)], capture_output=True, text=True, check=False) + deadline = time.monotonic() + self.unload_timeout + while self._print(name).returncode == 0: + if time.monotonic() >= deadline: + raise DeployError(f"{label_for(name)} is still loaded after {self.unload_timeout:.0f}s") + time.sleep(self.poll_interval) + + def install(self, unit: Unit) -> None: + self.directory.mkdir(parents=True, exist_ok=True) + self._plist(unit.name).write_text(launchd_plist(unit), encoding="utf-8") + + def start(self, name: str) -> None: + # bootout first: a reinstall must pick up the new plist rather than the loaded one. + self._bootout(name) + result = self._run( + ["launchctl", "bootstrap", f"gui/{os.getuid()}", str(self._plist(name))], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + raise DeployError(f"launchctl could not start {name}: {(result.stderr or '').strip()}") + + def stop(self, name: str) -> None: + self._bootout(name) + + def remove(self, name: str) -> None: + self.stop(name) + self._plist(name).unlink(missing_ok=True) + + def is_running(self, name: str) -> bool: + """A loaded job is not a running one: a crashed service still answers ``launchctl print``.""" + result = self._print(name) + return result.returncode == 0 and "state = running" in (result.stdout or "") + + +class SystemdServices: + """systemd user units in ~/.config/systemd/user (Linux).""" + + name = "systemd" + + def __init__(self, runner=subprocess.run, home: Optional[Path] = None) -> None: + self._run = runner + base = os.environ.get("XDG_CONFIG_HOME") + root = Path(base) if base else (home or Path.home()) / ".config" + self.directory = root / "systemd" / "user" + + def _unit_file(self, name: str) -> Path: + return self.directory / f"{name}.service" + + def _systemctl(self, *args: str, check: bool = False): + result = self._run(["systemctl", "--user", *args], capture_output=True, text=True, check=False) + if check and result.returncode != 0: + raise DeployError(f"systemctl --user {' '.join(args)} failed: {(result.stderr or '').strip()}") + return result + + def install(self, unit: Unit) -> None: + self.directory.mkdir(parents=True, exist_ok=True) + self._unit_file(unit.name).write_text(systemd_unit(unit), encoding="utf-8") + self._systemctl("daemon-reload") + + def start(self, name: str) -> None: + self._systemctl("enable", "--now", f"{name}.service", check=True) + + def stop(self, name: str) -> None: + self._systemctl("stop", f"{name}.service") + + def remove(self, name: str) -> None: + self._systemctl("disable", "--now", f"{name}.service") + self._unit_file(name).unlink(missing_ok=True) + self._systemctl("daemon-reload") + + def is_running(self, name: str) -> bool: + return self._systemctl("is-active", "--quiet", f"{name}.service").returncode == 0 + + +def services_for_platform(platform: str = sys.platform): + """The service manager for this machine, or a DeployError saying what to do instead.""" + if platform == "darwin": + return LaunchdServices() + if platform.startswith("linux"): + return SystemdServices() + raise DeployError( + "native mode supervises services with launchd or systemd, which Windows does not have. " + "Run DocsGPT on Docker with `docsgpt up`, or start `docsgpt api` and `docsgpt worker` yourself." + ) + + +def units_for(stack_directory: Path, executable: str, port: int, home: Path) -> list[Unit]: + """The API and worker services for a native install in ``stack_directory``.""" + environment = {"DOCSGPT_HOME": str(home)} + return [ + Unit( + name=API_SERVICE, + arguments=[executable, "api", "--host", "127.0.0.1", "--port", str(port)], + environment=dict(environment), + working_directory=str(stack_directory), + log_file=str(stack_directory / "logs" / "api.log"), + ), + Unit( + name=WORKER_SERVICE, + arguments=[executable, "worker"], + environment=dict(environment), + working_directory=str(stack_directory), + log_file=str(stack_directory / "logs" / "worker.log"), + ), + ] diff --git a/tests/deploy/test_native.py b/tests/deploy/test_native.py new file mode 100644 index 00000000..63646d98 --- /dev/null +++ b/tests/deploy/test_native.py @@ -0,0 +1,284 @@ +"""`docsgpt up --native`: run the server without Docker, supervised by the system's service manager.""" + +import json +import plistlib +import types + +import pytest + +from docsgpt.deploy import envfile, native +from docsgpt.deploy.docker import DeployError + +from .test_commands import FakePrompter, _context, _run + + +class FakeServices: + """A stand-in for launchd/systemd: records what would be installed and started.""" + + name = "fake" + + def __init__(self, running=()): + self.units = {} + self.started = [] + self.stopped = [] + self.removed = [] + self.running = set(running) + + def install(self, unit): + self.units[unit.name] = unit + + def start(self, name): + self.started.append(name) + self.running.add(name) + + def stop(self, name): + self.stopped.append(name) + self.running.discard(name) + + def remove(self, name): + self.removed.append(name) + self.running.discard(name) + + def is_running(self, name): + return name in self.running + + +def _native_context(services=None, migrated=None, **overrides): + context = _context(**overrides) + context.services = services or FakeServices() + context.run = lambda args, env=None: (migrated.append(args) if migrated is not None else None) or 0 + return context + + +class TestNativeUp: + def test_writes_settings_for_the_given_postgres_and_redis(self, tmp_path): + context = _native_context() + argv = ["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://docsgpt:pw@localhost:5432/docsgpt", + "--redis-url", "redis://localhost:6379"] + assert _run(argv, context) == 0 + + env = envfile.read(tmp_path / ".env") + assert env["POSTGRES_URI"] == "postgresql://docsgpt:pw@localhost:5432/docsgpt" + assert env["CELERY_BROKER_URL"] == "redis://localhost:6379/0" + assert env["CELERY_RESULT_BACKEND"] == "redis://localhost:6379/1" + assert env["CACHE_REDIS_URL"] == "redis://localhost:6379/2" + assert env["INTERNAL_KEY"], "the worker needs it to hand indexes to the API" + assert env["API_URL"] == "http://127.0.0.1:7091" + assert "DOCSGPT_IMAGE_TAG" not in env, "nothing here runs an image" + + def test_records_the_mode_so_the_other_commands_can_tell(self, tmp_path): + assert _run(["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://localhost/docsgpt"], _native_context()) == 0 + record = json.loads((tmp_path / "install.json").read_text()) + assert record["mode"] == "native" + assert record["version"] == "0.21.0" + + def test_migrates_before_starting_the_services(self, tmp_path): + migrated = [] + services = FakeServices() + context = _native_context(services=services, migrated=migrated) + assert _run(["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://localhost/docsgpt"], context) == 0 + assert any("migrate" in " ".join(call) for call in migrated), migrated + assert services.started == ["docsgpt-api", "docsgpt-worker"] + + def test_the_units_run_this_interpreter_with_the_stack_as_its_home(self, tmp_path): + services = FakeServices() + assert _run(["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://localhost/docsgpt"], _native_context(services)) == 0 + api = services.units["docsgpt-api"] + worker = services.units["docsgpt-worker"] + assert api.arguments[1:] == ["api", "--host", "127.0.0.1", "--port", "7091"] + assert worker.arguments[1:] == ["worker"] + for unit in (api, worker): + assert unit.environment["DOCSGPT_HOME"] == str(tmp_path) + assert unit.working_directory == str(tmp_path) + assert unit.log_file.endswith(".log") + + def test_the_migration_targets_the_stack_not_the_working_directory(self, tmp_path): + """`docsgpt up --native` run from a checkout must not migrate the checkout's database.""" + calls = [] + context = _native_context() + context.run = lambda args, env=None: calls.append((args, env)) or 0 + assert _run(["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://localhost/docsgpt"], context) == 0 + arguments, environment = calls[0] + assert arguments[1] == "migrate" + assert environment["DOCSGPT_HOME"] == str(tmp_path) + assert environment["DOCSGPT_ENV_FILE"] == str(tmp_path / ".env") + + def test_a_redis_database_in_the_url_moves_the_three_up(self, tmp_path): + """Sharing a Redis: `/5` puts the broker, the backend and the cache on 5, 6 and 7.""" + argv = ["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://localhost/docsgpt", "--redis-url", "redis://localhost:6379/5"] + assert _run(argv, _native_context()) == 0 + env = envfile.read(tmp_path / ".env") + assert env["CELERY_BROKER_URL"] == "redis://localhost:6379/5" + assert env["CELERY_RESULT_BACKEND"] == "redis://localhost:6379/6" + assert env["CACHE_REDIS_URL"] == "redis://localhost:6379/7" + + def test_a_later_up_keeps_the_port_the_install_was_given(self, tmp_path): + """`docsgpt up` with no --port must not move an install from its port back to 7091.""" + argv = ["up", "--native", "--dir", str(tmp_path), "--yes", "--port", "7099", + "--postgres-uri", "postgresql://localhost/docsgpt"] + assert _run(argv, _native_context()) == 0 + + services = FakeServices() + assert _run(["up", "--dir", str(tmp_path), "--yes"], _native_context(services)) == 0 + env = envfile.read(tmp_path / ".env") + assert env["DOCSGPT_PORT"] == "7099" + assert env["API_URL"] == "http://127.0.0.1:7099" + assert services.units["docsgpt-api"].arguments[-1] == "7099" + + def test_a_database_url_is_required(self, tmp_path): + with pytest.raises(DeployError, match="--postgres-uri"): + _run(["up", "--native", "--dir", str(tmp_path), "--yes"], _native_context()) + + def test_asks_for_the_urls_when_there_is_a_terminal(self, tmp_path): + prompter = FakePrompter(["postgresql://localhost/docsgpt", "redis://localhost:6379", "docsgpt"]) + context = _native_context(interactive=True, prompter=prompter) + assert _run(["up", "--native", "--dir", str(tmp_path)], context) == 0 + env = envfile.read(tmp_path / ".env") + assert env["POSTGRES_URI"] == "postgresql://localhost/docsgpt" + assert env["CELERY_BROKER_URL"] == "redis://localhost:6379/0" + assert env["LLM_PROVIDER"] == "docsgpt" + assert prompter.questions[0].startswith("PostgreSQL URL") + + def test_an_unhealthy_start_points_at_the_logs(self, tmp_path, capsys): + context = _native_context(healthy=False) + argv = ["up", "--native", "--dir", str(tmp_path), "--yes", "--postgres-uri", "postgresql://localhost/docsgpt"] + assert _run(argv, context) == 1 + assert "docsgpt logs" in capsys.readouterr().err + + +class TestNativeLifecycle: + def _installed(self, tmp_path, services): + argv = ["up", "--native", "--dir", str(tmp_path), "--yes", "--postgres-uri", "postgresql://localhost/docsgpt"] + assert _run(argv, _native_context(services)) == 0 + + def test_status_reports_the_native_services(self, tmp_path, capsys): + services = FakeServices() + self._installed(tmp_path, services) + assert _run(["status", "--dir", str(tmp_path)], _native_context(services)) == 0 + out = capsys.readouterr().out + assert "native" in out.lower() + assert "docsgpt-api" in out + + def test_down_stops_them_and_leaves_the_settings(self, tmp_path): + services = FakeServices() + self._installed(tmp_path, services) + assert _run(["down", "--dir", str(tmp_path)], _native_context(services)) == 0 + assert services.stopped == ["docsgpt-worker", "docsgpt-api"], "the worker goes first" + assert (tmp_path / ".env").is_file() + + def test_uninstall_removes_the_services(self, tmp_path): + services = FakeServices() + self._installed(tmp_path, services) + assert _run(["uninstall", "--yes", "--dir", str(tmp_path)], _native_context(services)) == 0 + assert sorted(services.removed) == ["docsgpt-api", "docsgpt-worker"] + + def test_docker_commands_refuse_a_native_install(self, tmp_path, capsys): + self._installed(tmp_path, FakeServices()) + assert _run(["logs", "--dir", str(tmp_path)], _native_context()) == 0, "logs work in both modes" + + +class FakeLaunchctl: + """launchctl, with a bootout that takes `unload_polls` polls to take effect.""" + + def __init__(self, unload_polls=1, state="running"): + self.calls = [] + self.loaded = True + self.unload_polls = unload_polls + self.state = state + + def __call__(self, args, capture_output=False, text=False, check=False): + self.calls.append(args) + verb = args[1] + if verb == "bootout": + self.pending = self.unload_polls + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + if verb == "print": + if not self.loaded: + return types.SimpleNamespace(returncode=113, stdout="", stderr="Could not find service") + if getattr(self, "pending", 0) > 0: + self.pending -= 1 + if self.pending == 0: + self.loaded = False + return types.SimpleNamespace(returncode=0, stdout="\tstate = running\n", stderr="") + return types.SimpleNamespace(returncode=0, stdout=f"\tstate = {self.state}\n", stderr="") + if verb == "bootstrap": + if self.loaded: + return types.SimpleNamespace(returncode=5, stdout="", stderr="Bootstrap failed: 5: Input/output error") + self.loaded = True + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + + +class TestLaunchd: + def _services(self, launchctl, tmp_path): + services = native.LaunchdServices(runner=launchctl, home=tmp_path) + services.poll_interval = 0 + return services + + def test_start_waits_for_the_old_job_to_unload_before_bootstrapping(self, tmp_path): + """launchctl bootout returns early; bootstrapping too soon fails with 'Bootstrap failed: 5'.""" + launchctl = FakeLaunchctl(unload_polls=3) + services = self._services(launchctl, tmp_path) + services.directory.mkdir(parents=True, exist_ok=True) + services.start("docsgpt-api") + verbs = [call[1] for call in launchctl.calls] + assert verbs[0] == "bootout" + assert verbs[-1] == "bootstrap" + assert verbs.count("print") >= 1, "it polled until the label stopped resolving" + assert verbs.index("bootstrap") > max(index for index, verb in enumerate(verbs) if verb == "print") + + def test_stop_returns_only_once_the_job_is_gone(self, tmp_path): + launchctl = FakeLaunchctl(unload_polls=2) + self._services(launchctl, tmp_path).stop("docsgpt-api") + assert not launchctl.loaded + + def test_a_job_that_will_not_unload_is_reported(self, tmp_path): + launchctl = FakeLaunchctl(unload_polls=10_000) + services = self._services(launchctl, tmp_path) + services.unload_timeout = 0 + with pytest.raises(DeployError, match="still loaded"): + services.stop("docsgpt-api") + + def test_a_loaded_but_not_running_job_is_not_running(self, tmp_path): + """`launchctl print` answers for a crashed service too, so status must read its state.""" + services = self._services(FakeLaunchctl(state="not running"), tmp_path) + assert services.is_running("docsgpt-api") is False + assert self._services(FakeLaunchctl(state="running"), tmp_path).is_running("docsgpt-api") is True + + +class TestUnitFiles: + def _unit(self, tmp_path): + return native.Unit( + name="docsgpt-api", + arguments=["/opt/venv/bin/docsgpt", "api", "--port", "7091"], + environment={"DOCSGPT_HOME": str(tmp_path)}, + working_directory=str(tmp_path), + log_file=str(tmp_path / "api.log"), + ) + + def test_launchd_plist_is_valid_and_keeps_the_service_alive(self, tmp_path): + body = native.launchd_plist(self._unit(tmp_path), label="cloud.docsgpt.api") + parsed = plistlib.loads(body.encode()) + assert parsed["Label"] == "cloud.docsgpt.api" + assert parsed["ProgramArguments"][1] == "api" + assert parsed["KeepAlive"] is True + assert parsed["EnvironmentVariables"]["DOCSGPT_HOME"] == str(tmp_path) + assert parsed["StandardErrorPath"].endswith("api.log") + + def test_systemd_unit_restarts_and_carries_the_environment(self, tmp_path): + body = native.systemd_unit(self._unit(tmp_path)) + assert "Restart=always" in body + assert f'Environment="DOCSGPT_HOME={tmp_path}"' in body + assert "ExecStart=/opt/venv/bin/docsgpt api --port 7091" in body + assert "WantedBy=default.target" in body + + def test_an_argument_with_spaces_survives_the_systemd_unit(self, tmp_path): + unit = self._unit(tmp_path) + unit.arguments = ["/opt/my venv/bin/docsgpt", "api"] + assert "ExecStart='/opt/my venv/bin/docsgpt' api" in native.systemd_unit(unit)