diff --git a/.github/styles/config/vocabularies/DocsGPT/accept.txt b/.github/styles/config/vocabularies/DocsGPT/accept.txt index 7605df49..48520858 100644 --- a/.github/styles/config/vocabularies/DocsGPT/accept.txt +++ b/.github/styles/config/vocabularies/DocsGPT/accept.txt @@ -3,9 +3,9 @@ Anthropic's api APIs Atlassian -automations autoescaping Autoescaping +automations backfill backfills bool @@ -21,9 +21,9 @@ diarization Docling docsgpt docstrings +enqueues Entra env -enqueues EOL ESLint feedbacks @@ -35,6 +35,7 @@ hardcoding Idempotency JSONPath kubectl +launchd Lightsail llama_cpp llm @@ -70,6 +71,7 @@ SGLang Shareability Signup Supabase +systemd UIs uncomment URl diff --git a/docs/content/Deploying/Pip-Install.mdx b/docs/content/Deploying/Pip-Install.mdx index 1b946687..04574490 100644 --- a/docs/content/Deploying/Pip-Install.mdx +++ b/docs/content/Deploying/Pip-Install.mdx @@ -99,6 +99,35 @@ Other commands: - `docsgpt verify-offline`: check that a prepared install starts with networking off. - `docsgpt reembed`: re-embed every index after changing `EMBEDDINGS_NAME` (see [Upgrading](/upgrading)). +## Run it as services, without Docker + +`docsgpt up --native` runs the API and the worker as services on the machine itself: launchd agents on macOS, systemd user units on Linux. It does not start PostgreSQL or Redis; point it at ones you already run. + +```bash +docsgpt up --native \ + --postgres-uri postgresql://docsgpt:@localhost:5432/docsgpt \ + --redis-url redis://localhost:6379 +``` + +Without a terminal both flags are required; with one, it asks for them and for the model provider. It writes the same `.env` a Docker install uses (minus the image settings), generates `INTERNAL_KEY` and `JWT_SECRET_KEY` on the first run, applies the migrations, then starts `docsgpt-api` and `docsgpt-worker` and waits for the API to answer. + +One Redis URL covers all three uses: the Celery broker, its result backend and the cache go on databases 0, 1 and 2 of it. Name a database in the URL and the three start there instead, so `redis://localhost:6379/5` puts them on 5, 6 and 7 — that is how you share a Redis that already holds something else. + +The same commands manage it: + +| Command | In native mode | +| --- | --- | +| `docsgpt status` | Which services run, the address, and whether the API answers | +| `docsgpt logs [api\|worker]` | The service log files under `/logs` | +| `docsgpt down` | Stops both services; settings stay | +| `docsgpt uninstall [--purge]` | Removes the services; `--purge` also deletes the stack directory | + +`uninstall` never touches the database or Redis: they were yours to begin with. + + + Windows has neither launchd nor systemd, so native mode is macOS and Linux only. On Windows, run DocsGPT on Docker with `docsgpt up`, or start `docsgpt api` and `docsgpt worker` yourself. + + ## Upgrade ```bash diff --git a/docs/content/changelog.mdx b/docs/content/changelog.mdx index 2c5f9501..107685e7 100644 --- a/docs/content/changelog.mdx +++ b/docs/content/changelog.mdx @@ -13,6 +13,14 @@ request, and [Upgrading](/upgrading) covers the steps an existing deployment has ## Unreleased +### Run DocsGPT without Docker + +`docsgpt up --native` runs the API and the worker as services on the machine itself, launchd on +macOS and systemd user units on Linux, against a PostgreSQL and Redis you already have +(`--postgres-uri`, `--redis-url`). `status`, `logs`, `down` and `uninstall` work on such an install +the same way they do on a Docker one, and never touch the database or Redis. See +[Run it as services, without Docker](/Deploying/Pip-Install#run-it-as-services-without-docker). + ### Back up and restore an install `docsgpt backup` writes a dump of the database and a tar of each data volume into one archive, and diff --git a/docsgpt/cli.py b/docsgpt/cli.py index e5a5af2a..1ca2a187 100644 --- a/docsgpt/cli.py +++ b/docsgpt/cli.py @@ -206,6 +206,10 @@ def _add_deploy_commands(commands) -> None: help="go back to the default image") up.set_defaults(docling=None) up.add_argument("--image-tag", help="image tag to run instead of this package's version, e.g. develop") + up.add_argument("--native", action="store_true", + help="run the API and worker as services on this machine instead of on Docker") + up.add_argument("--postgres-uri", help="native mode: the PostgreSQL DocsGPT should use") + up.add_argument("--redis-url", help="native mode: the Redis for the queue and the cache (default: localhost:6379)") up.add_argument("-y", "--yes", action="store_true", help="ask nothing: use the flags, then the defaults") up.add_argument("--reconfigure", action="store_true", help="ask the setup questions again") up.add_argument("--adopt", action="store_true", help="take over a DocsGPT stack started from another folder") diff --git a/docsgpt/deploy/commands.py b/docsgpt/deploy/commands.py index fb9c5e25..26c287fc 100644 --- a/docsgpt/deploy/commands.py +++ b/docsgpt/deploy/commands.py @@ -6,6 +6,7 @@ import getpass import json import os import re +import secrets import shutil import subprocess import sys @@ -18,7 +19,7 @@ from pathlib import Path from typing import Any, Optional from docsgpt.deploy import backup as backup_format -from docsgpt.deploy import envfile, stack +from docsgpt.deploy import envfile, native, stack from docsgpt.deploy.docker import DeployError, Docker, lan_ip, wait_healthy PROJECT = "docsgpt" @@ -78,9 +79,10 @@ def detect_installer() -> str: return "pip" -def _run_command(args: list[str]) -> int: +def _run_command(args: list[str], env: Optional[Mapping[str, str]] = None) -> int: + """Run ``args``; ``env`` adds to this process's environment rather than replacing it.""" try: - return subprocess.call(args) + return subprocess.call(args, env={**os.environ, **env} if env else None) except FileNotFoundError as exc: raise DeployError(f"{args[0]} is not on PATH") from exc @@ -106,8 +108,9 @@ class Context: wait: Callable[[str, float], bool] = wait_healthy open_browser: Callable[[str], Any] = webbrowser.open installer: Callable[[], str] = detect_installer - run: Callable[[list[str]], int] = _run_command + run: Callable[..., int] = _run_command exec_up: Callable[[list[str]], int] = _exec_command + services: Any = None @classmethod def default(cls, args) -> "Context": @@ -116,10 +119,33 @@ class Context: interactive = sys.stdin.isatty() and not getattr(args, "yes", False) return cls(docker=Docker(), prompter=Prompter(), interactive=interactive, version=__version__) + def service_manager(self): + """The launchd or systemd wrapper, made on first use so Docker installs never touch it.""" + if self.services is None: + self.services = native.services_for_platform() + return self.services + + +def _record(directory: Path) -> dict: + """What ``install.json`` says about this install (empty when there is none).""" + path = directory / stack.RECORD_FILE + if not path.is_file(): + return {} + try: + return json.loads(path.read_text(encoding="utf-8")) + except ValueError: + return {} + + +def _mode(directory: Path) -> str: + """``native`` or ``docker``, from the install record.""" + return "native" if _record(directory).get("mode") == "native" else "docker" + def _installed(directory: Path) -> Optional[dict[str, str]]: """The stack's settings, or None (with a message) when there is no install in ``directory``.""" - if not (directory / stack.COMPOSE_FILE).is_file(): + installed = (directory / stack.COMPOSE_FILE).is_file() or _mode(directory) == "native" + if not installed: print(f"No DocsGPT install in {directory}. Run `docsgpt up` first, or pass --dir.", file=sys.stderr) return None return envfile.read(directory / ".env") @@ -174,10 +200,110 @@ def _choose_provider(args, context: Context, existing: Mapping[str, str], ask: b raise DeployError(str(exc)) from exc +def _redis_urls(base: str) -> dict[str, str]: + """Celery's broker and result backend, and the cache, on three consecutive databases of one Redis. + + They start at database 0, or at the one the URL names: ``redis://host:6379/5`` puts them on 5, 6 + and 7, which is how one Redis is shared with something that already uses the first databases. + """ + trimmed = base.rstrip("/") + first = 0 + head, _, tail = trimmed.rpartition("/") + if tail.isdigit(): + trimmed, first = head, int(tail) + return { + "CELERY_BROKER_URL": f"{trimmed}/{first}", + "CELERY_RESULT_BACKEND": f"{trimmed}/{first + 1}", + "CACHE_REDIS_URL": f"{trimmed}/{first + 2}", + } + + +def _native_up(args, context: Context, directory: Path) -> int: + """Run the API and the worker as services on this machine, against an existing Postgres and Redis.""" + services = context.service_manager() + env_path = directory / ".env" + existing = envfile.read(env_path) + record = _record(directory) + configured = bool(record) + + postgres = args.postgres_uri or existing.get("POSTGRES_URI") + redis = args.redis_url or "" + ask = context.interactive and (not configured or args.reconfigure) + if not postgres and ask: + postgres = context.prompter.text("PostgreSQL URL (postgresql://user:password@host:5432/docsgpt)") + if not redis and ask: + redis = context.prompter.text("Redis URL", default="redis://localhost:6379") + if not postgres: + raise DeployError( + "native mode needs a database: pass --postgres-uri postgresql://user:password@host:5432/docsgpt " + "(Redis defaults to redis://localhost:6379)." + ) + + # An install keeps the port it was given: a later `up` with no --port must not move it back to the default. + port = int(args.port or existing.get("DOCSGPT_PORT") or stack.DEFAULT_PORT) + updates: dict[str, Optional[str]] = { + "POSTGRES_URI": postgres, + "API_URL": f"http://127.0.0.1:{port}", + "DOCSGPT_PORT": str(port), + } + if redis or "CELERY_BROKER_URL" not in existing: + updates.update(_redis_urls(redis or "redis://localhost:6379")) + for key in ("INTERNAL_KEY", "JWT_SECRET_KEY"): + if not existing.get(key): + updates[key] = secrets.token_hex(32) + if "VITE_API_STREAMING" not in existing: + updates["VITE_API_STREAMING"] = "true" + provider = _choose_provider(args, context, existing, ask) + if provider: + updates.update(provider) + elif "LLM_PROVIDER" not in existing: + updates.update(stack.provider_settings("docsgpt")) + + directory.mkdir(parents=True, exist_ok=True) + (directory / "logs").mkdir(exist_ok=True) + envfile.update(env_path, updates) + + executable = shutil.which("docsgpt") or sys.argv[0] + print("Applying database migrations ...") + # The child reads the stack's settings, not a .env in whatever directory this was run from. + stack_env = {"DOCSGPT_HOME": str(directory), "DOCSGPT_ENV_FILE": str(env_path)} + if context.run([executable, "migrate"], stack_env) != 0: + raise DeployError("`docsgpt migrate` failed; check the database URL and that the server is reachable.") + + for unit in native.units_for(directory, executable, port, directory): + services.install(unit) + for name in native.SERVICES: + print(f"Starting {name} ...") + services.start(name) + + now = datetime.now(timezone.utc).isoformat(timespec="seconds") + record = record or {"installed_at": now} + record.update(version=context.version, mode="native", updated_at=now) + (directory / stack.RECORD_FILE).write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + + health = f"http://127.0.0.1:{port}/api/health" + print("Waiting for DocsGPT to answer ...") + if not context.wait(health, args.timeout): + print( + f"DocsGPT did not answer at {health} within {args.timeout} seconds. " + "See what happened with `docsgpt logs`.", + file=sys.stderr, + ) + return 1 + + print(f"\nDocsGPT is running at http://localhost:{port}") + print(f"Services: {', '.join(native.SERVICES)} under {services.name}") + print(f"Settings: {env_path}") + print("Manage it with: docsgpt status | logs | down | uninstall") + return 0 + + def up(args, context: Optional[Context] = None) -> int: """Install or update the stack in its directory and start it.""" context = context or Context.default(args) directory = stack.stack_dir(args.dir) + if getattr(args, "native", False) or _mode(directory) == "native": + return _native_up(args, context, directory) env_path = directory / ".env" record_path = directory / stack.RECORD_FILE existing = envfile.read(env_path) @@ -269,6 +395,12 @@ def down(args, context: Optional[Context] = None) -> int: directory = stack.stack_dir(args.dir) if _installed(directory) is None: return 1 + if _mode(directory) == "native": + services = context.service_manager() + # The worker goes first: it talks to the API, not the other way round. + for name in reversed(native.SERVICES): + services.stop(name) + return 0 context.docker.compose(directory, *_EVERY_PROFILE, "down") return 0 @@ -280,6 +412,16 @@ def status(args, context: Optional[Context] = None) -> int: env = _installed(directory) if env is None: return 1 + if _mode(directory) == "native": + services = context.service_manager() + print(f"DocsGPT {_record(directory).get('version', 'unknown')} in {directory} (native, {services.name})") + print(f"Address: {stack.url(env, context.lan_ip())}") + for name in native.SERVICES: + print(f" {name}: {'running' if services.is_running(name) else 'stopped'}") + healthy = context.wait(stack.health_url(env), 0) + print("API: answering" if healthy else "API: not answering (see `docsgpt logs`)") + return 0 if healthy else 1 + tag = env.get("DOCSGPT_IMAGE_TAG", "unknown") + env.get("DOCSGPT_IMAGE_VARIANT", "") print(f"DocsGPT {tag} in {directory}") print(f"Address: {stack.url(env, context.lan_ip())} ({stack.exposure(env)})") @@ -295,6 +437,20 @@ def logs(args, context: Optional[Context] = None) -> int: directory = stack.stack_dir(args.dir) if _installed(directory) is None: return 1 + if _mode(directory) == "native": + logs_dir = directory / "logs" + wanted = args.services or [name.removeprefix("docsgpt-") for name in native.SERVICES] + for service in wanted: + path = logs_dir / f"{service}.log" + print(f"=== {path}") + if path.is_file(): + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + print("\n".join(lines[-args.tail:] if args.tail else lines)) + else: + print("(nothing logged yet)") + if args.follow: + print("Following is not supported in native mode; use `tail -f` on the files above.", file=sys.stderr) + return 0 options = (["--follow"] if args.follow else []) + (["--tail", str(args.tail)] if args.tail else []) return context.docker.compose(directory, "logs", *options, *args.services, check=False).returncode @@ -499,6 +655,20 @@ def uninstall(args, context: Optional[Context] = None) -> int: if not context.prompter.confirm(f"Remove the DocsGPT {what} in {directory}?", default=False): print("Nothing removed.") return 1 + if _mode(directory) == "native": + services = context.service_manager() + for name in reversed(native.SERVICES): + services.remove(name) + print(f"Removed the {', '.join(native.SERVICES)} services.") + if args.purge: + shutil.rmtree(directory) + print(f"Removed {directory}. The database and Redis it used are untouched.") + else: + (directory / stack.RECORD_FILE).unlink(missing_ok=True) + print(f"Settings stay in {directory / '.env'}; the database and Redis are untouched.") + print(f"To remove the docsgpt command too: {_uninstall_hint(context.installer())}") + return 0 + context.docker.compose(directory, *_EVERY_PROFILE, "down", "--remove-orphans", *(["-v"] if args.purge else [])) if args.purge: shutil.rmtree(directory) diff --git a/docsgpt/deploy/native.py b/docsgpt/deploy/native.py new file mode 100644 index 00000000..0ef0cf19 --- /dev/null +++ b/docsgpt/deploy/native.py @@ -0,0 +1,218 @@ +"""Running DocsGPT without Docker, supervised by the system's own service manager. + +The API and the worker each become one service: a launchd agent on macOS, a +systemd user unit on Linux. Both run the ``docsgpt`` command of this +installation with the stack directory as their data home, so a native install +keeps its settings in the same ``.env`` a Docker install would. + +Postgres and Redis are not started here; a native install points at ones that +already run (``--postgres-uri``, ``--redis-url``). +""" + +from __future__ import annotations + +import os +import plistlib +import shlex +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from docsgpt.deploy.docker import DeployError + +API_SERVICE = "docsgpt-api" +WORKER_SERVICE = "docsgpt-worker" +SERVICES = (API_SERVICE, WORKER_SERVICE) +LABEL_PREFIX = "cloud.docsgpt" + + +@dataclass +class Unit: + """One supervised process, in the terms every service manager needs.""" + + name: str + arguments: list[str] + environment: dict[str, str] + working_directory: str + log_file: str + + +def label_for(name: str) -> str: + """The launchd label for a service name (``docsgpt-api`` -> ``cloud.docsgpt.api``).""" + return f"{LABEL_PREFIX}.{name.removeprefix('docsgpt-')}" + + +def launchd_plist(unit: Unit, label: Optional[str] = None) -> str: + """The launchd agent for ``unit``: kept alive, with its output in the stack's log file.""" + body = { + "Label": label or label_for(unit.name), + "ProgramArguments": list(unit.arguments), + "EnvironmentVariables": dict(unit.environment), + "WorkingDirectory": unit.working_directory, + "StandardOutPath": unit.log_file, + "StandardErrorPath": unit.log_file, + "KeepAlive": True, + "RunAtLoad": True, + "ProcessType": "Background", + } + return plistlib.dumps(body).decode("utf-8") + + +def systemd_unit(unit: Unit) -> str: + """The systemd user unit for ``unit``; arguments are quoted, so a path with spaces survives.""" + environment = "\n".join(f'Environment="{key}={value}"' for key, value in sorted(unit.environment.items())) + command = " ".join(shlex.quote(argument) for argument in unit.arguments) + return f"""[Unit] +Description=DocsGPT ({unit.name}) +After=network-online.target + +[Service] +Type=simple +ExecStart={command} +WorkingDirectory={unit.working_directory} +{environment} +Restart=always +RestartSec=5 +StandardOutput=append:{unit.log_file} +StandardError=append:{unit.log_file} + +[Install] +WantedBy=default.target +""" + + +class LaunchdServices: + """launchd user agents in ~/Library/LaunchAgents (macOS).""" + + name = "launchd" + poll_interval = 0.2 + unload_timeout = 15.0 + + def __init__(self, runner=subprocess.run, home: Optional[Path] = None) -> None: + self._run = runner + self.directory = (home or Path.home()) / "Library" / "LaunchAgents" + + def _plist(self, name: str) -> Path: + return self.directory / f"{label_for(name)}.plist" + + def _target(self, name: str) -> str: + return f"gui/{os.getuid()}/{label_for(name)}" + + def _print(self, name: str): + return self._run(["launchctl", "print", self._target(name)], capture_output=True, text=True, check=False) + + def _bootout(self, name: str) -> None: + """Unload the job and wait for it to go. + + ``bootout`` returns before launchd has finished unloading, and bootstrapping a label that is + still on its way out fails with "Bootstrap failed: 5: Input/output error" — which is what a + restart used to hit. Waiting for the label to stop resolving makes stop and start ordered. + """ + self._run(["launchctl", "bootout", self._target(name)], capture_output=True, text=True, check=False) + deadline = time.monotonic() + self.unload_timeout + while self._print(name).returncode == 0: + if time.monotonic() >= deadline: + raise DeployError(f"{label_for(name)} is still loaded after {self.unload_timeout:.0f}s") + time.sleep(self.poll_interval) + + def install(self, unit: Unit) -> None: + self.directory.mkdir(parents=True, exist_ok=True) + self._plist(unit.name).write_text(launchd_plist(unit), encoding="utf-8") + + def start(self, name: str) -> None: + # bootout first: a reinstall must pick up the new plist rather than the loaded one. + self._bootout(name) + result = self._run( + ["launchctl", "bootstrap", f"gui/{os.getuid()}", str(self._plist(name))], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + raise DeployError(f"launchctl could not start {name}: {(result.stderr or '').strip()}") + + def stop(self, name: str) -> None: + self._bootout(name) + + def remove(self, name: str) -> None: + self.stop(name) + self._plist(name).unlink(missing_ok=True) + + def is_running(self, name: str) -> bool: + """A loaded job is not a running one: a crashed service still answers ``launchctl print``.""" + result = self._print(name) + return result.returncode == 0 and "state = running" in (result.stdout or "") + + +class SystemdServices: + """systemd user units in ~/.config/systemd/user (Linux).""" + + name = "systemd" + + def __init__(self, runner=subprocess.run, home: Optional[Path] = None) -> None: + self._run = runner + base = os.environ.get("XDG_CONFIG_HOME") + root = Path(base) if base else (home or Path.home()) / ".config" + self.directory = root / "systemd" / "user" + + def _unit_file(self, name: str) -> Path: + return self.directory / f"{name}.service" + + def _systemctl(self, *args: str, check: bool = False): + result = self._run(["systemctl", "--user", *args], capture_output=True, text=True, check=False) + if check and result.returncode != 0: + raise DeployError(f"systemctl --user {' '.join(args)} failed: {(result.stderr or '').strip()}") + return result + + def install(self, unit: Unit) -> None: + self.directory.mkdir(parents=True, exist_ok=True) + self._unit_file(unit.name).write_text(systemd_unit(unit), encoding="utf-8") + self._systemctl("daemon-reload") + + def start(self, name: str) -> None: + self._systemctl("enable", "--now", f"{name}.service", check=True) + + def stop(self, name: str) -> None: + self._systemctl("stop", f"{name}.service") + + def remove(self, name: str) -> None: + self._systemctl("disable", "--now", f"{name}.service") + self._unit_file(name).unlink(missing_ok=True) + self._systemctl("daemon-reload") + + def is_running(self, name: str) -> bool: + return self._systemctl("is-active", "--quiet", f"{name}.service").returncode == 0 + + +def services_for_platform(platform: str = sys.platform): + """The service manager for this machine, or a DeployError saying what to do instead.""" + if platform == "darwin": + return LaunchdServices() + if platform.startswith("linux"): + return SystemdServices() + raise DeployError( + "native mode supervises services with launchd or systemd, which Windows does not have. " + "Run DocsGPT on Docker with `docsgpt up`, or start `docsgpt api` and `docsgpt worker` yourself." + ) + + +def units_for(stack_directory: Path, executable: str, port: int, home: Path) -> list[Unit]: + """The API and worker services for a native install in ``stack_directory``.""" + environment = {"DOCSGPT_HOME": str(home)} + return [ + Unit( + name=API_SERVICE, + arguments=[executable, "api", "--host", "127.0.0.1", "--port", str(port)], + environment=dict(environment), + working_directory=str(stack_directory), + log_file=str(stack_directory / "logs" / "api.log"), + ), + Unit( + name=WORKER_SERVICE, + arguments=[executable, "worker"], + environment=dict(environment), + working_directory=str(stack_directory), + log_file=str(stack_directory / "logs" / "worker.log"), + ), + ] diff --git a/tests/deploy/test_native.py b/tests/deploy/test_native.py new file mode 100644 index 00000000..63646d98 --- /dev/null +++ b/tests/deploy/test_native.py @@ -0,0 +1,284 @@ +"""`docsgpt up --native`: run the server without Docker, supervised by the system's service manager.""" + +import json +import plistlib +import types + +import pytest + +from docsgpt.deploy import envfile, native +from docsgpt.deploy.docker import DeployError + +from .test_commands import FakePrompter, _context, _run + + +class FakeServices: + """A stand-in for launchd/systemd: records what would be installed and started.""" + + name = "fake" + + def __init__(self, running=()): + self.units = {} + self.started = [] + self.stopped = [] + self.removed = [] + self.running = set(running) + + def install(self, unit): + self.units[unit.name] = unit + + def start(self, name): + self.started.append(name) + self.running.add(name) + + def stop(self, name): + self.stopped.append(name) + self.running.discard(name) + + def remove(self, name): + self.removed.append(name) + self.running.discard(name) + + def is_running(self, name): + return name in self.running + + +def _native_context(services=None, migrated=None, **overrides): + context = _context(**overrides) + context.services = services or FakeServices() + context.run = lambda args, env=None: (migrated.append(args) if migrated is not None else None) or 0 + return context + + +class TestNativeUp: + def test_writes_settings_for_the_given_postgres_and_redis(self, tmp_path): + context = _native_context() + argv = ["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://docsgpt:pw@localhost:5432/docsgpt", + "--redis-url", "redis://localhost:6379"] + assert _run(argv, context) == 0 + + env = envfile.read(tmp_path / ".env") + assert env["POSTGRES_URI"] == "postgresql://docsgpt:pw@localhost:5432/docsgpt" + assert env["CELERY_BROKER_URL"] == "redis://localhost:6379/0" + assert env["CELERY_RESULT_BACKEND"] == "redis://localhost:6379/1" + assert env["CACHE_REDIS_URL"] == "redis://localhost:6379/2" + assert env["INTERNAL_KEY"], "the worker needs it to hand indexes to the API" + assert env["API_URL"] == "http://127.0.0.1:7091" + assert "DOCSGPT_IMAGE_TAG" not in env, "nothing here runs an image" + + def test_records_the_mode_so_the_other_commands_can_tell(self, tmp_path): + assert _run(["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://localhost/docsgpt"], _native_context()) == 0 + record = json.loads((tmp_path / "install.json").read_text()) + assert record["mode"] == "native" + assert record["version"] == "0.21.0" + + def test_migrates_before_starting_the_services(self, tmp_path): + migrated = [] + services = FakeServices() + context = _native_context(services=services, migrated=migrated) + assert _run(["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://localhost/docsgpt"], context) == 0 + assert any("migrate" in " ".join(call) for call in migrated), migrated + assert services.started == ["docsgpt-api", "docsgpt-worker"] + + def test_the_units_run_this_interpreter_with_the_stack_as_its_home(self, tmp_path): + services = FakeServices() + assert _run(["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://localhost/docsgpt"], _native_context(services)) == 0 + api = services.units["docsgpt-api"] + worker = services.units["docsgpt-worker"] + assert api.arguments[1:] == ["api", "--host", "127.0.0.1", "--port", "7091"] + assert worker.arguments[1:] == ["worker"] + for unit in (api, worker): + assert unit.environment["DOCSGPT_HOME"] == str(tmp_path) + assert unit.working_directory == str(tmp_path) + assert unit.log_file.endswith(".log") + + def test_the_migration_targets_the_stack_not_the_working_directory(self, tmp_path): + """`docsgpt up --native` run from a checkout must not migrate the checkout's database.""" + calls = [] + context = _native_context() + context.run = lambda args, env=None: calls.append((args, env)) or 0 + assert _run(["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://localhost/docsgpt"], context) == 0 + arguments, environment = calls[0] + assert arguments[1] == "migrate" + assert environment["DOCSGPT_HOME"] == str(tmp_path) + assert environment["DOCSGPT_ENV_FILE"] == str(tmp_path / ".env") + + def test_a_redis_database_in_the_url_moves_the_three_up(self, tmp_path): + """Sharing a Redis: `/5` puts the broker, the backend and the cache on 5, 6 and 7.""" + argv = ["up", "--native", "--dir", str(tmp_path), "--yes", + "--postgres-uri", "postgresql://localhost/docsgpt", "--redis-url", "redis://localhost:6379/5"] + assert _run(argv, _native_context()) == 0 + env = envfile.read(tmp_path / ".env") + assert env["CELERY_BROKER_URL"] == "redis://localhost:6379/5" + assert env["CELERY_RESULT_BACKEND"] == "redis://localhost:6379/6" + assert env["CACHE_REDIS_URL"] == "redis://localhost:6379/7" + + def test_a_later_up_keeps_the_port_the_install_was_given(self, tmp_path): + """`docsgpt up` with no --port must not move an install from its port back to 7091.""" + argv = ["up", "--native", "--dir", str(tmp_path), "--yes", "--port", "7099", + "--postgres-uri", "postgresql://localhost/docsgpt"] + assert _run(argv, _native_context()) == 0 + + services = FakeServices() + assert _run(["up", "--dir", str(tmp_path), "--yes"], _native_context(services)) == 0 + env = envfile.read(tmp_path / ".env") + assert env["DOCSGPT_PORT"] == "7099" + assert env["API_URL"] == "http://127.0.0.1:7099" + assert services.units["docsgpt-api"].arguments[-1] == "7099" + + def test_a_database_url_is_required(self, tmp_path): + with pytest.raises(DeployError, match="--postgres-uri"): + _run(["up", "--native", "--dir", str(tmp_path), "--yes"], _native_context()) + + def test_asks_for_the_urls_when_there_is_a_terminal(self, tmp_path): + prompter = FakePrompter(["postgresql://localhost/docsgpt", "redis://localhost:6379", "docsgpt"]) + context = _native_context(interactive=True, prompter=prompter) + assert _run(["up", "--native", "--dir", str(tmp_path)], context) == 0 + env = envfile.read(tmp_path / ".env") + assert env["POSTGRES_URI"] == "postgresql://localhost/docsgpt" + assert env["CELERY_BROKER_URL"] == "redis://localhost:6379/0" + assert env["LLM_PROVIDER"] == "docsgpt" + assert prompter.questions[0].startswith("PostgreSQL URL") + + def test_an_unhealthy_start_points_at_the_logs(self, tmp_path, capsys): + context = _native_context(healthy=False) + argv = ["up", "--native", "--dir", str(tmp_path), "--yes", "--postgres-uri", "postgresql://localhost/docsgpt"] + assert _run(argv, context) == 1 + assert "docsgpt logs" in capsys.readouterr().err + + +class TestNativeLifecycle: + def _installed(self, tmp_path, services): + argv = ["up", "--native", "--dir", str(tmp_path), "--yes", "--postgres-uri", "postgresql://localhost/docsgpt"] + assert _run(argv, _native_context(services)) == 0 + + def test_status_reports_the_native_services(self, tmp_path, capsys): + services = FakeServices() + self._installed(tmp_path, services) + assert _run(["status", "--dir", str(tmp_path)], _native_context(services)) == 0 + out = capsys.readouterr().out + assert "native" in out.lower() + assert "docsgpt-api" in out + + def test_down_stops_them_and_leaves_the_settings(self, tmp_path): + services = FakeServices() + self._installed(tmp_path, services) + assert _run(["down", "--dir", str(tmp_path)], _native_context(services)) == 0 + assert services.stopped == ["docsgpt-worker", "docsgpt-api"], "the worker goes first" + assert (tmp_path / ".env").is_file() + + def test_uninstall_removes_the_services(self, tmp_path): + services = FakeServices() + self._installed(tmp_path, services) + assert _run(["uninstall", "--yes", "--dir", str(tmp_path)], _native_context(services)) == 0 + assert sorted(services.removed) == ["docsgpt-api", "docsgpt-worker"] + + def test_docker_commands_refuse_a_native_install(self, tmp_path, capsys): + self._installed(tmp_path, FakeServices()) + assert _run(["logs", "--dir", str(tmp_path)], _native_context()) == 0, "logs work in both modes" + + +class FakeLaunchctl: + """launchctl, with a bootout that takes `unload_polls` polls to take effect.""" + + def __init__(self, unload_polls=1, state="running"): + self.calls = [] + self.loaded = True + self.unload_polls = unload_polls + self.state = state + + def __call__(self, args, capture_output=False, text=False, check=False): + self.calls.append(args) + verb = args[1] + if verb == "bootout": + self.pending = self.unload_polls + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + if verb == "print": + if not self.loaded: + return types.SimpleNamespace(returncode=113, stdout="", stderr="Could not find service") + if getattr(self, "pending", 0) > 0: + self.pending -= 1 + if self.pending == 0: + self.loaded = False + return types.SimpleNamespace(returncode=0, stdout="\tstate = running\n", stderr="") + return types.SimpleNamespace(returncode=0, stdout=f"\tstate = {self.state}\n", stderr="") + if verb == "bootstrap": + if self.loaded: + return types.SimpleNamespace(returncode=5, stdout="", stderr="Bootstrap failed: 5: Input/output error") + self.loaded = True + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + + +class TestLaunchd: + def _services(self, launchctl, tmp_path): + services = native.LaunchdServices(runner=launchctl, home=tmp_path) + services.poll_interval = 0 + return services + + def test_start_waits_for_the_old_job_to_unload_before_bootstrapping(self, tmp_path): + """launchctl bootout returns early; bootstrapping too soon fails with 'Bootstrap failed: 5'.""" + launchctl = FakeLaunchctl(unload_polls=3) + services = self._services(launchctl, tmp_path) + services.directory.mkdir(parents=True, exist_ok=True) + services.start("docsgpt-api") + verbs = [call[1] for call in launchctl.calls] + assert verbs[0] == "bootout" + assert verbs[-1] == "bootstrap" + assert verbs.count("print") >= 1, "it polled until the label stopped resolving" + assert verbs.index("bootstrap") > max(index for index, verb in enumerate(verbs) if verb == "print") + + def test_stop_returns_only_once_the_job_is_gone(self, tmp_path): + launchctl = FakeLaunchctl(unload_polls=2) + self._services(launchctl, tmp_path).stop("docsgpt-api") + assert not launchctl.loaded + + def test_a_job_that_will_not_unload_is_reported(self, tmp_path): + launchctl = FakeLaunchctl(unload_polls=10_000) + services = self._services(launchctl, tmp_path) + services.unload_timeout = 0 + with pytest.raises(DeployError, match="still loaded"): + services.stop("docsgpt-api") + + def test_a_loaded_but_not_running_job_is_not_running(self, tmp_path): + """`launchctl print` answers for a crashed service too, so status must read its state.""" + services = self._services(FakeLaunchctl(state="not running"), tmp_path) + assert services.is_running("docsgpt-api") is False + assert self._services(FakeLaunchctl(state="running"), tmp_path).is_running("docsgpt-api") is True + + +class TestUnitFiles: + def _unit(self, tmp_path): + return native.Unit( + name="docsgpt-api", + arguments=["/opt/venv/bin/docsgpt", "api", "--port", "7091"], + environment={"DOCSGPT_HOME": str(tmp_path)}, + working_directory=str(tmp_path), + log_file=str(tmp_path / "api.log"), + ) + + def test_launchd_plist_is_valid_and_keeps_the_service_alive(self, tmp_path): + body = native.launchd_plist(self._unit(tmp_path), label="cloud.docsgpt.api") + parsed = plistlib.loads(body.encode()) + assert parsed["Label"] == "cloud.docsgpt.api" + assert parsed["ProgramArguments"][1] == "api" + assert parsed["KeepAlive"] is True + assert parsed["EnvironmentVariables"]["DOCSGPT_HOME"] == str(tmp_path) + assert parsed["StandardErrorPath"].endswith("api.log") + + def test_systemd_unit_restarts_and_carries_the_environment(self, tmp_path): + body = native.systemd_unit(self._unit(tmp_path)) + assert "Restart=always" in body + assert f'Environment="DOCSGPT_HOME={tmp_path}"' in body + assert "ExecStart=/opt/venv/bin/docsgpt api --port 7091" in body + assert "WantedBy=default.target" in body + + def test_an_argument_with_spaces_survives_the_systemd_unit(self, tmp_path): + unit = self._unit(tmp_path) + unit.arguments = ["/opt/my venv/bin/docsgpt", "api"] + assert "ExecStart='/opt/my venv/bin/docsgpt' api" in native.systemd_unit(unit)