From 7ca33b2b72c1b761b810148bfcb98fdf99ea5bee Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 25 Apr 2026 13:38:03 +0100 Subject: [PATCH] feat: OTEL --- application/core/logging_config.py | 58 ++++++++++-- application/requirements.txt | 12 ++- docs/content/Deploying/Observability.mdx | 111 +++++++++++++++++++++++ docs/content/Deploying/_meta.js | 4 + tests/core/test_logging_config.py | 104 +++++++++++++++++++++ 5 files changed, 279 insertions(+), 10 deletions(-) create mode 100644 docs/content/Deploying/Observability.mdx create mode 100644 tests/core/test_logging_config.py diff --git a/application/core/logging_config.py b/application/core/logging_config.py index e693cb91..7f1339ac 100644 --- a/application/core/logging_config.py +++ b/application/core/logging_config.py @@ -1,11 +1,45 @@ +import logging +import os from logging.config import dictConfig -def setup_logging(): + +def _otlp_logs_enabled() -> bool: + """Return True when the user has opted in to OTLP log export. + + Gated by the standard OTEL env vars so no project-specific knob is needed: + set ``OTEL_LOGS_EXPORTER=otlp`` (and leave ``OTEL_SDK_DISABLED`` unset or + false) to flip it on. When false, ``setup_logging`` keeps its original + console-only behavior. + """ + exporter = os.getenv("OTEL_LOGS_EXPORTER", "").strip().lower() + disabled = os.getenv("OTEL_SDK_DISABLED", "false").strip().lower() == "true" + return exporter == "otlp" and not disabled + + +def setup_logging() -> None: + """Configure the root logger with a stdout console handler. + + When OTLP log export is enabled, ``opentelemetry-instrument`` attaches a + ``LoggingHandler`` to the root logger before this function runs. The + ``dictConfig`` call below replaces ``root.handlers`` with the console + handler, which would silently drop the OTEL handler. To make OTLP log + export work without forcing every contributor to opt in, snapshot the + OTEL handlers up front and re-attach them after ``dictConfig``. + """ + preserved_handlers: list[logging.Handler] = [] + if _otlp_logs_enabled(): + preserved_handlers = [ + h + for h in logging.getLogger().handlers + if h.__class__.__module__.startswith("opentelemetry") + ] + dictConfig({ - 'version': 1, - 'formatters': { - 'default': { - 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s', + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "default": { + "format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s", } }, "handlers": { @@ -15,8 +49,14 @@ def setup_logging(): "formatter": "default", } }, - 'root': { - 'level': 'INFO', - 'handlers': ['console'], + "root": { + "level": "INFO", + "handlers": ["console"], }, - }) \ No newline at end of file + }) + + if preserved_handlers: + root = logging.getLogger() + for handler in preserved_handlers: + if handler not in root.handlers: + root.addHandler(handler) diff --git a/application/requirements.txt b/application/requirements.txt index 57597811..c1258b1e 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -51,6 +51,16 @@ networkx==3.6.1 numpy==2.4.4 openai==2.32.0 openapi3-parser==1.1.22 +opentelemetry-distro>=0.50b0,<1 +opentelemetry-exporter-otlp>=1.29.0,<2 +opentelemetry-instrumentation-celery>=0.50b0,<1 +opentelemetry-instrumentation-flask>=0.50b0,<1 +opentelemetry-instrumentation-logging>=0.50b0,<1 +opentelemetry-instrumentation-psycopg>=0.50b0,<1 +opentelemetry-instrumentation-redis>=0.50b0,<1 +opentelemetry-instrumentation-requests>=0.50b0,<1 +opentelemetry-instrumentation-sqlalchemy>=0.50b0,<1 +opentelemetry-instrumentation-starlette>=0.50b0,<1 orjson==3.11.7 packaging==26.0 pandas==3.0.2 @@ -60,7 +70,7 @@ pdf2image>=1.17.0 pillow portalocker>=2.7.0,<4.0.0 prompt-toolkit==3.0.52 -protobuf==7.34.1 +protobuf==6.33.6 psycopg[binary,pool]>=3.1,<4 py==1.11.0 pydantic diff --git a/docs/content/Deploying/Observability.mdx b/docs/content/Deploying/Observability.mdx new file mode 100644 index 00000000..451fafe6 --- /dev/null +++ b/docs/content/Deploying/Observability.mdx @@ -0,0 +1,111 @@ +--- +title: Observability +description: Send traces, metrics, and logs from DocsGPT to any OpenTelemetry-compatible backend (Axiom, Honeycomb, Grafana, Datadog, Jaeger, etc.). +--- + +import { Callout } from 'nextra/components' + +# Observability + +DocsGPT bundles the OpenTelemetry SDK and auto-instrumentation packages +in `application/requirements.txt` — they install with the rest of the +backend deps. Telemetry is **off by default**; opt in by prefixing the +launch command with `opentelemetry-instrument` and setting OTLP env +vars. + +Auto-instrumentation covers Flask, Starlette, Celery, SQLAlchemy, +psycopg, Redis, requests, and Python logging. LLM/retriever calls are +not captured at this layer — see *Going further* below. + +## Enabling + +Set these env vars in your `.env` (or compose `environment:` block): + +```bash +OTEL_SDK_DISABLED=false +OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +OTEL_EXPORTER_OTLP_ENDPOINT=https://your-collector.example.com +OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20 +OTEL_TRACES_EXPORTER=otlp +OTEL_METRICS_EXPORTER=otlp +OTEL_LOGS_EXPORTER=otlp +OTEL_PYTHON_LOG_CORRELATION=true +OTEL_RESOURCE_ATTRIBUTES=service.name=docsgpt-backend,deployment.environment=prod +``` + +Then prefix the process command with `opentelemetry-instrument`. The +simplest way is a compose override (no image rebuild): + +```yaml +# deployment/docker-compose.override.yaml +services: + backend: + command: > + opentelemetry-instrument gunicorn -w 1 -k uvicorn_worker.UvicornWorker + --bind 0.0.0.0:7091 --config application/gunicorn_conf.py + application.asgi:asgi_app + environment: + - OTEL_SERVICE_NAME=docsgpt-backend + worker: + command: opentelemetry-instrument celery -A application.app.celery worker -l INFO -B + environment: + - OTEL_SERVICE_NAME=docsgpt-celery-worker +``` + +For local dev, prepend `dotenv run --` so the `OTEL_*` vars from `.env` +reach `opentelemetry-instrument` before it boots the SDK: + +```bash +dotenv run -- opentelemetry-instrument flask --app application/app.py run --port=7091 +dotenv run -- opentelemetry-instrument celery -A application.app.celery worker -l INFO --pool=solo +``` + + + + Logs are exported in-process when `OTEL_LOGS_EXPORTER=otlp` is set — + `application/core/logging_config.py` detects the flag and preserves + the OTEL log handler. Without it, `logging` writes only to stdout. + + +## Backend examples + +### Axiom + +```bash +OTEL_EXPORTER_OTLP_ENDPOINT=https://api.axiom.co +OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20xaat-XXXX,X-Axiom-Dataset=docsgpt +OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +``` + +`%20` is the URL-encoded space between `Bearer` and the token. Create +the dataset in the Axiom UI before sending. + +### Self-hosted OTLP collector / Jaeger / Tempo + +```bash +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +OTEL_EXPORTER_OTLP_PROTOCOL=grpc +``` + +### Honeycomb / Grafana Cloud / Datadog + +Each vendor publishes a single-line `OTEL_EXPORTER_OTLP_ENDPOINT` plus +`OTEL_EXPORTER_OTLP_HEADERS` recipe — drop them in alongside the +service-name override. + +## Caveats + +- The Dockerfile uses `gunicorn -w 1`. If you raise worker count, move + SDK init into a `post_worker_init` hook to avoid one-thread-per-process + exporter contention. +- `asgi.py` wraps Flask in Starlette's `WSGIMiddleware`. Both + instrumentors are installed, so each request produces a Starlette + span enclosing a Flask span. Drop + `opentelemetry-instrumentation-flask` from `requirements.txt` if the + duplication is noisy. +- OTEL packages add ~50 MB to the image. They install on every build — + the runtime cost is zero unless you set `opentelemetry-instrument` on + the command and set the OTLP env vars. +- The OTEL exporter ecosystem currently caps `protobuf` at `<7`, so the + backend runs on protobuf 6.x. This will catch up in a future OTEL + release. \ No newline at end of file diff --git a/docs/content/Deploying/_meta.js b/docs/content/Deploying/_meta.js index 71527117..19130bde 100644 --- a/docs/content/Deploying/_meta.js +++ b/docs/content/Deploying/_meta.js @@ -23,6 +23,10 @@ export default { "title": "🐘 PostgreSQL for User Data", "href": "/Deploying/Postgres-Migration" }, + "Observability": { + "title": "🔭 Observability", + "href": "/Deploying/Observability" + }, "Amazon-Lightsail": { "title": "Hosting DocsGPT on Amazon Lightsail", "href": "/Deploying/Amazon-Lightsail", diff --git a/tests/core/test_logging_config.py b/tests/core/test_logging_config.py new file mode 100644 index 00000000..cff033c5 --- /dev/null +++ b/tests/core/test_logging_config.py @@ -0,0 +1,104 @@ +"""Tests for setup_logging — in particular the OTEL log-handler hand-off. + +`opentelemetry-instrument` attaches an OTEL `LoggingHandler` to the root +logger before our module-level `setup_logging()` runs in `application/app.py`. +The default `dictConfig` call replaces `root.handlers`, which would silently +drop the OTEL handler. setup_logging snapshots and re-attaches OTEL handlers +when OTLP log export is enabled. +""" + +from __future__ import annotations + +import logging +import sys +import types + +import pytest + +from application.core.logging_config import setup_logging + + +@pytest.fixture(autouse=True) +def _reset_root_logger(): + """Snapshot/restore the root logger so tests don't leak handlers.""" + root = logging.getLogger() + saved_handlers = list(root.handlers) + saved_level = root.level + yield + root.handlers = saved_handlers + root.setLevel(saved_level) + + +def _make_fake_otel_handler() -> logging.Handler: + """Build a Handler whose class lives in a module starting with 'opentelemetry'. + + Mirrors how the real `opentelemetry.sdk._logs.LoggingHandler` would be + detected without needing the OTEL SDK installed in the test env. + """ + fake_module = types.ModuleType("opentelemetry.fake_sdk._logs") + sys.modules.setdefault(fake_module.__name__, fake_module) + + class FakeOtelHandler(logging.Handler): + pass + + FakeOtelHandler.__module__ = fake_module.__name__ + return FakeOtelHandler() + + +@pytest.mark.unit +class TestSetupLogging: + + def test_default_keeps_only_console_handler(self, monkeypatch): + monkeypatch.delenv("OTEL_LOGS_EXPORTER", raising=False) + monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False) + logging.getLogger().handlers = [] + + setup_logging() + + handlers = logging.getLogger().handlers + assert len(handlers) == 1 + assert isinstance(handlers[0], logging.StreamHandler) + + def test_preserves_otel_handler_when_otlp_logs_enabled(self, monkeypatch): + monkeypatch.setenv("OTEL_LOGS_EXPORTER", "otlp") + monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False) + + otel_handler = _make_fake_otel_handler() + logging.getLogger().handlers = [otel_handler] + + setup_logging() + + handlers = logging.getLogger().handlers + assert otel_handler in handlers, ( + "OTEL handler must survive setup_logging when OTLP log export is on" + ) + assert any( + isinstance(h, logging.StreamHandler) and not isinstance(h, type(otel_handler)) + for h in handlers + ), "console handler should still be installed alongside the OTEL handler" + + def test_does_not_preserve_when_sdk_disabled(self, monkeypatch): + monkeypatch.setenv("OTEL_LOGS_EXPORTER", "otlp") + monkeypatch.setenv("OTEL_SDK_DISABLED", "true") + + otel_handler = _make_fake_otel_handler() + logging.getLogger().handlers = [otel_handler] + + setup_logging() + + handlers = logging.getLogger().handlers + assert otel_handler not in handlers, ( + "When OTEL_SDK_DISABLED=true the handler should not be preserved" + ) + + def test_does_not_preserve_when_logs_exporter_unset(self, monkeypatch): + monkeypatch.delenv("OTEL_LOGS_EXPORTER", raising=False) + monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False) + + otel_handler = _make_fake_otel_handler() + logging.getLogger().handlers = [otel_handler] + + setup_logging() + + handlers = logging.getLogger().handlers + assert otel_handler not in handlers