feat: OTEL

application/core/logging_config.py

@@ -1,11 +1,45 @@
+import logging
+import os
 from logging.config import dictConfig
 
-def setup_logging():
+
+def _otlp_logs_enabled() -> bool:
+    """Return True when the user has opted in to OTLP log export.
+
+    Gated by the standard OTEL env vars so no project-specific knob is needed:
+    set ``OTEL_LOGS_EXPORTER=otlp`` (and leave ``OTEL_SDK_DISABLED`` unset or
+    false) to flip it on. When false, ``setup_logging`` keeps its original
+    console-only behavior.
+    """
+    exporter = os.getenv("OTEL_LOGS_EXPORTER", "").strip().lower()
+    disabled = os.getenv("OTEL_SDK_DISABLED", "false").strip().lower() == "true"
+    return exporter == "otlp" and not disabled
+
+
+def setup_logging() -> None:
+    """Configure the root logger with a stdout console handler.
+
+    When OTLP log export is enabled, ``opentelemetry-instrument`` attaches a
+    ``LoggingHandler`` to the root logger before this function runs. The
+    ``dictConfig`` call below replaces ``root.handlers`` with the console
+    handler, which would silently drop the OTEL handler. To make OTLP log
+    export work without forcing every contributor to opt in, snapshot the
+    OTEL handlers up front and re-attach them after ``dictConfig``.
+    """
+    preserved_handlers: list[logging.Handler] = []
+    if _otlp_logs_enabled():
+        preserved_handlers = [
+            h
+            for h in logging.getLogger().handlers
+            if h.__class__.__module__.startswith("opentelemetry")
+        ]
+
     dictConfig({
-        'version': 1,
-        'formatters': {
-            'default': {
-                'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "default": {
+                "format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s",
             }
         },
         "handlers": {
@@ -15,8 +49,14 @@ def setup_logging():
                 "formatter": "default",
             }
         },
-        'root': {
-            'level': 'INFO',
-            'handlers': ['console'],
+        "root": {
+            "level": "INFO",
+            "handlers": ["console"],
         },
-    })
+    })
+
+    if preserved_handlers:
+        root = logging.getLogger()
+        for handler in preserved_handlers:
+            if handler not in root.handlers:
+                root.addHandler(handler)

application/requirements.txt

@@ -51,6 +51,16 @@ networkx==3.6.1
 numpy==2.4.4
 openai==2.32.0
 openapi3-parser==1.1.22
+opentelemetry-distro>=0.50b0,<1
+opentelemetry-exporter-otlp>=1.29.0,<2
+opentelemetry-instrumentation-celery>=0.50b0,<1
+opentelemetry-instrumentation-flask>=0.50b0,<1
+opentelemetry-instrumentation-logging>=0.50b0,<1
+opentelemetry-instrumentation-psycopg>=0.50b0,<1
+opentelemetry-instrumentation-redis>=0.50b0,<1
+opentelemetry-instrumentation-requests>=0.50b0,<1
+opentelemetry-instrumentation-sqlalchemy>=0.50b0,<1
+opentelemetry-instrumentation-starlette>=0.50b0,<1
 orjson==3.11.7
 packaging==26.0
 pandas==3.0.2
@@ -60,7 +70,7 @@ pdf2image>=1.17.0
 pillow
 portalocker>=2.7.0,<4.0.0
 prompt-toolkit==3.0.52
-protobuf==7.34.1
+protobuf==6.33.6
 psycopg[binary,pool]>=3.1,<4
 py==1.11.0
 pydantic

docs/content/Deploying/Observability.mdx

@@ -0,0 +1,111 @@
+---
+title: Observability
+description: Send traces, metrics, and logs from DocsGPT to any OpenTelemetry-compatible backend (Axiom, Honeycomb, Grafana, Datadog, Jaeger, etc.).
+---
+
+import { Callout } from 'nextra/components'
+
+# Observability
+
+DocsGPT bundles the OpenTelemetry SDK and auto-instrumentation packages
+in `application/requirements.txt` — they install with the rest of the
+backend deps. Telemetry is **off by default**; opt in by prefixing the
+launch command with `opentelemetry-instrument` and setting OTLP env
+vars.
+
+Auto-instrumentation covers Flask, Starlette, Celery, SQLAlchemy,
+psycopg, Redis, requests, and Python logging. LLM/retriever calls are
+not captured at this layer — see *Going further* below.
+
+## Enabling
+
+Set these env vars in your `.env` (or compose `environment:` block):
+
+```bash
+OTEL_SDK_DISABLED=false
+OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+OTEL_EXPORTER_OTLP_ENDPOINT=https://your-collector.example.com
+OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20<token>
+OTEL_TRACES_EXPORTER=otlp
+OTEL_METRICS_EXPORTER=otlp
+OTEL_LOGS_EXPORTER=otlp
+OTEL_PYTHON_LOG_CORRELATION=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=docsgpt-backend,deployment.environment=prod
+```
+
+Then prefix the process command with `opentelemetry-instrument`. The
+simplest way is a compose override (no image rebuild):
+
+```yaml
+# deployment/docker-compose.override.yaml
+services:
+  backend:
+    command: >
+      opentelemetry-instrument gunicorn -w 1 -k uvicorn_worker.UvicornWorker
+      --bind 0.0.0.0:7091 --config application/gunicorn_conf.py
+      application.asgi:asgi_app
+    environment:
+      - OTEL_SERVICE_NAME=docsgpt-backend
+  worker:
+    command: opentelemetry-instrument celery -A application.app.celery worker -l INFO -B
+    environment:
+      - OTEL_SERVICE_NAME=docsgpt-celery-worker
+```
+
+For local dev, prepend `dotenv run --` so the `OTEL_*` vars from `.env`
+reach `opentelemetry-instrument` before it boots the SDK:
+
+```bash
+dotenv run -- opentelemetry-instrument flask --app application/app.py run --port=7091
+dotenv run -- opentelemetry-instrument celery -A application.app.celery worker -l INFO --pool=solo
+```
+
+
+<Callout type="info" emoji="ℹ️">
+  Logs are exported in-process when `OTEL_LOGS_EXPORTER=otlp` is set —
+  `application/core/logging_config.py` detects the flag and preserves
+  the OTEL log handler. Without it, `logging` writes only to stdout.
+</Callout>
+
+## Backend examples
+
+### Axiom
+
+```bash
+OTEL_EXPORTER_OTLP_ENDPOINT=https://api.axiom.co
+OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer%20xaat-XXXX,X-Axiom-Dataset=docsgpt
+OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+```
+
+`%20` is the URL-encoded space between `Bearer` and the token. Create
+the dataset in the Axiom UI before sending.
+
+### Self-hosted OTLP collector / Jaeger / Tempo
+
+```bash
+OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+OTEL_EXPORTER_OTLP_PROTOCOL=grpc
+```
+
+### Honeycomb / Grafana Cloud / Datadog
+
+Each vendor publishes a single-line `OTEL_EXPORTER_OTLP_ENDPOINT` plus
+`OTEL_EXPORTER_OTLP_HEADERS` recipe — drop them in alongside the
+service-name override.
+
+## Caveats
+
+- The Dockerfile uses `gunicorn -w 1`. If you raise worker count, move
+  SDK init into a `post_worker_init` hook to avoid one-thread-per-process
+  exporter contention.
+- `asgi.py` wraps Flask in Starlette's `WSGIMiddleware`. Both
+  instrumentors are installed, so each request produces a Starlette
+  span enclosing a Flask span. Drop
+  `opentelemetry-instrumentation-flask` from `requirements.txt` if the
+  duplication is noisy.
+- OTEL packages add ~50 MB to the image. They install on every build —
+  the runtime cost is zero unless you set `opentelemetry-instrument` on
+  the command and set the OTLP env vars.
+- The OTEL exporter ecosystem currently caps `protobuf` at `<7`, so the
+  backend runs on protobuf 6.x. This will catch up in a future OTEL
+  release.

docs/content/Deploying/_meta.js

@@ -23,6 +23,10 @@ export default {
     "title": "🐘 PostgreSQL for User Data",
     "href": "/Deploying/Postgres-Migration"
   },
+  "Observability": {
+    "title": "🔭 Observability",
+    "href": "/Deploying/Observability"
+  },
   "Amazon-Lightsail": {
     "title": "Hosting DocsGPT on Amazon Lightsail",
     "href": "/Deploying/Amazon-Lightsail",

tests/core/test_logging_config.py

@@ -0,0 +1,104 @@
+"""Tests for setup_logging — in particular the OTEL log-handler hand-off.
+
+`opentelemetry-instrument` attaches an OTEL `LoggingHandler` to the root
+logger before our module-level `setup_logging()` runs in `application/app.py`.
+The default `dictConfig` call replaces `root.handlers`, which would silently
+drop the OTEL handler. setup_logging snapshots and re-attaches OTEL handlers
+when OTLP log export is enabled.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+import types
+
+import pytest
+
+from application.core.logging_config import setup_logging
+
+
+@pytest.fixture(autouse=True)
+def _reset_root_logger():
+    """Snapshot/restore the root logger so tests don't leak handlers."""
+    root = logging.getLogger()
+    saved_handlers = list(root.handlers)
+    saved_level = root.level
+    yield
+    root.handlers = saved_handlers
+    root.setLevel(saved_level)
+
+
+def _make_fake_otel_handler() -> logging.Handler:
+    """Build a Handler whose class lives in a module starting with 'opentelemetry'.
+
+    Mirrors how the real `opentelemetry.sdk._logs.LoggingHandler` would be
+    detected without needing the OTEL SDK installed in the test env.
+    """
+    fake_module = types.ModuleType("opentelemetry.fake_sdk._logs")
+    sys.modules.setdefault(fake_module.__name__, fake_module)
+
+    class FakeOtelHandler(logging.Handler):
+        pass
+
+    FakeOtelHandler.__module__ = fake_module.__name__
+    return FakeOtelHandler()
+
+
+@pytest.mark.unit
+class TestSetupLogging:
+
+    def test_default_keeps_only_console_handler(self, monkeypatch):
+        monkeypatch.delenv("OTEL_LOGS_EXPORTER", raising=False)
+        monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)
+        logging.getLogger().handlers = []
+
+        setup_logging()
+
+        handlers = logging.getLogger().handlers
+        assert len(handlers) == 1
+        assert isinstance(handlers[0], logging.StreamHandler)
+
+    def test_preserves_otel_handler_when_otlp_logs_enabled(self, monkeypatch):
+        monkeypatch.setenv("OTEL_LOGS_EXPORTER", "otlp")
+        monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)
+
+        otel_handler = _make_fake_otel_handler()
+        logging.getLogger().handlers = [otel_handler]
+
+        setup_logging()
+
+        handlers = logging.getLogger().handlers
+        assert otel_handler in handlers, (
+            "OTEL handler must survive setup_logging when OTLP log export is on"
+        )
+        assert any(
+            isinstance(h, logging.StreamHandler) and not isinstance(h, type(otel_handler))
+            for h in handlers
+        ), "console handler should still be installed alongside the OTEL handler"
+
+    def test_does_not_preserve_when_sdk_disabled(self, monkeypatch):
+        monkeypatch.setenv("OTEL_LOGS_EXPORTER", "otlp")
+        monkeypatch.setenv("OTEL_SDK_DISABLED", "true")
+
+        otel_handler = _make_fake_otel_handler()
+        logging.getLogger().handlers = [otel_handler]
+
+        setup_logging()
+
+        handlers = logging.getLogger().handlers
+        assert otel_handler not in handlers, (
+            "When OTEL_SDK_DISABLED=true the handler should not be preserved"
+        )
+
+    def test_does_not_preserve_when_logs_exporter_unset(self, monkeypatch):
+        monkeypatch.delenv("OTEL_LOGS_EXPORTER", raising=False)
+        monkeypatch.delenv("OTEL_SDK_DISABLED", raising=False)
+
+        otel_handler = _make_fake_otel_handler()
+        logging.getLogger().handlers = [otel_handler]
+
+        setup_logging()
+
+        handlers = logging.getLogger().handlers
+        assert otel_handler not in handlers