diff --git a/docs/content/Deploying/Settings-Reference.mdx b/docs/content/Deploying/Settings-Reference.mdx index b08af68a..4f44bb47 100644 --- a/docs/content/Deploying/Settings-Reference.mdx +++ b/docs/content/Deploying/Settings-Reference.mdx @@ -34,7 +34,13 @@ Signing key for session tokens and other signed capabilities. Required on every Type `str`, default `default-docsgpt-encryption-key`. -Key used to encrypt stored credentials such as tool and connector secrets. +Key used to encrypt stored credentials such as tool and connector secrets. Set your own value before connecting services on a multi-user install; the default is public. + +### `ENCRYPTION_SECRET_KEY_PREVIOUS` + +Type `str`, default unset. + +Previous ENCRYPTION_SECRET_KEY, tried when a stored credential was encrypted with it. Set it while rotating the key, run `docsgpt connectors reencrypt`, then remove it. ### `INTERNAL_KEY` diff --git a/docsgpt/app.py b/docsgpt/app.py index 5a45ab31..1e018bf8 100644 --- a/docsgpt/app.py +++ b/docsgpt/app.py @@ -184,6 +184,27 @@ if settings.AUTH_TYPE == "simple_jwt": print(f"Generated Simple JWT Token: {SIMPLE_JWT_TOKEN}") +def _warn_default_encryption_key() -> None: + """Say when stored credentials are sealed with the public default key.""" + from docsgpt.security.encryption import is_default_encryption_key + + if not is_default_encryption_key(): + return + if settings.AUTH_TYPE: + logging.getLogger(__name__).warning( + "ENCRYPTION_SECRET_KEY is the public default: connecting services is refused until you set your " + "own value (then run `docsgpt connectors reencrypt`)." + ) + else: + logging.getLogger(__name__).warning( + "ENCRYPTION_SECRET_KEY is the public default. Stored connector credentials are only as safe as " + "that key; set your own value before exposing this install." + ) + + +_warn_default_encryption_key() + + @app.route("/") def home(): if request.remote_addr in ("0.0.0.0", "127.0.0.1", "localhost", "172.18.0.1"): diff --git a/docsgpt/core/settings/auth.py b/docsgpt/core/settings/auth.py index 706a3339..9c6deacb 100644 --- a/docsgpt/core/settings/auth.py +++ b/docsgpt/core/settings/auth.py @@ -29,7 +29,17 @@ class AuthSettings(SettingsGroup): ) ENCRYPTION_SECRET_KEY: str = Field( default="default-docsgpt-encryption-key", - description="Key used to encrypt stored credentials such as tool and connector secrets.", + description=( + "Key used to encrypt stored credentials such as tool and connector secrets. Set your own value before " + "connecting services on a multi-user install; the default is public." + ), + ) + ENCRYPTION_SECRET_KEY_PREVIOUS: Optional[str] = Field( + default=None, + description=( + "Previous ENCRYPTION_SECRET_KEY, tried when a stored credential was encrypted with it. Set it while " + "rotating the key, run `docsgpt connectors reencrypt`, then remove it." + ), ) INTERNAL_KEY: Optional[str] = Field( default=None, description="Internal API key for worker-to-backend authentication." diff --git a/docsgpt/security/encryption.py b/docsgpt/security/encryption.py index aa77069d..5793fc7c 100644 --- a/docsgpt/security/encryption.py +++ b/docsgpt/security/encryption.py @@ -1,11 +1,17 @@ import base64 +import functools +import hashlib +import hmac import json import logging import os +from typing import Optional from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.ciphers import algorithms, Cipher, modes +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.hkdf import HKDF from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC from docsgpt.core.settings import settings @@ -86,3 +92,134 @@ def _pad_data(data: bytes) -> bytes: def _unpad_data(data: bytes) -> bytes: padding_len = data[-1] return data[:-padding_len] + + +# --------------------------------------------------------------------------- +# Envelope v2: connection credentials +# --------------------------------------------------------------------------- +# +# ``v2::`` +# +# AES-256-GCM, so a tampered blob fails to decrypt instead of returning +# garbage. The master key is derived once per process from +# ENCRYPTION_SECRET_KEY (PBKDF2, cached); each record gets its own key from +# HKDF(master, salt, owner id), which keeps the v1 owner binding without +# paying 100k PBKDF2 iterations on every token read in the worker. The owner +# id is also the GCM associated data, so a blob copied onto another user's +# row does not decrypt. ``key_id`` names the master key, so a blob written +# under ENCRYPTION_SECRET_KEY_PREVIOUS is still readable during a rotation. + +_V2_PREFIX = "v2" +_V2_MASTER_SALT = b"docsgpt-credentials-v2" +_V2_ITERATIONS = 200_000 +_V2_SALT_BYTES = 16 +_V2_NONCE_BYTES = 12 +DEFAULT_ENCRYPTION_KEY = "default-docsgpt-encryption-key" + + +class CredentialDecryptionError(Exception): + """A stored credential could not be decrypted (wrong key, tampering, bad format).""" + + +@functools.lru_cache(maxsize=8) +def _master_key(secret: str) -> bytes: + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=32, + salt=_V2_MASTER_SALT, + iterations=_V2_ITERATIONS, + backend=default_backend(), + ) + return kdf.derive(secret.encode()) + + +def _key_id(master: bytes) -> str: + return hmac.new(master, b"docsgpt-key-id", hashlib.sha256).hexdigest()[:8] + + +def _record_key(master: bytes, owner_id: str, salt: bytes) -> bytes: + return HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + info=b"docsgpt-v2|" + owner_id.encode(), + backend=default_backend(), + ).derive(master) + + +def _candidate_keys() -> dict[str, bytes]: + """Master keys this process can decrypt with, by key id (current first).""" + keys: dict[str, bytes] = {} + for secret in (settings.ENCRYPTION_SECRET_KEY, settings.ENCRYPTION_SECRET_KEY_PREVIOUS): + if secret: + master = _master_key(secret) + keys.setdefault(_key_id(master), master) + return keys + + +def current_key_id() -> str: + """Key id of ENCRYPTION_SECRET_KEY, as written into new v2 blobs.""" + return _key_id(_master_key(settings.ENCRYPTION_SECRET_KEY)) + + +def is_default_encryption_key() -> bool: + """Whether ENCRYPTION_SECRET_KEY is still the public default.""" + return settings.ENCRYPTION_SECRET_KEY == DEFAULT_ENCRYPTION_KEY + + +def encrypt_json(data: dict, owner_id: str) -> str: + """Encrypt ``data`` for ``owner_id`` into a v2 envelope. + + Args: + data: JSON-serialisable credentials. + owner_id: The user the credentials belong to; decryption needs it. + + Returns: + The ``v2::`` string. + """ + master = _master_key(settings.ENCRYPTION_SECRET_KEY) + key_id = _key_id(master) + salt = os.urandom(_V2_SALT_BYTES) + nonce = os.urandom(_V2_NONCE_BYTES) + key = _record_key(master, owner_id, salt) + plaintext = json.dumps(data, separators=(",", ":")).encode() + ciphertext = AESGCM(key).encrypt(nonce, plaintext, owner_id.encode()) + payload = base64.b64encode(salt + nonce + ciphertext).decode() + return f"{_V2_PREFIX}:{key_id}:{payload}" + + +def envelope_key_id(blob: str) -> Optional[str]: + """The key id a v2 blob was written with, or None for anything else.""" + parts = (blob or "").split(":", 2) + if len(parts) != 3 or parts[0] != _V2_PREFIX: + return None + return parts[1] + + +def decrypt_json(blob: str, owner_id: str) -> dict: + """Decrypt a v2 envelope written for ``owner_id``. + + Raises: + CredentialDecryptionError: The blob is malformed, was written with a + key this process does not have, belongs to another owner, or was + tampered with. + """ + key_id = envelope_key_id(blob) + if key_id is None: + raise CredentialDecryptionError("Not a v2 credential envelope") + master = _candidate_keys().get(key_id) + if master is None: + raise CredentialDecryptionError("Credential was encrypted with an unknown key") + try: + raw = base64.b64decode(blob.split(":", 2)[2].encode(), validate=True) + salt = raw[:_V2_SALT_BYTES] + nonce = raw[_V2_SALT_BYTES:_V2_SALT_BYTES + _V2_NONCE_BYTES] + ciphertext = raw[_V2_SALT_BYTES + _V2_NONCE_BYTES:] + key = _record_key(master, owner_id, salt) + plaintext = AESGCM(key).decrypt(nonce, ciphertext, owner_id.encode()) + data = json.loads(plaintext.decode()) + except Exception as exc: + raise CredentialDecryptionError("Credential could not be decrypted") from exc + if not isinstance(data, dict): + raise CredentialDecryptionError("Credential payload is not an object") + return data diff --git a/docsgpt/storage/db/redaction.py b/docsgpt/storage/db/redaction.py index cb1e4754..483c7564 100644 --- a/docsgpt/storage/db/redaction.py +++ b/docsgpt/storage/db/redaction.py @@ -33,13 +33,17 @@ _SECRET_SUBSTRINGS = ( "credential", "authorization", "bearer", + # Connection secrets: an OAuth token_info blob, MCP token and client + # registration dicts (``client_secret`` is covered by ``secret``). + "token_info", + "client_info", ) def is_secret_key(key: str) -> bool: """True when ``key`` names a credential that must not be persisted/returned.""" k = key.lower() - if k == "token": + if k in ("token", "tokens"): return True return any(s in k for s in _SECRET_SUBSTRINGS) diff --git a/tests/security/test_encryption_v2.py b/tests/security/test_encryption_v2.py new file mode 100644 index 00000000..1551c45b --- /dev/null +++ b/tests/security/test_encryption_v2.py @@ -0,0 +1,94 @@ +"""Tests for the v2 credential envelope.""" + +from __future__ import annotations + +import base64 + +import pytest + +from docsgpt.security import encryption as enc + + +@pytest.fixture(autouse=True) +def _keys(monkeypatch): + from docsgpt.core.settings import settings + + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", "current-key-for-tests") + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY_PREVIOUS", None) + yield + + +class TestEnvelope: + def test_round_trip(self): + blob = enc.encrypt_json({"access_token": "at", "n": 1}, "alice") + assert blob.startswith("v2:") + assert enc.decrypt_json(blob, "alice") == {"access_token": "at", "n": 1} + + def test_plaintext_never_in_blob(self): + blob = enc.encrypt_json({"refresh_token": "very-secret-refresh"}, "alice") + assert "very-secret-refresh" not in blob + assert "very-secret-refresh".encode() not in base64.b64decode(blob.split(":", 2)[2]) + + def test_each_blob_is_unique(self): + assert enc.encrypt_json({"a": 1}, "alice") != enc.encrypt_json({"a": 1}, "alice") + + def test_bound_to_owner(self): + blob = enc.encrypt_json({"a": 1}, "alice") + with pytest.raises(enc.CredentialDecryptionError): + enc.decrypt_json(blob, "bob") + + def test_tampering_is_detected(self): + blob = enc.encrypt_json({"a": 1}, "alice") + prefix, key_id, payload = blob.split(":", 2) + raw = bytearray(base64.b64decode(payload)) + raw[-1] ^= 0x01 + tampered = f"{prefix}:{key_id}:{base64.b64encode(bytes(raw)).decode()}" + with pytest.raises(enc.CredentialDecryptionError): + enc.decrypt_json(tampered, "alice") + + @pytest.mark.parametrize("blob", ["", "v1:abc", "v2:only-two", "v2:deadbeef:%%%not-base64"]) + def test_malformed(self, blob): + with pytest.raises(enc.CredentialDecryptionError): + enc.decrypt_json(blob, "alice") + + def test_key_id_names_the_key(self): + blob = enc.encrypt_json({"a": 1}, "alice") + assert enc.envelope_key_id(blob) == enc.current_key_id() + assert enc.envelope_key_id("not an envelope") is None + + +class TestRotation: + def test_unknown_key_fails(self, monkeypatch): + from docsgpt.core.settings import settings + + blob = enc.encrypt_json({"a": 1}, "alice") + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", "a-new-key") + with pytest.raises(enc.CredentialDecryptionError): + enc.decrypt_json(blob, "alice") + + def test_previous_key_still_decrypts(self, monkeypatch): + from docsgpt.core.settings import settings + + blob = enc.encrypt_json({"a": 1}, "alice") + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", "a-new-key") + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY_PREVIOUS", "current-key-for-tests") + assert enc.decrypt_json(blob, "alice") == {"a": 1} + assert enc.envelope_key_id(enc.encrypt_json({"a": 1}, "alice")) == enc.current_key_id() + assert enc.envelope_key_id(blob) != enc.current_key_id() + + +class TestDefaultKey: + def test_detects_default(self, monkeypatch): + from docsgpt.core.settings import settings + + monkeypatch.setattr(settings, "ENCRYPTION_SECRET_KEY", enc.DEFAULT_ENCRYPTION_KEY) + assert enc.is_default_encryption_key() + + def test_custom_key(self): + assert not enc.is_default_encryption_key() + + +class TestLegacyV1Unchanged: + def test_v1_round_trip_still_works(self): + blob = enc.encrypt_credentials({"token": "t"}, "alice") + assert enc.decrypt_credentials(blob, "alice") == {"token": "t"}