mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 12:16:20 +00:00
fix: sanitise on request
This commit is contained in:
1 parent
9fb59785b2
commit
cd6f40471a
5 files changed
+157
-28
No files matched your search
@@ -12,6 +12,7 @@ from application.api.user.base import (
|
||||
generate_hourly_range,
|
||||
generate_minute_range,
|
||||
)
|
||||
from application.storage.db.redaction import redact_secrets
|
||||
from application.storage.db.repositories.agents import AgentsRepository
|
||||
from application.storage.db.repositories.token_usage import TokenUsageRepository
|
||||
from application.storage.db.session import db_readonly
|
||||
@@ -1059,7 +1060,10 @@ class GetUserLogs(Resource):
|
||||
item.update(
|
||||
{
|
||||
"endpoint": payload.get("endpoint"),
|
||||
"stacks": payload.get("stacks"),
|
||||
# Redact on the way out too: rows written
|
||||
# before write-time redaction still carry the
|
||||
# reflected provider/user secrets in ``stacks``.
|
||||
"stacks": redact_secrets(payload.get("stacks")),
|
||||
}
|
||||
)
|
||||
elif m["event_type"] == "workflow":
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Secret redaction for reflected ``stack_logs`` data.
|
||||
|
||||
``stacks`` is built by reflecting every public attribute of runtime
|
||||
objects (the ``llm`` component carries the deployment provider
|
||||
``api_key`` and the caller's ``user_api_key``). The unified-logs endpoint
|
||||
returns ``stacks`` to the client, so credentials must be scrubbed — at
|
||||
write time for new rows, and at read time for rows written before
|
||||
redaction existed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
REDACTED = "[REDACTED]"
|
||||
|
||||
# Substrings marking a key as a credential. Compound token-count fields
|
||||
# (``prompt_tokens`` / ``generated_tokens`` / ``token_budget`` / ...) are
|
||||
# intentionally not matched: secret token forms are spelled out
|
||||
# (``access_token`` etc.) and a bare ``token`` is handled separately.
|
||||
_SECRET_SUBSTRINGS = (
|
||||
"api_key",
|
||||
"apikey",
|
||||
"api_token",
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"auth_token",
|
||||
"id_token",
|
||||
"session_token",
|
||||
"secret",
|
||||
"password",
|
||||
"passwd",
|
||||
"passphrase",
|
||||
"private_key",
|
||||
"credential",
|
||||
"authorization",
|
||||
"bearer",
|
||||
)
|
||||
|
||||
|
||||
def is_secret_key(key: str) -> bool:
|
||||
"""True when ``key`` names a credential that must not be persisted/returned."""
|
||||
k = key.lower()
|
||||
if k == "token":
|
||||
return True
|
||||
return any(s in k for s in _SECRET_SUBSTRINGS)
|
||||
|
||||
|
||||
def redact_secrets(obj):
|
||||
"""Recursively replace secret-keyed values with ``[REDACTED]``.
|
||||
|
||||
Walks dicts and lists; leaf scalars pass through unchanged. ``None``
|
||||
returns ``None`` so callers can redact an optional payload directly.
|
||||
"""
|
||||
if isinstance(obj, dict):
|
||||
return {
|
||||
k: (
|
||||
REDACTED
|
||||
if isinstance(k, str) and is_secret_key(k)
|
||||
else redact_secrets(v)
|
||||
)
|
||||
for k, v in obj.items()
|
||||
}
|
||||
if isinstance(obj, list):
|
||||
return [redact_secrets(v) for v in obj]
|
||||
return obj
|
||||
@@ -13,36 +13,11 @@ import json
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from application.storage.db.redaction import redact_secrets
|
||||
from application.storage.db.serialization import PGNativeJSONEncoder
|
||||
|
||||
from sqlalchemy import Connection, text
|
||||
|
||||
_REDACTED = "[REDACTED]"
|
||||
|
||||
|
||||
def _is_secret_key(key: str) -> bool:
|
||||
k = key.lower()
|
||||
return k.endswith("api_key") or "secret" in k or "password" in k
|
||||
|
||||
|
||||
def _redact_secrets(obj):
|
||||
"""Strip provider/agent keys from ``stacks`` before they persist.
|
||||
|
||||
The ``llm`` stack component is built from every public attribute of
|
||||
the LLM instance, which includes ``api_key`` (the deployment provider
|
||||
secret) and ``user_api_key``. The unified-logs endpoint returns
|
||||
``stacks`` verbatim, so these must never reach the table.
|
||||
"""
|
||||
if isinstance(obj, dict):
|
||||
return {
|
||||
k: (_REDACTED if isinstance(k, str) and _is_secret_key(k)
|
||||
else _redact_secrets(v))
|
||||
for k, v in obj.items()
|
||||
}
|
||||
if isinstance(obj, list):
|
||||
return [_redact_secrets(v) for v in obj]
|
||||
return obj
|
||||
|
||||
|
||||
class StackLogsRepository:
|
||||
"""Postgres-backed replacement for Mongo ``stack_logs`` collection."""
|
||||
@@ -81,7 +56,7 @@ class StackLogsRepository:
|
||||
"api_key": api_key,
|
||||
"query": query,
|
||||
"stacks": json.dumps(
|
||||
_redact_secrets(stacks or []), cls=PGNativeJSONEncoder
|
||||
redact_secrets(stacks or []), cls=PGNativeJSONEncoder
|
||||
),
|
||||
"timestamp": timestamp,
|
||||
},
|
||||
|
||||
@@ -731,6 +731,49 @@ class TestUnifiedLogsBranches:
|
||||
assert response.status_code == 200
|
||||
assert [log["level"] for log in response.json["logs"]] == ["error"]
|
||||
|
||||
def test_system_stacks_redacted_on_read(self, app, pg_conn):
|
||||
# A row written before write-time redaction existed still holds
|
||||
# the reflected provider secret in ``stacks``. The endpoint must
|
||||
# scrub it on the way out, not hand it back to the client.
|
||||
import json
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
pg_conn.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO stack_logs
|
||||
(activity_id, endpoint, level, user_id, stacks)
|
||||
VALUES
|
||||
('act-leak', 'stream', 'error', 'u-redact',
|
||||
CAST(:stacks AS jsonb))
|
||||
"""
|
||||
),
|
||||
{
|
||||
"stacks": json.dumps(
|
||||
[
|
||||
{
|
||||
"component": "llm",
|
||||
"data": {
|
||||
"api_key": "sk-deployment-secret",
|
||||
"user_api_key": "agent-key",
|
||||
"model": "gpt-x",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
response = _post_logs(app, pg_conn, "u-redact", {"event_type": "system"})
|
||||
assert response.status_code == 200
|
||||
logs = response.json["logs"]
|
||||
assert len(logs) == 1
|
||||
data = logs[0]["stacks"][0]["data"]
|
||||
assert data["api_key"] == "[REDACTED]"
|
||||
assert data["user_api_key"] == "[REDACTED]"
|
||||
assert data["model"] == "gpt-x"
|
||||
|
||||
def test_non_numeric_page_returns_400(self, app, pg_conn):
|
||||
response = _post_logs(app, pg_conn, "u", {"page": "abc"})
|
||||
assert response.status_code == 400
|
||||
|
||||
@@ -83,3 +83,46 @@ class TestInsert:
|
||||
# Non-secret fields (incl. token *counts*) are untouched.
|
||||
assert data["model"] == "gpt-x"
|
||||
assert data["prompt_tokens"] == 42
|
||||
|
||||
def test_redacts_broadened_secret_keys(self, pg_conn):
|
||||
# Credentials beyond ``*api_key`` — OAuth tokens, bearer/auth
|
||||
# headers, private keys — must also be scrubbed, while token
|
||||
# *count* fields are preserved.
|
||||
repo = _repo(pg_conn)
|
||||
repo.insert(
|
||||
activity_id="act-broad",
|
||||
level="error",
|
||||
stacks=[
|
||||
{
|
||||
"component": "tool",
|
||||
"data": {
|
||||
"access_token": "ya29.secret",
|
||||
"api_token": "r8_secret",
|
||||
"token": "hf_secret",
|
||||
"Authorization": "Bearer xyz",
|
||||
"private_key": "-----BEGIN-----",
|
||||
"client_secret": "cs_secret",
|
||||
"prompt_tokens": 10,
|
||||
"generated_tokens": 5,
|
||||
"token_budget": 1000,
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
row = pg_conn.execute(
|
||||
text("SELECT stacks FROM stack_logs WHERE activity_id = 'act-broad'")
|
||||
).fetchone()
|
||||
data = dict(row._mapping)["stacks"][0]["data"]
|
||||
for key in (
|
||||
"access_token",
|
||||
"api_token",
|
||||
"token",
|
||||
"Authorization",
|
||||
"private_key",
|
||||
"client_secret",
|
||||
):
|
||||
assert data[key] == "[REDACTED]", key
|
||||
# Token *counts* / budgets are not credentials and must survive.
|
||||
assert data["prompt_tokens"] == 10
|
||||
assert data["generated_tokens"] == 5
|
||||
assert data["token_budget"] == 1000
|
||||
Reference in new issue
Block a user