feat(usage): persist LLM call latency

The wrappers in docsgpt/usage.py already measured how long each call took --
for the llm_gen_finished / llm_stream_finished log lines -- and then threw the
number away. Nothing in the schema recorded it, so an operator could see what
an instance spent but never how slow it was.

Adds token_usage.duration_ms and token_usage.ttft_ms, and threads the
measurements the wrappers already take into the insert. The stream wrapper now
also stamps the moment of the first yielded chunk.

Both columns are nullable, and deliberately so: ttft_ms is NULL for a
non-streaming call and for a stream that failed before yielding anything, and
every row written before this migration is NULL too. A 0 there would drag a
p50 toward an instant first token that never happened.
This commit is contained in:
arc53-machine committed 2026-09-22 10:28:20 +01:00
1 parent e30631a060
commit b17f40a993
6 files changed
+261 -5

No files matched your search

+5
View File
@@ -250,6 +250,11 @@ token_usage_table = Table(
# Added in ``0033_quotas``. USD cost of the call at write time; 0 for
# unpriced and bring-your-own models.
Column("cost", Numeric(12, 8), nullable=False, server_default="0"),
# Added in ``0035_token_usage_latency``. Wall-clock for the call, and time
# to the first streamed chunk. ``ttft_ms`` is NULL for non-streaming calls
# and for streams that failed before yielding -- "no first token", not 0.
Column("duration_ms", Integer),
Column("ttft_ms", Integer),
)
user_logs_table = Table(
@@ -38,6 +38,8 @@ class TokenUsageRepository:
cached_tokens: Optional[int] = None,
cache_write_tokens: Optional[int] = None,
cost: float = 0.0,
duration_ms: Optional[int] = None,
ttft_ms: Optional[int] = None,
) -> None:
# Attribution guard: the ``token_usage_attribution_chk`` CHECK
# constraint requires at least one of ``user_id`` / ``api_key``
@@ -64,6 +66,7 @@ class TokenUsageRepository:
user_id, api_key, agent_id,
prompt_tokens, generated_tokens,
cached_tokens, cache_write_tokens, cost,
duration_ms, ttft_ms,
source, request_id, model_id, timestamp
)
VALUES (
@@ -71,6 +74,7 @@ class TokenUsageRepository:
CAST(:agent_id AS uuid),
:prompt_tokens, :generated_tokens,
:cached_tokens, :cache_write_tokens, :cost,
:duration_ms, :ttft_ms,
:source, :request_id, :model_id, COALESCE(:timestamp, now())
)
"""
@@ -84,6 +88,8 @@ class TokenUsageRepository:
"cached_tokens": cached_tokens,
"cache_write_tokens": cache_write_tokens,
"cost": cost,
"duration_ms": duration_ms,
"ttft_ms": ttft_ms,
"source": source,
"request_id": request_id,
"model_id": model_id,