feat(admin): cost, group-by and latency in usage

Three gaps in one view.

Cost: token_usage.cost has been written on every call since quotas landed, and
tokens_by_model already returned it, but bucketed_totals and top_token_users
selected tokens only. An admin could set a USD quota in /admin/quotas and had
no way to see the spend it was capping. Buckets and top users now carry cost,
the endpoint reports a window total and a per-model split, and the chart takes
a Tokens/Cost toggle with a currency axis.

Group-by: the endpoint has supported group_by=model|agent|source from the
start and the UI only ever sent bucket=day. The selector is now wired, and a
grouped series pads missing buckets so a model that was idle on Tuesday plots
a zero instead of shifting its whole row one bar left.

Latency: surfaced from the columns the previous commit added, as p50/p95 with
a median time-to-first-token underneath. Unmeasured rows are excluded rather
than counted as zero, and the card says so when nothing was measured.

Also surfaces the prompt-cache hit rate, computed only over rows whose
provider reported a cache breakdown -- NULL means "not reported", and folding
those in as 0% would understate it.

Per-user drill-down: GET /api/admin/users/<id>/usage returns a daily series
plus splits by model and by flow, reachable from the top-users table and from
the user detail dialog. The detail dialog previously showed one tokens_30d
number, which answers neither "what is this person costing" nor "what is
driving it".
This commit is contained in:
arc53-machine committed 2026-09-22 10:33:56 +01:00
1 parent b17f40a993
commit a56f9df122
14 files changed
+1019 -61

No files matched your search

+76 -3
View File
@@ -228,14 +228,37 @@ class TestUserLifecycle:
@pytest.mark.unit
class TestUsageAndAudit:
def test_usage(self, client):
@staticmethod
def _usage_repos():
usage = Mock()
usage.bucketed_totals.return_value = [
{"bucket": "2026-06-14", "prompt_tokens": 5, "generated_tokens": 3}
{
"bucket": "2026-06-14",
"prompt_tokens": 5,
"generated_tokens": 3,
"cost": 0.25,
"cached_tokens": None,
}
]
usage.sum_tokens_in_range.return_value = 8
usage.tokens_by_model.return_value = [
{"model_id": "gpt-x", "tokens": 8, "cost": 0.25}
]
stats = Mock()
stats.top_token_users.return_value = [{"user_id": "a", "tokens": 8}]
stats.top_token_users.return_value = [
{"user_id": "a", "tokens": 8, "cost": 0.25}
]
stats.latency_summary.return_value = {
"samples": 2,
"p50_ms": 120,
"p95_ms": 400,
"ttft_samples": 2,
"ttft_p50_ms": 30,
}
return usage, stats
def test_usage(self, client):
usage, stats = self._usage_repos()
with _admin(
TokenUsageRepository=Mock(return_value=usage),
AdminStatsRepository=Mock(return_value=stats),
@@ -247,6 +270,56 @@ class TestUsageAndAudit:
assert len(data["series"]) == 1
assert data["top_users"][0]["user_id"] == "a"
def test_usage_reports_spend_and_latency(self, client):
"""Cost quotas were settable long before spend was visible."""
usage, stats = self._usage_repos()
with _admin(
TokenUsageRepository=Mock(return_value=usage),
AdminStatsRepository=Mock(return_value=stats),
):
data = _body(client.get("/api/admin/usage"))
assert data["total_cost"] == 0.25
assert data["series"][0]["cost"] == 0.25
assert data["by_model"][0]["model_id"] == "gpt-x"
assert data["latency"]["p95_ms"] == 400
assert data["top_users"][0]["cost"] == 0.25
def test_usage_group_by_reaches_the_repository(self, client):
usage, stats = self._usage_repos()
with _admin(
TokenUsageRepository=Mock(return_value=usage),
AdminStatsRepository=Mock(return_value=stats),
):
assert client.get("/api/admin/usage?group_by=model").status_code == 200
assert usage.bucketed_totals.call_args.kwargs["group_by"] == "model"
def test_usage_rejects_an_unknown_grouping(self, client):
with _admin():
assert client.get("/api/admin/usage?group_by=nonsense").status_code == 400
def test_per_user_usage_drilldown(self, client):
usage, stats = self._usage_repos()
stats.user_usage_breakdown.return_value = {
"totals": {"tokens": 8, "cost": 0.25, "calls": 1},
"by_model": [{"key": "gpt-x", "tokens": 8, "cost": 0.25}],
"by_source": [{"key": "agent_stream", "tokens": 8, "cost": 0.25}],
}
with _admin(
TokenUsageRepository=Mock(return_value=usage),
AdminStatsRepository=Mock(return_value=stats),
):
data = _body(client.get("/api/admin/users/u1/usage?days=7"))
assert data["totals"]["cost"] == 0.25
assert data["by_source"][0]["key"] == "agent_stream"
assert len(data["series"]) == 1
assert usage.bucketed_totals.call_args.kwargs["user_id"] == "u1"
def test_per_user_usage_requires_admin(self, client):
with patch("docsgpt.app.handle_auth", return_value={"sub": "u1"}), patch(
"docsgpt.app.resolve_roles", return_value=["user"]
):
assert client.get("/api/admin/users/u1/usage").status_code == 403
def test_usage_invalid_bucket(self, client):
with _admin():
resp = client.get("/api/admin/usage?bucket=year")