mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-15 04:17:56 +00:00
feat(admin): cost, group-by and latency in usage
Three gaps in one view. Cost: token_usage.cost has been written on every call since quotas landed, and tokens_by_model already returned it, but bucketed_totals and top_token_users selected tokens only. An admin could set a USD quota in /admin/quotas and had no way to see the spend it was capping. Buckets and top users now carry cost, the endpoint reports a window total and a per-model split, and the chart takes a Tokens/Cost toggle with a currency axis. Group-by: the endpoint has supported group_by=model|agent|source from the start and the UI only ever sent bucket=day. The selector is now wired, and a grouped series pads missing buckets so a model that was idle on Tuesday plots a zero instead of shifting its whole row one bar left. Latency: surfaced from the columns the previous commit added, as p50/p95 with a median time-to-first-token underneath. Unmeasured rows are excluded rather than counted as zero, and the card says so when nothing was measured. Also surfaces the prompt-cache hit rate, computed only over rows whose provider reported a cache breakdown -- NULL means "not reported", and folding those in as 0% would understate it. Per-user drill-down: GET /api/admin/users/<id>/usage returns a daily series plus splits by model and by flow, reachable from the top-users table and from the user detail dialog. The detail dialog previously showed one tokens_30d number, which answers neither "what is this person costing" nor "what is driving it".
This commit is contained in:
1 parent
b17f40a993
commit
a56f9df122
14 files changed
+1019
-61
No files matched your search
@@ -228,14 +228,37 @@ class TestUserLifecycle:
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestUsageAndAudit:
|
||||
def test_usage(self, client):
|
||||
@staticmethod
|
||||
def _usage_repos():
|
||||
usage = Mock()
|
||||
usage.bucketed_totals.return_value = [
|
||||
{"bucket": "2026-06-14", "prompt_tokens": 5, "generated_tokens": 3}
|
||||
{
|
||||
"bucket": "2026-06-14",
|
||||
"prompt_tokens": 5,
|
||||
"generated_tokens": 3,
|
||||
"cost": 0.25,
|
||||
"cached_tokens": None,
|
||||
}
|
||||
]
|
||||
usage.sum_tokens_in_range.return_value = 8
|
||||
usage.tokens_by_model.return_value = [
|
||||
{"model_id": "gpt-x", "tokens": 8, "cost": 0.25}
|
||||
]
|
||||
stats = Mock()
|
||||
stats.top_token_users.return_value = [{"user_id": "a", "tokens": 8}]
|
||||
stats.top_token_users.return_value = [
|
||||
{"user_id": "a", "tokens": 8, "cost": 0.25}
|
||||
]
|
||||
stats.latency_summary.return_value = {
|
||||
"samples": 2,
|
||||
"p50_ms": 120,
|
||||
"p95_ms": 400,
|
||||
"ttft_samples": 2,
|
||||
"ttft_p50_ms": 30,
|
||||
}
|
||||
return usage, stats
|
||||
|
||||
def test_usage(self, client):
|
||||
usage, stats = self._usage_repos()
|
||||
with _admin(
|
||||
TokenUsageRepository=Mock(return_value=usage),
|
||||
AdminStatsRepository=Mock(return_value=stats),
|
||||
@@ -247,6 +270,56 @@ class TestUsageAndAudit:
|
||||
assert len(data["series"]) == 1
|
||||
assert data["top_users"][0]["user_id"] == "a"
|
||||
|
||||
def test_usage_reports_spend_and_latency(self, client):
|
||||
"""Cost quotas were settable long before spend was visible."""
|
||||
usage, stats = self._usage_repos()
|
||||
with _admin(
|
||||
TokenUsageRepository=Mock(return_value=usage),
|
||||
AdminStatsRepository=Mock(return_value=stats),
|
||||
):
|
||||
data = _body(client.get("/api/admin/usage"))
|
||||
assert data["total_cost"] == 0.25
|
||||
assert data["series"][0]["cost"] == 0.25
|
||||
assert data["by_model"][0]["model_id"] == "gpt-x"
|
||||
assert data["latency"]["p95_ms"] == 400
|
||||
assert data["top_users"][0]["cost"] == 0.25
|
||||
|
||||
def test_usage_group_by_reaches_the_repository(self, client):
|
||||
usage, stats = self._usage_repos()
|
||||
with _admin(
|
||||
TokenUsageRepository=Mock(return_value=usage),
|
||||
AdminStatsRepository=Mock(return_value=stats),
|
||||
):
|
||||
assert client.get("/api/admin/usage?group_by=model").status_code == 200
|
||||
assert usage.bucketed_totals.call_args.kwargs["group_by"] == "model"
|
||||
|
||||
def test_usage_rejects_an_unknown_grouping(self, client):
|
||||
with _admin():
|
||||
assert client.get("/api/admin/usage?group_by=nonsense").status_code == 400
|
||||
|
||||
def test_per_user_usage_drilldown(self, client):
|
||||
usage, stats = self._usage_repos()
|
||||
stats.user_usage_breakdown.return_value = {
|
||||
"totals": {"tokens": 8, "cost": 0.25, "calls": 1},
|
||||
"by_model": [{"key": "gpt-x", "tokens": 8, "cost": 0.25}],
|
||||
"by_source": [{"key": "agent_stream", "tokens": 8, "cost": 0.25}],
|
||||
}
|
||||
with _admin(
|
||||
TokenUsageRepository=Mock(return_value=usage),
|
||||
AdminStatsRepository=Mock(return_value=stats),
|
||||
):
|
||||
data = _body(client.get("/api/admin/users/u1/usage?days=7"))
|
||||
assert data["totals"]["cost"] == 0.25
|
||||
assert data["by_source"][0]["key"] == "agent_stream"
|
||||
assert len(data["series"]) == 1
|
||||
assert usage.bucketed_totals.call_args.kwargs["user_id"] == "u1"
|
||||
|
||||
def test_per_user_usage_requires_admin(self, client):
|
||||
with patch("docsgpt.app.handle_auth", return_value={"sub": "u1"}), patch(
|
||||
"docsgpt.app.resolve_roles", return_value=["user"]
|
||||
):
|
||||
assert client.get("/api/admin/users/u1/usage").status_code == 403
|
||||
|
||||
def test_usage_invalid_bucket(self, client):
|
||||
with _admin():
|
||||
resp = client.get("/api/admin/usage?bucket=year")
|
||||
|
||||
Reference in new issue
Block a user