feat(admin): cost, group-by and latency in usage

Three gaps in one view.

Cost: token_usage.cost has been written on every call since quotas landed, and
tokens_by_model already returned it, but bucketed_totals and top_token_users
selected tokens only. An admin could set a USD quota in /admin/quotas and had
no way to see the spend it was capping. Buckets and top users now carry cost,
the endpoint reports a window total and a per-model split, and the chart takes
a Tokens/Cost toggle with a currency axis.

Group-by: the endpoint has supported group_by=model|agent|source from the
start and the UI only ever sent bucket=day. The selector is now wired, and a
grouped series pads missing buckets so a model that was idle on Tuesday plots
a zero instead of shifting its whole row one bar left.

Latency: surfaced from the columns the previous commit added, as p50/p95 with
a median time-to-first-token underneath. Unmeasured rows are excluded rather
than counted as zero, and the card says so when nothing was measured.

Also surfaces the prompt-cache hit rate, computed only over rows whose
provider reported a cache breakdown -- NULL means "not reported", and folding
those in as 0% would understate it.

Per-user drill-down: GET /api/admin/users/<id>/usage returns a daily series
plus splits by model and by flow, reachable from the top-users table and from
the user detail dialog. The detail dialog previously showed one tokens_30d
number, which answers neither "what is this person costing" nor "what is
driving it".
This commit is contained in:
arc53-machine committed 2026-09-22 10:33:56 +01:00
1 parent b17f40a993
commit a56f9df122
14 files changed
+1019 -61

No files matched your search

+33 -2
View File
@@ -287,6 +287,26 @@ class AdminUserSessionsResource(Resource):
return make_response(jsonify({"success": True, "revoked": ok}), 200)
@admin_ns.route("/admin/users/<string:user_id>/usage")
class AdminUserUsageResource(Resource):
@admin_required
def get(self, user_id):
"""One user's spend: daily series plus a split by model and by flow."""
days = max(1, min(365, _int_arg("days", 30)))
start = datetime.now(timezone.utc) - timedelta(days=days)
with db_readonly() as conn:
series = TokenUsageRepository(conn).bucketed_totals(
bucket_unit="day", user_id=user_id, timestamp_gte=start
)
breakdown = AdminStatsRepository(conn).user_usage_breakdown(
user_id, since=start
)
return make_response(
jsonify({"success": True, "days": days, "series": series, **breakdown}),
200,
)
@admin_ns.route("/admin/admins")
class AdminAdminsResource(Resource):
@admin_required
@@ -304,7 +324,12 @@ class AdminUsageResource(Resource):
@admin_required
def get(self):
"""Global token usage: time-bucketed series + total + top users."""
"""Global usage: time-bucketed series, totals, spend, latency, top users.
Every bucket carries both tokens and USD cost. Quota policies have
always been settable in dollars; until now nothing showed the spend
they were capping.
"""
days = max(1, min(365, _int_arg("days", 30)))
bucket = request.args.get("bucket", "day")
group_by = request.args.get("group_by", "none")
@@ -313,6 +338,7 @@ class AdminUsageResource(Resource):
start = datetime.now(timezone.utc) - timedelta(days=days)
with db_readonly() as conn:
usage_repo = TokenUsageRepository(conn)
stats_repo = AdminStatsRepository(conn)
series = usage_repo.bucketed_totals(
bucket_unit=bucket,
timestamp_gte=start,
@@ -321,7 +347,9 @@ class AdminUsageResource(Resource):
total = usage_repo.sum_tokens_in_range(
start=start, end=datetime.now(timezone.utc)
)
top_users = AdminStatsRepository(conn).top_token_users(since=start, limit=10)
by_model = usage_repo.tokens_by_model(start=start)
latency = stats_repo.latency_summary(since=start)
top_users = stats_repo.top_token_users(since=start, limit=10)
return make_response(
jsonify(
{
@@ -331,6 +359,9 @@ class AdminUsageResource(Resource):
"group_by": group_by,
"series": series,
"total_tokens": int(total),
"total_cost": round(sum(row["cost"] for row in by_model), 4),
"by_model": by_model,
"latency": latency,
"top_users": top_users,
}
),
+113 -3
View File
@@ -16,6 +16,11 @@ from sqlalchemy import Connection, text
from docsgpt.storage.db.base_repository import row_to_dict
def _round_ms(value) -> Optional[int]:
"""Round a percentile to whole milliseconds, preserving NULL as None."""
return int(round(float(value))) if value is not None else None
class AdminStatsRepository:
"""Global aggregates for the admin overview and per-user drill-down."""
@@ -66,11 +71,18 @@ class AdminStatsRepository:
}
def top_token_users(self, *, since: datetime, limit: int = 10) -> list[dict]:
"""Highest token consumers since ``since`` (admin usage view)."""
"""Highest consumers since ``since``, with spend (admin usage view).
Ordered by tokens, not cost: a cheap model can dominate token volume
while a costly one dominates the bill, and the operator wants to see
both columns rather than have the ranking pick for them.
"""
result = self._conn.execute(
text(
"""
SELECT user_id, SUM(prompt_tokens + generated_tokens) AS tokens
SELECT user_id,
SUM(prompt_tokens + generated_tokens) AS tokens,
COALESCE(SUM(cost), 0) AS cost
FROM token_usage
WHERE timestamp >= :since AND user_id IS NOT NULL
GROUP BY user_id
@@ -80,7 +92,105 @@ class AdminStatsRepository:
),
{"since": since, "limit": int(limit)},
)
return [{"user_id": r[0], "tokens": int(r[1])} for r in result.fetchall()]
return [
{"user_id": r[0], "tokens": int(r[1]), "cost": float(r[2])}
for r in result.fetchall()
]
def latency_summary(self, *, since: datetime) -> dict:
"""Call-latency percentiles since ``since``.
Rows written before the latency columns existed, and streams that
never yielded, hold NULL; percentile_cont skips them, so the summary
describes the calls that were actually measured. ``samples`` says how
many those were, so a percentile from three calls is not read as a
service-level figure.
"""
row = self._conn.execute(
text(
"""
SELECT count(duration_ms) AS samples,
percentile_cont(0.5) WITHIN GROUP (ORDER BY duration_ms)
AS p50_ms,
percentile_cont(0.95) WITHIN GROUP (ORDER BY duration_ms)
AS p95_ms,
count(ttft_ms) AS ttft_samples,
percentile_cont(0.5) WITHIN GROUP (ORDER BY ttft_ms)
AS ttft_p50_ms
FROM token_usage
WHERE timestamp >= :since
"""
),
{"since": since},
).one()
return {
"samples": int(row.samples or 0),
"p50_ms": _round_ms(row.p50_ms),
"p95_ms": _round_ms(row.p95_ms),
"ttft_samples": int(row.ttft_samples or 0),
"ttft_p50_ms": _round_ms(row.ttft_p50_ms),
}
def user_usage_breakdown(
self, user_id: str, *, since: datetime, limit: int = 10
) -> dict:
"""Per-user spend detail: totals, and a split by model and by flow.
The admin user drill-down showed a single ``tokens_30d`` number, which
cannot answer either question an operator actually has -- what is this
person costing, and what is driving it.
Args:
user_id: The billable user (auth ``sub``).
since: Inclusive window start.
limit: How many models / flows to return, busiest first.
Returns:
``{"totals": {...}, "by_model": [...], "by_source": [...]}``.
"""
totals = self._conn.execute(
text(
"""
SELECT COALESCE(SUM(prompt_tokens + generated_tokens), 0) AS tokens,
COALESCE(SUM(cost), 0) AS cost,
count(*) AS calls
FROM token_usage
WHERE user_id = :u AND timestamp >= :since
"""
),
{"u": user_id, "since": since},
).one()
def _breakdown(key_expr: str) -> list[dict]:
rows = self._conn.execute(
text(
f"""
SELECT {key_expr} AS key,
COALESCE(SUM(prompt_tokens + generated_tokens), 0) AS tokens,
COALESCE(SUM(cost), 0) AS cost
FROM token_usage
WHERE user_id = :u AND timestamp >= :since
GROUP BY key
ORDER BY tokens DESC, key
LIMIT :limit
"""
),
{"u": user_id, "since": since, "limit": int(limit)},
)
return [
{"key": r.key, "tokens": int(r.tokens), "cost": float(r.cost)}
for r in rows.fetchall()
]
return {
"totals": {
"tokens": int(totals.tokens),
"cost": float(totals.cost),
"calls": int(totals.calls),
},
"by_model": _breakdown("COALESCE(model_id, 'unknown')"),
"by_source": _breakdown("COALESCE(source, 'agent_stream')"),
}
def list_users(
self, user_id_filter: Optional[str], offset: int, limit: int
+11 -1
View File
@@ -302,7 +302,11 @@ class TokenUsageRepository:
f"""
SELECT to_char(tu.timestamp AT TIME ZONE 'UTC', :fmt) AS bucket,
COALESCE(SUM(tu.prompt_tokens), 0) AS prompt_tokens,
COALESCE(SUM(tu.generated_tokens), 0) AS generated_tokens
COALESCE(SUM(tu.generated_tokens), 0) AS generated_tokens,
COALESCE(SUM(tu.cost), 0) AS cost,
-- NULL cache bins mean "provider did not report", so
-- they stay out of the sum rather than reading as 0.
SUM(tu.cached_tokens) AS cached_tokens
{group_select}
FROM token_usage tu
{join}
@@ -318,6 +322,12 @@ class TokenUsageRepository:
"bucket": row._mapping["bucket"],
"prompt_tokens": int(row._mapping["prompt_tokens"]),
"generated_tokens": int(row._mapping["generated_tokens"]),
"cost": float(row._mapping["cost"]),
"cached_tokens": (
int(row._mapping["cached_tokens"])
if row._mapping["cached_tokens"] is not None
else None
),
**(
{"group_key": row._mapping["group_key"]}
if group_by is not None
+22
View File
@@ -97,6 +97,28 @@ export function fmtNumber(n?: number | null): string {
return new Intl.NumberFormat().format(n ?? 0);
}
/**
* USD, with enough precision that a fraction of a cent is not rendered as $0.
* An instance on a cheap model can run whole days under a dollar, and "$0"
* next to a cost quota reads as "nothing is being counted".
*/
export function fmtUsd(n?: number | null): string {
const value = n ?? 0;
const digits = value !== 0 && Math.abs(value) < 0.01 ? 4 : 2;
return new Intl.NumberFormat(undefined, {
style: 'currency',
currency: 'USD',
minimumFractionDigits: 2,
maximumFractionDigits: digits,
}).format(value);
}
/** Milliseconds as ms or s, or an em dash when nothing was measured. */
export function fmtMs(n?: number | null): string {
if (n === null || n === undefined) return '—';
return n < 1000 ? `${Math.round(n)}ms` : `${(n / 1000).toFixed(1)}s`;
}
export function fmtCompact(n?: number | null): string {
return new Intl.NumberFormat(undefined, {
notation: 'compact',
+143 -51
View File
@@ -4,6 +4,13 @@ import { useSelector } from 'react-redux';
import adminService from '../api/services/adminService';
import SkeletonLoader from '../components/SkeletonLoader';
import { Button } from '../components/ui/button';
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from '../components/ui/select';
import {
Table,
TableBody,
@@ -15,16 +22,33 @@ import {
} from '../components/ui/table';
import { useDarkTheme } from '../hooks';
import { selectToken } from '../preferences/preferenceSlice';
import { formatDate } from '../utils/dateTimeUtils';
import { Loading, LoadError, StatCard, fmtNumber } from './AdminUI';
import UsageChart, { usageColors } from './UsageChart';
import {
Loading,
LoadError,
StatCard,
fmtMs,
fmtNumber,
fmtUsd,
} from './AdminUI';
import UsageChart from './UsageChart';
import {
GROUP_OPTIONS,
buildUsageChart,
cacheHitRate as computeCacheHitRate,
type GroupBy,
type Metric,
type UsageBucket,
} from './usageChartData';
import UserUsageModal from './UserUsageModal';
type Bucket = {
bucket: string;
prompt_tokens: number;
generated_tokens: number;
type TopUser = { user_id: string; tokens: number; cost: number };
type Latency = {
samples: number;
p50_ms: number | null;
p95_ms: number | null;
ttft_samples: number;
ttft_p50_ms: number | null;
};
type TopUser = { user_id: string; tokens: number };
const RANGES = [7, 30, 90];
@@ -32,47 +56,40 @@ export default function Usage() {
const token = useSelector(selectToken);
const [isDarkTheme] = useDarkTheme();
const [days, setDays] = useState(30);
const [groupBy, setGroupBy] = useState<GroupBy>('none');
const [metric, setMetric] = useState<Metric>('tokens');
const [data, setData] = useState<any | null>(null);
const [loading, setLoading] = useState(true);
const [drilldown, setDrilldown] = useState<string | null>(null);
const load = useCallback(async () => {
setLoading(true);
try {
const res = await adminService.getUsage({ days, bucket: 'day' }, token);
const res = await adminService.getUsage(
{ days, bucket: 'day', group_by: groupBy },
token,
);
setData(await res.json().catch(() => ({})));
} finally {
setLoading(false);
}
}, [token, days]);
}, [token, days, groupBy]);
useEffect(() => {
load();
}, [load]);
const series: Bucket[] = data?.series ?? [];
const series: UsageBucket[] = data?.series ?? [];
const topUsers: TopUser[] = data?.top_users ?? [];
const promptTotal = series.reduce((sum, b) => sum + b.prompt_tokens, 0);
const generatedTotal = series.reduce((sum, b) => sum + b.generated_tokens, 0);
const latency: Latency | undefined = data?.latency;
const chartData = useMemo(() => {
const colors = usageColors();
return {
labels: series.map((b) => formatDate(b.bucket)),
datasets: [
{
label: 'Prompt',
data: series.map((b) => b.prompt_tokens),
backgroundColor: colors.prompt,
},
{
label: 'Generated',
data: series.map((b) => b.generated_tokens),
backgroundColor: colors.generated,
},
],
};
const chartData = useMemo(
() => buildUsageChart(series, groupBy, metric),
// isDarkTheme re-resolves the canvas colors when the theme toggles.
}, [series, isDarkTheme]);
[series, groupBy, metric, isDarkTheme],
);
const cacheHitRate = useMemo(() => computeCacheHitRate(series), [series]);
if (data === null && loading) return <Loading />;
if (data && !data.success)
@@ -80,34 +97,89 @@ export default function Usage() {
return (
<div className="mt-6">
<div className="mb-4 flex items-center gap-1">
{RANGES.map((r) => (
<div className="mb-4 flex flex-wrap items-center gap-2">
<div className="flex items-center gap-1">
{RANGES.map((range) => (
<Button
key={range}
variant={range === days ? 'default' : 'outline'}
size="sm"
className="rounded-3xl"
onClick={() => setDays(range)}
>
{range}d
</Button>
))}
</div>
<Select
value={groupBy}
onValueChange={(value) => setGroupBy(value as GroupBy)}
>
<SelectTrigger className="w-40">
<SelectValue />
</SelectTrigger>
<SelectContent>
{GROUP_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
))}
</SelectContent>
</Select>
<div className="flex items-center gap-1">
<Button
key={r}
variant={r === days ? 'default' : 'outline'}
variant={metric === 'tokens' ? 'default' : 'outline'}
size="sm"
className="rounded-3xl"
onClick={() => setDays(r)}
onClick={() => setMetric('tokens')}
>
{r}d
Tokens
</Button>
))}
<Button
variant={metric === 'cost' ? 'default' : 'outline'}
size="sm"
className="rounded-3xl"
onClick={() => setMetric('cost')}
>
Cost
</Button>
</div>
</div>
<div className="grid grid-cols-2 gap-3 md:grid-cols-4">
<StatCard
label={`Total tokens (${days}d)`}
label={`Spend (${days}d)`}
value={fmtUsd(data?.total_cost)}
sub="Unpriced and BYO models record $0"
/>
<StatCard
label={`Tokens (${days}d)`}
value={fmtNumber(data?.total_tokens)}
/>
<StatCard label="Prompt tokens" value={fmtNumber(promptTotal)} />
<StatCard label="Generated tokens" value={fmtNumber(generatedTotal)} />
<StatCard label="Days with usage" value={fmtNumber(series.length)} />
<StatCard
label="Latency p50 / p95"
value={`${fmtMs(latency?.p50_ms)} / ${fmtMs(latency?.p95_ms)}`}
sub={
latency?.ttft_p50_ms !== null && latency?.ttft_p50_ms !== undefined
? `First token ${fmtMs(latency.ttft_p50_ms)} (p50)`
: 'No streamed calls measured'
}
/>
<StatCard
label="Prompt cache hits"
value={cacheHitRate === null ? '—' : `${cacheHitRate.toFixed(1)}%`}
sub={
cacheHitRate === null
? 'Provider reported no cache data'
: 'Of prompt tokens the provider reported'
}
/>
</div>
<div className="border-border dark:border-border mt-4 h-[345px] w-full overflow-hidden rounded-2xl border px-6 py-5">
<div className="flex flex-row items-center justify-between gap-3">
<p className="text-foreground dark:text-foreground font-bold">
Token usage
{metric === 'cost' ? 'Spend' : 'Token usage'}
</p>
<div
id="admin-usage-legend"
@@ -122,14 +194,18 @@ export default function Usage() {
No usage in this period.
</p>
) : (
<UsageChart data={chartData} legendID="admin-usage-legend" />
<UsageChart
data={chartData}
legendID="admin-usage-legend"
currency={metric === 'cost'}
/>
)}
</div>
</div>
<div className="border-border dark:border-border mt-4 w-full overflow-hidden rounded-2xl border px-6 py-5">
<p className="text-foreground dark:text-foreground mb-3 font-bold">
Top users by tokens
Top users
</p>
{topUsers.length === 0 ? (
<p className="text-muted-foreground text-sm">No usage.</p>
@@ -139,17 +215,31 @@ export default function Usage() {
<TableHead>
<TableRow>
<TableHeader>User</TableHeader>
<TableHeader className="text-right">Tokens</TableHeader>
<TableHeader align="right">Tokens</TableHeader>
<TableHeader align="right">Cost</TableHeader>
</TableRow>
</TableHead>
<TableBody>
{topUsers.map((u) => (
<TableRow key={u.user_id}>
{topUsers.map((user) => (
<TableRow
key={user.user_id}
className="hover:bg-muted/40 cursor-pointer"
onClick={() => setDrilldown(user.user_id)}
>
<TableCell className="font-mono text-[13px] break-all">
{u.user_id}
{user.user_id}
</TableCell>
<TableCell className="text-right whitespace-nowrap tabular-nums">
{fmtNumber(u.tokens)}
<TableCell
align="right"
className="whitespace-nowrap tabular-nums"
>
{fmtNumber(user.tokens)}
</TableCell>
<TableCell
align="right"
className="whitespace-nowrap tabular-nums"
>
{fmtUsd(user.cost)}
</TableCell>
</TableRow>
))}
@@ -158,6 +248,8 @@ export default function Usage() {
</TableContainer>
)}
</div>
<UserUsageModal userId={drilldown} onClose={() => setDrilldown(null)} />
</div>
);
}
+32 -1
View File
@@ -39,6 +39,25 @@ export function usageColors(): { prompt: string; generated: string } {
};
}
// Follow-on colors for a grouped chart (by model / agent / source), matching
// the palette settings/Analytics already uses for its stacked series.
const SERIES_COLORS = [
'#FF6384',
'#36A2EB',
'#FFCE56',
'#4BC0C0',
'#9966FF',
'#FF9F40',
'#2BC596',
];
/** Color for dataset ``index``; the first uses the resolved brand color. */
export function seriesColor(index: number): string {
return index === 0
? readCssVar('--primary', '#7D54D1')
: SERIES_COLORS[(index - 1) % SERIES_COLORS.length];
}
function compactTick(value: number | string): string {
const n = typeof value === 'number' ? value : Number(value);
if (Number.isNaN(n)) return String(value);
@@ -48,16 +67,28 @@ function compactTick(value: number | string): string {
}).format(n);
}
function currencyTick(value: number | string): string {
const n = typeof value === 'number' ? value : Number(value);
if (Number.isNaN(n)) return String(value);
return `$${new Intl.NumberFormat(undefined, {
notation: 'compact',
maximumFractionDigits: 2,
}).format(n)}`;
}
type UsageChartProps = {
data: ChartData<'bar'>;
legendID: string;
maxTicksLimitInX?: number;
/** Format the y axis as USD rather than a token count. */
currency?: boolean;
};
export default function UsageChart({
data,
legendID,
maxTicksLimitInX = 8,
currency = false,
}: UsageChartProps) {
const options = {
responsive: true,
@@ -76,7 +107,7 @@ export default function UsageChart({
y: {
grid: { lineWidth: 0.2, color: '#C4C4C4' },
border: { width: 0.2, color: '#C4C4C4' },
ticks: { callback: compactTick },
ticks: { callback: currency ? currencyTick : compactTick },
stacked: true,
},
},
+201
View File
@@ -0,0 +1,201 @@
import { useEffect, useMemo, useState } from 'react';
import { useSelector } from 'react-redux';
import adminService from '../api/services/adminService';
import { Button } from '../components/ui/button';
import { Modal } from '../components/ui/modal';
import {
Table,
TableBody,
TableCell,
TableContainer,
TableHead,
TableHeader,
TableRow,
} from '../components/ui/table';
import { useDarkTheme } from '../hooks';
import { selectToken } from '../preferences/preferenceSlice';
import { formatDate } from '../utils/dateTimeUtils';
import { Loading, LoadError, StatCard, fmtNumber, fmtUsd } from './AdminUI';
import UsageChart, { usageColors } from './UsageChart';
type Bucket = {
bucket: string;
prompt_tokens: number;
generated_tokens: number;
cost: number;
};
type Split = { key: string; tokens: number; cost: number };
const RANGES = [7, 30, 90];
/** Label for a token_usage.source value — what the spend was for. */
const FLOW_LABELS: Record<string, string> = {
agent_stream: 'Chat',
schedule: 'Scheduled run',
webhook: 'Webhook',
workflow: 'Workflow',
title: 'Title generation',
compression: 'History compression',
rag_condense: 'Question condensing',
fallback: 'Provider fallback',
};
function SplitTable({ title, rows }: { title: string; rows: Split[] }) {
return (
<div>
<p className="text-muted-foreground mb-2 text-sm font-medium">{title}</p>
{rows.length === 0 ? (
<p className="text-muted-foreground text-sm">No usage.</p>
) : (
<TableContainer>
<Table>
<TableHead>
<TableRow>
<TableHeader>{title}</TableHeader>
<TableHeader align="right">Tokens</TableHeader>
<TableHeader align="right">Cost</TableHeader>
</TableRow>
</TableHead>
<TableBody>
{rows.map((row) => (
<TableRow key={row.key}>
<TableCell className="text-[13px] break-all">
{FLOW_LABELS[row.key] ?? row.key}
</TableCell>
<TableCell align="right" className="tabular-nums">
{fmtNumber(row.tokens)}
</TableCell>
<TableCell align="right" className="tabular-nums">
{fmtUsd(row.cost)}
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</TableContainer>
)}
</div>
);
}
/**
* Per-user spend drill-down. The user detail view carried a single
* `tokens_30d` number, which answers neither question an operator has: what is
* this person costing, and what is driving it.
*/
export default function UserUsageModal({
userId,
onClose,
}: {
userId: string | null;
onClose: () => void;
}) {
const token = useSelector(selectToken);
const [isDarkTheme] = useDarkTheme();
const [days, setDays] = useState(30);
const [data, setData] = useState<any | null>(null);
const [loading, setLoading] = useState(false);
useEffect(() => {
if (!userId) {
setData(null);
return;
}
let cancelled = false;
setLoading(true);
adminService
.getUserUsage(userId, { days }, token)
.then((res) => res.json())
.then((json) => {
if (!cancelled) setData(json);
})
.catch(() => !cancelled && setData({ success: false }))
.finally(() => !cancelled && setLoading(false));
return () => {
cancelled = true;
};
}, [userId, days, token]);
const series: Bucket[] = data?.series ?? [];
const chartData = useMemo(() => {
const colors = usageColors();
return {
labels: series.map((bucket) => formatDate(bucket.bucket)),
datasets: [
{
label: 'Prompt',
data: series.map((bucket) => bucket.prompt_tokens),
backgroundColor: colors.prompt,
},
{
label: 'Generated',
data: series.map((bucket) => bucket.generated_tokens),
backgroundColor: colors.generated,
},
],
};
// isDarkTheme re-resolves the canvas colors when the theme toggles.
}, [series, isDarkTheme]);
const totals = data?.totals ?? {};
return (
<Modal
open={userId !== null}
onOpenChange={(open) => !open && onClose()}
title={userId ? `Usage · ${userId}` : 'Usage'}
size="lg"
>
<div className="mb-4 flex items-center gap-1">
{RANGES.map((range) => (
<Button
key={range}
variant={range === days ? 'default' : 'outline'}
size="sm"
className="rounded-3xl"
onClick={() => setDays(range)}
>
{range}d
</Button>
))}
</div>
{loading && data === null ? (
<Loading />
) : data && !data.success ? (
<LoadError message="Failed to load usage." />
) : (
<div className="space-y-4">
<div className="grid grid-cols-3 gap-3">
<StatCard label="Cost" value={fmtUsd(totals.cost)} />
<StatCard label="Tokens" value={fmtNumber(totals.tokens)} />
<StatCard label="Calls" value={fmtNumber(totals.calls)} />
</div>
<div className="border-border h-64 rounded-2xl border px-4 py-3">
<div className="flex items-center justify-between">
<p className="text-foreground text-sm font-bold">Daily tokens</p>
<div id="admin-user-usage-legend" className="flex" />
</div>
<div className="relative mt-px h-48 w-full">
{series.length === 0 ? (
<p className="text-muted-foreground mt-8 text-sm">
No usage in this period.
</p>
) : (
<UsageChart
data={chartData}
legendID="admin-user-usage-legend"
/>
)}
</div>
</div>
<SplitTable title="Model" rows={data?.by_model ?? []} />
<SplitTable title="Flow" rows={data?.by_source ?? []} />
</div>
)}
</Modal>
);
}
+16
View File
@@ -42,6 +42,7 @@ import {
fmtRelative,
} from './AdminUI';
import UserQuotaModal from './UserQuotaModal';
import UserUsageModal from './UserUsageModal';
type AdminUser = {
user_id: string;
@@ -72,6 +73,7 @@ export default function Users() {
const [busy, setBusy] = useState<string | null>(null);
const [menuUserId, setMenuUserId] = useState<string | null>(null);
const [detail, setDetail] = useState<any | null>(null);
const [usageFor, setUsageFor] = useState<string | null>(null);
const [quotaUserId, setQuotaUserId] = useState<string | null>(null);
const [feedback, setFeedback] = useState<{
ok: boolean;
@@ -443,6 +445,7 @@ export default function Users() {
/>
) : null}
<UserUsageModal userId={usageFor} onClose={() => setUsageFor(null)} />
<UserQuotaModal
userId={quotaUserId}
onClose={() => setQuotaUserId(null)}
@@ -528,6 +531,19 @@ export default function Users() {
</span>
</div>
</div>
<Button
variant="outline"
size="sm"
onClick={() => {
// Close the detail dialog before the usage dialog opens, so
// the two never stack.
const userId = detail.user.user_id;
setDetail(null);
setUsageFor(userId);
}}
>
View spend breakdown
</Button>
<div>
<p className="text-muted-foreground mb-1 text-xs">
Recent auth events
+101
View File
@@ -0,0 +1,101 @@
import { describe, expect, it, vi } from 'vitest';
import {
buildUsageChart,
cacheHitRate,
type UsageBucket,
} from './usageChartData';
// Chart.js colors resolve against a live DOM; the shapes under test do not.
vi.mock('./UsageChart', () => ({
seriesColor: (index: number) => `color-${index}`,
usageColors: () => ({ prompt: 'prompt-color', generated: 'generated-color' }),
}));
const bucket = (overrides: Partial<UsageBucket>): UsageBucket => ({
bucket: '2026-01-01',
prompt_tokens: 100,
generated_tokens: 20,
cost: 1,
cached_tokens: null,
...overrides,
});
describe('buildUsageChart', () => {
it('splits prompt and generated tokens when ungrouped', () => {
const chart = buildUsageChart([bucket({})], 'none', 'tokens');
expect(chart.datasets.map((d) => d.label)).toEqual(['Prompt', 'Generated']);
expect(chart.datasets[0].data).toEqual([100]);
});
it('plots a single series for cost', () => {
const chart = buildUsageChart([bucket({ cost: 2.5 })], 'none', 'cost');
expect(chart.datasets).toHaveLength(1);
expect(chart.datasets[0].data).toEqual([2.5]);
});
it('gives each group its own dataset', () => {
const chart = buildUsageChart(
[
bucket({ group_key: 'gpt-a' }),
bucket({ group_key: 'gpt-b', prompt_tokens: 5, generated_tokens: 5 }),
],
'model',
'tokens',
);
expect(chart.datasets.map((d) => d.label)).toEqual(['gpt-a', 'gpt-b']);
expect(chart.datasets[1].data).toEqual([10]);
});
it('pads a group that is missing a bucket so the axes stay aligned', () => {
const chart = buildUsageChart(
[
bucket({ bucket: '2026-01-01', group_key: 'a' }),
bucket({ bucket: '2026-01-02', group_key: 'b' }),
],
'model',
'tokens',
);
expect(chart.labels).toHaveLength(2);
// 'a' has no row on the second day: a zero, not a shifted bar.
expect(chart.datasets[0].data).toEqual([120, 0]);
expect(chart.datasets[1].data).toEqual([0, 120]);
});
it('labels a group with no key rather than dropping the row', () => {
const chart = buildUsageChart([bucket({})], 'model', 'tokens');
expect(chart.datasets[0].label).toBe('unknown');
});
it('handles an empty series', () => {
const chart = buildUsageChart([], 'model', 'cost');
expect(chart.labels).toEqual([]);
expect(chart.datasets).toEqual([]);
});
});
describe('cacheHitRate', () => {
it('is unknown when no provider reported cache bins', () => {
expect(cacheHitRate([bucket({ cached_tokens: null })])).toBeNull();
});
it('is unknown for an empty series', () => {
expect(cacheHitRate([])).toBeNull();
});
it('measures only the rows that reported', () => {
const rate = cacheHitRate([
bucket({ prompt_tokens: 100, cached_tokens: 25 }),
// Unreported: excluded entirely rather than counted as a 0% row, which
// would halve the rate.
bucket({ prompt_tokens: 100, cached_tokens: null }),
]);
expect(rate).toBeCloseTo(25);
});
it('avoids dividing by zero prompt tokens', () => {
expect(
cacheHitRate([bucket({ prompt_tokens: 0, cached_tokens: 0 })]),
).toBeNull();
});
});
+113
View File
@@ -0,0 +1,113 @@
import { formatDate } from '../utils/dateTimeUtils';
import { seriesColor, usageColors } from './UsageChart';
export type UsageBucket = {
bucket: string;
prompt_tokens: number;
generated_tokens: number;
cost: number;
cached_tokens: number | null;
group_key?: string;
};
export type GroupBy = 'none' | 'model' | 'agent' | 'source';
export type Metric = 'tokens' | 'cost';
export const GROUP_OPTIONS: { value: GroupBy; label: string }[] = [
{ value: 'none', label: 'Total' },
{ value: 'model', label: 'By model' },
{ value: 'agent', label: 'By agent' },
{ value: 'source', label: 'By flow' },
];
function valueOf(row: UsageBucket, metric: Metric): number {
return metric === 'cost'
? row.cost
: row.prompt_tokens + row.generated_tokens;
}
/**
* Shape the admin usage series for the chart.
*
* Ungrouped gives one dataset per token kind (or a single cost series);
* grouped gives one dataset per group key, each aligned to the shared bucket
* axis so a group missing a day plots 0 rather than shifting the whole series
* left by one bar.
*/
export function buildUsageChart(
series: UsageBucket[],
groupBy: GroupBy,
metric: Metric,
) {
const buckets = Array.from(new Set(series.map((row) => row.bucket))).sort();
const labels = buckets.map((bucket) => formatDate(bucket));
if (groupBy === 'none') {
if (metric === 'cost') {
return {
labels,
datasets: [
{
label: 'Cost',
data: series.map((row) => row.cost),
backgroundColor: seriesColor(0),
},
],
};
}
const colors = usageColors();
return {
labels,
datasets: [
{
label: 'Prompt',
data: series.map((row) => row.prompt_tokens),
backgroundColor: colors.prompt,
},
{
label: 'Generated',
data: series.map((row) => row.generated_tokens),
backgroundColor: colors.generated,
},
],
};
}
const keys = Array.from(
new Set(series.map((row) => row.group_key ?? 'unknown')),
);
return {
labels,
datasets: keys.map((key, index) => {
const byBucket = new Map(
series
.filter((row) => (row.group_key ?? 'unknown') === key)
.map((row) => [row.bucket, valueOf(row, metric)]),
);
return {
label: key,
data: buckets.map((bucket) => byBucket.get(bucket) ?? 0),
backgroundColor: seriesColor(index),
};
}),
};
}
/**
* Prompt-cache hit rate over the rows whose provider reported a breakdown.
*
* NULL cached_tokens means "the provider said nothing", which is not the same
* as "no cache hits" — counting those rows as 0 would understate the rate.
* Returns null when nothing reported.
*/
export function cacheHitRate(series: UsageBucket[]): number | null {
const reported = series.filter((row) => row.cached_tokens !== null);
if (reported.length === 0) return null;
const prompt = reported.reduce((sum, row) => sum + row.prompt_tokens, 0);
if (prompt === 0) return null;
const cached = reported.reduce(
(sum, row) => sum + (row.cached_tokens ?? 0),
0,
);
return (cached / prompt) * 100;
}
+2
View File
@@ -170,6 +170,8 @@ const endpoints = {
`/api/admin/users/${encodeURIComponent(id)}/role`,
USER_REVOKE_SESSIONS: (id: string) =>
`/api/admin/users/${encodeURIComponent(id)}/revoke-sessions`,
USER_USAGE: (id: string) =>
`/api/admin/users/${encodeURIComponent(id)}/usage`,
ADMINS: '/api/admin/admins',
USAGE: '/api/admin/usage',
ACTIVITY: '/api/admin/activity',
@@ -68,6 +68,12 @@ const adminService = {
token: string | null,
): Promise<any> =>
apiClient.get(`${endpoints.ADMIN.USAGE}${qs(params)}`, token),
getUserUsage: (
userId: string,
params: { days?: number },
token: string | null,
): Promise<any> =>
apiClient.get(`${endpoints.ADMIN.USER_USAGE(userId)}${qs(params)}`, token),
getActivity: (filters: ActivityFilters, token: string | null): Promise<any> =>
apiClient.get(`${endpoints.ADMIN.ACTIVITY}${qs(filters)}`, token),
getActivityEvents: (token: string | null): Promise<any> =>
+76 -3
View File
@@ -228,14 +228,37 @@ class TestUserLifecycle:
@pytest.mark.unit
class TestUsageAndAudit:
def test_usage(self, client):
@staticmethod
def _usage_repos():
usage = Mock()
usage.bucketed_totals.return_value = [
{"bucket": "2026-06-14", "prompt_tokens": 5, "generated_tokens": 3}
{
"bucket": "2026-06-14",
"prompt_tokens": 5,
"generated_tokens": 3,
"cost": 0.25,
"cached_tokens": None,
}
]
usage.sum_tokens_in_range.return_value = 8
usage.tokens_by_model.return_value = [
{"model_id": "gpt-x", "tokens": 8, "cost": 0.25}
]
stats = Mock()
stats.top_token_users.return_value = [{"user_id": "a", "tokens": 8}]
stats.top_token_users.return_value = [
{"user_id": "a", "tokens": 8, "cost": 0.25}
]
stats.latency_summary.return_value = {
"samples": 2,
"p50_ms": 120,
"p95_ms": 400,
"ttft_samples": 2,
"ttft_p50_ms": 30,
}
return usage, stats
def test_usage(self, client):
usage, stats = self._usage_repos()
with _admin(
TokenUsageRepository=Mock(return_value=usage),
AdminStatsRepository=Mock(return_value=stats),
@@ -247,6 +270,56 @@ class TestUsageAndAudit:
assert len(data["series"]) == 1
assert data["top_users"][0]["user_id"] == "a"
def test_usage_reports_spend_and_latency(self, client):
"""Cost quotas were settable long before spend was visible."""
usage, stats = self._usage_repos()
with _admin(
TokenUsageRepository=Mock(return_value=usage),
AdminStatsRepository=Mock(return_value=stats),
):
data = _body(client.get("/api/admin/usage"))
assert data["total_cost"] == 0.25
assert data["series"][0]["cost"] == 0.25
assert data["by_model"][0]["model_id"] == "gpt-x"
assert data["latency"]["p95_ms"] == 400
assert data["top_users"][0]["cost"] == 0.25
def test_usage_group_by_reaches_the_repository(self, client):
usage, stats = self._usage_repos()
with _admin(
TokenUsageRepository=Mock(return_value=usage),
AdminStatsRepository=Mock(return_value=stats),
):
assert client.get("/api/admin/usage?group_by=model").status_code == 200
assert usage.bucketed_totals.call_args.kwargs["group_by"] == "model"
def test_usage_rejects_an_unknown_grouping(self, client):
with _admin():
assert client.get("/api/admin/usage?group_by=nonsense").status_code == 400
def test_per_user_usage_drilldown(self, client):
usage, stats = self._usage_repos()
stats.user_usage_breakdown.return_value = {
"totals": {"tokens": 8, "cost": 0.25, "calls": 1},
"by_model": [{"key": "gpt-x", "tokens": 8, "cost": 0.25}],
"by_source": [{"key": "agent_stream", "tokens": 8, "cost": 0.25}],
}
with _admin(
TokenUsageRepository=Mock(return_value=usage),
AdminStatsRepository=Mock(return_value=stats),
):
data = _body(client.get("/api/admin/users/u1/usage?days=7"))
assert data["totals"]["cost"] == 0.25
assert data["by_source"][0]["key"] == "agent_stream"
assert len(data["series"]) == 1
assert usage.bucketed_totals.call_args.kwargs["user_id"] == "u1"
def test_per_user_usage_requires_admin(self, client):
with patch("docsgpt.app.handle_auth", return_value={"sub": "u1"}), patch(
"docsgpt.app.resolve_roles", return_value=["user"]
):
assert client.get("/api/admin/users/u1/usage").status_code == 403
def test_usage_invalid_bucket(self, client):
with _admin():
resp = client.get("/api/admin/usage?bucket=year")
@@ -0,0 +1,150 @@
"""Repository tests for admin spend, latency and the per-user drill-down."""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import text
from docsgpt.storage.db.repositories.admin_stats import AdminStatsRepository
from docsgpt.storage.db.repositories.token_usage import TokenUsageRepository
pytestmark = pytest.mark.integration
def _insert(conn, **fields):
defaults = {
"user_id": "u1",
"prompt_tokens": 100,
"generated_tokens": 20,
"cost": 0.5,
"source": "agent_stream",
"model_id": "gpt-x",
"cached_tokens": None,
"duration_ms": None,
"ttft_ms": None,
"timestamp": datetime.now(timezone.utc),
}
defaults.update(fields)
conn.execute(
text(
"INSERT INTO token_usage (user_id, prompt_tokens, generated_tokens, "
"cost, source, model_id, cached_tokens, duration_ms, ttft_ms, timestamp) "
"VALUES (:user_id, :prompt_tokens, :generated_tokens, :cost, :source, "
":model_id, :cached_tokens, :duration_ms, :ttft_ms, :timestamp)"
),
defaults,
)
@pytest.fixture()
def since():
return datetime.now(timezone.utc) - timedelta(days=7)
class TestBucketedCost:
def test_buckets_carry_cost_alongside_tokens(self, pg_conn, since):
_insert(pg_conn, cost=1.25)
_insert(pg_conn, cost=0.75)
series = TokenUsageRepository(pg_conn).bucketed_totals(
bucket_unit="day", timestamp_gte=since
)
assert len(series) == 1
assert series[0]["cost"] == pytest.approx(2.0)
assert series[0]["prompt_tokens"] == 200
def test_unreported_cache_bins_stay_unknown(self, pg_conn, since):
"""NULL means the provider said nothing, which is not the same as 0."""
_insert(pg_conn, cached_tokens=None)
series = TokenUsageRepository(pg_conn).bucketed_totals(
bucket_unit="day", timestamp_gte=since
)
assert series[0]["cached_tokens"] is None
def test_reported_cache_bins_sum(self, pg_conn, since):
_insert(pg_conn, cached_tokens=30)
_insert(pg_conn, cached_tokens=None)
series = TokenUsageRepository(pg_conn).bucketed_totals(
bucket_unit="day", timestamp_gte=since
)
assert series[0]["cached_tokens"] == 30
def test_grouping_keeps_cost_per_group(self, pg_conn, since):
_insert(pg_conn, model_id="cheap", cost=0.1)
_insert(pg_conn, model_id="pricey", cost=9.9)
series = TokenUsageRepository(pg_conn).bucketed_totals(
bucket_unit="day", timestamp_gte=since, group_by="model"
)
costs = {row["group_key"]: row["cost"] for row in series}
assert costs == {"cheap": pytest.approx(0.1), "pricey": pytest.approx(9.9)}
class TestTopUsers:
def test_reports_spend_next_to_tokens(self, pg_conn, since):
_insert(pg_conn, user_id="heavy", prompt_tokens=1000, cost=2.0)
_insert(pg_conn, user_id="light", prompt_tokens=10, cost=0.1)
rows = AdminStatsRepository(pg_conn).top_token_users(since=since)
assert [row["user_id"] for row in rows] == ["heavy", "light"]
assert rows[0]["cost"] == pytest.approx(2.0)
class TestLatencySummary:
def test_percentiles_over_measured_calls(self, pg_conn, since):
for duration in (100, 200, 300, 400):
_insert(pg_conn, duration_ms=duration, ttft_ms=duration // 10)
summary = AdminStatsRepository(pg_conn).latency_summary(since=since)
assert summary["samples"] == 4
assert summary["p50_ms"] == 250
assert summary["p95_ms"] == 385
assert summary["ttft_p50_ms"] == 25
def test_unmeasured_rows_are_skipped_not_counted_as_zero(self, pg_conn, since):
_insert(pg_conn, duration_ms=1000, ttft_ms=None)
_insert(pg_conn, duration_ms=None, ttft_ms=None)
summary = AdminStatsRepository(pg_conn).latency_summary(since=since)
assert summary["samples"] == 1
assert summary["p50_ms"] == 1000
# A stream that never yielded contributes no first-token sample.
assert summary["ttft_samples"] == 0
assert summary["ttft_p50_ms"] is None
def test_no_data_reports_no_samples(self, pg_conn, since):
summary = AdminStatsRepository(pg_conn).latency_summary(since=since)
assert summary == {
"samples": 0,
"p50_ms": None,
"p95_ms": None,
"ttft_samples": 0,
"ttft_p50_ms": None,
}
class TestUserUsageBreakdown:
def test_totals_and_splits(self, pg_conn, since):
_insert(pg_conn, user_id="u1", model_id="a", source="agent_stream", cost=1.0)
_insert(pg_conn, user_id="u1", model_id="b", source="schedule", cost=2.0)
_insert(pg_conn, user_id="other", model_id="a", cost=99.0)
detail = AdminStatsRepository(pg_conn).user_usage_breakdown("u1", since=since)
assert detail["totals"]["calls"] == 2
assert detail["totals"]["cost"] == pytest.approx(3.0)
assert {row["key"] for row in detail["by_model"]} == {"a", "b"}
assert {row["key"] for row in detail["by_source"]} == {
"agent_stream",
"schedule",
}
def test_a_user_with_no_usage_reports_zeroes(self, pg_conn, since):
detail = AdminStatsRepository(pg_conn).user_usage_breakdown("ghost", since=since)
assert detail["totals"] == {"tokens": 0, "cost": 0.0, "calls": 0}
assert detail["by_model"] == [] and detail["by_source"] == []
def test_window_is_respected(self, pg_conn, since):
_insert(
pg_conn,
user_id="u1",
timestamp=datetime.now(timezone.utc) - timedelta(days=30),
)
detail = AdminStatsRepository(pg_conn).user_usage_breakdown("u1", since=since)
assert detail["totals"]["calls"] == 0