mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 02:15:06 +00:00
feat(admin): cost, group-by and latency in usage
Three gaps in one view. Cost: token_usage.cost has been written on every call since quotas landed, and tokens_by_model already returned it, but bucketed_totals and top_token_users selected tokens only. An admin could set a USD quota in /admin/quotas and had no way to see the spend it was capping. Buckets and top users now carry cost, the endpoint reports a window total and a per-model split, and the chart takes a Tokens/Cost toggle with a currency axis. Group-by: the endpoint has supported group_by=model|agent|source from the start and the UI only ever sent bucket=day. The selector is now wired, and a grouped series pads missing buckets so a model that was idle on Tuesday plots a zero instead of shifting its whole row one bar left. Latency: surfaced from the columns the previous commit added, as p50/p95 with a median time-to-first-token underneath. Unmeasured rows are excluded rather than counted as zero, and the card says so when nothing was measured. Also surfaces the prompt-cache hit rate, computed only over rows whose provider reported a cache breakdown -- NULL means "not reported", and folding those in as 0% would understate it. Per-user drill-down: GET /api/admin/users/<id>/usage returns a daily series plus splits by model and by flow, reachable from the top-users table and from the user detail dialog. The detail dialog previously showed one tokens_30d number, which answers neither "what is this person costing" nor "what is driving it".
This commit is contained in:
1 parent
b17f40a993
commit
a56f9df122
14 files changed
+1019
-61
No files matched your search
@@ -287,6 +287,26 @@ class AdminUserSessionsResource(Resource):
|
||||
return make_response(jsonify({"success": True, "revoked": ok}), 200)
|
||||
|
||||
|
||||
@admin_ns.route("/admin/users/<string:user_id>/usage")
|
||||
class AdminUserUsageResource(Resource):
|
||||
@admin_required
|
||||
def get(self, user_id):
|
||||
"""One user's spend: daily series plus a split by model and by flow."""
|
||||
days = max(1, min(365, _int_arg("days", 30)))
|
||||
start = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
with db_readonly() as conn:
|
||||
series = TokenUsageRepository(conn).bucketed_totals(
|
||||
bucket_unit="day", user_id=user_id, timestamp_gte=start
|
||||
)
|
||||
breakdown = AdminStatsRepository(conn).user_usage_breakdown(
|
||||
user_id, since=start
|
||||
)
|
||||
return make_response(
|
||||
jsonify({"success": True, "days": days, "series": series, **breakdown}),
|
||||
200,
|
||||
)
|
||||
|
||||
|
||||
@admin_ns.route("/admin/admins")
|
||||
class AdminAdminsResource(Resource):
|
||||
@admin_required
|
||||
@@ -304,7 +324,12 @@ class AdminUsageResource(Resource):
|
||||
|
||||
@admin_required
|
||||
def get(self):
|
||||
"""Global token usage: time-bucketed series + total + top users."""
|
||||
"""Global usage: time-bucketed series, totals, spend, latency, top users.
|
||||
|
||||
Every bucket carries both tokens and USD cost. Quota policies have
|
||||
always been settable in dollars; until now nothing showed the spend
|
||||
they were capping.
|
||||
"""
|
||||
days = max(1, min(365, _int_arg("days", 30)))
|
||||
bucket = request.args.get("bucket", "day")
|
||||
group_by = request.args.get("group_by", "none")
|
||||
@@ -313,6 +338,7 @@ class AdminUsageResource(Resource):
|
||||
start = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
with db_readonly() as conn:
|
||||
usage_repo = TokenUsageRepository(conn)
|
||||
stats_repo = AdminStatsRepository(conn)
|
||||
series = usage_repo.bucketed_totals(
|
||||
bucket_unit=bucket,
|
||||
timestamp_gte=start,
|
||||
@@ -321,7 +347,9 @@ class AdminUsageResource(Resource):
|
||||
total = usage_repo.sum_tokens_in_range(
|
||||
start=start, end=datetime.now(timezone.utc)
|
||||
)
|
||||
top_users = AdminStatsRepository(conn).top_token_users(since=start, limit=10)
|
||||
by_model = usage_repo.tokens_by_model(start=start)
|
||||
latency = stats_repo.latency_summary(since=start)
|
||||
top_users = stats_repo.top_token_users(since=start, limit=10)
|
||||
return make_response(
|
||||
jsonify(
|
||||
{
|
||||
@@ -331,6 +359,9 @@ class AdminUsageResource(Resource):
|
||||
"group_by": group_by,
|
||||
"series": series,
|
||||
"total_tokens": int(total),
|
||||
"total_cost": round(sum(row["cost"] for row in by_model), 4),
|
||||
"by_model": by_model,
|
||||
"latency": latency,
|
||||
"top_users": top_users,
|
||||
}
|
||||
),
|
||||
|
||||
@@ -16,6 +16,11 @@ from sqlalchemy import Connection, text
|
||||
from docsgpt.storage.db.base_repository import row_to_dict
|
||||
|
||||
|
||||
def _round_ms(value) -> Optional[int]:
|
||||
"""Round a percentile to whole milliseconds, preserving NULL as None."""
|
||||
return int(round(float(value))) if value is not None else None
|
||||
|
||||
|
||||
class AdminStatsRepository:
|
||||
"""Global aggregates for the admin overview and per-user drill-down."""
|
||||
|
||||
@@ -66,11 +71,18 @@ class AdminStatsRepository:
|
||||
}
|
||||
|
||||
def top_token_users(self, *, since: datetime, limit: int = 10) -> list[dict]:
|
||||
"""Highest token consumers since ``since`` (admin usage view)."""
|
||||
"""Highest consumers since ``since``, with spend (admin usage view).
|
||||
|
||||
Ordered by tokens, not cost: a cheap model can dominate token volume
|
||||
while a costly one dominates the bill, and the operator wants to see
|
||||
both columns rather than have the ranking pick for them.
|
||||
"""
|
||||
result = self._conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT user_id, SUM(prompt_tokens + generated_tokens) AS tokens
|
||||
SELECT user_id,
|
||||
SUM(prompt_tokens + generated_tokens) AS tokens,
|
||||
COALESCE(SUM(cost), 0) AS cost
|
||||
FROM token_usage
|
||||
WHERE timestamp >= :since AND user_id IS NOT NULL
|
||||
GROUP BY user_id
|
||||
@@ -80,7 +92,105 @@ class AdminStatsRepository:
|
||||
),
|
||||
{"since": since, "limit": int(limit)},
|
||||
)
|
||||
return [{"user_id": r[0], "tokens": int(r[1])} for r in result.fetchall()]
|
||||
return [
|
||||
{"user_id": r[0], "tokens": int(r[1]), "cost": float(r[2])}
|
||||
for r in result.fetchall()
|
||||
]
|
||||
|
||||
def latency_summary(self, *, since: datetime) -> dict:
|
||||
"""Call-latency percentiles since ``since``.
|
||||
|
||||
Rows written before the latency columns existed, and streams that
|
||||
never yielded, hold NULL; percentile_cont skips them, so the summary
|
||||
describes the calls that were actually measured. ``samples`` says how
|
||||
many those were, so a percentile from three calls is not read as a
|
||||
service-level figure.
|
||||
"""
|
||||
row = self._conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT count(duration_ms) AS samples,
|
||||
percentile_cont(0.5) WITHIN GROUP (ORDER BY duration_ms)
|
||||
AS p50_ms,
|
||||
percentile_cont(0.95) WITHIN GROUP (ORDER BY duration_ms)
|
||||
AS p95_ms,
|
||||
count(ttft_ms) AS ttft_samples,
|
||||
percentile_cont(0.5) WITHIN GROUP (ORDER BY ttft_ms)
|
||||
AS ttft_p50_ms
|
||||
FROM token_usage
|
||||
WHERE timestamp >= :since
|
||||
"""
|
||||
),
|
||||
{"since": since},
|
||||
).one()
|
||||
return {
|
||||
"samples": int(row.samples or 0),
|
||||
"p50_ms": _round_ms(row.p50_ms),
|
||||
"p95_ms": _round_ms(row.p95_ms),
|
||||
"ttft_samples": int(row.ttft_samples or 0),
|
||||
"ttft_p50_ms": _round_ms(row.ttft_p50_ms),
|
||||
}
|
||||
|
||||
def user_usage_breakdown(
|
||||
self, user_id: str, *, since: datetime, limit: int = 10
|
||||
) -> dict:
|
||||
"""Per-user spend detail: totals, and a split by model and by flow.
|
||||
|
||||
The admin user drill-down showed a single ``tokens_30d`` number, which
|
||||
cannot answer either question an operator actually has -- what is this
|
||||
person costing, and what is driving it.
|
||||
|
||||
Args:
|
||||
user_id: The billable user (auth ``sub``).
|
||||
since: Inclusive window start.
|
||||
limit: How many models / flows to return, busiest first.
|
||||
|
||||
Returns:
|
||||
``{"totals": {...}, "by_model": [...], "by_source": [...]}``.
|
||||
"""
|
||||
totals = self._conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT COALESCE(SUM(prompt_tokens + generated_tokens), 0) AS tokens,
|
||||
COALESCE(SUM(cost), 0) AS cost,
|
||||
count(*) AS calls
|
||||
FROM token_usage
|
||||
WHERE user_id = :u AND timestamp >= :since
|
||||
"""
|
||||
),
|
||||
{"u": user_id, "since": since},
|
||||
).one()
|
||||
|
||||
def _breakdown(key_expr: str) -> list[dict]:
|
||||
rows = self._conn.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT {key_expr} AS key,
|
||||
COALESCE(SUM(prompt_tokens + generated_tokens), 0) AS tokens,
|
||||
COALESCE(SUM(cost), 0) AS cost
|
||||
FROM token_usage
|
||||
WHERE user_id = :u AND timestamp >= :since
|
||||
GROUP BY key
|
||||
ORDER BY tokens DESC, key
|
||||
LIMIT :limit
|
||||
"""
|
||||
),
|
||||
{"u": user_id, "since": since, "limit": int(limit)},
|
||||
)
|
||||
return [
|
||||
{"key": r.key, "tokens": int(r.tokens), "cost": float(r.cost)}
|
||||
for r in rows.fetchall()
|
||||
]
|
||||
|
||||
return {
|
||||
"totals": {
|
||||
"tokens": int(totals.tokens),
|
||||
"cost": float(totals.cost),
|
||||
"calls": int(totals.calls),
|
||||
},
|
||||
"by_model": _breakdown("COALESCE(model_id, 'unknown')"),
|
||||
"by_source": _breakdown("COALESCE(source, 'agent_stream')"),
|
||||
}
|
||||
|
||||
def list_users(
|
||||
self, user_id_filter: Optional[str], offset: int, limit: int
|
||||
|
||||
@@ -302,7 +302,11 @@ class TokenUsageRepository:
|
||||
f"""
|
||||
SELECT to_char(tu.timestamp AT TIME ZONE 'UTC', :fmt) AS bucket,
|
||||
COALESCE(SUM(tu.prompt_tokens), 0) AS prompt_tokens,
|
||||
COALESCE(SUM(tu.generated_tokens), 0) AS generated_tokens
|
||||
COALESCE(SUM(tu.generated_tokens), 0) AS generated_tokens,
|
||||
COALESCE(SUM(tu.cost), 0) AS cost,
|
||||
-- NULL cache bins mean "provider did not report", so
|
||||
-- they stay out of the sum rather than reading as 0.
|
||||
SUM(tu.cached_tokens) AS cached_tokens
|
||||
{group_select}
|
||||
FROM token_usage tu
|
||||
{join}
|
||||
@@ -318,6 +322,12 @@ class TokenUsageRepository:
|
||||
"bucket": row._mapping["bucket"],
|
||||
"prompt_tokens": int(row._mapping["prompt_tokens"]),
|
||||
"generated_tokens": int(row._mapping["generated_tokens"]),
|
||||
"cost": float(row._mapping["cost"]),
|
||||
"cached_tokens": (
|
||||
int(row._mapping["cached_tokens"])
|
||||
if row._mapping["cached_tokens"] is not None
|
||||
else None
|
||||
),
|
||||
**(
|
||||
{"group_key": row._mapping["group_key"]}
|
||||
if group_by is not None
|
||||
|
||||
@@ -97,6 +97,28 @@ export function fmtNumber(n?: number | null): string {
|
||||
return new Intl.NumberFormat().format(n ?? 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* USD, with enough precision that a fraction of a cent is not rendered as $0.
|
||||
* An instance on a cheap model can run whole days under a dollar, and "$0"
|
||||
* next to a cost quota reads as "nothing is being counted".
|
||||
*/
|
||||
export function fmtUsd(n?: number | null): string {
|
||||
const value = n ?? 0;
|
||||
const digits = value !== 0 && Math.abs(value) < 0.01 ? 4 : 2;
|
||||
return new Intl.NumberFormat(undefined, {
|
||||
style: 'currency',
|
||||
currency: 'USD',
|
||||
minimumFractionDigits: 2,
|
||||
maximumFractionDigits: digits,
|
||||
}).format(value);
|
||||
}
|
||||
|
||||
/** Milliseconds as ms or s, or an em dash when nothing was measured. */
|
||||
export function fmtMs(n?: number | null): string {
|
||||
if (n === null || n === undefined) return '—';
|
||||
return n < 1000 ? `${Math.round(n)}ms` : `${(n / 1000).toFixed(1)}s`;
|
||||
}
|
||||
|
||||
export function fmtCompact(n?: number | null): string {
|
||||
return new Intl.NumberFormat(undefined, {
|
||||
notation: 'compact',
|
||||
|
||||
+143
-51
@@ -4,6 +4,13 @@ import { useSelector } from 'react-redux';
|
||||
import adminService from '../api/services/adminService';
|
||||
import SkeletonLoader from '../components/SkeletonLoader';
|
||||
import { Button } from '../components/ui/button';
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from '../components/ui/select';
|
||||
import {
|
||||
Table,
|
||||
TableBody,
|
||||
@@ -15,16 +22,33 @@ import {
|
||||
} from '../components/ui/table';
|
||||
import { useDarkTheme } from '../hooks';
|
||||
import { selectToken } from '../preferences/preferenceSlice';
|
||||
import { formatDate } from '../utils/dateTimeUtils';
|
||||
import { Loading, LoadError, StatCard, fmtNumber } from './AdminUI';
|
||||
import UsageChart, { usageColors } from './UsageChart';
|
||||
import {
|
||||
Loading,
|
||||
LoadError,
|
||||
StatCard,
|
||||
fmtMs,
|
||||
fmtNumber,
|
||||
fmtUsd,
|
||||
} from './AdminUI';
|
||||
import UsageChart from './UsageChart';
|
||||
import {
|
||||
GROUP_OPTIONS,
|
||||
buildUsageChart,
|
||||
cacheHitRate as computeCacheHitRate,
|
||||
type GroupBy,
|
||||
type Metric,
|
||||
type UsageBucket,
|
||||
} from './usageChartData';
|
||||
import UserUsageModal from './UserUsageModal';
|
||||
|
||||
type Bucket = {
|
||||
bucket: string;
|
||||
prompt_tokens: number;
|
||||
generated_tokens: number;
|
||||
type TopUser = { user_id: string; tokens: number; cost: number };
|
||||
type Latency = {
|
||||
samples: number;
|
||||
p50_ms: number | null;
|
||||
p95_ms: number | null;
|
||||
ttft_samples: number;
|
||||
ttft_p50_ms: number | null;
|
||||
};
|
||||
type TopUser = { user_id: string; tokens: number };
|
||||
|
||||
const RANGES = [7, 30, 90];
|
||||
|
||||
@@ -32,47 +56,40 @@ export default function Usage() {
|
||||
const token = useSelector(selectToken);
|
||||
const [isDarkTheme] = useDarkTheme();
|
||||
const [days, setDays] = useState(30);
|
||||
const [groupBy, setGroupBy] = useState<GroupBy>('none');
|
||||
const [metric, setMetric] = useState<Metric>('tokens');
|
||||
const [data, setData] = useState<any | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [drilldown, setDrilldown] = useState<string | null>(null);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const res = await adminService.getUsage({ days, bucket: 'day' }, token);
|
||||
const res = await adminService.getUsage(
|
||||
{ days, bucket: 'day', group_by: groupBy },
|
||||
token,
|
||||
);
|
||||
setData(await res.json().catch(() => ({})));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [token, days]);
|
||||
}, [token, days, groupBy]);
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
}, [load]);
|
||||
|
||||
const series: Bucket[] = data?.series ?? [];
|
||||
const series: UsageBucket[] = data?.series ?? [];
|
||||
const topUsers: TopUser[] = data?.top_users ?? [];
|
||||
const promptTotal = series.reduce((sum, b) => sum + b.prompt_tokens, 0);
|
||||
const generatedTotal = series.reduce((sum, b) => sum + b.generated_tokens, 0);
|
||||
const latency: Latency | undefined = data?.latency;
|
||||
|
||||
const chartData = useMemo(() => {
|
||||
const colors = usageColors();
|
||||
return {
|
||||
labels: series.map((b) => formatDate(b.bucket)),
|
||||
datasets: [
|
||||
{
|
||||
label: 'Prompt',
|
||||
data: series.map((b) => b.prompt_tokens),
|
||||
backgroundColor: colors.prompt,
|
||||
},
|
||||
{
|
||||
label: 'Generated',
|
||||
data: series.map((b) => b.generated_tokens),
|
||||
backgroundColor: colors.generated,
|
||||
},
|
||||
],
|
||||
};
|
||||
const chartData = useMemo(
|
||||
() => buildUsageChart(series, groupBy, metric),
|
||||
// isDarkTheme re-resolves the canvas colors when the theme toggles.
|
||||
}, [series, isDarkTheme]);
|
||||
[series, groupBy, metric, isDarkTheme],
|
||||
);
|
||||
|
||||
const cacheHitRate = useMemo(() => computeCacheHitRate(series), [series]);
|
||||
|
||||
if (data === null && loading) return <Loading />;
|
||||
if (data && !data.success)
|
||||
@@ -80,34 +97,89 @@ export default function Usage() {
|
||||
|
||||
return (
|
||||
<div className="mt-6">
|
||||
<div className="mb-4 flex items-center gap-1">
|
||||
{RANGES.map((r) => (
|
||||
<div className="mb-4 flex flex-wrap items-center gap-2">
|
||||
<div className="flex items-center gap-1">
|
||||
{RANGES.map((range) => (
|
||||
<Button
|
||||
key={range}
|
||||
variant={range === days ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
className="rounded-3xl"
|
||||
onClick={() => setDays(range)}
|
||||
>
|
||||
{range}d
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
<Select
|
||||
value={groupBy}
|
||||
onValueChange={(value) => setGroupBy(value as GroupBy)}
|
||||
>
|
||||
<SelectTrigger className="w-40">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{GROUP_OPTIONS.map((option) => (
|
||||
<SelectItem key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<div className="flex items-center gap-1">
|
||||
<Button
|
||||
key={r}
|
||||
variant={r === days ? 'default' : 'outline'}
|
||||
variant={metric === 'tokens' ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
className="rounded-3xl"
|
||||
onClick={() => setDays(r)}
|
||||
onClick={() => setMetric('tokens')}
|
||||
>
|
||||
{r}d
|
||||
Tokens
|
||||
</Button>
|
||||
))}
|
||||
<Button
|
||||
variant={metric === 'cost' ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
className="rounded-3xl"
|
||||
onClick={() => setMetric('cost')}
|
||||
>
|
||||
Cost
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-3 md:grid-cols-4">
|
||||
<StatCard
|
||||
label={`Total tokens (${days}d)`}
|
||||
label={`Spend (${days}d)`}
|
||||
value={fmtUsd(data?.total_cost)}
|
||||
sub="Unpriced and BYO models record $0"
|
||||
/>
|
||||
<StatCard
|
||||
label={`Tokens (${days}d)`}
|
||||
value={fmtNumber(data?.total_tokens)}
|
||||
/>
|
||||
<StatCard label="Prompt tokens" value={fmtNumber(promptTotal)} />
|
||||
<StatCard label="Generated tokens" value={fmtNumber(generatedTotal)} />
|
||||
<StatCard label="Days with usage" value={fmtNumber(series.length)} />
|
||||
<StatCard
|
||||
label="Latency p50 / p95"
|
||||
value={`${fmtMs(latency?.p50_ms)} / ${fmtMs(latency?.p95_ms)}`}
|
||||
sub={
|
||||
latency?.ttft_p50_ms !== null && latency?.ttft_p50_ms !== undefined
|
||||
? `First token ${fmtMs(latency.ttft_p50_ms)} (p50)`
|
||||
: 'No streamed calls measured'
|
||||
}
|
||||
/>
|
||||
<StatCard
|
||||
label="Prompt cache hits"
|
||||
value={cacheHitRate === null ? '—' : `${cacheHitRate.toFixed(1)}%`}
|
||||
sub={
|
||||
cacheHitRate === null
|
||||
? 'Provider reported no cache data'
|
||||
: 'Of prompt tokens the provider reported'
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="border-border dark:border-border mt-4 h-[345px] w-full overflow-hidden rounded-2xl border px-6 py-5">
|
||||
<div className="flex flex-row items-center justify-between gap-3">
|
||||
<p className="text-foreground dark:text-foreground font-bold">
|
||||
Token usage
|
||||
{metric === 'cost' ? 'Spend' : 'Token usage'}
|
||||
</p>
|
||||
<div
|
||||
id="admin-usage-legend"
|
||||
@@ -122,14 +194,18 @@ export default function Usage() {
|
||||
No usage in this period.
|
||||
</p>
|
||||
) : (
|
||||
<UsageChart data={chartData} legendID="admin-usage-legend" />
|
||||
<UsageChart
|
||||
data={chartData}
|
||||
legendID="admin-usage-legend"
|
||||
currency={metric === 'cost'}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="border-border dark:border-border mt-4 w-full overflow-hidden rounded-2xl border px-6 py-5">
|
||||
<p className="text-foreground dark:text-foreground mb-3 font-bold">
|
||||
Top users by tokens
|
||||
Top users
|
||||
</p>
|
||||
{topUsers.length === 0 ? (
|
||||
<p className="text-muted-foreground text-sm">No usage.</p>
|
||||
@@ -139,17 +215,31 @@ export default function Usage() {
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableHeader>User</TableHeader>
|
||||
<TableHeader className="text-right">Tokens</TableHeader>
|
||||
<TableHeader align="right">Tokens</TableHeader>
|
||||
<TableHeader align="right">Cost</TableHeader>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
<TableBody>
|
||||
{topUsers.map((u) => (
|
||||
<TableRow key={u.user_id}>
|
||||
{topUsers.map((user) => (
|
||||
<TableRow
|
||||
key={user.user_id}
|
||||
className="hover:bg-muted/40 cursor-pointer"
|
||||
onClick={() => setDrilldown(user.user_id)}
|
||||
>
|
||||
<TableCell className="font-mono text-[13px] break-all">
|
||||
{u.user_id}
|
||||
{user.user_id}
|
||||
</TableCell>
|
||||
<TableCell className="text-right whitespace-nowrap tabular-nums">
|
||||
{fmtNumber(u.tokens)}
|
||||
<TableCell
|
||||
align="right"
|
||||
className="whitespace-nowrap tabular-nums"
|
||||
>
|
||||
{fmtNumber(user.tokens)}
|
||||
</TableCell>
|
||||
<TableCell
|
||||
align="right"
|
||||
className="whitespace-nowrap tabular-nums"
|
||||
>
|
||||
{fmtUsd(user.cost)}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
@@ -158,6 +248,8 @@ export default function Usage() {
|
||||
</TableContainer>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<UserUsageModal userId={drilldown} onClose={() => setDrilldown(null)} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -39,6 +39,25 @@ export function usageColors(): { prompt: string; generated: string } {
|
||||
};
|
||||
}
|
||||
|
||||
// Follow-on colors for a grouped chart (by model / agent / source), matching
|
||||
// the palette settings/Analytics already uses for its stacked series.
|
||||
const SERIES_COLORS = [
|
||||
'#FF6384',
|
||||
'#36A2EB',
|
||||
'#FFCE56',
|
||||
'#4BC0C0',
|
||||
'#9966FF',
|
||||
'#FF9F40',
|
||||
'#2BC596',
|
||||
];
|
||||
|
||||
/** Color for dataset ``index``; the first uses the resolved brand color. */
|
||||
export function seriesColor(index: number): string {
|
||||
return index === 0
|
||||
? readCssVar('--primary', '#7D54D1')
|
||||
: SERIES_COLORS[(index - 1) % SERIES_COLORS.length];
|
||||
}
|
||||
|
||||
function compactTick(value: number | string): string {
|
||||
const n = typeof value === 'number' ? value : Number(value);
|
||||
if (Number.isNaN(n)) return String(value);
|
||||
@@ -48,16 +67,28 @@ function compactTick(value: number | string): string {
|
||||
}).format(n);
|
||||
}
|
||||
|
||||
function currencyTick(value: number | string): string {
|
||||
const n = typeof value === 'number' ? value : Number(value);
|
||||
if (Number.isNaN(n)) return String(value);
|
||||
return `$${new Intl.NumberFormat(undefined, {
|
||||
notation: 'compact',
|
||||
maximumFractionDigits: 2,
|
||||
}).format(n)}`;
|
||||
}
|
||||
|
||||
type UsageChartProps = {
|
||||
data: ChartData<'bar'>;
|
||||
legendID: string;
|
||||
maxTicksLimitInX?: number;
|
||||
/** Format the y axis as USD rather than a token count. */
|
||||
currency?: boolean;
|
||||
};
|
||||
|
||||
export default function UsageChart({
|
||||
data,
|
||||
legendID,
|
||||
maxTicksLimitInX = 8,
|
||||
currency = false,
|
||||
}: UsageChartProps) {
|
||||
const options = {
|
||||
responsive: true,
|
||||
@@ -76,7 +107,7 @@ export default function UsageChart({
|
||||
y: {
|
||||
grid: { lineWidth: 0.2, color: '#C4C4C4' },
|
||||
border: { width: 0.2, color: '#C4C4C4' },
|
||||
ticks: { callback: compactTick },
|
||||
ticks: { callback: currency ? currencyTick : compactTick },
|
||||
stacked: true,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
import { useEffect, useMemo, useState } from 'react';
|
||||
import { useSelector } from 'react-redux';
|
||||
|
||||
import adminService from '../api/services/adminService';
|
||||
import { Button } from '../components/ui/button';
|
||||
import { Modal } from '../components/ui/modal';
|
||||
import {
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableContainer,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
} from '../components/ui/table';
|
||||
import { useDarkTheme } from '../hooks';
|
||||
import { selectToken } from '../preferences/preferenceSlice';
|
||||
import { formatDate } from '../utils/dateTimeUtils';
|
||||
import { Loading, LoadError, StatCard, fmtNumber, fmtUsd } from './AdminUI';
|
||||
import UsageChart, { usageColors } from './UsageChart';
|
||||
|
||||
type Bucket = {
|
||||
bucket: string;
|
||||
prompt_tokens: number;
|
||||
generated_tokens: number;
|
||||
cost: number;
|
||||
};
|
||||
type Split = { key: string; tokens: number; cost: number };
|
||||
|
||||
const RANGES = [7, 30, 90];
|
||||
|
||||
/** Label for a token_usage.source value — what the spend was for. */
|
||||
const FLOW_LABELS: Record<string, string> = {
|
||||
agent_stream: 'Chat',
|
||||
schedule: 'Scheduled run',
|
||||
webhook: 'Webhook',
|
||||
workflow: 'Workflow',
|
||||
title: 'Title generation',
|
||||
compression: 'History compression',
|
||||
rag_condense: 'Question condensing',
|
||||
fallback: 'Provider fallback',
|
||||
};
|
||||
|
||||
function SplitTable({ title, rows }: { title: string; rows: Split[] }) {
|
||||
return (
|
||||
<div>
|
||||
<p className="text-muted-foreground mb-2 text-sm font-medium">{title}</p>
|
||||
{rows.length === 0 ? (
|
||||
<p className="text-muted-foreground text-sm">No usage.</p>
|
||||
) : (
|
||||
<TableContainer>
|
||||
<Table>
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableHeader>{title}</TableHeader>
|
||||
<TableHeader align="right">Tokens</TableHeader>
|
||||
<TableHeader align="right">Cost</TableHeader>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
<TableBody>
|
||||
{rows.map((row) => (
|
||||
<TableRow key={row.key}>
|
||||
<TableCell className="text-[13px] break-all">
|
||||
{FLOW_LABELS[row.key] ?? row.key}
|
||||
</TableCell>
|
||||
<TableCell align="right" className="tabular-nums">
|
||||
{fmtNumber(row.tokens)}
|
||||
</TableCell>
|
||||
<TableCell align="right" className="tabular-nums">
|
||||
{fmtUsd(row.cost)}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-user spend drill-down. The user detail view carried a single
|
||||
* `tokens_30d` number, which answers neither question an operator has: what is
|
||||
* this person costing, and what is driving it.
|
||||
*/
|
||||
export default function UserUsageModal({
|
||||
userId,
|
||||
onClose,
|
||||
}: {
|
||||
userId: string | null;
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const token = useSelector(selectToken);
|
||||
const [isDarkTheme] = useDarkTheme();
|
||||
const [days, setDays] = useState(30);
|
||||
const [data, setData] = useState<any | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!userId) {
|
||||
setData(null);
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
setLoading(true);
|
||||
adminService
|
||||
.getUserUsage(userId, { days }, token)
|
||||
.then((res) => res.json())
|
||||
.then((json) => {
|
||||
if (!cancelled) setData(json);
|
||||
})
|
||||
.catch(() => !cancelled && setData({ success: false }))
|
||||
.finally(() => !cancelled && setLoading(false));
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [userId, days, token]);
|
||||
|
||||
const series: Bucket[] = data?.series ?? [];
|
||||
const chartData = useMemo(() => {
|
||||
const colors = usageColors();
|
||||
return {
|
||||
labels: series.map((bucket) => formatDate(bucket.bucket)),
|
||||
datasets: [
|
||||
{
|
||||
label: 'Prompt',
|
||||
data: series.map((bucket) => bucket.prompt_tokens),
|
||||
backgroundColor: colors.prompt,
|
||||
},
|
||||
{
|
||||
label: 'Generated',
|
||||
data: series.map((bucket) => bucket.generated_tokens),
|
||||
backgroundColor: colors.generated,
|
||||
},
|
||||
],
|
||||
};
|
||||
// isDarkTheme re-resolves the canvas colors when the theme toggles.
|
||||
}, [series, isDarkTheme]);
|
||||
|
||||
const totals = data?.totals ?? {};
|
||||
|
||||
return (
|
||||
<Modal
|
||||
open={userId !== null}
|
||||
onOpenChange={(open) => !open && onClose()}
|
||||
title={userId ? `Usage · ${userId}` : 'Usage'}
|
||||
size="lg"
|
||||
>
|
||||
<div className="mb-4 flex items-center gap-1">
|
||||
{RANGES.map((range) => (
|
||||
<Button
|
||||
key={range}
|
||||
variant={range === days ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
className="rounded-3xl"
|
||||
onClick={() => setDays(range)}
|
||||
>
|
||||
{range}d
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{loading && data === null ? (
|
||||
<Loading />
|
||||
) : data && !data.success ? (
|
||||
<LoadError message="Failed to load usage." />
|
||||
) : (
|
||||
<div className="space-y-4">
|
||||
<div className="grid grid-cols-3 gap-3">
|
||||
<StatCard label="Cost" value={fmtUsd(totals.cost)} />
|
||||
<StatCard label="Tokens" value={fmtNumber(totals.tokens)} />
|
||||
<StatCard label="Calls" value={fmtNumber(totals.calls)} />
|
||||
</div>
|
||||
|
||||
<div className="border-border h-64 rounded-2xl border px-4 py-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<p className="text-foreground text-sm font-bold">Daily tokens</p>
|
||||
<div id="admin-user-usage-legend" className="flex" />
|
||||
</div>
|
||||
<div className="relative mt-px h-48 w-full">
|
||||
{series.length === 0 ? (
|
||||
<p className="text-muted-foreground mt-8 text-sm">
|
||||
No usage in this period.
|
||||
</p>
|
||||
) : (
|
||||
<UsageChart
|
||||
data={chartData}
|
||||
legendID="admin-user-usage-legend"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<SplitTable title="Model" rows={data?.by_model ?? []} />
|
||||
<SplitTable title="Flow" rows={data?.by_source ?? []} />
|
||||
</div>
|
||||
)}
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
@@ -42,6 +42,7 @@ import {
|
||||
fmtRelative,
|
||||
} from './AdminUI';
|
||||
import UserQuotaModal from './UserQuotaModal';
|
||||
import UserUsageModal from './UserUsageModal';
|
||||
|
||||
type AdminUser = {
|
||||
user_id: string;
|
||||
@@ -72,6 +73,7 @@ export default function Users() {
|
||||
const [busy, setBusy] = useState<string | null>(null);
|
||||
const [menuUserId, setMenuUserId] = useState<string | null>(null);
|
||||
const [detail, setDetail] = useState<any | null>(null);
|
||||
const [usageFor, setUsageFor] = useState<string | null>(null);
|
||||
const [quotaUserId, setQuotaUserId] = useState<string | null>(null);
|
||||
const [feedback, setFeedback] = useState<{
|
||||
ok: boolean;
|
||||
@@ -443,6 +445,7 @@ export default function Users() {
|
||||
/>
|
||||
) : null}
|
||||
|
||||
<UserUsageModal userId={usageFor} onClose={() => setUsageFor(null)} />
|
||||
<UserQuotaModal
|
||||
userId={quotaUserId}
|
||||
onClose={() => setQuotaUserId(null)}
|
||||
@@ -528,6 +531,19 @@ export default function Users() {
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
// Close the detail dialog before the usage dialog opens, so
|
||||
// the two never stack.
|
||||
const userId = detail.user.user_id;
|
||||
setDetail(null);
|
||||
setUsageFor(userId);
|
||||
}}
|
||||
>
|
||||
View spend breakdown
|
||||
</Button>
|
||||
<div>
|
||||
<p className="text-muted-foreground mb-1 text-xs">
|
||||
Recent auth events
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import {
|
||||
buildUsageChart,
|
||||
cacheHitRate,
|
||||
type UsageBucket,
|
||||
} from './usageChartData';
|
||||
|
||||
// Chart.js colors resolve against a live DOM; the shapes under test do not.
|
||||
vi.mock('./UsageChart', () => ({
|
||||
seriesColor: (index: number) => `color-${index}`,
|
||||
usageColors: () => ({ prompt: 'prompt-color', generated: 'generated-color' }),
|
||||
}));
|
||||
|
||||
const bucket = (overrides: Partial<UsageBucket>): UsageBucket => ({
|
||||
bucket: '2026-01-01',
|
||||
prompt_tokens: 100,
|
||||
generated_tokens: 20,
|
||||
cost: 1,
|
||||
cached_tokens: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('buildUsageChart', () => {
|
||||
it('splits prompt and generated tokens when ungrouped', () => {
|
||||
const chart = buildUsageChart([bucket({})], 'none', 'tokens');
|
||||
expect(chart.datasets.map((d) => d.label)).toEqual(['Prompt', 'Generated']);
|
||||
expect(chart.datasets[0].data).toEqual([100]);
|
||||
});
|
||||
|
||||
it('plots a single series for cost', () => {
|
||||
const chart = buildUsageChart([bucket({ cost: 2.5 })], 'none', 'cost');
|
||||
expect(chart.datasets).toHaveLength(1);
|
||||
expect(chart.datasets[0].data).toEqual([2.5]);
|
||||
});
|
||||
|
||||
it('gives each group its own dataset', () => {
|
||||
const chart = buildUsageChart(
|
||||
[
|
||||
bucket({ group_key: 'gpt-a' }),
|
||||
bucket({ group_key: 'gpt-b', prompt_tokens: 5, generated_tokens: 5 }),
|
||||
],
|
||||
'model',
|
||||
'tokens',
|
||||
);
|
||||
expect(chart.datasets.map((d) => d.label)).toEqual(['gpt-a', 'gpt-b']);
|
||||
expect(chart.datasets[1].data).toEqual([10]);
|
||||
});
|
||||
|
||||
it('pads a group that is missing a bucket so the axes stay aligned', () => {
|
||||
const chart = buildUsageChart(
|
||||
[
|
||||
bucket({ bucket: '2026-01-01', group_key: 'a' }),
|
||||
bucket({ bucket: '2026-01-02', group_key: 'b' }),
|
||||
],
|
||||
'model',
|
||||
'tokens',
|
||||
);
|
||||
expect(chart.labels).toHaveLength(2);
|
||||
// 'a' has no row on the second day: a zero, not a shifted bar.
|
||||
expect(chart.datasets[0].data).toEqual([120, 0]);
|
||||
expect(chart.datasets[1].data).toEqual([0, 120]);
|
||||
});
|
||||
|
||||
it('labels a group with no key rather than dropping the row', () => {
|
||||
const chart = buildUsageChart([bucket({})], 'model', 'tokens');
|
||||
expect(chart.datasets[0].label).toBe('unknown');
|
||||
});
|
||||
|
||||
it('handles an empty series', () => {
|
||||
const chart = buildUsageChart([], 'model', 'cost');
|
||||
expect(chart.labels).toEqual([]);
|
||||
expect(chart.datasets).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cacheHitRate', () => {
|
||||
it('is unknown when no provider reported cache bins', () => {
|
||||
expect(cacheHitRate([bucket({ cached_tokens: null })])).toBeNull();
|
||||
});
|
||||
|
||||
it('is unknown for an empty series', () => {
|
||||
expect(cacheHitRate([])).toBeNull();
|
||||
});
|
||||
|
||||
it('measures only the rows that reported', () => {
|
||||
const rate = cacheHitRate([
|
||||
bucket({ prompt_tokens: 100, cached_tokens: 25 }),
|
||||
// Unreported: excluded entirely rather than counted as a 0% row, which
|
||||
// would halve the rate.
|
||||
bucket({ prompt_tokens: 100, cached_tokens: null }),
|
||||
]);
|
||||
expect(rate).toBeCloseTo(25);
|
||||
});
|
||||
|
||||
it('avoids dividing by zero prompt tokens', () => {
|
||||
expect(
|
||||
cacheHitRate([bucket({ prompt_tokens: 0, cached_tokens: 0 })]),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,113 @@
|
||||
import { formatDate } from '../utils/dateTimeUtils';
|
||||
import { seriesColor, usageColors } from './UsageChart';
|
||||
|
||||
export type UsageBucket = {
|
||||
bucket: string;
|
||||
prompt_tokens: number;
|
||||
generated_tokens: number;
|
||||
cost: number;
|
||||
cached_tokens: number | null;
|
||||
group_key?: string;
|
||||
};
|
||||
|
||||
export type GroupBy = 'none' | 'model' | 'agent' | 'source';
|
||||
export type Metric = 'tokens' | 'cost';
|
||||
|
||||
export const GROUP_OPTIONS: { value: GroupBy; label: string }[] = [
|
||||
{ value: 'none', label: 'Total' },
|
||||
{ value: 'model', label: 'By model' },
|
||||
{ value: 'agent', label: 'By agent' },
|
||||
{ value: 'source', label: 'By flow' },
|
||||
];
|
||||
|
||||
function valueOf(row: UsageBucket, metric: Metric): number {
|
||||
return metric === 'cost'
|
||||
? row.cost
|
||||
: row.prompt_tokens + row.generated_tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shape the admin usage series for the chart.
|
||||
*
|
||||
* Ungrouped gives one dataset per token kind (or a single cost series);
|
||||
* grouped gives one dataset per group key, each aligned to the shared bucket
|
||||
* axis so a group missing a day plots 0 rather than shifting the whole series
|
||||
* left by one bar.
|
||||
*/
|
||||
export function buildUsageChart(
|
||||
series: UsageBucket[],
|
||||
groupBy: GroupBy,
|
||||
metric: Metric,
|
||||
) {
|
||||
const buckets = Array.from(new Set(series.map((row) => row.bucket))).sort();
|
||||
const labels = buckets.map((bucket) => formatDate(bucket));
|
||||
|
||||
if (groupBy === 'none') {
|
||||
if (metric === 'cost') {
|
||||
return {
|
||||
labels,
|
||||
datasets: [
|
||||
{
|
||||
label: 'Cost',
|
||||
data: series.map((row) => row.cost),
|
||||
backgroundColor: seriesColor(0),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
const colors = usageColors();
|
||||
return {
|
||||
labels,
|
||||
datasets: [
|
||||
{
|
||||
label: 'Prompt',
|
||||
data: series.map((row) => row.prompt_tokens),
|
||||
backgroundColor: colors.prompt,
|
||||
},
|
||||
{
|
||||
label: 'Generated',
|
||||
data: series.map((row) => row.generated_tokens),
|
||||
backgroundColor: colors.generated,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const keys = Array.from(
|
||||
new Set(series.map((row) => row.group_key ?? 'unknown')),
|
||||
);
|
||||
return {
|
||||
labels,
|
||||
datasets: keys.map((key, index) => {
|
||||
const byBucket = new Map(
|
||||
series
|
||||
.filter((row) => (row.group_key ?? 'unknown') === key)
|
||||
.map((row) => [row.bucket, valueOf(row, metric)]),
|
||||
);
|
||||
return {
|
||||
label: key,
|
||||
data: buckets.map((bucket) => byBucket.get(bucket) ?? 0),
|
||||
backgroundColor: seriesColor(index),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Prompt-cache hit rate over the rows whose provider reported a breakdown.
|
||||
*
|
||||
* NULL cached_tokens means "the provider said nothing", which is not the same
|
||||
* as "no cache hits" — counting those rows as 0 would understate the rate.
|
||||
* Returns null when nothing reported.
|
||||
*/
|
||||
export function cacheHitRate(series: UsageBucket[]): number | null {
|
||||
const reported = series.filter((row) => row.cached_tokens !== null);
|
||||
if (reported.length === 0) return null;
|
||||
const prompt = reported.reduce((sum, row) => sum + row.prompt_tokens, 0);
|
||||
if (prompt === 0) return null;
|
||||
const cached = reported.reduce(
|
||||
(sum, row) => sum + (row.cached_tokens ?? 0),
|
||||
0,
|
||||
);
|
||||
return (cached / prompt) * 100;
|
||||
}
|
||||
@@ -170,6 +170,8 @@ const endpoints = {
|
||||
`/api/admin/users/${encodeURIComponent(id)}/role`,
|
||||
USER_REVOKE_SESSIONS: (id: string) =>
|
||||
`/api/admin/users/${encodeURIComponent(id)}/revoke-sessions`,
|
||||
USER_USAGE: (id: string) =>
|
||||
`/api/admin/users/${encodeURIComponent(id)}/usage`,
|
||||
ADMINS: '/api/admin/admins',
|
||||
USAGE: '/api/admin/usage',
|
||||
ACTIVITY: '/api/admin/activity',
|
||||
|
||||
@@ -68,6 +68,12 @@ const adminService = {
|
||||
token: string | null,
|
||||
): Promise<any> =>
|
||||
apiClient.get(`${endpoints.ADMIN.USAGE}${qs(params)}`, token),
|
||||
getUserUsage: (
|
||||
userId: string,
|
||||
params: { days?: number },
|
||||
token: string | null,
|
||||
): Promise<any> =>
|
||||
apiClient.get(`${endpoints.ADMIN.USER_USAGE(userId)}${qs(params)}`, token),
|
||||
getActivity: (filters: ActivityFilters, token: string | null): Promise<any> =>
|
||||
apiClient.get(`${endpoints.ADMIN.ACTIVITY}${qs(filters)}`, token),
|
||||
getActivityEvents: (token: string | null): Promise<any> =>
|
||||
|
||||
@@ -228,14 +228,37 @@ class TestUserLifecycle:
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestUsageAndAudit:
|
||||
def test_usage(self, client):
|
||||
@staticmethod
|
||||
def _usage_repos():
|
||||
usage = Mock()
|
||||
usage.bucketed_totals.return_value = [
|
||||
{"bucket": "2026-06-14", "prompt_tokens": 5, "generated_tokens": 3}
|
||||
{
|
||||
"bucket": "2026-06-14",
|
||||
"prompt_tokens": 5,
|
||||
"generated_tokens": 3,
|
||||
"cost": 0.25,
|
||||
"cached_tokens": None,
|
||||
}
|
||||
]
|
||||
usage.sum_tokens_in_range.return_value = 8
|
||||
usage.tokens_by_model.return_value = [
|
||||
{"model_id": "gpt-x", "tokens": 8, "cost": 0.25}
|
||||
]
|
||||
stats = Mock()
|
||||
stats.top_token_users.return_value = [{"user_id": "a", "tokens": 8}]
|
||||
stats.top_token_users.return_value = [
|
||||
{"user_id": "a", "tokens": 8, "cost": 0.25}
|
||||
]
|
||||
stats.latency_summary.return_value = {
|
||||
"samples": 2,
|
||||
"p50_ms": 120,
|
||||
"p95_ms": 400,
|
||||
"ttft_samples": 2,
|
||||
"ttft_p50_ms": 30,
|
||||
}
|
||||
return usage, stats
|
||||
|
||||
def test_usage(self, client):
|
||||
usage, stats = self._usage_repos()
|
||||
with _admin(
|
||||
TokenUsageRepository=Mock(return_value=usage),
|
||||
AdminStatsRepository=Mock(return_value=stats),
|
||||
@@ -247,6 +270,56 @@ class TestUsageAndAudit:
|
||||
assert len(data["series"]) == 1
|
||||
assert data["top_users"][0]["user_id"] == "a"
|
||||
|
||||
def test_usage_reports_spend_and_latency(self, client):
|
||||
"""Cost quotas were settable long before spend was visible."""
|
||||
usage, stats = self._usage_repos()
|
||||
with _admin(
|
||||
TokenUsageRepository=Mock(return_value=usage),
|
||||
AdminStatsRepository=Mock(return_value=stats),
|
||||
):
|
||||
data = _body(client.get("/api/admin/usage"))
|
||||
assert data["total_cost"] == 0.25
|
||||
assert data["series"][0]["cost"] == 0.25
|
||||
assert data["by_model"][0]["model_id"] == "gpt-x"
|
||||
assert data["latency"]["p95_ms"] == 400
|
||||
assert data["top_users"][0]["cost"] == 0.25
|
||||
|
||||
def test_usage_group_by_reaches_the_repository(self, client):
|
||||
usage, stats = self._usage_repos()
|
||||
with _admin(
|
||||
TokenUsageRepository=Mock(return_value=usage),
|
||||
AdminStatsRepository=Mock(return_value=stats),
|
||||
):
|
||||
assert client.get("/api/admin/usage?group_by=model").status_code == 200
|
||||
assert usage.bucketed_totals.call_args.kwargs["group_by"] == "model"
|
||||
|
||||
def test_usage_rejects_an_unknown_grouping(self, client):
|
||||
with _admin():
|
||||
assert client.get("/api/admin/usage?group_by=nonsense").status_code == 400
|
||||
|
||||
def test_per_user_usage_drilldown(self, client):
|
||||
usage, stats = self._usage_repos()
|
||||
stats.user_usage_breakdown.return_value = {
|
||||
"totals": {"tokens": 8, "cost": 0.25, "calls": 1},
|
||||
"by_model": [{"key": "gpt-x", "tokens": 8, "cost": 0.25}],
|
||||
"by_source": [{"key": "agent_stream", "tokens": 8, "cost": 0.25}],
|
||||
}
|
||||
with _admin(
|
||||
TokenUsageRepository=Mock(return_value=usage),
|
||||
AdminStatsRepository=Mock(return_value=stats),
|
||||
):
|
||||
data = _body(client.get("/api/admin/users/u1/usage?days=7"))
|
||||
assert data["totals"]["cost"] == 0.25
|
||||
assert data["by_source"][0]["key"] == "agent_stream"
|
||||
assert len(data["series"]) == 1
|
||||
assert usage.bucketed_totals.call_args.kwargs["user_id"] == "u1"
|
||||
|
||||
def test_per_user_usage_requires_admin(self, client):
|
||||
with patch("docsgpt.app.handle_auth", return_value={"sub": "u1"}), patch(
|
||||
"docsgpt.app.resolve_roles", return_value=["user"]
|
||||
):
|
||||
assert client.get("/api/admin/users/u1/usage").status_code == 403
|
||||
|
||||
def test_usage_invalid_bucket(self, client):
|
||||
with _admin():
|
||||
resp = client.get("/api/admin/usage?bucket=year")
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Repository tests for admin spend, latency and the per-user drill-down."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
|
||||
from docsgpt.storage.db.repositories.admin_stats import AdminStatsRepository
|
||||
from docsgpt.storage.db.repositories.token_usage import TokenUsageRepository
|
||||
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
def _insert(conn, **fields):
|
||||
defaults = {
|
||||
"user_id": "u1",
|
||||
"prompt_tokens": 100,
|
||||
"generated_tokens": 20,
|
||||
"cost": 0.5,
|
||||
"source": "agent_stream",
|
||||
"model_id": "gpt-x",
|
||||
"cached_tokens": None,
|
||||
"duration_ms": None,
|
||||
"ttft_ms": None,
|
||||
"timestamp": datetime.now(timezone.utc),
|
||||
}
|
||||
defaults.update(fields)
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO token_usage (user_id, prompt_tokens, generated_tokens, "
|
||||
"cost, source, model_id, cached_tokens, duration_ms, ttft_ms, timestamp) "
|
||||
"VALUES (:user_id, :prompt_tokens, :generated_tokens, :cost, :source, "
|
||||
":model_id, :cached_tokens, :duration_ms, :ttft_ms, :timestamp)"
|
||||
),
|
||||
defaults,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def since():
|
||||
return datetime.now(timezone.utc) - timedelta(days=7)
|
||||
|
||||
|
||||
class TestBucketedCost:
|
||||
def test_buckets_carry_cost_alongside_tokens(self, pg_conn, since):
|
||||
_insert(pg_conn, cost=1.25)
|
||||
_insert(pg_conn, cost=0.75)
|
||||
series = TokenUsageRepository(pg_conn).bucketed_totals(
|
||||
bucket_unit="day", timestamp_gte=since
|
||||
)
|
||||
assert len(series) == 1
|
||||
assert series[0]["cost"] == pytest.approx(2.0)
|
||||
assert series[0]["prompt_tokens"] == 200
|
||||
|
||||
def test_unreported_cache_bins_stay_unknown(self, pg_conn, since):
|
||||
"""NULL means the provider said nothing, which is not the same as 0."""
|
||||
_insert(pg_conn, cached_tokens=None)
|
||||
series = TokenUsageRepository(pg_conn).bucketed_totals(
|
||||
bucket_unit="day", timestamp_gte=since
|
||||
)
|
||||
assert series[0]["cached_tokens"] is None
|
||||
|
||||
def test_reported_cache_bins_sum(self, pg_conn, since):
|
||||
_insert(pg_conn, cached_tokens=30)
|
||||
_insert(pg_conn, cached_tokens=None)
|
||||
series = TokenUsageRepository(pg_conn).bucketed_totals(
|
||||
bucket_unit="day", timestamp_gte=since
|
||||
)
|
||||
assert series[0]["cached_tokens"] == 30
|
||||
|
||||
def test_grouping_keeps_cost_per_group(self, pg_conn, since):
|
||||
_insert(pg_conn, model_id="cheap", cost=0.1)
|
||||
_insert(pg_conn, model_id="pricey", cost=9.9)
|
||||
series = TokenUsageRepository(pg_conn).bucketed_totals(
|
||||
bucket_unit="day", timestamp_gte=since, group_by="model"
|
||||
)
|
||||
costs = {row["group_key"]: row["cost"] for row in series}
|
||||
assert costs == {"cheap": pytest.approx(0.1), "pricey": pytest.approx(9.9)}
|
||||
|
||||
|
||||
class TestTopUsers:
|
||||
def test_reports_spend_next_to_tokens(self, pg_conn, since):
|
||||
_insert(pg_conn, user_id="heavy", prompt_tokens=1000, cost=2.0)
|
||||
_insert(pg_conn, user_id="light", prompt_tokens=10, cost=0.1)
|
||||
rows = AdminStatsRepository(pg_conn).top_token_users(since=since)
|
||||
assert [row["user_id"] for row in rows] == ["heavy", "light"]
|
||||
assert rows[0]["cost"] == pytest.approx(2.0)
|
||||
|
||||
|
||||
class TestLatencySummary:
|
||||
def test_percentiles_over_measured_calls(self, pg_conn, since):
|
||||
for duration in (100, 200, 300, 400):
|
||||
_insert(pg_conn, duration_ms=duration, ttft_ms=duration // 10)
|
||||
summary = AdminStatsRepository(pg_conn).latency_summary(since=since)
|
||||
assert summary["samples"] == 4
|
||||
assert summary["p50_ms"] == 250
|
||||
assert summary["p95_ms"] == 385
|
||||
assert summary["ttft_p50_ms"] == 25
|
||||
|
||||
def test_unmeasured_rows_are_skipped_not_counted_as_zero(self, pg_conn, since):
|
||||
_insert(pg_conn, duration_ms=1000, ttft_ms=None)
|
||||
_insert(pg_conn, duration_ms=None, ttft_ms=None)
|
||||
summary = AdminStatsRepository(pg_conn).latency_summary(since=since)
|
||||
assert summary["samples"] == 1
|
||||
assert summary["p50_ms"] == 1000
|
||||
# A stream that never yielded contributes no first-token sample.
|
||||
assert summary["ttft_samples"] == 0
|
||||
assert summary["ttft_p50_ms"] is None
|
||||
|
||||
def test_no_data_reports_no_samples(self, pg_conn, since):
|
||||
summary = AdminStatsRepository(pg_conn).latency_summary(since=since)
|
||||
assert summary == {
|
||||
"samples": 0,
|
||||
"p50_ms": None,
|
||||
"p95_ms": None,
|
||||
"ttft_samples": 0,
|
||||
"ttft_p50_ms": None,
|
||||
}
|
||||
|
||||
|
||||
class TestUserUsageBreakdown:
|
||||
def test_totals_and_splits(self, pg_conn, since):
|
||||
_insert(pg_conn, user_id="u1", model_id="a", source="agent_stream", cost=1.0)
|
||||
_insert(pg_conn, user_id="u1", model_id="b", source="schedule", cost=2.0)
|
||||
_insert(pg_conn, user_id="other", model_id="a", cost=99.0)
|
||||
detail = AdminStatsRepository(pg_conn).user_usage_breakdown("u1", since=since)
|
||||
assert detail["totals"]["calls"] == 2
|
||||
assert detail["totals"]["cost"] == pytest.approx(3.0)
|
||||
assert {row["key"] for row in detail["by_model"]} == {"a", "b"}
|
||||
assert {row["key"] for row in detail["by_source"]} == {
|
||||
"agent_stream",
|
||||
"schedule",
|
||||
}
|
||||
|
||||
def test_a_user_with_no_usage_reports_zeroes(self, pg_conn, since):
|
||||
detail = AdminStatsRepository(pg_conn).user_usage_breakdown("ghost", since=since)
|
||||
assert detail["totals"] == {"tokens": 0, "cost": 0.0, "calls": 0}
|
||||
assert detail["by_model"] == [] and detail["by_source"] == []
|
||||
|
||||
def test_window_is_respected(self, pg_conn, since):
|
||||
_insert(
|
||||
pg_conn,
|
||||
user_id="u1",
|
||||
timestamp=datetime.now(timezone.utc) - timedelta(days=30),
|
||||
)
|
||||
detail = AdminStatsRepository(pg_conn).user_usage_breakdown("u1", since=since)
|
||||
assert detail["totals"]["calls"] == 0
|
||||
Reference in new issue
Block a user