diff --git a/docsgpt/api/admin/routes.py b/docsgpt/api/admin/routes.py index d0dbf2d1..ff51149d 100644 --- a/docsgpt/api/admin/routes.py +++ b/docsgpt/api/admin/routes.py @@ -287,6 +287,26 @@ class AdminUserSessionsResource(Resource): return make_response(jsonify({"success": True, "revoked": ok}), 200) +@admin_ns.route("/admin/users//usage") +class AdminUserUsageResource(Resource): + @admin_required + def get(self, user_id): + """One user's spend: daily series plus a split by model and by flow.""" + days = max(1, min(365, _int_arg("days", 30))) + start = datetime.now(timezone.utc) - timedelta(days=days) + with db_readonly() as conn: + series = TokenUsageRepository(conn).bucketed_totals( + bucket_unit="day", user_id=user_id, timestamp_gte=start + ) + breakdown = AdminStatsRepository(conn).user_usage_breakdown( + user_id, since=start + ) + return make_response( + jsonify({"success": True, "days": days, "series": series, **breakdown}), + 200, + ) + + @admin_ns.route("/admin/admins") class AdminAdminsResource(Resource): @admin_required @@ -304,7 +324,12 @@ class AdminUsageResource(Resource): @admin_required def get(self): - """Global token usage: time-bucketed series + total + top users.""" + """Global usage: time-bucketed series, totals, spend, latency, top users. + + Every bucket carries both tokens and USD cost. Quota policies have + always been settable in dollars; until now nothing showed the spend + they were capping. + """ days = max(1, min(365, _int_arg("days", 30))) bucket = request.args.get("bucket", "day") group_by = request.args.get("group_by", "none") @@ -313,6 +338,7 @@ class AdminUsageResource(Resource): start = datetime.now(timezone.utc) - timedelta(days=days) with db_readonly() as conn: usage_repo = TokenUsageRepository(conn) + stats_repo = AdminStatsRepository(conn) series = usage_repo.bucketed_totals( bucket_unit=bucket, timestamp_gte=start, @@ -321,7 +347,9 @@ class AdminUsageResource(Resource): total = usage_repo.sum_tokens_in_range( start=start, end=datetime.now(timezone.utc) ) - top_users = AdminStatsRepository(conn).top_token_users(since=start, limit=10) + by_model = usage_repo.tokens_by_model(start=start) + latency = stats_repo.latency_summary(since=start) + top_users = stats_repo.top_token_users(since=start, limit=10) return make_response( jsonify( { @@ -331,6 +359,9 @@ class AdminUsageResource(Resource): "group_by": group_by, "series": series, "total_tokens": int(total), + "total_cost": round(sum(row["cost"] for row in by_model), 4), + "by_model": by_model, + "latency": latency, "top_users": top_users, } ), diff --git a/docsgpt/storage/db/repositories/admin_stats.py b/docsgpt/storage/db/repositories/admin_stats.py index cec29e54..23fdd3d9 100644 --- a/docsgpt/storage/db/repositories/admin_stats.py +++ b/docsgpt/storage/db/repositories/admin_stats.py @@ -16,6 +16,11 @@ from sqlalchemy import Connection, text from docsgpt.storage.db.base_repository import row_to_dict +def _round_ms(value) -> Optional[int]: + """Round a percentile to whole milliseconds, preserving NULL as None.""" + return int(round(float(value))) if value is not None else None + + class AdminStatsRepository: """Global aggregates for the admin overview and per-user drill-down.""" @@ -66,11 +71,18 @@ class AdminStatsRepository: } def top_token_users(self, *, since: datetime, limit: int = 10) -> list[dict]: - """Highest token consumers since ``since`` (admin usage view).""" + """Highest consumers since ``since``, with spend (admin usage view). + + Ordered by tokens, not cost: a cheap model can dominate token volume + while a costly one dominates the bill, and the operator wants to see + both columns rather than have the ranking pick for them. + """ result = self._conn.execute( text( """ - SELECT user_id, SUM(prompt_tokens + generated_tokens) AS tokens + SELECT user_id, + SUM(prompt_tokens + generated_tokens) AS tokens, + COALESCE(SUM(cost), 0) AS cost FROM token_usage WHERE timestamp >= :since AND user_id IS NOT NULL GROUP BY user_id @@ -80,7 +92,105 @@ class AdminStatsRepository: ), {"since": since, "limit": int(limit)}, ) - return [{"user_id": r[0], "tokens": int(r[1])} for r in result.fetchall()] + return [ + {"user_id": r[0], "tokens": int(r[1]), "cost": float(r[2])} + for r in result.fetchall() + ] + + def latency_summary(self, *, since: datetime) -> dict: + """Call-latency percentiles since ``since``. + + Rows written before the latency columns existed, and streams that + never yielded, hold NULL; percentile_cont skips them, so the summary + describes the calls that were actually measured. ``samples`` says how + many those were, so a percentile from three calls is not read as a + service-level figure. + """ + row = self._conn.execute( + text( + """ + SELECT count(duration_ms) AS samples, + percentile_cont(0.5) WITHIN GROUP (ORDER BY duration_ms) + AS p50_ms, + percentile_cont(0.95) WITHIN GROUP (ORDER BY duration_ms) + AS p95_ms, + count(ttft_ms) AS ttft_samples, + percentile_cont(0.5) WITHIN GROUP (ORDER BY ttft_ms) + AS ttft_p50_ms + FROM token_usage + WHERE timestamp >= :since + """ + ), + {"since": since}, + ).one() + return { + "samples": int(row.samples or 0), + "p50_ms": _round_ms(row.p50_ms), + "p95_ms": _round_ms(row.p95_ms), + "ttft_samples": int(row.ttft_samples or 0), + "ttft_p50_ms": _round_ms(row.ttft_p50_ms), + } + + def user_usage_breakdown( + self, user_id: str, *, since: datetime, limit: int = 10 + ) -> dict: + """Per-user spend detail: totals, and a split by model and by flow. + + The admin user drill-down showed a single ``tokens_30d`` number, which + cannot answer either question an operator actually has -- what is this + person costing, and what is driving it. + + Args: + user_id: The billable user (auth ``sub``). + since: Inclusive window start. + limit: How many models / flows to return, busiest first. + + Returns: + ``{"totals": {...}, "by_model": [...], "by_source": [...]}``. + """ + totals = self._conn.execute( + text( + """ + SELECT COALESCE(SUM(prompt_tokens + generated_tokens), 0) AS tokens, + COALESCE(SUM(cost), 0) AS cost, + count(*) AS calls + FROM token_usage + WHERE user_id = :u AND timestamp >= :since + """ + ), + {"u": user_id, "since": since}, + ).one() + + def _breakdown(key_expr: str) -> list[dict]: + rows = self._conn.execute( + text( + f""" + SELECT {key_expr} AS key, + COALESCE(SUM(prompt_tokens + generated_tokens), 0) AS tokens, + COALESCE(SUM(cost), 0) AS cost + FROM token_usage + WHERE user_id = :u AND timestamp >= :since + GROUP BY key + ORDER BY tokens DESC, key + LIMIT :limit + """ + ), + {"u": user_id, "since": since, "limit": int(limit)}, + ) + return [ + {"key": r.key, "tokens": int(r.tokens), "cost": float(r.cost)} + for r in rows.fetchall() + ] + + return { + "totals": { + "tokens": int(totals.tokens), + "cost": float(totals.cost), + "calls": int(totals.calls), + }, + "by_model": _breakdown("COALESCE(model_id, 'unknown')"), + "by_source": _breakdown("COALESCE(source, 'agent_stream')"), + } def list_users( self, user_id_filter: Optional[str], offset: int, limit: int diff --git a/docsgpt/storage/db/repositories/token_usage.py b/docsgpt/storage/db/repositories/token_usage.py index ec380b85..b0b29e0d 100644 --- a/docsgpt/storage/db/repositories/token_usage.py +++ b/docsgpt/storage/db/repositories/token_usage.py @@ -302,7 +302,11 @@ class TokenUsageRepository: f""" SELECT to_char(tu.timestamp AT TIME ZONE 'UTC', :fmt) AS bucket, COALESCE(SUM(tu.prompt_tokens), 0) AS prompt_tokens, - COALESCE(SUM(tu.generated_tokens), 0) AS generated_tokens + COALESCE(SUM(tu.generated_tokens), 0) AS generated_tokens, + COALESCE(SUM(tu.cost), 0) AS cost, + -- NULL cache bins mean "provider did not report", so + -- they stay out of the sum rather than reading as 0. + SUM(tu.cached_tokens) AS cached_tokens {group_select} FROM token_usage tu {join} @@ -318,6 +322,12 @@ class TokenUsageRepository: "bucket": row._mapping["bucket"], "prompt_tokens": int(row._mapping["prompt_tokens"]), "generated_tokens": int(row._mapping["generated_tokens"]), + "cost": float(row._mapping["cost"]), + "cached_tokens": ( + int(row._mapping["cached_tokens"]) + if row._mapping["cached_tokens"] is not None + else None + ), **( {"group_key": row._mapping["group_key"]} if group_by is not None diff --git a/frontend/src/admin/AdminUI.tsx b/frontend/src/admin/AdminUI.tsx index df8709de..6efc5010 100644 --- a/frontend/src/admin/AdminUI.tsx +++ b/frontend/src/admin/AdminUI.tsx @@ -97,6 +97,28 @@ export function fmtNumber(n?: number | null): string { return new Intl.NumberFormat().format(n ?? 0); } +/** + * USD, with enough precision that a fraction of a cent is not rendered as $0. + * An instance on a cheap model can run whole days under a dollar, and "$0" + * next to a cost quota reads as "nothing is being counted". + */ +export function fmtUsd(n?: number | null): string { + const value = n ?? 0; + const digits = value !== 0 && Math.abs(value) < 0.01 ? 4 : 2; + return new Intl.NumberFormat(undefined, { + style: 'currency', + currency: 'USD', + minimumFractionDigits: 2, + maximumFractionDigits: digits, + }).format(value); +} + +/** Milliseconds as ms or s, or an em dash when nothing was measured. */ +export function fmtMs(n?: number | null): string { + if (n === null || n === undefined) return '—'; + return n < 1000 ? `${Math.round(n)}ms` : `${(n / 1000).toFixed(1)}s`; +} + export function fmtCompact(n?: number | null): string { return new Intl.NumberFormat(undefined, { notation: 'compact', diff --git a/frontend/src/admin/Usage.tsx b/frontend/src/admin/Usage.tsx index edd78743..85e85704 100644 --- a/frontend/src/admin/Usage.tsx +++ b/frontend/src/admin/Usage.tsx @@ -4,6 +4,13 @@ import { useSelector } from 'react-redux'; import adminService from '../api/services/adminService'; import SkeletonLoader from '../components/SkeletonLoader'; import { Button } from '../components/ui/button'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '../components/ui/select'; import { Table, TableBody, @@ -15,16 +22,33 @@ import { } from '../components/ui/table'; import { useDarkTheme } from '../hooks'; import { selectToken } from '../preferences/preferenceSlice'; -import { formatDate } from '../utils/dateTimeUtils'; -import { Loading, LoadError, StatCard, fmtNumber } from './AdminUI'; -import UsageChart, { usageColors } from './UsageChart'; +import { + Loading, + LoadError, + StatCard, + fmtMs, + fmtNumber, + fmtUsd, +} from './AdminUI'; +import UsageChart from './UsageChart'; +import { + GROUP_OPTIONS, + buildUsageChart, + cacheHitRate as computeCacheHitRate, + type GroupBy, + type Metric, + type UsageBucket, +} from './usageChartData'; +import UserUsageModal from './UserUsageModal'; -type Bucket = { - bucket: string; - prompt_tokens: number; - generated_tokens: number; +type TopUser = { user_id: string; tokens: number; cost: number }; +type Latency = { + samples: number; + p50_ms: number | null; + p95_ms: number | null; + ttft_samples: number; + ttft_p50_ms: number | null; }; -type TopUser = { user_id: string; tokens: number }; const RANGES = [7, 30, 90]; @@ -32,47 +56,40 @@ export default function Usage() { const token = useSelector(selectToken); const [isDarkTheme] = useDarkTheme(); const [days, setDays] = useState(30); + const [groupBy, setGroupBy] = useState('none'); + const [metric, setMetric] = useState('tokens'); const [data, setData] = useState(null); const [loading, setLoading] = useState(true); + const [drilldown, setDrilldown] = useState(null); const load = useCallback(async () => { setLoading(true); try { - const res = await adminService.getUsage({ days, bucket: 'day' }, token); + const res = await adminService.getUsage( + { days, bucket: 'day', group_by: groupBy }, + token, + ); setData(await res.json().catch(() => ({}))); } finally { setLoading(false); } - }, [token, days]); + }, [token, days, groupBy]); useEffect(() => { load(); }, [load]); - const series: Bucket[] = data?.series ?? []; + const series: UsageBucket[] = data?.series ?? []; const topUsers: TopUser[] = data?.top_users ?? []; - const promptTotal = series.reduce((sum, b) => sum + b.prompt_tokens, 0); - const generatedTotal = series.reduce((sum, b) => sum + b.generated_tokens, 0); + const latency: Latency | undefined = data?.latency; - const chartData = useMemo(() => { - const colors = usageColors(); - return { - labels: series.map((b) => formatDate(b.bucket)), - datasets: [ - { - label: 'Prompt', - data: series.map((b) => b.prompt_tokens), - backgroundColor: colors.prompt, - }, - { - label: 'Generated', - data: series.map((b) => b.generated_tokens), - backgroundColor: colors.generated, - }, - ], - }; + const chartData = useMemo( + () => buildUsageChart(series, groupBy, metric), // isDarkTheme re-resolves the canvas colors when the theme toggles. - }, [series, isDarkTheme]); + [series, groupBy, metric, isDarkTheme], + ); + + const cacheHitRate = useMemo(() => computeCacheHitRate(series), [series]); if (data === null && loading) return ; if (data && !data.success) @@ -80,34 +97,89 @@ export default function Usage() { return (
-
- {RANGES.map((r) => ( +
+
+ {RANGES.map((range) => ( + + ))} +
+ +
- ))} + +
+ - - - + +

- Token usage + {metric === 'cost' ? 'Spend' : 'Token usage'}

) : ( - + )}

- Top users by tokens + Top users

{topUsers.length === 0 ? (

No usage.

@@ -139,17 +215,31 @@ export default function Usage() { User - Tokens + Tokens + Cost - {topUsers.map((u) => ( - + {topUsers.map((user) => ( + setDrilldown(user.user_id)} + > - {u.user_id} + {user.user_id} - - {fmtNumber(u.tokens)} + + {fmtNumber(user.tokens)} + + + {fmtUsd(user.cost)} ))} @@ -158,6 +248,8 @@ export default function Usage() { )}
+ + setDrilldown(null)} />
); } diff --git a/frontend/src/admin/UsageChart.tsx b/frontend/src/admin/UsageChart.tsx index d9c27f4d..f7d58496 100644 --- a/frontend/src/admin/UsageChart.tsx +++ b/frontend/src/admin/UsageChart.tsx @@ -39,6 +39,25 @@ export function usageColors(): { prompt: string; generated: string } { }; } +// Follow-on colors for a grouped chart (by model / agent / source), matching +// the palette settings/Analytics already uses for its stacked series. +const SERIES_COLORS = [ + '#FF6384', + '#36A2EB', + '#FFCE56', + '#4BC0C0', + '#9966FF', + '#FF9F40', + '#2BC596', +]; + +/** Color for dataset ``index``; the first uses the resolved brand color. */ +export function seriesColor(index: number): string { + return index === 0 + ? readCssVar('--primary', '#7D54D1') + : SERIES_COLORS[(index - 1) % SERIES_COLORS.length]; +} + function compactTick(value: number | string): string { const n = typeof value === 'number' ? value : Number(value); if (Number.isNaN(n)) return String(value); @@ -48,16 +67,28 @@ function compactTick(value: number | string): string { }).format(n); } +function currencyTick(value: number | string): string { + const n = typeof value === 'number' ? value : Number(value); + if (Number.isNaN(n)) return String(value); + return `$${new Intl.NumberFormat(undefined, { + notation: 'compact', + maximumFractionDigits: 2, + }).format(n)}`; +} + type UsageChartProps = { data: ChartData<'bar'>; legendID: string; maxTicksLimitInX?: number; + /** Format the y axis as USD rather than a token count. */ + currency?: boolean; }; export default function UsageChart({ data, legendID, maxTicksLimitInX = 8, + currency = false, }: UsageChartProps) { const options = { responsive: true, @@ -76,7 +107,7 @@ export default function UsageChart({ y: { grid: { lineWidth: 0.2, color: '#C4C4C4' }, border: { width: 0.2, color: '#C4C4C4' }, - ticks: { callback: compactTick }, + ticks: { callback: currency ? currencyTick : compactTick }, stacked: true, }, }, diff --git a/frontend/src/admin/UserUsageModal.tsx b/frontend/src/admin/UserUsageModal.tsx new file mode 100644 index 00000000..14d812cf --- /dev/null +++ b/frontend/src/admin/UserUsageModal.tsx @@ -0,0 +1,201 @@ +import { useEffect, useMemo, useState } from 'react'; +import { useSelector } from 'react-redux'; + +import adminService from '../api/services/adminService'; +import { Button } from '../components/ui/button'; +import { Modal } from '../components/ui/modal'; +import { + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableHeader, + TableRow, +} from '../components/ui/table'; +import { useDarkTheme } from '../hooks'; +import { selectToken } from '../preferences/preferenceSlice'; +import { formatDate } from '../utils/dateTimeUtils'; +import { Loading, LoadError, StatCard, fmtNumber, fmtUsd } from './AdminUI'; +import UsageChart, { usageColors } from './UsageChart'; + +type Bucket = { + bucket: string; + prompt_tokens: number; + generated_tokens: number; + cost: number; +}; +type Split = { key: string; tokens: number; cost: number }; + +const RANGES = [7, 30, 90]; + +/** Label for a token_usage.source value — what the spend was for. */ +const FLOW_LABELS: Record = { + agent_stream: 'Chat', + schedule: 'Scheduled run', + webhook: 'Webhook', + workflow: 'Workflow', + title: 'Title generation', + compression: 'History compression', + rag_condense: 'Question condensing', + fallback: 'Provider fallback', +}; + +function SplitTable({ title, rows }: { title: string; rows: Split[] }) { + return ( +
+

{title}

+ {rows.length === 0 ? ( +

No usage.

+ ) : ( + + + + + {title} + Tokens + Cost + + + + {rows.map((row) => ( + + + {FLOW_LABELS[row.key] ?? row.key} + + + {fmtNumber(row.tokens)} + + + {fmtUsd(row.cost)} + + + ))} + +
+ + )} +
+ ); +} + +/** + * Per-user spend drill-down. The user detail view carried a single + * `tokens_30d` number, which answers neither question an operator has: what is + * this person costing, and what is driving it. + */ +export default function UserUsageModal({ + userId, + onClose, +}: { + userId: string | null; + onClose: () => void; +}) { + const token = useSelector(selectToken); + const [isDarkTheme] = useDarkTheme(); + const [days, setDays] = useState(30); + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + + useEffect(() => { + if (!userId) { + setData(null); + return; + } + let cancelled = false; + setLoading(true); + adminService + .getUserUsage(userId, { days }, token) + .then((res) => res.json()) + .then((json) => { + if (!cancelled) setData(json); + }) + .catch(() => !cancelled && setData({ success: false })) + .finally(() => !cancelled && setLoading(false)); + return () => { + cancelled = true; + }; + }, [userId, days, token]); + + const series: Bucket[] = data?.series ?? []; + const chartData = useMemo(() => { + const colors = usageColors(); + return { + labels: series.map((bucket) => formatDate(bucket.bucket)), + datasets: [ + { + label: 'Prompt', + data: series.map((bucket) => bucket.prompt_tokens), + backgroundColor: colors.prompt, + }, + { + label: 'Generated', + data: series.map((bucket) => bucket.generated_tokens), + backgroundColor: colors.generated, + }, + ], + }; + // isDarkTheme re-resolves the canvas colors when the theme toggles. + }, [series, isDarkTheme]); + + const totals = data?.totals ?? {}; + + return ( + !open && onClose()} + title={userId ? `Usage · ${userId}` : 'Usage'} + size="lg" + > +
+ {RANGES.map((range) => ( + + ))} +
+ + {loading && data === null ? ( + + ) : data && !data.success ? ( + + ) : ( +
+
+ + + +
+ +
+
+

Daily tokens

+
+
+
+ {series.length === 0 ? ( +

+ No usage in this period. +

+ ) : ( + + )} +
+
+ + + +
+ )} + + ); +} diff --git a/frontend/src/admin/Users.tsx b/frontend/src/admin/Users.tsx index d7ae7f8e..242e678e 100644 --- a/frontend/src/admin/Users.tsx +++ b/frontend/src/admin/Users.tsx @@ -42,6 +42,7 @@ import { fmtRelative, } from './AdminUI'; import UserQuotaModal from './UserQuotaModal'; +import UserUsageModal from './UserUsageModal'; type AdminUser = { user_id: string; @@ -72,6 +73,7 @@ export default function Users() { const [busy, setBusy] = useState(null); const [menuUserId, setMenuUserId] = useState(null); const [detail, setDetail] = useState(null); + const [usageFor, setUsageFor] = useState(null); const [quotaUserId, setQuotaUserId] = useState(null); const [feedback, setFeedback] = useState<{ ok: boolean; @@ -443,6 +445,7 @@ export default function Users() { /> ) : null} + setUsageFor(null)} /> setQuotaUserId(null)} @@ -528,6 +531,19 @@ export default function Users() {
+

Recent auth events diff --git a/frontend/src/admin/usageChartData.test.ts b/frontend/src/admin/usageChartData.test.ts new file mode 100644 index 00000000..b596910c --- /dev/null +++ b/frontend/src/admin/usageChartData.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { + buildUsageChart, + cacheHitRate, + type UsageBucket, +} from './usageChartData'; + +// Chart.js colors resolve against a live DOM; the shapes under test do not. +vi.mock('./UsageChart', () => ({ + seriesColor: (index: number) => `color-${index}`, + usageColors: () => ({ prompt: 'prompt-color', generated: 'generated-color' }), +})); + +const bucket = (overrides: Partial): UsageBucket => ({ + bucket: '2026-01-01', + prompt_tokens: 100, + generated_tokens: 20, + cost: 1, + cached_tokens: null, + ...overrides, +}); + +describe('buildUsageChart', () => { + it('splits prompt and generated tokens when ungrouped', () => { + const chart = buildUsageChart([bucket({})], 'none', 'tokens'); + expect(chart.datasets.map((d) => d.label)).toEqual(['Prompt', 'Generated']); + expect(chart.datasets[0].data).toEqual([100]); + }); + + it('plots a single series for cost', () => { + const chart = buildUsageChart([bucket({ cost: 2.5 })], 'none', 'cost'); + expect(chart.datasets).toHaveLength(1); + expect(chart.datasets[0].data).toEqual([2.5]); + }); + + it('gives each group its own dataset', () => { + const chart = buildUsageChart( + [ + bucket({ group_key: 'gpt-a' }), + bucket({ group_key: 'gpt-b', prompt_tokens: 5, generated_tokens: 5 }), + ], + 'model', + 'tokens', + ); + expect(chart.datasets.map((d) => d.label)).toEqual(['gpt-a', 'gpt-b']); + expect(chart.datasets[1].data).toEqual([10]); + }); + + it('pads a group that is missing a bucket so the axes stay aligned', () => { + const chart = buildUsageChart( + [ + bucket({ bucket: '2026-01-01', group_key: 'a' }), + bucket({ bucket: '2026-01-02', group_key: 'b' }), + ], + 'model', + 'tokens', + ); + expect(chart.labels).toHaveLength(2); + // 'a' has no row on the second day: a zero, not a shifted bar. + expect(chart.datasets[0].data).toEqual([120, 0]); + expect(chart.datasets[1].data).toEqual([0, 120]); + }); + + it('labels a group with no key rather than dropping the row', () => { + const chart = buildUsageChart([bucket({})], 'model', 'tokens'); + expect(chart.datasets[0].label).toBe('unknown'); + }); + + it('handles an empty series', () => { + const chart = buildUsageChart([], 'model', 'cost'); + expect(chart.labels).toEqual([]); + expect(chart.datasets).toEqual([]); + }); +}); + +describe('cacheHitRate', () => { + it('is unknown when no provider reported cache bins', () => { + expect(cacheHitRate([bucket({ cached_tokens: null })])).toBeNull(); + }); + + it('is unknown for an empty series', () => { + expect(cacheHitRate([])).toBeNull(); + }); + + it('measures only the rows that reported', () => { + const rate = cacheHitRate([ + bucket({ prompt_tokens: 100, cached_tokens: 25 }), + // Unreported: excluded entirely rather than counted as a 0% row, which + // would halve the rate. + bucket({ prompt_tokens: 100, cached_tokens: null }), + ]); + expect(rate).toBeCloseTo(25); + }); + + it('avoids dividing by zero prompt tokens', () => { + expect( + cacheHitRate([bucket({ prompt_tokens: 0, cached_tokens: 0 })]), + ).toBeNull(); + }); +}); diff --git a/frontend/src/admin/usageChartData.ts b/frontend/src/admin/usageChartData.ts new file mode 100644 index 00000000..7bc5a2ff --- /dev/null +++ b/frontend/src/admin/usageChartData.ts @@ -0,0 +1,113 @@ +import { formatDate } from '../utils/dateTimeUtils'; +import { seriesColor, usageColors } from './UsageChart'; + +export type UsageBucket = { + bucket: string; + prompt_tokens: number; + generated_tokens: number; + cost: number; + cached_tokens: number | null; + group_key?: string; +}; + +export type GroupBy = 'none' | 'model' | 'agent' | 'source'; +export type Metric = 'tokens' | 'cost'; + +export const GROUP_OPTIONS: { value: GroupBy; label: string }[] = [ + { value: 'none', label: 'Total' }, + { value: 'model', label: 'By model' }, + { value: 'agent', label: 'By agent' }, + { value: 'source', label: 'By flow' }, +]; + +function valueOf(row: UsageBucket, metric: Metric): number { + return metric === 'cost' + ? row.cost + : row.prompt_tokens + row.generated_tokens; +} + +/** + * Shape the admin usage series for the chart. + * + * Ungrouped gives one dataset per token kind (or a single cost series); + * grouped gives one dataset per group key, each aligned to the shared bucket + * axis so a group missing a day plots 0 rather than shifting the whole series + * left by one bar. + */ +export function buildUsageChart( + series: UsageBucket[], + groupBy: GroupBy, + metric: Metric, +) { + const buckets = Array.from(new Set(series.map((row) => row.bucket))).sort(); + const labels = buckets.map((bucket) => formatDate(bucket)); + + if (groupBy === 'none') { + if (metric === 'cost') { + return { + labels, + datasets: [ + { + label: 'Cost', + data: series.map((row) => row.cost), + backgroundColor: seriesColor(0), + }, + ], + }; + } + const colors = usageColors(); + return { + labels, + datasets: [ + { + label: 'Prompt', + data: series.map((row) => row.prompt_tokens), + backgroundColor: colors.prompt, + }, + { + label: 'Generated', + data: series.map((row) => row.generated_tokens), + backgroundColor: colors.generated, + }, + ], + }; + } + + const keys = Array.from( + new Set(series.map((row) => row.group_key ?? 'unknown')), + ); + return { + labels, + datasets: keys.map((key, index) => { + const byBucket = new Map( + series + .filter((row) => (row.group_key ?? 'unknown') === key) + .map((row) => [row.bucket, valueOf(row, metric)]), + ); + return { + label: key, + data: buckets.map((bucket) => byBucket.get(bucket) ?? 0), + backgroundColor: seriesColor(index), + }; + }), + }; +} + +/** + * Prompt-cache hit rate over the rows whose provider reported a breakdown. + * + * NULL cached_tokens means "the provider said nothing", which is not the same + * as "no cache hits" — counting those rows as 0 would understate the rate. + * Returns null when nothing reported. + */ +export function cacheHitRate(series: UsageBucket[]): number | null { + const reported = series.filter((row) => row.cached_tokens !== null); + if (reported.length === 0) return null; + const prompt = reported.reduce((sum, row) => sum + row.prompt_tokens, 0); + if (prompt === 0) return null; + const cached = reported.reduce( + (sum, row) => sum + (row.cached_tokens ?? 0), + 0, + ); + return (cached / prompt) * 100; +} diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index ddfc4345..28a95dbc 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -170,6 +170,8 @@ const endpoints = { `/api/admin/users/${encodeURIComponent(id)}/role`, USER_REVOKE_SESSIONS: (id: string) => `/api/admin/users/${encodeURIComponent(id)}/revoke-sessions`, + USER_USAGE: (id: string) => + `/api/admin/users/${encodeURIComponent(id)}/usage`, ADMINS: '/api/admin/admins', USAGE: '/api/admin/usage', ACTIVITY: '/api/admin/activity', diff --git a/frontend/src/api/services/adminService.ts b/frontend/src/api/services/adminService.ts index ad44e97e..aef39370 100644 --- a/frontend/src/api/services/adminService.ts +++ b/frontend/src/api/services/adminService.ts @@ -68,6 +68,12 @@ const adminService = { token: string | null, ): Promise => apiClient.get(`${endpoints.ADMIN.USAGE}${qs(params)}`, token), + getUserUsage: ( + userId: string, + params: { days?: number }, + token: string | null, + ): Promise => + apiClient.get(`${endpoints.ADMIN.USER_USAGE(userId)}${qs(params)}`, token), getActivity: (filters: ActivityFilters, token: string | null): Promise => apiClient.get(`${endpoints.ADMIN.ACTIVITY}${qs(filters)}`, token), getActivityEvents: (token: string | null): Promise => diff --git a/tests/api/test_admin_dashboard.py b/tests/api/test_admin_dashboard.py index 59c67e4c..ad76ad9f 100644 --- a/tests/api/test_admin_dashboard.py +++ b/tests/api/test_admin_dashboard.py @@ -228,14 +228,37 @@ class TestUserLifecycle: @pytest.mark.unit class TestUsageAndAudit: - def test_usage(self, client): + @staticmethod + def _usage_repos(): usage = Mock() usage.bucketed_totals.return_value = [ - {"bucket": "2026-06-14", "prompt_tokens": 5, "generated_tokens": 3} + { + "bucket": "2026-06-14", + "prompt_tokens": 5, + "generated_tokens": 3, + "cost": 0.25, + "cached_tokens": None, + } ] usage.sum_tokens_in_range.return_value = 8 + usage.tokens_by_model.return_value = [ + {"model_id": "gpt-x", "tokens": 8, "cost": 0.25} + ] stats = Mock() - stats.top_token_users.return_value = [{"user_id": "a", "tokens": 8}] + stats.top_token_users.return_value = [ + {"user_id": "a", "tokens": 8, "cost": 0.25} + ] + stats.latency_summary.return_value = { + "samples": 2, + "p50_ms": 120, + "p95_ms": 400, + "ttft_samples": 2, + "ttft_p50_ms": 30, + } + return usage, stats + + def test_usage(self, client): + usage, stats = self._usage_repos() with _admin( TokenUsageRepository=Mock(return_value=usage), AdminStatsRepository=Mock(return_value=stats), @@ -247,6 +270,56 @@ class TestUsageAndAudit: assert len(data["series"]) == 1 assert data["top_users"][0]["user_id"] == "a" + def test_usage_reports_spend_and_latency(self, client): + """Cost quotas were settable long before spend was visible.""" + usage, stats = self._usage_repos() + with _admin( + TokenUsageRepository=Mock(return_value=usage), + AdminStatsRepository=Mock(return_value=stats), + ): + data = _body(client.get("/api/admin/usage")) + assert data["total_cost"] == 0.25 + assert data["series"][0]["cost"] == 0.25 + assert data["by_model"][0]["model_id"] == "gpt-x" + assert data["latency"]["p95_ms"] == 400 + assert data["top_users"][0]["cost"] == 0.25 + + def test_usage_group_by_reaches_the_repository(self, client): + usage, stats = self._usage_repos() + with _admin( + TokenUsageRepository=Mock(return_value=usage), + AdminStatsRepository=Mock(return_value=stats), + ): + assert client.get("/api/admin/usage?group_by=model").status_code == 200 + assert usage.bucketed_totals.call_args.kwargs["group_by"] == "model" + + def test_usage_rejects_an_unknown_grouping(self, client): + with _admin(): + assert client.get("/api/admin/usage?group_by=nonsense").status_code == 400 + + def test_per_user_usage_drilldown(self, client): + usage, stats = self._usage_repos() + stats.user_usage_breakdown.return_value = { + "totals": {"tokens": 8, "cost": 0.25, "calls": 1}, + "by_model": [{"key": "gpt-x", "tokens": 8, "cost": 0.25}], + "by_source": [{"key": "agent_stream", "tokens": 8, "cost": 0.25}], + } + with _admin( + TokenUsageRepository=Mock(return_value=usage), + AdminStatsRepository=Mock(return_value=stats), + ): + data = _body(client.get("/api/admin/users/u1/usage?days=7")) + assert data["totals"]["cost"] == 0.25 + assert data["by_source"][0]["key"] == "agent_stream" + assert len(data["series"]) == 1 + assert usage.bucketed_totals.call_args.kwargs["user_id"] == "u1" + + def test_per_user_usage_requires_admin(self, client): + with patch("docsgpt.app.handle_auth", return_value={"sub": "u1"}), patch( + "docsgpt.app.resolve_roles", return_value=["user"] + ): + assert client.get("/api/admin/users/u1/usage").status_code == 403 + def test_usage_invalid_bucket(self, client): with _admin(): resp = client.get("/api/admin/usage?bucket=year") diff --git a/tests/storage/db/repositories/test_admin_usage_stats.py b/tests/storage/db/repositories/test_admin_usage_stats.py new file mode 100644 index 00000000..29f1eced --- /dev/null +++ b/tests/storage/db/repositories/test_admin_usage_stats.py @@ -0,0 +1,150 @@ +"""Repository tests for admin spend, latency and the per-user drill-down.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest +from sqlalchemy import text + +from docsgpt.storage.db.repositories.admin_stats import AdminStatsRepository +from docsgpt.storage.db.repositories.token_usage import TokenUsageRepository + + +pytestmark = pytest.mark.integration + + +def _insert(conn, **fields): + defaults = { + "user_id": "u1", + "prompt_tokens": 100, + "generated_tokens": 20, + "cost": 0.5, + "source": "agent_stream", + "model_id": "gpt-x", + "cached_tokens": None, + "duration_ms": None, + "ttft_ms": None, + "timestamp": datetime.now(timezone.utc), + } + defaults.update(fields) + conn.execute( + text( + "INSERT INTO token_usage (user_id, prompt_tokens, generated_tokens, " + "cost, source, model_id, cached_tokens, duration_ms, ttft_ms, timestamp) " + "VALUES (:user_id, :prompt_tokens, :generated_tokens, :cost, :source, " + ":model_id, :cached_tokens, :duration_ms, :ttft_ms, :timestamp)" + ), + defaults, + ) + + +@pytest.fixture() +def since(): + return datetime.now(timezone.utc) - timedelta(days=7) + + +class TestBucketedCost: + def test_buckets_carry_cost_alongside_tokens(self, pg_conn, since): + _insert(pg_conn, cost=1.25) + _insert(pg_conn, cost=0.75) + series = TokenUsageRepository(pg_conn).bucketed_totals( + bucket_unit="day", timestamp_gte=since + ) + assert len(series) == 1 + assert series[0]["cost"] == pytest.approx(2.0) + assert series[0]["prompt_tokens"] == 200 + + def test_unreported_cache_bins_stay_unknown(self, pg_conn, since): + """NULL means the provider said nothing, which is not the same as 0.""" + _insert(pg_conn, cached_tokens=None) + series = TokenUsageRepository(pg_conn).bucketed_totals( + bucket_unit="day", timestamp_gte=since + ) + assert series[0]["cached_tokens"] is None + + def test_reported_cache_bins_sum(self, pg_conn, since): + _insert(pg_conn, cached_tokens=30) + _insert(pg_conn, cached_tokens=None) + series = TokenUsageRepository(pg_conn).bucketed_totals( + bucket_unit="day", timestamp_gte=since + ) + assert series[0]["cached_tokens"] == 30 + + def test_grouping_keeps_cost_per_group(self, pg_conn, since): + _insert(pg_conn, model_id="cheap", cost=0.1) + _insert(pg_conn, model_id="pricey", cost=9.9) + series = TokenUsageRepository(pg_conn).bucketed_totals( + bucket_unit="day", timestamp_gte=since, group_by="model" + ) + costs = {row["group_key"]: row["cost"] for row in series} + assert costs == {"cheap": pytest.approx(0.1), "pricey": pytest.approx(9.9)} + + +class TestTopUsers: + def test_reports_spend_next_to_tokens(self, pg_conn, since): + _insert(pg_conn, user_id="heavy", prompt_tokens=1000, cost=2.0) + _insert(pg_conn, user_id="light", prompt_tokens=10, cost=0.1) + rows = AdminStatsRepository(pg_conn).top_token_users(since=since) + assert [row["user_id"] for row in rows] == ["heavy", "light"] + assert rows[0]["cost"] == pytest.approx(2.0) + + +class TestLatencySummary: + def test_percentiles_over_measured_calls(self, pg_conn, since): + for duration in (100, 200, 300, 400): + _insert(pg_conn, duration_ms=duration, ttft_ms=duration // 10) + summary = AdminStatsRepository(pg_conn).latency_summary(since=since) + assert summary["samples"] == 4 + assert summary["p50_ms"] == 250 + assert summary["p95_ms"] == 385 + assert summary["ttft_p50_ms"] == 25 + + def test_unmeasured_rows_are_skipped_not_counted_as_zero(self, pg_conn, since): + _insert(pg_conn, duration_ms=1000, ttft_ms=None) + _insert(pg_conn, duration_ms=None, ttft_ms=None) + summary = AdminStatsRepository(pg_conn).latency_summary(since=since) + assert summary["samples"] == 1 + assert summary["p50_ms"] == 1000 + # A stream that never yielded contributes no first-token sample. + assert summary["ttft_samples"] == 0 + assert summary["ttft_p50_ms"] is None + + def test_no_data_reports_no_samples(self, pg_conn, since): + summary = AdminStatsRepository(pg_conn).latency_summary(since=since) + assert summary == { + "samples": 0, + "p50_ms": None, + "p95_ms": None, + "ttft_samples": 0, + "ttft_p50_ms": None, + } + + +class TestUserUsageBreakdown: + def test_totals_and_splits(self, pg_conn, since): + _insert(pg_conn, user_id="u1", model_id="a", source="agent_stream", cost=1.0) + _insert(pg_conn, user_id="u1", model_id="b", source="schedule", cost=2.0) + _insert(pg_conn, user_id="other", model_id="a", cost=99.0) + detail = AdminStatsRepository(pg_conn).user_usage_breakdown("u1", since=since) + assert detail["totals"]["calls"] == 2 + assert detail["totals"]["cost"] == pytest.approx(3.0) + assert {row["key"] for row in detail["by_model"]} == {"a", "b"} + assert {row["key"] for row in detail["by_source"]} == { + "agent_stream", + "schedule", + } + + def test_a_user_with_no_usage_reports_zeroes(self, pg_conn, since): + detail = AdminStatsRepository(pg_conn).user_usage_breakdown("ghost", since=since) + assert detail["totals"] == {"tokens": 0, "cost": 0.0, "calls": 0} + assert detail["by_model"] == [] and detail["by_source"] == [] + + def test_window_is_respected(self, pg_conn, since): + _insert( + pg_conn, + user_id="u1", + timestamp=datetime.now(timezone.utc) - timedelta(days=30), + ) + detail = AdminStatsRepository(pg_conn).user_usage_breakdown("u1", since=since) + assert detail["totals"]["calls"] == 0