fix(admin): keyset export paging, rollup-free spend, consistent totals

Review pass over the preceding commits.

- The activity export paged by OFFSET. The journals are append-only and the
  feed is newest-first, so rows written mid-export shift the window down and
  repeat rows already emitted. Pages by keyset on (created_at, feed, id) now.
- top_token_users and the per-user breakdown counted run-level rollup rows.
  A scheduled run already has a row per LLM call, so its spend was billed
  twice -- invisible while the column was tokens, obvious once it was dollars.
  Both now exclude them, matching every other spend query.
- total_cost was summed from the per-model split, which drops rows with no
  model_id and so undercounted the figure printed above the series. Summed
  from the series instead.
- The CSV export encoded its detail cell without the fallback the NDJSON
  branch had, so a non-JSON-native value would have failed the stream.
- The activity view reset pagination in an effect, which fetched the stale
  page against the new filters before fetching again. Reset in the setters.
- The audit taxonomy is no longer re-exported from the helper module; the one
  caller that wanted it imports from where it lives.

docsgpt/storage/db/repositories/activity.py

@@ -46,6 +46,10 @@ ACTIVITY_COLUMNS = (
 # the category filter cannot match it.
 _FEED_CATEGORIES = {"auth": None, "device": "device", "guardrail": "safety"}
 
+# ``feed`` and ``id`` break ties so a page boundary is deterministic and the
+# keyset cursor below has something total to compare against.
+_ORDER_BY = "ORDER BY created_at DESC, feed DESC, id DESC"
+
 _AUTH_BRANCH = f"""
     SELECT 'auth' AS feed,
            id::text AS id,
@@ -158,14 +162,14 @@ class ActivityRepository:
     def _query(
         self,
         *,
-        feeds: Optional[Sequence[str]],
-        categories: Optional[Sequence[str]],
-        events: Optional[Sequence[str]],
-        actor_id: Optional[str],
-        user_id: Optional[str],
-        since: Optional[datetime],
-        until: Optional[datetime],
-        search: Optional[str],
+        feeds: Optional[Sequence[str]] = None,
+        categories: Optional[Sequence[str]] = None,
+        events: Optional[Sequence[str]] = None,
+        actor_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+        search: Optional[str] = None,
     ) -> tuple[str, dict]:
         selected = self._feeds_for(feeds, categories)
         if not selected:
@@ -247,11 +251,33 @@ class ActivityRepository:
             return []
         params.update({"limit": int(limit), "offset": int(offset)})
         result = self._conn.execute(
-            text(f"{sql} ORDER BY created_at DESC, feed, id DESC LIMIT :limit OFFSET :offset"),
+            text(f"{sql} {_ORDER_BY} LIMIT :limit OFFSET :offset"),
             params,
         )
         return [dict(row._mapping) for row in result.fetchall()]
 
+    def _page_after(
+        self, cursor: Optional[tuple], limit: int, filters: dict
+    ) -> list[dict]:
+        """One keyset page strictly older than ``cursor``.
+
+        OFFSET paging would drift here: the feed is ordered newest-first and
+        the journals are append-only, so rows inserted mid-export push the
+        window down and repeat rows that were already emitted.
+        """
+        sql, params = self._query(**filters)
+        if not sql:
+            return []
+        if cursor is not None:
+            sql += " AND " if "WHERE" in sql else " WHERE "
+            sql += "(created_at, feed, id) < (:cur_at, :cur_feed, :cur_id)"
+            params.update(
+                {"cur_at": cursor[0], "cur_feed": cursor[1], "cur_id": cursor[2]}
+            )
+        params["limit"] = int(limit)
+        result = self._conn.execute(text(f"{sql} {_ORDER_BY} LIMIT :limit"), params)
+        return [dict(row._mapping) for row in result.fetchall()]
+
     def count(
         self,
         *,
@@ -291,8 +317,9 @@ class ActivityRepository:
     ) -> Iterator[dict]:
         """Yield every matching row, newest first, for the export endpoint.
 
-        Pages through the feed rather than materializing it, so an export of a
-        large instance streams instead of building the whole result in memory.
+        Pages by keyset, not offset: the journals are append-only and the feed
+        is newest-first, so rows written mid-export would otherwise shift the
+        window and repeat rows already emitted.
 
         Args:
             chunk_size: Rows fetched per round trip.
@@ -304,17 +331,18 @@ class ActivityRepository:
             Row dicts carrying :data:`ACTIVITY_COLUMNS`.
         """
         emitted = 0
-        offset = 0
+        cursor: Optional[tuple] = None
         while emitted < max_rows:
-            page = self.list(
-                limit=min(chunk_size, max_rows - emitted), offset=offset, **filters
+            page = self._page_after(
+                cursor, min(chunk_size, max_rows - emitted), filters
             )
             if not page:
                 return
             for row in page:
                 yield row
+            last = page[-1]
+            cursor = (last["created_at"], last["feed"], last["id"])
             emitted += len(page)
-            offset += len(page)
 
     def event_names(self) -> list[dict]:
         """Distinct ``(event, category)`` pairs across all three journals.

docsgpt/storage/db/repositories/admin_stats.py

@@ -14,6 +14,7 @@ from typing import Optional
 from sqlalchemy import Connection, text
 
 from docsgpt.storage.db.base_repository import row_to_dict
+from docsgpt.storage.db.repositories.token_usage import TokenUsageRepository
 
 
 def _round_ms(value) -> Optional[int]:
@@ -76,6 +77,10 @@ class AdminStatsRepository:
         Ordered by tokens, not cost: a cheap model can dominate token volume
         while a costly one dominates the bill, and the operator wants to see
         both columns rather than have the ranking pick for them.
+
+        Run-level rollup rows are excluded, matching every other spend query:
+        a scheduled run already has a row per LLM call, so counting the rollup
+        too would bill it twice.
         """
         result = self._conn.execute(
             text(
@@ -85,12 +90,17 @@ class AdminStatsRepository:
                        COALESCE(SUM(cost), 0) AS cost
                 FROM token_usage
                 WHERE timestamp >= :since AND user_id IS NOT NULL
+                  AND source <> ALL(:rollup_sources)
                 GROUP BY user_id
                 ORDER BY tokens DESC
                 LIMIT :limit
                 """
             ),
-            {"since": since, "limit": int(limit)},
+            {
+                "since": since,
+                "limit": int(limit),
+                "rollup_sources": list(TokenUsageRepository.ROLLUP_SOURCES),
+            },
         )
         return [
             {"user_id": r[0], "tokens": int(r[1]), "cost": float(r[2])}
@@ -140,6 +150,9 @@ class AdminStatsRepository:
         cannot answer either question an operator actually has -- what is this
         person costing, and what is driving it.
 
+        Run-level rollup rows are excluded, so a scheduled run is counted once
+        (by its per-call rows), not twice.
+
         Args:
             user_id: The billable user (auth ``sub``).
             since: Inclusive window start.
@@ -148,6 +161,7 @@ class AdminStatsRepository:
         Returns:
             ``{"totals": {...}, "by_model": [...], "by_source": [...]}``.
         """
+        rollups = list(TokenUsageRepository.ROLLUP_SOURCES)
         totals = self._conn.execute(
             text(
                 """
@@ -156,9 +170,10 @@ class AdminStatsRepository:
                        count(*) AS calls
                 FROM token_usage
                 WHERE user_id = :u AND timestamp >= :since
+                  AND source <> ALL(:rollup_sources)
                 """
             ),
-            {"u": user_id, "since": since},
+            {"u": user_id, "since": since, "rollup_sources": rollups},
         ).one()
 
         def _breakdown(key_expr: str) -> list[dict]:
@@ -170,12 +185,18 @@ class AdminStatsRepository:
                            COALESCE(SUM(cost), 0) AS cost
                     FROM token_usage
                     WHERE user_id = :u AND timestamp >= :since
+                      AND source <> ALL(:rollup_sources)
                     GROUP BY key
                     ORDER BY tokens DESC, key
                     LIMIT :limit
                     """
                 ),
-                {"u": user_id, "since": since, "limit": int(limit)},
+                {
+                    "u": user_id,
+                    "since": since,
+                    "limit": int(limit),
+                    "rollup_sources": rollups,
+                },
             )
             return [
                 {"key": r.key, "tokens": int(r.tokens), "cost": float(r.cost)}