fix(admin): keyset export paging, rollup-free spend, consistent totals

Review pass over the preceding commits.

- The activity export paged by OFFSET. The journals are append-only and the
  feed is newest-first, so rows written mid-export shift the window down and
  repeat rows already emitted. Pages by keyset on (created_at, feed, id) now.
- top_token_users and the per-user breakdown counted run-level rollup rows.
  A scheduled run already has a row per LLM call, so its spend was billed
  twice -- invisible while the column was tokens, obvious once it was dollars.
  Both now exclude them, matching every other spend query.
- total_cost was summed from the per-model split, which drops rows with no
  model_id and so undercounted the figure printed above the series. Summed
  from the series instead.
- The CSV export encoded its detail cell without the fallback the NDJSON
  branch had, so a non-JSON-native value would have failed the stream.
- The activity view reset pagination in an effect, which fetched the stale
  page against the new filters before fetching again. Reset in the setters.
- The audit taxonomy is no longer re-exported from the helper module; the one
  caller that wanted it imports from where it lives.

docsgpt/api/admin/activity.py

@@ -43,7 +43,7 @@ _EXPORT_CHUNK = 1_000
 _MAX_SEARCH_LENGTH = 200
 
 
-def _csv_list(name: str, allowed: tuple[str, ...]) -> Optional[list[str]]:
+def _facet_list(name: str, allowed: tuple[str, ...]) -> Optional[list[str]]:
     """Parse a repeated/comma-separated query arg, dropping unknown values.
 
     Args:
@@ -96,8 +96,8 @@ def _search_arg() -> Optional[str]:
 def _filters() -> dict:
     """The filter set shared by the feed and the export."""
     return {
-        "feeds": _csv_list("feed", _FEEDS),
-        "categories": _csv_list("category", ACTIVITY_CATEGORIES),
+        "feeds": _facet_list("feed", _FEEDS),
+        "categories": _facet_list("category", ACTIVITY_CATEGORIES),
         "events": _event_list(),
         "actor_id": request.args.get("actor_id") or None,
         "user_id": request.args.get("user_id") or None,
@@ -216,7 +216,9 @@ def _csv_rows(filters: dict, limit: int) -> Iterator[str]:
     for row in _export_rows(filters, limit):
         serialized = _serialize(row)
         # ``detail`` is a JSON object; a CSV cell holds its compact encoding.
-        serialized["detail"] = json.dumps(serialized.get("detail") or {})
+        serialized["detail"] = json.dumps(
+            serialized.get("detail") or {}, default=str
+        )
         writer.writerow(serialized)
         yield _drain(buffer)
 

docsgpt/api/admin/routes.py

@@ -359,7 +359,9 @@ class AdminUsageResource(Resource):
                     "group_by": group_by,
                     "series": series,
                     "total_tokens": int(total),
-                    "total_cost": round(sum(row["cost"] for row in by_model), 4),
+                    # Summed from the series, not from ``by_model``: the latter
+                    # drops rows with no model_id, so it would undercount.
+                    "total_cost": round(sum(row["cost"] for row in series), 4),
                     "by_model": by_model,
                     "latency": latency,
                     "top_users": top_users,

docsgpt/api/audit.py

@@ -14,6 +14,9 @@ Two properties matter and are enforced here rather than at every call site:
 2. **Request context is optional.** Celery tasks (ingestion finishing, a
    scheduled run) have no Flask request; the row simply records without an IP
    or user agent instead of raising.
+
+The event → category taxonomy the admin activity feed filters on lives in
+``docsgpt/audit_events.py``, which the storage layer needs too.
 """
 
 from __future__ import annotations
@@ -23,14 +26,10 @@ from typing import Any, Optional
 
 from flask import has_request_context, request
 
-from docsgpt.audit_events import ACTIVITY_CATEGORIES, category_for
 from docsgpt.storage.db.repositories.auth_events import AuthEventsRepository
 
 logger = logging.getLogger(__name__)
 
-# Re-exported so route modules have one import for recording and classifying.
-__all__ = ["ACTIVITY_CATEGORIES", "category_for", "record_event"]
-
 
 def record_event(
     conn,

docsgpt/api/user/sources/upload.py

@@ -397,10 +397,6 @@ class UploadFile(Resource):
             return make_response(jsonify({"success": False}), 400)
         # Predetermined id matches the dedup-claim row; loser GET sees same.
         response_task_id = predetermined_task_id or task.id
-        # ``source_uuid`` was minted above and passed to the worker as
-        # ``source_id``; the worker uses it verbatim for every SSE event,
-        # so the frontend can correlate inbound ``source.ingest.*`` to
-        # this upload regardless of whether an idempotency key was set.
         # Audited here, not in the worker: this is the user action. The
         # ingest may still fail, which the source's own status records.
         try:
@@ -416,9 +412,15 @@ class UploadFile(Resource):
                 )
         except Exception as err:
             current_app.logger.warning(
-                "Could not audit upload for source %s: %s", source_uuid, err,
+                "Could not audit upload for source %s: %s",
+                source_uuid,
+                err,
                 exc_info=True,
             )
+        # ``source_uuid`` was minted above and passed to the worker as
+        # ``source_id``; the worker uses it verbatim for every SSE event,
+        # so the frontend can correlate inbound ``source.ingest.*`` to
+        # this upload regardless of whether an idempotency key was set.
         response_payload: dict = {
             "success": True,
             "task_id": response_task_id,