Add tool analytics and dashboard charts

src/serena/agent.py

@@ -36,6 +36,7 @@ from sensai.util.logging import FallbackHandler, LogTime
 from sensai.util.string import ToStringMixin, dict_string
 
 from serena import serena_version
+from serena.analytics import record_tool_usage
 from serena.config import SerenaAgentContext, SerenaAgentMode
 from serena.constants import (
     DEFAULT_ENCODING,
@@ -1409,6 +1410,9 @@ class Tool(Component, ToolInterface):
 
                 # apply the actual tool
                 result = apply_fn(**kwargs)
+                input_chars = sum(len(str(v)) for v in kwargs.values())
+                output_chars = len(result) if isinstance(result, str) else len(str(result))
+                record_tool_usage(self.get_name_from_cls(), input_chars, output_chars)
 
             except Exception as e:
                 if not catch_exceptions:

src/serena/analytics.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import threading
+from collections import defaultdict
+
+
+class ToolStatsEntry:
+    def __init__(self) -> None:
+        self.count = 0
+        self.input_chars = 0
+        self.output_chars = 0
+
+    def to_dict(self) -> dict[str, int]:
+        return {
+            "count": self.count,
+            "input_chars": self.input_chars,
+            "output_chars": self.output_chars,
+        }
+
+_lock = threading.Lock()
+_tool_stats: dict[str, ToolStatsEntry] = defaultdict(ToolStatsEntry)
+
+
+def record_tool_usage(tool_name: str, input_chars: int, output_chars: int) -> None:
+    with _lock:
+        entry = _tool_stats[tool_name]
+        entry.count += 1
+        entry.input_chars += input_chars
+        entry.output_chars += output_chars
+
+
+def get_tool_stats() -> dict[str, dict[str, int]]:
+    with _lock:
+        return {name: entry.to_dict() for name, entry in _tool_stats.items()}
+
+
+def clear_tool_stats() -> None:
+    with _lock:
+        _tool_stats.clear()

src/serena/dashboard.py

@@ -9,6 +9,7 @@ from flask import Flask, Response, request, send_from_directory
 from pydantic import BaseModel
 from sensai.util import logging
 
+from serena.analytics import clear_tool_stats, get_tool_stats
 from serena.constants import SERENA_DASHBOARD_DIR, SERENA_LOG_FORMAT
 
 log = logging.getLogger(__name__)
@@ -67,6 +68,10 @@ class ResponseToolNames(BaseModel):
     tool_names: list[str]
 
 
+class ResponseToolStats(BaseModel):
+    stats: dict[str, dict[str, int]]
+
+
 class SerenaDashboardAPI:
     log = logging.getLogger(__qualname__)
 
@@ -110,6 +115,16 @@ class SerenaDashboardAPI:
             result = self._get_tool_names()
             return result.model_dump()
 
+        @self._app.route("/get_tool_stats", methods=["GET"])
+        def get_tool_stats_route() -> dict[str, Any]:
+            result = self._get_tool_stats()
+            return result.model_dump()
+
+        @self._app.route("/clear_tool_stats", methods=["POST"])
+        def clear_tool_stats_route() -> dict[str, str]:
+            self._clear_tool_stats()
+            return {"status": "cleared"}
+
         @self._app.route("/shutdown", methods=["PUT"])
         def shutdown() -> dict[str, str]:
             self._shutdown()
@@ -123,6 +138,13 @@ class SerenaDashboardAPI:
     def _get_tool_names(self) -> ResponseToolNames:
         return ResponseToolNames(tool_names=self._tool_names)
 
+    def _get_tool_stats(self) -> ResponseToolStats:
+        return ResponseToolStats(stats=get_tool_stats())
+
+    @staticmethod
+    def _clear_tool_stats() -> None:
+        clear_tool_stats()
+
     def _shutdown(self) -> None:
         log.info("Shutting down Serena")
         if self._shutdown_callback:

src/serena/resources/dashboard/dashboard.js

@@ -40,10 +40,19 @@ class Dashboard {
         this.$errorContainer = $('#error-container');
         this.$loadButton = $('#load-logs');
         this.$shutdownButton = $('#shutdown');
+        this.$toggleStats = $('#toggle-stats');
+        this.$statsSection = $('#stats-section');
+        this.$clearStats = $('#clear-stats');
+
+        this.countChart = null;
+        this.inputChart = null;
+        this.outputChart = null;
 
         // register event handlers
         this.$loadButton.click(this.loadLogs.bind(this));
         this.$shutdownButton.click(this.shutdown.bind(this));
+        this.$toggleStats.click(this.toggleStats.bind(this));
+        this.$clearStats.click(this.clearStats.bind(this));
 
         // initialize the application
         this.loadToolNames().then(function() {
@@ -176,6 +185,87 @@ class Dashboard {
         this.pollInterval = setInterval(this.pollForNewLogs.bind(this), 1000);
     }
 
+    toggleStats() {
+        if (this.$statsSection.is(':visible')) {
+            this.$statsSection.hide();
+            this.$toggleStats.text('Show Stats');
+        } else {
+            this.$statsSection.show();
+            this.$toggleStats.text('Hide Stats');
+            this.loadStats();
+        }
+    }
+
+    loadStats() {
+        let self = this;
+        $.ajax({
+            url: '/get_tool_stats',
+            type: 'GET',
+            success: function(response) {
+                self.displayStats(response.stats || {});
+            },
+            error: function(xhr, status, error) {
+                console.error('Error loading stats:', error);
+            }
+        });
+    }
+
+    clearStats() {
+        let self = this;
+        $.ajax({
+            url: '/clear_tool_stats',
+            type: 'POST',
+            success: function() {
+                self.loadStats();
+            },
+            error: function(xhr, status, error) {
+                console.error('Error clearing stats:', error);
+            }
+        });
+    }
+
+    displayStats(stats) {
+        const names = Object.keys(stats);
+        const counts = names.map(n => stats[n].count);
+        const inputChars = names.map(n => stats[n].input_chars);
+        const outputChars = names.map(n => stats[n].output_chars);
+
+        const countCtx = document.getElementById('count-chart');
+        const inputCtx = document.getElementById('input-chart');
+        const outputCtx = document.getElementById('output-chart');
+
+        if (this.countChart) this.countChart.destroy();
+        if (this.inputChart) this.inputChart.destroy();
+        if (this.outputChart) this.outputChart.destroy();
+
+        if (names.length === 0) {
+            this.countChart = null;
+            this.inputChart = null;
+            this.outputChart = null;
+            countCtx.getContext('2d').clearRect(0,0,countCtx.width,countCtx.height);
+            inputCtx.getContext('2d').clearRect(0,0,inputCtx.width,inputCtx.height);
+            outputCtx.getContext('2d').clearRect(0,0,outputCtx.width,outputCtx.height);
+            return;
+        }
+
+        this.countChart = new Chart(countCtx, {
+            type: 'pie',
+            data: { labels: names, datasets: [{ data: counts }] },
+        });
+
+        this.inputChart = new Chart(inputCtx, {
+            type: 'bar',
+            data: { labels: names, datasets: [{ label: 'Input Chars', data: inputChars }] },
+            options: { scales: { y: { beginAtZero: true } } }
+        });
+
+        this.outputChart = new Chart(outputCtx, {
+            type: 'bar',
+            data: { labels: names, datasets: [{ label: 'Output Chars', data: outputChars }] },
+            options: { scales: { y: { beginAtZero: true } } }
+        });
+    }
+
     shutdown() {
         const self = this;
         const _shutdown = function () {
@@ -198,4 +288,4 @@ class Dashboard {
             console.log("Shutdown cancelled");
         }
     }
-}
+}

src/serena/resources/dashboard/index.html

@@ -8,6 +8,7 @@
     <link rel="icon" type="image/png" sizes="32x32" href="serena-icon-32.png">
     <link rel="icon" type="image/png" sizes="48x48" href="serena-icon-48.png">
     <script src="jquery.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
     <script src="dashboard.js"></script>
     <style>
         body {
@@ -107,15 +108,25 @@
     <div class="controls">
         <button id="load-logs" class="btn">Reload Log</button>
         <button id="shutdown" class="btn">Shutdown Server</button>
+        <button id="toggle-stats" class="btn">Show Stats</button>
     </div>
-    
+
     <div id="error-container"></div>
     <div id="log-container" class="log-container"></div>
 
+    <div id="stats-section" style="display:none; margin-top:20px; text-align:center;">
+        <button id="clear-stats" class="btn" style="margin-bottom:10px;">Clear Stats</button>
+        <div style="max-width:600px;margin:0 auto;">
+            <canvas id="count-chart" height="200"></canvas>
+            <canvas id="input-chart" height="200"></canvas>
+            <canvas id="output-chart" height="200"></canvas>
+        </div>
+    </div>
+
     <script>
         $(document).ready(function() {
             const dashboard = new Dashboard();
         });
     </script>
 </body>
-</html>
+</html>