Merge pull request #264 from oraios/codex/add-analytics-to-serena-dashboard

Add basic tool analytics to dashboard

CHANGELOG.md

@@ -22,6 +22,9 @@ Status of the `main` branch. Changes prior to the next official version change w
    * **Add support for Clojure**
 * Configuration:
    * Add option `web_dashboard_open_on_launch` (allowing the dashboard to be enabled without opening a browser window) 
+   * Add options `record_tool_usage_stats` and `token_count_estimator`
+* Dashboard:
+  * Displaying tool usage statistics if enabled in the config
 
 Fixes:
 * Fix `ExecuteShellCommandTool` and `GetCurrentConfigTool` hanging on Windows

README.md

@@ -706,10 +706,11 @@ Serena provides two convenient ways of accessing the logs of the current session
 
     This is mainly supported on Windows, but it may also work on Linux; macOS is unsupported.
 
-Both can be enabled or disabled in Serena's configuration file (`serena_config.yml`, see above).
+Both can be enabled, configured or disabled in Serena's configuration file (`serena_config.yml`, see above).
 If enabled, they will automatically be opened as soon as the Serena agent/MCP server is started.
+The web dashboard will display usage statistics of Serena's tools if you set  `record_tool_usage_stats: True` in your config.
 
-In addition to viewing logs, both tools allow to shut down the Serena agent.
+In addition to viewing logs, both tools allow to shut down the Serena agent. 
 This function is provided, because clients like Claude Desktop may fail to terminate the MCP server subprocess 
 when they themselves are closed.
 

pyproject.toml

@@ -32,6 +32,8 @@ dependencies = [
   "docstring_parser>=0.16",
   "joblib>=1.5.1",
   "tqdm>=4.67.1",
+  "tiktoken>=0.9.0",
+  "anthropic>=0.54.0",
 ]
 
 [project.scripts]
@@ -46,7 +48,7 @@ dev = [
   "black[jupyter]>=23.7.0",
   "jinja2",
   # In version 1.0.4 we get a NoneType error related to some config conversion (yml_analytics is None and should be a list)
-  "mypy>=1.4.1",
+  "mypy>=1.16.1",
   "poethepoet>=0.20.0",
   "pytest>=8.0.2",
   "pytest-xdist>=3.5.0",

src/serena/agent.py

@@ -21,6 +21,7 @@ from sensai.util.logging import LogTime
 from tqdm import tqdm
 
 from serena import serena_version
+from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats
 from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
 from serena.config.serena_config import SerenaConfig, ToolSet, get_serena_managed_dir
 from serena.constants import (
@@ -179,11 +180,19 @@ class SerenaAgent:
         if self._gui_log_handler is not None:
             self._gui_log_handler.log_viewer.set_tool_names(tool_names)
 
+        self._tool_usage_stats: ToolUsageStats | None = None
+        if self.serena_config.record_tool_usage_stats:
+            token_count_estimator = RegisteredTokenCountEstimator[self.serena_config.token_count_estimator]
+            log.info(f"Tool usage statistics recording is enabled with token count estimator: {token_count_estimator.name}.")
+            self._tool_usage_stats = ToolUsageStats(token_count_estimator)
+
         # start the dashboard (web frontend), registering its log handler
         if self.serena_config.web_dashboard:
             dashboard_log_handler = MemoryLogHandler(level=serena_log_level)
             Logger.root.addHandler(dashboard_log_handler)
-            self._dashboard_thread, port = SerenaDashboardAPI(dashboard_log_handler, tool_names).run_in_thread()
+            self._dashboard_thread, port = SerenaDashboardAPI(
+                dashboard_log_handler, tool_names, tool_usage_stats=self._tool_usage_stats
+            ).run_in_thread()
             if self.serena_config.web_dashboard_open_on_launch:
                 # open the dashboard URL in the default web browser (using a separate process to control
                 # output redirection)
@@ -240,6 +249,19 @@ class SerenaAgent:
                     "You should now pass either --project <project_name> or --project <project_root>."
                 )
 
+    def record_tool_usage_if_enabled(self, input_kwargs: dict, tool_result: str | dict, tool: Tool) -> None:
+        """
+        Record the usage of a tool with the given input and output strings if tool usage statistics recording is enabled.
+        """
+        tool_name = tool.get_name()
+        if self._tool_usage_stats is not None:
+            input_str = str(input_kwargs)
+            output_str = str(tool_result)
+            log.debug(f"Recording tool usage for tool '{tool_name}'")
+            self._tool_usage_stats.record_tool_usage(tool_name, input_str, output_str)
+        else:
+            log.debug(f"Tool usage statistics recording is disabled, not recording usage of '{tool_name}'.")
+
     @staticmethod
     def _open_dashboard(port: int) -> None:
         # Redirect stdout and stderr file descriptors to /dev/null,

src/serena/analytics.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+import logging
+import threading
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from copy import copy
+from dataclasses import asdict, dataclass
+from enum import Enum
+
+from anthropic.types import MessageParam, MessageTokensCount
+from dotenv import load_dotenv
+
+log = logging.getLogger(__name__)
+
+
+class TokenCountEstimator(ABC):
+    @abstractmethod
+    def estimate_token_count(self, text: str) -> int:
+        """
+        Estimate the number of tokens in the given text.
+        This is an abstract method that should be implemented by subclasses.
+        """
+
+
+class TiktokenCountEstimator(TokenCountEstimator):
+    """
+    Approximate token count using tiktoken.
+    """
+
+    def __init__(self, model_name: str = "gpt-4o"):
+        """
+        The tokenizer will be downloaded on the first initialization, which may take some time.
+
+        :param model_name: see `tiktoken.model` to see available models.
+        """
+        import tiktoken
+
+        log.info(f"Loading tiktoken encoding for model {model_name}, this may take a while on the first run.")
+        self._encoding = tiktoken.encoding_for_model(model_name)
+
+    def estimate_token_count(self, text: str) -> int:
+        return len(self._encoding.encode(text))
+
+
+class AnthropicTokenCount(TokenCountEstimator):
+    """
+    The exact count using the Anthropic API.
+    Counting is free, but has a rate limit and will require an API key,
+    (typically, set through an env variable).
+    See https://docs.anthropic.com/en/docs/build-with-claude/token-counting
+    """
+
+    def __init__(self, model_name: str = "claude-sonnet-4-20250514", api_key: str | None = None):
+        import anthropic
+
+        self._model_name = model_name
+        if api_key is None:
+            load_dotenv()
+        self._anthropic_client = anthropic.Anthropic(api_key=api_key)
+
+    def _send_count_tokens_request(self, text: str) -> MessageTokensCount:
+        return self._anthropic_client.messages.count_tokens(
+            model=self._model_name,
+            messages=[MessageParam(role="user", content=text)],
+        )
+
+    def estimate_token_count(self, text: str) -> int:
+        return self._send_count_tokens_request(text).input_tokens
+
+
+_registered_token_estimator_instances_cache: dict[RegisteredTokenCountEstimator, TokenCountEstimator] = {}
+
+
+class RegisteredTokenCountEstimator(Enum):
+    TIKTOKEN_GPT4O = "TIKTOKEN_GPT4O"
+    ANTHROPIC_CLAUDE_SONNET_4 = "ANTHROPIC_CLAUDE_SONNET_4"
+
+    @classmethod
+    def get_valid_names(cls) -> list[str]:
+        """
+        Get a list of all registered token count estimator names.
+        """
+        return [estimator.name for estimator in cls]
+
+    def _create_estimator(self) -> TokenCountEstimator:
+        match self:
+            case RegisteredTokenCountEstimator.TIKTOKEN_GPT4O:
+                return TiktokenCountEstimator(model_name="gpt-4o")
+            case RegisteredTokenCountEstimator.ANTHROPIC_CLAUDE_SONNET_4:
+                return AnthropicTokenCount(model_name="claude-sonnet-4-20250514")
+            case _:
+                raise ValueError(f"Unknown token count estimator: {self.value}")
+
+    def load_estimator(self) -> TokenCountEstimator:
+        estimator_instance = _registered_token_estimator_instances_cache.get(self)
+        if estimator_instance is None:
+            estimator_instance = self._create_estimator()
+            _registered_token_estimator_instances_cache[self] = estimator_instance
+        return estimator_instance
+
+
+class ToolUsageStats:
+    """
+    A class to record and manage tool usage statistics.
+    """
+
+    def __init__(self, token_count_estimator: RegisteredTokenCountEstimator = RegisteredTokenCountEstimator.TIKTOKEN_GPT4O):
+        self._token_count_estimator = token_count_estimator.load_estimator()
+        self._token_estimator_name = token_count_estimator.value
+        self._tool_stats: dict[str, ToolUsageStats.Entry] = defaultdict(ToolUsageStats.Entry)
+        self._tool_stats_lock = threading.Lock()
+
+    @property
+    def token_estimator_name(self) -> str:
+        """
+        Get the name of the registered token count estimator used.
+        """
+        return self._token_estimator_name
+
+    @dataclass(kw_only=True)
+    class Entry:
+        num_times_called: int = 0
+        input_tokens: int = 0
+        output_tokens: int = 0
+
+        def update_on_call(self, input_tokens: int, output_tokens: int) -> None:
+            """
+            Update the entry with the number of tokens used for a single call.
+            """
+            self.num_times_called += 1
+            self.input_tokens += input_tokens
+            self.output_tokens += output_tokens
+
+    def _estimate_token_count(self, text: str) -> int:
+        return self._token_count_estimator.estimate_token_count(text)
+
+    def get_stats(self, tool_name: str) -> ToolUsageStats.Entry:
+        """
+        Get (a copy of) the current usage statistics for a specific tool.
+        """
+        with self._tool_stats_lock:
+            return copy(self._tool_stats[tool_name])
+
+    def record_tool_usage(self, tool_name: str, input_str: str, output_str: str) -> None:
+        input_tokens = self._estimate_token_count(input_str)
+        output_tokens = self._estimate_token_count(output_str)
+        with self._tool_stats_lock:
+            entry = self._tool_stats[tool_name]
+            entry.update_on_call(input_tokens, output_tokens)
+
+    def get_tool_stats_dict(self) -> dict[str, dict[str, int]]:
+        with self._tool_stats_lock:
+            return {name: asdict(entry) for name, entry in self._tool_stats.items()}
+
+    def clear(self) -> None:
+        with self._tool_stats_lock:
+            self._tool_stats.clear()

src/serena/config/serena_config.py

@@ -27,6 +27,8 @@ from serena.util.general import load_yaml, save_yaml
 from serena.util.inspection import determine_programming_language_composition
 from solidlsp.ls_config import Language
 
+from ..analytics import RegisteredTokenCountEstimator
+
 if TYPE_CHECKING:
     from ..project import Project
 
@@ -254,6 +256,17 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
     """
     whether to apply JetBrains mode
     """
+    record_tool_usage_stats: bool = False
+    """Whether to record tool usage statistics, they will be shown in the web dashboard if recording is active. 
+    """
+    token_count_estimator: str = RegisteredTokenCountEstimator.TIKTOKEN_GPT4O.name
+    """Only relevant if `record_tool_usage` is True; the name of the token count estimator to use for tool usage statistics.
+    See the `RegisteredTokenCountEstimator` enum for available options.
+    
+    Note: some token estimators (like tiktoken) may require downloading data files
+    on the first run, which can take some time and require internet access. Others, like the Anthropic ones, may require an API key
+    and rate limits may apply.
+    """
 
     CONFIG_FILE = "serena_config.yml"
     CONFIG_FILE_DOCKER = "serena_config.docker.yml"  # Docker-specific config file; auto-generated if missing, mounted via docker-compose for user customization
@@ -351,6 +364,10 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
         instance.excluded_tools = loaded_commented_yaml.get("excluded_tools", [])
         instance.included_optional_tools = loaded_commented_yaml.get("included_optional_tools", [])
         instance.jetbrains = loaded_commented_yaml.get("jetbrains", False)
+        instance.record_tool_usage_stats = loaded_commented_yaml.get("record_tool_usage_stats", False)
+        instance.token_count_estimator = loaded_commented_yaml.get(
+            "token_count_estimator", RegisteredTokenCountEstimator.TIKTOKEN_GPT4O.name
+        )
 
         # re-save the configuration file if any migrations were performed
         if num_project_migrations > 0:

src/serena/dashboard.py

@@ -9,6 +9,7 @@ from flask import Flask, Response, request, send_from_directory
 from pydantic import BaseModel
 from sensai.util import logging
 
+from serena.analytics import ToolUsageStats
 from serena.constants import SERENA_DASHBOARD_DIR, SERENA_LOG_FORMAT
 
 log = logging.getLogger(__name__)
@@ -67,16 +68,25 @@ class ResponseToolNames(BaseModel):
     tool_names: list[str]
 
 
+class ResponseToolStats(BaseModel):
+    stats: dict[str, dict[str, int]]
+
+
 class SerenaDashboardAPI:
     log = logging.getLogger(__qualname__)
 
     def __init__(
-        self, memory_log_handler: MemoryLogHandler, tool_names: list[str], shutdown_callback: Callable[[], None] | None = None
+        self,
+        memory_log_handler: MemoryLogHandler,
+        tool_names: list[str],
+        shutdown_callback: Callable[[], None] | None = None,
+        tool_usage_stats: ToolUsageStats | None = None,
     ) -> None:
         self._memory_log_handler = memory_log_handler
         self._tool_names = tool_names
         self._shutdown_callback = shutdown_callback
         self._app = Flask(__name__)
+        self._tool_usage_stats = tool_usage_stats
         self._setup_routes()
 
     @property
@@ -110,6 +120,21 @@ class SerenaDashboardAPI:
             result = self._get_tool_names()
             return result.model_dump()
 
+        @self._app.route("/get_tool_stats", methods=["GET"])
+        def get_tool_stats_route() -> dict[str, Any]:
+            result = self._get_tool_stats()
+            return result.model_dump()
+
+        @self._app.route("/clear_tool_stats", methods=["POST"])
+        def clear_tool_stats_route() -> dict[str, str]:
+            self._clear_tool_stats()
+            return {"status": "cleared"}
+
+        @self._app.route("/get_token_count_estimator_name", methods=["GET"])
+        def get_token_count_estimator_name() -> dict[str, str]:
+            estimator_name = self._tool_usage_stats.token_estimator_name if self._tool_usage_stats else "unknown"
+            return {"token_count_estimator_name": estimator_name}
+
         @self._app.route("/shutdown", methods=["PUT"])
         def shutdown() -> dict[str, str]:
             self._shutdown()
@@ -123,6 +148,16 @@ class SerenaDashboardAPI:
     def _get_tool_names(self) -> ResponseToolNames:
         return ResponseToolNames(tool_names=self._tool_names)
 
+    def _get_tool_stats(self) -> ResponseToolStats:
+        if self._tool_usage_stats is not None:
+            return ResponseToolStats(stats=self._tool_usage_stats.get_tool_stats_dict())
+        else:
+            return ResponseToolStats(stats={})
+
+    def _clear_tool_stats(self) -> None:
+        if self._tool_usage_stats is not None:
+            self._tool_usage_stats.clear()
+
     def _shutdown(self) -> None:
         log.info("Shutting down Serena")
         if self._shutdown_callback:

src/serena/mcp.py

@@ -103,6 +103,7 @@ class SerenaMCPFactory:
             is_async=is_async,
             context_kwarg=None,
             annotations=None,
+            title=None,
         )
 
     @abstractmethod

src/serena/resources/config/modes/editing.yml

@@ -19,6 +19,8 @@ prompt: |
   You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file
   or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible
   while still solving your task, meaning you only read the bodies when you need to, and after you have found the symbol you want to edit.
+  Before calling symbolic reading tools, you should have a basic understanding of the repository structure that you can get from memories
+  or by using the `list_dir` and `find_file` tools (or similar).
   For example, if you are working with python code and already know that you need to read the body of the constructor of the class Foo, you can directly
   use `find_symbol` with the name path `Foo/__init__` and `include_body=True`. If you don't know yet which methods in `Foo` you need to read or edit,
   you can use `find_symbol` with the name path `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding

src/serena/resources/dashboard/dashboard.js

@@ -40,10 +40,22 @@ class Dashboard {
         this.$errorContainer = $('#error-container');
         this.$loadButton = $('#load-logs');
         this.$shutdownButton = $('#shutdown');
+        this.$toggleStats = $('#toggle-stats');
+        this.$statsSection = $('#stats-section');
+        this.$refreshStats = $('#refresh-stats');
+        this.$clearStats = $('#clear-stats');
+
+        this.countChart = null;
+        this.tokensChart = null;
+        this.inputChart = null;
+        this.outputChart = null;
 
         // register event handlers
         this.$loadButton.click(this.loadLogs.bind(this));
         this.$shutdownButton.click(this.shutdown.bind(this));
+        this.$toggleStats.click(this.toggleStats.bind(this));
+        this.$refreshStats.click(this.loadStats.bind(this));
+        this.$clearStats.click(this.clearStats.bind(this));
 
         // initialize the application
         this.loadToolNames().then(function() {
@@ -176,6 +188,232 @@ class Dashboard {
         this.pollInterval = setInterval(this.pollForNewLogs.bind(this), 1000);
     }
 
+    toggleStats() {
+        if (this.$statsSection.is(':visible')) {
+            this.$statsSection.hide();
+            this.$toggleStats.text('Show Stats');
+        } else {
+            this.$statsSection.show();
+            this.$toggleStats.text('Hide Stats');
+            this.loadStats();
+        }
+    }
+
+    loadStats() {
+        let self = this;
+        $.when(
+            $.ajax({ url: '/get_tool_stats', type: 'GET' }),
+            $.ajax({ url: '/get_token_count_estimator_name', type: 'GET' })
+        ).done(function(statsResp, estimatorResp) {
+            const stats = statsResp[0].stats;
+            const tokenCountEstimatorName = estimatorResp[0].token_count_estimator_name;
+            self.displayStats(stats, tokenCountEstimatorName);
+        }).fail(function() {
+            console.error('Error loading stats or estimator name');
+        });
+    }
+
+
+    clearStats() {
+        let self = this;
+        $.ajax({
+            url: '/clear_tool_stats',
+            type: 'POST',
+            success: function() {
+                self.loadStats();
+            },
+            error: function(xhr, status, error) {
+                console.error('Error clearing stats:', error);
+            }
+        });
+    }
+
+    displayStats(stats, tokenCountEstimatorName) {
+        const names = Object.keys(stats);
+      // If no stats collected
+        if (names.length === 0) {
+            // hide summary, charts, estimator name
+            $('#stats-summary').hide();
+            $('#estimator-name').hide();
+            $('.charts-container').hide();
+            // show no-stats message
+            $('#no-stats-message').show();
+            return;
+        } else {
+            // Ensure everything is visible
+            $('#estimator-name').show();
+            $('#stats-summary').show();
+            $('.charts-container').show();
+            $('#no-stats-message').hide();
+        }
+
+        $('#estimator-name').html(`<strong>Token count estimator:</strong> ${tokenCountEstimatorName}`);
+
+        const counts = names.map(n => stats[n].num_times_called);
+        const inputTokens = names.map(n => stats[n].input_tokens);
+        const outputTokens = names.map(n => stats[n].output_tokens);
+        const totalTokens = names.map(n => stats[n].input_tokens + stats[n].output_tokens);
+        
+        // Calculate totals for summary table
+        const totalCalls = counts.reduce((sum, count) => sum + count, 0);
+        const totalInputTokens = inputTokens.reduce((sum, tokens) => sum + tokens, 0);
+        const totalOutputTokens = outputTokens.reduce((sum, tokens) => sum + tokens, 0);
+        
+        // Generate consistent colors for tools
+        const colors = this.generateColors(names.length);
+
+        const countCtx = document.getElementById('count-chart');
+        const tokensCtx = document.getElementById('tokens-chart');
+        const inputCtx = document.getElementById('input-chart');
+        const outputCtx = document.getElementById('output-chart');
+
+        if (this.countChart) this.countChart.destroy();
+        if (this.tokensChart) this.tokensChart.destroy();
+        if (this.inputChart) this.inputChart.destroy();
+        if (this.outputChart) this.outputChart.destroy();
+
+        // Update summary table
+        this.updateSummaryTable(totalCalls, totalInputTokens, totalOutputTokens);
+
+        // Register datalabels plugin
+        Chart.register(ChartDataLabels);
+
+        // Tool calls pie chart
+        this.countChart = new Chart(countCtx, {
+            type: 'pie',
+            data: { 
+                labels: names, 
+                datasets: [{ 
+                    data: counts,
+                    backgroundColor: colors
+                }] 
+            },
+            options: {
+                plugins: {
+                    legend: { display: true },
+                    datalabels: {
+                        display: true,
+                        color: 'white',
+                        font: { weight: 'bold' },
+                        formatter: (value) => value
+                    }
+                }
+            }
+        });
+
+        // Input tokens pie chart
+        this.inputChart = new Chart(inputCtx, {
+            type: 'pie',
+            data: { 
+                labels: names, 
+                datasets: [{ 
+                    data: inputTokens,
+                    backgroundColor: colors
+                }] 
+            },
+            options: {
+                plugins: {
+                    legend: { display: true },
+                    datalabels: {
+                        display: true,
+                        color: 'white',
+                        font: { weight: 'bold' },
+                        formatter: (value) => value
+                    }
+                }
+            }
+        });
+
+        // Output tokens pie chart
+        this.outputChart = new Chart(outputCtx, {
+            type: 'pie',
+            data: { 
+                labels: names, 
+                datasets: [{ 
+                    data: outputTokens,
+                    backgroundColor: colors
+                }] 
+            },
+            options: {
+                plugins: {
+                    legend: { display: true },
+                    datalabels: {
+                        display: true,
+                        color: 'white',
+                        font: { weight: 'bold' },
+                        formatter: (value) => value
+                    }
+                }
+            }
+        });
+
+        // Combined input/output tokens bar chart
+        this.tokensChart = new Chart(tokensCtx, {
+            type: 'bar',
+            data: { 
+                labels: names, 
+                datasets: [
+                    { 
+                        label: 'Input Tokens', 
+                        data: inputTokens,
+                        backgroundColor: colors.map(color => color + '80'), // Semi-transparent
+                        borderColor: colors,
+                        borderWidth: 2,
+                        borderSkipped: false,
+                        yAxisID: 'y'
+                    },
+                    { 
+                        label: 'Output Tokens', 
+                        data: outputTokens,
+                        backgroundColor: colors,
+                        yAxisID: 'y1'
+                    }
+                ]
+            },
+            options: {
+                responsive: true,
+                scales: {
+                    y: {
+                        type: 'linear',
+                        display: true,
+                        position: 'left',
+                        beginAtZero: true,
+                        title: { display: true, text: 'Input Tokens' }
+                    },
+                    y1: {
+                        type: 'linear',
+                        display: true,
+                        position: 'right',
+                        beginAtZero: true,
+                        title: { display: true, text: 'Output Tokens' },
+                        grid: { drawOnChartArea: false }
+                    }
+                }
+            }
+        });
+    }
+
+    generateColors(count) {
+        const colors = [
+            '#FF6384', '#36A2EB', '#FFCE56', '#4BC0C0', '#9966FF',
+            '#FF9F40', '#FF6384', '#C9CBCF', '#4BC0C0', '#FF6384'
+        ];
+        return Array.from({length: count}, (_, i) => colors[i % colors.length]);
+    }
+
+    updateSummaryTable(totalCalls, totalInputTokens, totalOutputTokens) {
+        const tableHtml = `
+            <table class="stats-summary">
+                <tr><th>Metric</th><th>Total</th></tr>
+                <tr><td>Tool Calls</td><td>${totalCalls}</td></tr>
+                <tr><td>Input Tokens</td><td>${totalInputTokens}</td></tr>
+                <tr><td>Output Tokens</td><td>${totalOutputTokens}</td></tr>
+                <tr><td>Total Tokens</td><td>${totalInputTokens + totalOutputTokens}</td></tr>
+            </table>
+        `;
+        $('#stats-summary').html(tableHtml);
+    }
+
     shutdown() {
         const self = this;
         const _shutdown = function () {
@@ -198,4 +436,4 @@ class Dashboard {
             console.log("Shutdown cancelled");
         }
     }
-}
+}

src/serena/resources/dashboard/index.html

@@ -8,6 +8,8 @@
     <link rel="icon" type="image/png" sizes="32x32" href="serena-icon-32.png">
     <link rel="icon" type="image/png" sizes="48x48" href="serena-icon-48.png">
     <script src="jquery.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-datalabels@2"></script>
     <script src="dashboard.js"></script>
     <style>
         body {
@@ -97,6 +99,70 @@
             text-align: center;
             margin: 10px 0;
         }
+        
+        .charts-container {
+            display: flex;
+            flex-wrap: wrap;
+            gap: 15px;
+            justify-content: space-between;
+            max-width: 1400px;
+            margin: 0 auto;
+        }
+        
+        .chart-group {
+            flex: 1;
+            min-width: 280px;
+            max-width: 320px;
+            text-align: center;
+        }
+        
+        .chart-wide {
+            flex: 0 0 100%;
+            min-width: 100%;
+            margin-top: 10px;
+        }
+        
+        .chart-group h3 {
+            margin: 0 0 10px 0;
+            color: #333;
+        }
+        
+        .stats-summary {
+            margin: 0 auto;
+            border-collapse: collapse;
+            background: white;
+            border-radius: 5px;
+            overflow: hidden;
+            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+        }
+        
+        .stats-summary th,
+        .stats-summary td {
+            padding: 10px 20px;
+            text-align: left;
+            border-bottom: 1px solid #ddd;
+        }
+        
+        .stats-summary th {
+            background-color: #f8f9fa;
+            font-weight: bold;
+        }
+        
+        .stats-summary tr:last-child td {
+            border-bottom: none;
+        }
+        
+        @media (max-width: 768px) {
+            .charts-container {
+                flex-direction: column;
+            }
+            
+            .chart-group,
+            .chart-wide {
+                min-width: auto;
+                max-width: none;
+            }
+        }
     </style>
 </head>
 <body>
@@ -107,15 +173,49 @@
     <div class="controls">
         <button id="load-logs" class="btn">Reload Log</button>
         <button id="shutdown" class="btn">Shutdown Server</button>
+        <button id="toggle-stats" class="btn">Show Stats</button>
     </div>
-    
+
     <div id="error-container"></div>
     <div id="log-container" class="log-container"></div>
 
+    <div id="stats-section" style="display:none; margin-top:20px;">
+        <div style="text-align:center; margin-bottom:20px;">
+            <button id="refresh-stats" class="btn">Refresh Stats</button>
+            <button id="clear-stats" class="btn">Clear Stats</button>
+        </div>
+        
+        <div id="stats-summary" style="margin-bottom:20px; text-align:center;"></div>
+        <div id="estimator-name" style="text-align:center; margin-bottom:10px;"></div>
+        <div id="no-stats-message" style="text-align:center; color:#666; font-style:italic; display:none;">
+          No tool stats collected. Have you enabled tool stats collection in the configuration?
+        </div>
+
+        
+        <div class="charts-container">
+            <div class="chart-group">
+                <h3>Tool Calls</h3>
+                <canvas id="count-chart" height="200"></canvas>
+            </div>
+            <div class="chart-group">
+                <h3>Input Tokens</h3>
+                <canvas id="input-chart" height="200"></canvas>
+            </div>
+            <div class="chart-group">
+                <h3>Output Tokens</h3>
+                <canvas id="output-chart" height="200"></canvas>
+            </div>
+            <div class="chart-group chart-wide">
+                <h3>Input vs Output Tokens</h3>
+                <canvas id="tokens-chart" height="120"></canvas>
+            </div>
+        </div>
+    </div>
+
     <script>
         $(document).ready(function() {
             const dashboard = new Dashboard();
         });
     </script>
 </body>
-</html>
+</html>

src/serena/resources/serena_config.template.yml

@@ -48,6 +48,18 @@ jetbrains: False
 # NOTE: The plugin is yet unreleased. This is for Serena developers only.
 
 
+record_tool_usage_stats:  False
+# whether to record tool usage statistics, they will be shown in the web dashboard if recording is active.
+
+token_count_estimator: TIKTOKEN_GPT4O
+# Only relevant if `record_tool_usage` is True; the name of the token count estimator to use for tool usage statistics.
+# See the `RegisteredTokenCountEstimator` enum for available options.
+#
+# Note: some token estimators (like tiktoken) may require downloading data files
+# on the first run, which can take some time and require internet access. Others, like the Anthropic ones, may require an API key
+# and rate limits may apply.
+
+
 # MANAGED BY SERENA, KEEP AT THE BOTTOM OF THE YAML AND DON'T EDIT WITHOUT NEED
 # The list of registered projects.
 # To add a project, within a chat, simply ask Serena to "activate the project /path/to/project" or,

src/serena/tools/jetbrains_tools.py

@@ -113,6 +113,9 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerOptional):
         """
         Gets an overview of the top-level symbols in the given file.
         Calling this is often a good idea before more targeted reading, searching or editing operations on the code symbols.
+        Before requesting a symbol overview, it is usually a good idea to narrow down the scope of the overview
+        by first understanding the basic directory structure of the repository that you can get from memories
+        or by using the `list_dir` and `find_file` tools (or similar).
 
         :param relative_path: the relative path to the file to get the overview of
         :param max_answer_chars: max characters for the JSON result. If exceeded, no content is returned.

src/serena/tools/symbol_tools.py

@@ -53,6 +53,9 @@ class GetSymbolsOverviewTool(Tool):
         For each analyzed file, we list the top-level symbols in the file (name_path, kind).
         Use this tool to get a high-level understanding of the code symbols.
         Calling this is often a good idea before more targeted reading, searching or editing operations on the code symbols.
+        Before requesting a symbol overview, it is usually a good idea to narrow down the scope of the overview
+        by first understanding the basic directory structure of the repository that you can get from memories
+        or by using the `list_dir` and `find_file` tools (or similar).
 
         :param relative_path: the relative path to the file or directory to get the overview of
         :param max_answer_chars: if the overview is longer than this number of characters,

src/serena/tools/tools_base.py

@@ -2,10 +2,10 @@ import inspect
 import os
 import traceback
 from abc import ABC
-from collections.abc import Callable, Iterable
+from collections.abc import Iterable
 from dataclasses import dataclass
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, Self, TypeVar
+from typing import TYPE_CHECKING, Any, Protocol, Self, TypeVar
 
 from mcp.server.fastmcp.utilities.func_metadata import FuncMetadata, func_metadata
 from sensai.util import logging
@@ -89,6 +89,13 @@ class ToolMarkerOptional:
     """
 
 
+class ApplyMethodProtocol(Protocol):
+    """Callable protocol for the apply method of a tool."""
+
+    def __call__(self, *args: Any, **kwargs: Any) -> str:
+        pass
+
+
 class Tool(Component):
     # NOTE: each tool should implement the apply method, which is then used in
     # the central method of the Tool class `apply_ex`.
@@ -112,7 +119,7 @@ class Tool(Component):
     def get_name(self) -> str:
         return self.get_name_from_cls()
 
-    def get_apply_fn(self) -> Callable:
+    def get_apply_fn(self) -> ApplyMethodProtocol:
         apply_fn = getattr(self, "apply")
         if apply_fn is None:
             raise RuntimeError(f"apply not defined in {self}. Did you forget to implement it?")
@@ -231,6 +238,7 @@ class Tool(Component):
 
                 # apply the actual tool
                 result = apply_fn(**kwargs)
+                self.agent.record_tool_usage_if_enabled(kwargs, result, self)
 
             except Exception as e:
                 if not catch_exceptions: