Add Serena Dashboard: Web service with log viewer

(supported by memory logger and backend service)

dashboard/index.html

@@ -0,0 +1,324 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Serena Log Viewer</title>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.7.1/jquery.min.js"></script>
+    <style>
+        body {
+            font-family: 'Consolas', 'Monaco', 'Courier New', monospace;
+            margin: 0;
+            padding: 20px;
+            background-color: #f5f5f5;
+        }
+        
+        .header {
+            text-align: center;
+            margin-bottom: 20px;
+        }
+        
+        .header h1 {
+            color: #333;
+            margin: 0;
+        }
+        
+        .log-container {
+            background-color: white;
+            border: 1px solid #ddd;
+            border-radius: 5px;
+            height: 600px;
+            overflow-y: auto;
+            overflow-x: auto;
+            padding: 10px;
+            white-space: pre-wrap;
+            font-size: 12px;
+            line-height: 1.4;
+        }
+        
+        .controls {
+            margin-bottom: 10px;
+            text-align: center;
+        }
+
+        .logo {
+            margin-bottom: 10px;
+            text-align: center;
+        }
+
+        .btn {
+            background-color: #eaa45d;
+            color: white;
+            border: none;
+            padding: 8px 16px;
+            border-radius: 4px;
+            cursor: pointer;
+            font-size: 14px;
+        }
+        
+        .btn:hover {
+            background-color: #dca662;
+        }
+        
+        .btn:disabled {
+            background-color: #6c757d;
+            cursor: not-allowed;
+        }
+        
+        /* Log level colors matching the GUI viewer */
+        .log-debug {
+            color: #808080; /* Gray */
+        }
+        
+        .log-info {
+            color: #000000; /* Black */
+        }
+        
+        .log-warning {
+            color: #FF8C00; /* Dark Orange */
+        }
+        
+        .log-error {
+            color: #FF0000; /* Red */
+        }
+        
+        .log-default {
+            color: #000000; /* Black */
+        }
+        
+        /* Tool name highlighting */
+        .tool-name {
+            background-color: #ffff00; /* Yellow background */
+            font-weight: bold;
+        }
+        
+        .loading {
+            text-align: center;
+            color: #666;
+            font-style: italic;
+        }
+        
+        .error-message {
+            color: #FF0000;
+            text-align: center;
+            margin: 10px 0;
+        }
+    </style>
+</head>
+<body>
+    <div class="header">
+        <img src="serena-logs.png" alt="Serena" style="max-width: 400px; height: auto;">
+    </div>
+    
+    <div class="controls">
+        <button id="load-logs" class="btn">Load Logs</button>
+        <button id="clear-logs" class="btn">Clear Display</button>
+    </div>
+    
+    <div id="error-container"></div>
+    <div id="log-container" class="log-container"></div>
+
+    <script>
+        $(document).ready(function() {
+            // Variables to track state
+            let toolNames = [];
+            let currentMaxIdx = -1;
+            let pollInterval = null;
+            
+            // Determine log level from message
+            function determineLogLevel(message) {
+                const messageUpper = message.toUpperCase();
+                if (messageUpper.startsWith('DEBUG')) {
+                    return 'debug';
+                } else if (messageUpper.startsWith('INFO')) {
+                    return 'info';
+                } else if (messageUpper.startsWith('WARNING')) {
+                    return 'warning';
+                } else if (messageUpper.startsWith('ERROR')) {
+                    return 'error';
+                } else {
+                    return 'default';
+                }
+            }
+            
+            // Highlight tool names in a message
+            function highlightToolNames(message) {
+                let highlightedMessage = message;
+                toolNames.forEach(function(toolName) {
+                    const regex = new RegExp('\\b' + toolName + '\\b', 'gi');
+                    highlightedMessage = highlightedMessage.replace(regex, '<span class="tool-name">' + toolName + '</span>');
+                });
+                return highlightedMessage;
+            }
+            
+            // Format and display a log message
+            function displayLogMessage(message) {
+                const logLevel = determineLogLevel(message);
+                const highlightedMessage = highlightToolNames(message);
+                const logEntry = $('<div>').addClass('log-' + logLevel).html(highlightedMessage + '\n');
+                $('#log-container').append(logEntry);
+            }
+            
+            // Load tool names from the API
+            function loadToolNames() {
+                return $.ajax({
+                    url: '/get_tool_names',
+                    type: 'GET',
+                    success: function(response) {
+                        toolNames = response.tool_names || [];
+                        console.log('Loaded tool names:', toolNames);
+                    },
+                    error: function(xhr, status, error) {
+                        console.error('Error loading tool names:', error);
+                        // Use fallback tool names if API fails
+                        toolNames = [
+                            'find_symbol', 'read_file', 'create_text_file', 'replace_symbol_body',
+                            'insert_after_symbol', 'insert_before_symbol', 'search_for_pattern',
+                            'execute_shell_command', 'get_symbols_overview', 'find_referencing_symbols'
+                        ];
+                    }
+                });
+            }
+            
+            // Load logs from the API (full reload)
+            function loadLogs() {
+                const $loadBtn = $('#load-logs');
+                const $errorContainer = $('#error-container');
+                
+                // Disable button and show loading state
+                $loadBtn.prop('disabled', true).text('Loading...');
+                $errorContainer.empty();
+                
+                // Make API call
+                $.ajax({
+                    url: '/get_log_messages',
+                    type: 'POST',
+                    contentType: 'application/json',
+                    data: JSON.stringify({
+                        start_idx: 0
+                    }),
+                    success: function(response) {
+                        // Clear existing logs
+                        $('#log-container').empty();
+                        
+                        // Update max_idx
+                        currentMaxIdx = response.max_idx || -1;
+                        
+                        // Display each log message
+                        if (response.messages && response.messages.length > 0) {
+                            response.messages.forEach(function(message) {
+                                displayLogMessage(message);
+                            });
+                            
+                            // Auto-scroll to bottom
+                            const logContainer = $('#log-container')[0];
+                            logContainer.scrollTop = logContainer.scrollHeight;
+                        } else {
+                            $('#log-container').html('<div class="loading">No log messages found.</div>');
+                        }
+                        
+                        // Start periodic polling for new logs
+                        startPeriodicPolling();
+                    },
+                    error: function(xhr, status, error) {
+                        console.error('Error loading logs:', error);
+                        $errorContainer.html('<div class="error-message">Error loading logs: ' + 
+                                           (xhr.responseJSON ? xhr.responseJSON.detail : error) + '</div>');
+                    },
+                    complete: function() {
+                        // Re-enable button
+                        $loadBtn.prop('disabled', false).text('Load Logs');
+                    }
+                });
+            }
+            
+            // Poll for new log messages
+            function pollForNewLogs() {
+                console.log("poll");
+                $.ajax({
+                    url: '/get_log_messages',
+                    type: 'POST',
+                    contentType: 'application/json',
+                    data: JSON.stringify({
+                        start_idx: currentMaxIdx + 1
+                    }),
+                    success: function(response) {
+                        // Only append new messages if we have any
+                        if (response.messages && response.messages.length > 0) {
+                            let wasAtBottom = false;
+                            const logContainer = $('#log-container')[0];
+                            
+                            // Check if user was at the bottom before adding new logs
+                            if (logContainer.scrollHeight > 0) {
+                                wasAtBottom = (logContainer.scrollTop + logContainer.clientHeight) >= (logContainer.scrollHeight - 10);
+                            }
+                            
+                            // Append new messages
+                            response.messages.forEach(function(message) {
+                                displayLogMessage(message);
+                            });
+                            
+                            // Update max_idx
+                            currentMaxIdx = response.max_idx || currentMaxIdx;
+                            
+                            // Auto-scroll to bottom if user was already at bottom
+                            if (wasAtBottom) {
+                                logContainer.scrollTop = logContainer.scrollHeight;
+                            }
+                        } else {
+                            // Update max_idx even if no new messages
+                            currentMaxIdx = response.max_idx || currentMaxIdx;
+                        }
+                    },
+                    error: function(xhr, status, error) {
+                        console.error('Error polling for new logs:', error);
+                        // Continue polling even on error (API might be temporarily unavailable)
+                    }
+                });
+            }
+            
+            // Start periodic polling
+            function startPeriodicPolling() {
+                // Clear any existing interval
+                if (pollInterval) {
+                    clearInterval(pollInterval);
+                }
+                
+                // Start polling every second (1000ms)
+                pollInterval = setInterval(pollForNewLogs, 1000);
+            }
+            
+            // Stop periodic polling
+            function stopPeriodicPolling() {
+                if (pollInterval) {
+                    clearInterval(pollInterval);
+                    pollInterval = null;
+                }
+            }
+            
+            // Clear the log display
+            function clearLogs() {
+                stopPeriodicPolling();
+                $('#log-container').empty();
+                $('#error-container').empty();
+                currentMaxIdx = -1;
+            }
+            
+            // Event handlers
+            $('#load-logs').click(loadLogs);
+            $('#clear-logs').click(clearLogs);
+            
+            // Initialize the application
+            loadToolNames().then(function() {
+                // Load logs on page load after tool names are loaded
+                loadLogs();
+            });
+            
+            // Stop polling when page is about to unload
+            $(window).on('beforeunload', function() {
+                stopPeriodicPolling();
+            });
+        });
+    </script>
+</body>
+</html>

src/serena/agent.py

@@ -9,6 +9,7 @@ import platform
 import shutil
 import sys
 import traceback
+import webbrowser
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from collections.abc import Callable, Generator, Iterable, Sequence
@@ -32,6 +33,7 @@ from multilspy.multilspy_types import SymbolKind
 from serena import serena_root_path, serena_version
 from serena.config import SerenaAgentContext, SerenaAgentMode
 from serena.constants import SERENA_MANAGED_DIR_NAME
+from serena.dashboard import MemoryLogHandler, SerenaDashboardAPI
 from serena.prompt_factory import PromptFactory, SerenaPromptFactory
 from serena.symbol import SymbolLocation, SymbolManager
 from serena.text_utils import search_files
@@ -480,6 +482,16 @@ class SerenaAgent:
                 self._gui_log_handler = GuiLogViewerHandler(GuiLogViewer(title="Serena Logs"), level=log_level, format_string=LOG_FORMAT)
                 Logger.root.addHandler(self._gui_log_handler)
 
+        # instantiate all tool classes
+        self._all_tools: dict[type[Tool], Tool] = {tool_class: tool_class(self) for tool_class in ToolRegistry.get_all_tool_classes()}
+
+        # start the dashboard (web frontend) and register its log handler
+        memory_log_handler = MemoryLogHandler()
+        Logger.root.addHandler(memory_log_handler)
+        tool_names = [tool.get_name() for tool in self._all_tools.values()]
+        self._dashboard_thread, port = SerenaDashboardAPI(memory_log_handler, tool_names).run_in_thread()
+        webbrowser.open(f"http://localhost:{port}/dashboard/index.html")
+
         log.info(f"Starting Serena server (version={serena_version()}, process id={os.getpid()}, parent process id={os.getppid()})")
         log.info("Available projects: {}".format(", ".join(self.serena_config.project_names)))
 
@@ -495,9 +507,6 @@ class SerenaAgent:
         self.memories_manager: MemoriesManager | None = None
         self.lines_read: LinesRead | None = None
 
-        # instantiate all tool classes
-        self._all_tools: dict[type[Tool], Tool] = {tool_class: tool_class(self) for tool_class in ToolRegistry.get_all_tool_classes()}
-
         # Apply context and mode tool configurations
         if context is None:
             context = SerenaAgentContext.load_default()

src/serena/dashboard.py

@@ -0,0 +1,112 @@
+import os
+import queue
+import socket
+import threading
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel
+from sensai.util import logging
+
+from serena import serena_root_path
+
+log = logging.getLogger(__name__)
+
+
+class MemoryLogHandler(logging.Handler):
+    def __init__(self, level=logging.NOTSET):
+        super().__init__(level=level)
+        self.setFormatter(logging.Formatter(logging.LOG_DEFAULT_FORMAT))
+        self._log_buffer = LogBuffer()
+        self._log_queue = queue.Queue()
+        self._stop_event = threading.Event()
+
+        # start background thread to process logs
+        self.worker_thread = threading.Thread(target=self._process_queue, daemon=True)
+        self.worker_thread.start()
+
+    def emit(self, record):
+        msg = self.format(record)
+        self._log_queue.put_nowait(msg)
+
+    def _process_queue(self):
+        while not self._stop_event.is_set():
+            try:
+                msg = self._log_queue.get(timeout=1)
+                self._log_buffer.append(msg)
+                self._log_queue.task_done()
+            except queue.Empty:
+                continue
+
+    def get_log_messages(self) -> list[str]:
+        return self._log_buffer.logs
+
+
+class LogBuffer:
+    def __init__(self):
+        self.logs = []
+
+    def append(self, msg: str):
+        self.logs.append(msg)
+
+
+class RequestLog(BaseModel):
+    start_idx: int = 0
+
+
+class ResponseLog(BaseModel):
+    messages: list[str]
+    max_idx: int
+
+
+class ResponseToolNames(BaseModel):
+    tool_names: list[str]
+
+
+class SerenaDashboardAPI:
+    log = logging.getLogger(__qualname__)
+
+    def __init__(self, memory_log_handler: MemoryLogHandler, tool_names: list[str]):
+        self._memory_log_handler = memory_log_handler
+        self._tool_names = tool_names
+        self._app = FastAPI(title="Serena Dashboard")
+        self._setup_routes()
+
+    def _setup_routes(self):
+        static_dir = os.path.join(serena_root_path(), "dashboard")
+        self._app.mount("/dashboard", StaticFiles(directory=static_dir), name="dashboard")
+
+        self._app.add_api_route("/get_log_messages", self._get_log_messages, methods=["POST"], response_model=ResponseLog)
+        self._app.add_api_route("/get_tool_names", self._get_tool_names, methods=["GET"], response_model=ResponseToolNames)
+
+    async def _get_log_messages(self, request: RequestLog) -> ResponseLog:
+        all_messages = self._memory_log_handler.get_log_messages()
+        requested_messages = all_messages[request.start_idx :] if request.start_idx <= len(all_messages) else []
+        return ResponseLog(messages=requested_messages, max_idx=len(all_messages) - 1)
+
+    async def _get_tool_names(self) -> ResponseToolNames:
+        return ResponseToolNames(tool_names=self._tool_names)
+
+    @staticmethod
+    def _find_first_free_port(start_port: int) -> int:
+        port = start_port
+        while port <= 65535:
+            try:
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+                    sock.bind(("localhost", port))
+                    return port
+            except OSError:
+                port += 1
+
+        raise RuntimeError(f"No free ports found starting from {start_port}")
+
+    def run(self, host="127.0.0.1", port=0x5EDA):
+        uvicorn.run(self._app, host=host, port=port, workers=1, log_config=None, log_level="critical")
+        return port
+
+    def run_in_thread(self) -> tuple[threading.Thread, int]:
+        port = self._find_first_free_port(0x5EDA)
+        thread = threading.Thread(target=lambda: self.run(port=port), daemon=True)
+        thread.start()
+        return thread, port