Limit the size of the log buffer

Fixes #1034

src/serena/constants.py

@@ -28,3 +28,6 @@ PROJECT_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.template.y
 SERENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml")
 
 SERENA_LOG_FORMAT = "%(levelname)-5s %(asctime)-15s [%(threadName)s] %(name)s:%(funcName)s:%(lineno)d - %(message)s"
+
+LOG_MESSAGES_BUFFER_SIZE = 2500
+"""The maximum number of log messages to keep in the buffer (for the dashboard)."""

src/serena/dashboard.py

@@ -363,11 +363,10 @@ class SerenaDashboardAPI:
                 return {"status": "error", "message": str(e)}
 
     def _get_log_messages(self, request_log: RequestLog) -> ResponseLog:
-        all_messages = self._memory_log_handler.get_log_messages()
-        requested_messages = all_messages[request_log.start_idx :] if request_log.start_idx <= len(all_messages) else []
+        messages = self._memory_log_handler.get_log_messages(from_idx=request_log.start_idx)
         project = self._agent.get_active_project()
         project_name = project.project_name if project else None
-        return ResponseLog(messages=requested_messages, max_idx=len(all_messages) - 1, active_project=project_name)
+        return ResponseLog(messages=messages.messages, max_idx=messages.max_idx, active_project=project_name)
 
     def _get_tool_names(self) -> ResponseToolNames:
         return ResponseToolNames(tool_names=self._tool_names)

src/serena/gui_log_viewer.py

@@ -68,7 +68,7 @@ class GuiLogViewer:
         }
 
         if memory_log_handler is not None:
-            for msg in memory_log_handler.get_log_messages():
+            for msg in memory_log_handler.get_log_messages().messages:
                 self.message_queue.put(msg)
             memory_log_handler.add_emit_callback(lambda msg: self.message_queue.put(msg))
 

src/serena/util/logging.py

@@ -1,20 +1,33 @@
 import queue
 import threading
 from collections.abc import Callable
+from dataclasses import dataclass
 from typing import Optional
 
 from sensai.util import logging
 
-from serena.constants import SERENA_LOG_FORMAT
+from serena.constants import LOG_MESSAGES_BUFFER_SIZE, SERENA_LOG_FORMAT
 
 lg = logging
 
 
+@dataclass
+class LogMessages:
+    messages: list[str]
+    """
+    the list of log messages, ordered from oldest to newest
+    """
+    max_idx: int
+    """
+    the 0-based index of the last message in `messages` (in the full log history)
+    """
+
+
 class MemoryLogHandler(logging.Handler):
-    def __init__(self, level: int = logging.NOTSET) -> None:
+    def __init__(self, level: int = logging.NOTSET, max_messages: int | None = LOG_MESSAGES_BUFFER_SIZE) -> None:
         super().__init__(level=level)
         self.setFormatter(logging.Formatter(SERENA_LOG_FORMAT))
-        self._log_buffer = LogBuffer()
+        self._log_buffer = LogBuffer(max_messages=max_messages)
         self._log_queue: queue.Queue[str] = queue.Queue()
         self._stop_event = threading.Event()
         self._emit_callbacks: list[Callable[[str], None]] = []
@@ -48,26 +61,48 @@ class MemoryLogHandler(logging.Handler):
             except queue.Empty:
                 continue
 
-    def get_log_messages(self) -> list[str]:
-        return self._log_buffer.get_log_messages()
+    def get_log_messages(self, from_idx: int = 0) -> LogMessages:
+        return self._log_buffer.get_log_messages(from_idx=from_idx)
 
 
 class LogBuffer:
     """
-    A thread-safe buffer for storing log messages.
+    A thread-safe buffer for storing (an optionally limited number of) log messages.
     """
 
-    def __init__(self) -> None:
-        self._log_messages: list[str] = []
+    def __init__(self, max_messages: int | None = None) -> None:
+        self._max_messages = max_messages
+        self._log_messages = []
         self._lock = threading.Lock()
+        self._max_idx = -1
+        """
+        the 0-based index of the most recently added log message
+        """
 
     def append(self, msg: str) -> None:
         with self._lock:
             self._log_messages.append(msg)
+            self._max_idx += 1
+            if self._max_messages is not None and len(self._log_messages) > self._max_messages:
+                excess = len(self._log_messages) - self._max_messages
+                self._log_messages = self._log_messages[excess:]
 
-    def get_log_messages(self) -> list[str]:
+    def get_log_messages(self, from_idx: int = 0) -> LogMessages:
+        """
+        :param from_idx: the 0-based index of the first log message to return.
+            If from_idx is less than or equal to the index of the oldest message in the buffer,
+            then all messages in the buffer will be returned.
+        :return: the list of messages
+        """
+        from_idx = max(from_idx, 0)
         with self._lock:
-            return self._log_messages.copy()
+            first_stored_idx = self._max_idx - len(self._log_messages) + 1
+            if from_idx <= first_stored_idx:
+                messages = self._log_messages.copy()
+            else:
+                start_idx = from_idx - first_stored_idx
+                messages = self._log_messages[start_idx:].copy()
+            return LogMessages(messages=messages, max_idx=self._max_idx)
 
 
 class SuspendedLoggersContext: