Support the dynamic addition of languages while a project is already activated

Move LanguageServerManager ownership from SerenaAgent to Project

Add Project.save_config to support saving updated configurations

src/serena/agent.py

@@ -28,6 +28,7 @@ from serena.prompt_factory import SerenaPromptFactory
 from serena.tools import ActivateProjectTool, GetCurrentConfigTool, Tool, ToolMarker, ToolRegistry
 from serena.util.inspection import iter_subclasses
 from serena.util.logging import MemoryLogHandler
+from solidlsp.ls_config import Language
 
 if TYPE_CHECKING:
     from serena.gui_log_viewer import GuiLogViewer
@@ -86,7 +87,6 @@ class SerenaAgent:
 
         # project-specific instances, which will be initialized upon project activation
         self._active_project: Project | None = None
-        self._language_server_manager: LanguageServerManager | None = None
 
         # adjust log level
         serena_log_level = self.serena_config.log_level
@@ -193,16 +193,19 @@ class SerenaAgent:
                 process.join(timeout=1)
 
     def get_language_server_manager(self) -> LanguageServerManager | None:
-        return self._language_server_manager
+        if self._active_project is not None:
+            return self._active_project.language_server_manager
+        return None
 
     def get_language_server_manager_or_raise(self) -> LanguageServerManager:
-        if self._language_server_manager is None:
+        language_server_manager = self.get_language_server_manager()
+        if language_server_manager is None:
             raise Exception(
                 "The language server manager is not initialized, indicating a problem during project activation. "
                 "Inform the user, telling them to inspect Serena's logs in order to determine the issue. "
                 "IMPORTANT: Wait for further instructions before you continue!"
             )
-        return self._language_server_manager
+        return language_server_manager
 
     def get_context(self) -> SerenaAgentContext:
         return self._context
@@ -381,7 +384,7 @@ class SerenaAgent:
             log.info(f"Scheduling {task_name}")
             return self._task_executor.submit(task_execution_wrapper)
 
-    def execute_task(self, task: Callable[[], T]) -> T:
+    def execute_task(self, task: Callable[[], T], name: str | None = None) -> T:
         """
         Executes the given task synchronously via the agent's task executor.
         This is useful for tasks that need to be executed immediately and whose results are needed right away.
@@ -389,7 +392,7 @@ class SerenaAgent:
         :param task: the task to execute
         :return: the result of the task execution
         """
-        future = self.issue_task(task)
+        future = self.issue_task(task, name=name)
         return future.result()
 
     def is_using_language_server(self) -> bool:
@@ -519,7 +522,7 @@ class SerenaAgent:
 
     def reset_language_server_manager(self) -> None:
         """
-        Starts/resets the language server for the current project
+        Starts/resets the language server manager for the current project
         """
         tool_timeout = self.serena_config.tool_timeout
         if tool_timeout is None or tool_timeout < 0:
@@ -529,20 +532,24 @@ class SerenaAgent:
                 raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds")
             ls_timeout = tool_timeout - 5  # the LS timeout is for a single call, it should be smaller than the tool timeout
 
-        # stop the language server if it is running
-        if self._language_server_manager is not None:
-            self._language_server_manager.stop_all()
-            self._language_server_manager = None
-
         # instantiate and start the necessary language servers
-        assert self._active_project is not None, "Active project required"
-        self._language_server_manager = self._active_project.create_language_server_manager(
+        self.get_active_project_or_raise().create_language_server_manager(
             log_level=self.serena_config.log_level,
             ls_timeout=ls_timeout,
             trace_lsp_communication=self.serena_config.trace_lsp_communication,
             ls_specific_settings=self.serena_config.ls_specific_settings,
         )
 
+    def add_language(self, language: Language) -> None:
+        """
+        Adds a new language to the active project, spawning the respective language server and updating the project configuration.
+        The addition is scheduled via the agent's task executor and executed synchronously, i.e. the method returns
+        when the addition is complete.
+
+        :param language: the language to add
+        """
+        self.execute_task(lambda: self.get_active_project_or_raise().add_language(language), name=f"AddLanguage:{language.value}")
+
     def get_tool(self, tool_class: type[TTool]) -> TTool:
         return self._all_tools[tool_class]  # type: ignore
 
@@ -556,9 +563,9 @@ class SerenaAgent:
         if not hasattr(self, "_is_initialized"):
             return
         log.info("SerenaAgent is shutting down ...")
-        if self._language_server_manager is not None:
-            self._language_server_manager.stop_all(save_cache=True)
-            self._language_server_manager = None
+        if self._active_project is not None:
+            self._active_project.shutdown()
+            self._active_project = None
         if self._gui_log_viewer:
             log.info("Stopping the GUI log window ...")
             self._gui_log_viewer.stop()

src/serena/config/serena_config.py

@@ -2,6 +2,7 @@
 The Serena Model Context Protocol (MCP) Server
 """
 
+import dataclasses
 import os
 import shutil
 from collections.abc import Iterable
@@ -258,6 +259,14 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
             encoding=data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING),
         )
 
+    def to_yaml_dict(self) -> dict:
+        """
+        :return: a yaml-serializable dictionary representation of this configuration
+        """
+        d = dataclasses.asdict(self)
+        d["languages"] = [lang.value for lang in self.languages]
+        return d
+
     @classmethod
     def load(cls, project_root: Path | str, autogenerate: bool = False) -> Self:
         """

src/serena/ls_manager.py

@@ -1,12 +1,54 @@
 import logging
-from collections.abc import Callable, Iterator
+from collections.abc import Iterator
+from dataclasses import dataclass
 
+from serena.constants import SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME
 from solidlsp import SolidLanguageServer
-from solidlsp.ls_config import Language
+from solidlsp.ls_config import Language, LanguageServerConfig
+from solidlsp.ls_logger import LanguageServerLogger
+from solidlsp.settings import SolidLSPSettings
 
 log = logging.getLogger(__name__)
 
 
+@dataclass
+class LanguageServerFactoryParams:
+    project_root: str
+    encoding: str
+    ignored_patterns: list[str]
+    ls_timeout: float | None = None
+    ls_specific_settings: dict | None = None
+    log_level: int = logging.INFO
+    trace_lsp_communication: bool = False
+
+
+class LanguageServerFactory:
+    def __init__(self, language: Language):
+        self.language = language
+
+    def create_language_server(self, params: LanguageServerFactoryParams) -> SolidLanguageServer:
+        ls_config = LanguageServerConfig(
+            code_language=self.language,
+            ignored_paths=params.ignored_patterns,
+            trace_lsp_communication=params.trace_lsp_communication,
+            encoding=params.encoding,
+        )
+        ls_logger = LanguageServerLogger(log_level=params.log_level)
+
+        log.info(f"Creating language server instance for {params.project_root}, language={self.language}.")
+        return SolidLanguageServer.create(
+            ls_config,
+            ls_logger,
+            params.project_root,
+            timeout=params.ls_timeout,
+            solidlsp_settings=SolidLSPSettings(
+                solidlsp_dir=SERENA_MANAGED_DIR_IN_HOME,
+                project_data_relative_path=SERENA_MANAGED_DIR_NAME,
+                ls_specific_settings=params.ls_specific_settings or {},
+            ),
+        )
+
+
 class LanguageServerManager:
     """
     Manages one or more language servers
@@ -15,16 +57,19 @@ class LanguageServerManager:
     def __init__(
         self,
         language_servers: dict[Language, SolidLanguageServer],
-        language_server_factories: dict[Language, Callable[[], SolidLanguageServer]] | None = None,
+        language_server_factories: dict[Language, LanguageServerFactory] | None = None,
+        language_server_factory_params: LanguageServerFactoryParams | None = None,
     ) -> None:
         """
         :param language_servers: a mapping from language to language server; the servers are assumed to be already started.
             The first server in the iteration order is used as the default server.
             All servers are assumed to serve the same project root.
         :param language_server_factories: factories for the language servers, to be used for restarting them if needed
+        :param language_server_factory_params: parameters to be used when (re-)creating language servers via factories
         """
         self._language_servers = language_servers
         self._language_server_factories = language_server_factories or {}
+        self._language_server_factory_params = language_server_factory_params
         self._default_language_server = next(iter(language_servers.values()))
         self._root_path = self._default_language_server.repository_root_path
 
@@ -48,19 +93,44 @@ class LanguageServerManager:
             ls = self._default_language_server
         return self._ensure_functional_ls(ls)
 
+    def _create_and_start_language_server(self, language: Language) -> SolidLanguageServer:
+        factory = self._language_server_factories.get(language)
+        if factory is None:
+            raise ValueError(f"No language server factory found for language {language}; cannot create new instance")
+        if self._language_server_factory_params is None:
+            raise ValueError("No language server factory parameters provided; cannot instantiate new language server instance")
+        self._language_servers[language] = factory.create_language_server(self._language_server_factory_params)
+        self._language_servers[language].start()
+        return self._language_servers[language]
+
     def restart_language_server(self, language: Language) -> SolidLanguageServer:
         """
         Forces recreation and restart of the language server for the given language.
+        It is assumed that the language server for the given language is no longer running.
 
         :param language: the language
         :return: the newly created language server
         """
-        factory = self._language_server_factories.get(language)
-        if factory is None:
-            raise ValueError(f"No language server factory found for language {language}; cannot recreate and restart")
-        self._language_servers[language] = factory()
-        self._language_servers[language].start()
-        return self._language_servers[language]
+        return self._create_and_start_language_server(language)
+
+    def add_language_server(self, language: Language) -> SolidLanguageServer:
+        """
+        Dynamically adds a new language server for the given language.
+
+        :param language: the language
+        :param factory: the factory to create the language server
+        :return: the newly created language server
+        """
+        if language in self._language_servers:
+            raise ValueError(f"Language server for language {language} already present")
+        self._language_server_factories[language] = LanguageServerFactory(language)
+        try:
+            return self._create_and_start_language_server(language)
+        except Exception as e:
+            del self._language_server_factories[language]
+            if language in self._language_servers:
+                del self._language_servers[language]
+            raise Exception(f"Addition of language server for {language.value} failed") from e
 
     def iter_language_servers(self) -> Iterator[SolidLanguageServer]:
         for ls in self._language_servers.values():

src/serena/project.py

@@ -2,7 +2,6 @@ import json
 import logging
 import os
 import threading
-from collections.abc import Callable
 from pathlib import Path
 from typing import Any
 
@@ -11,15 +10,14 @@ from sensai.util.logging import LogTime
 from sensai.util.string import ToStringMixin
 
 from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig, get_serena_managed_in_project_dir
-from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME
-from serena.ls_manager import LanguageServerManager
+from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME
+from serena.ls_manager import LanguageServerFactory, LanguageServerFactoryParams, LanguageServerManager
 from serena.text_utils import MatchedConsecutiveLines, search_files
 from serena.util.file_system import GitignoreParser, match_path
+from serena.util.general import load_yaml, save_yaml
 from solidlsp import SolidLanguageServer
-from solidlsp.ls_config import Language, LanguageServerConfig
-from solidlsp.ls_logger import LanguageServerLogger
+from solidlsp.ls_config import Language
 from solidlsp.ls_utils import FileUtils
-from solidlsp.settings import SolidLSPSettings
 
 log = logging.getLogger(__name__)
 
@@ -63,6 +61,7 @@ class Project(ToStringMixin):
         self.project_root = project_root
         self.project_config = project_config
         self.memories_manager = MemoriesManager(project_root)
+        self.language_server_manager: LanguageServerManager | None = None
         self._is_newly_created = is_newly_created
 
         # create .gitignore file in the project's Serena data folder if not yet present
@@ -113,6 +112,16 @@ class Project(ToStringMixin):
         project_config = ProjectConfig.load(project_root, autogenerate=autogenerate)
         return Project(project_root=str(project_root), project_config=project_config)
 
+    def save_config(self) -> None:
+        """
+        Saves the current project configuration to disk.
+        """
+        config_path = os.path.join(self.project_root, self.project_config.rel_path_to_project_yml())
+        log.info("Saving updated project configuration to %s", config_path)
+        config_with_comments = load_yaml(config_path, preserve_comments=True)
+        config_with_comments.update(self.project_config.to_yaml_dict())
+        save_yaml(config_path, config_with_comments, preserve_comments=True)
+
     def path_to_serena_data_folder(self) -> str:
         return os.path.join(self.project_root, SERENA_MANAGED_DIR_NAME)
 
@@ -351,39 +360,6 @@ class Project(ToStringMixin):
             source_file_path=relative_file_path,
         )
 
-    def _create_language_server_factory(
-        self,
-        language: Language,
-        ls_timeout: float | None,
-        ls_specific_settings: dict | None,
-        log_level: int,
-        trace_lsp_communication: bool,
-    ) -> Callable[[], SolidLanguageServer]:
-
-        def factory() -> SolidLanguageServer:
-            ls_config = LanguageServerConfig(
-                code_language=language,
-                ignored_paths=self._ignored_patterns,
-                trace_lsp_communication=trace_lsp_communication,
-                encoding=self.project_config.encoding,
-            )
-            ls_logger = LanguageServerLogger(log_level=log_level)
-
-            log.info(f"Creating language server instance for {self.project_root}, language={language}.")
-            return SolidLanguageServer.create(
-                ls_config,
-                ls_logger,
-                self.project_root,
-                timeout=ls_timeout,
-                solidlsp_settings=SolidLSPSettings(
-                    solidlsp_dir=SERENA_MANAGED_DIR_IN_HOME,
-                    project_data_relative_path=SERENA_MANAGED_DIR_NAME,
-                    ls_specific_settings=ls_specific_settings or {},
-                ),
-            )
-
-        return factory
-
     def create_language_server_manager(
         self,
         log_level: int = logging.INFO,
@@ -392,29 +368,41 @@ class Project(ToStringMixin):
         ls_specific_settings: dict[Language, Any] | None = None,
     ) -> LanguageServerManager:
         """
-        Create a language server for a project. Note that you will have to start it
-        before performing any LS operations.
+        Creates the language server manager for the project, starting one language server per configured programming language.
 
-        :param project: either a path to the project root or a ProjectConfig instance.
-            If no project.yml is found, the default project configuration will be used.
         :param log_level: the log level for the language server
         :param ls_timeout: the timeout for the language server
         :param trace_lsp_communication: whether to trace LSP communication
         :param ls_specific_settings: optional LS specific configuration of the language server,
             see docstrings in the inits of subclasses of SolidLanguageServer to see what values may be passed.
-        :return: the language server
+        :return: the language server manager, which is also stored in the project instance
         """
+        # if there is an existing instance, stop its language servers first
+        if self.language_server_manager is not None:
+            log.info("Stopping existing language server manager ...")
+            self.language_server_manager.stop_all()
+            self.language_server_manager = None
+
         log.info(f"Creating language server manager for {self.project_root}")
+        language_server_factory_params = LanguageServerFactoryParams(
+            project_root=self.project_root,
+            encoding=self.project_config.encoding,
+            ignored_patterns=self._ignored_patterns,
+            ls_timeout=ls_timeout,
+            ls_specific_settings=ls_specific_settings,
+            log_level=log_level,
+            trace_lsp_communication=trace_lsp_communication,
+        )
         language_servers: dict[Language, SolidLanguageServer] = {}
-        language_server_factories: dict[Language, Callable[[], SolidLanguageServer]] = {}
+        language_server_factories: dict[Language, LanguageServerFactory] = {}
         threads = []
         exceptions = {}
         lock = threading.Lock()
 
-        def start_language_server(language: Language, factory: Callable[[], SolidLanguageServer]):
+        def start_language_server(language: Language, factory: LanguageServerFactory) -> None:
             try:
                 with LogTime(f"Language server startup (language={language.value})"):
-                    language_server = factory()
+                    language_server = factory.create_language_server(language_server_factory_params)
                     language_server.start()
                     if not language_server.is_running():
                         raise RuntimeError(
@@ -429,7 +417,7 @@ class Project(ToStringMixin):
 
         # start language servers in parallel threads
         for language in self.project_config.languages:
-            factory = self._create_language_server_factory(language, ls_timeout, ls_specific_settings, log_level, trace_lsp_communication)
+            factory = LanguageServerFactory(language)
             language_server_factories[language] = factory
             thread = threading.Thread(target=start_language_server, args=(language, factory), name="StartLS:" + language.value)
             thread.start()
@@ -444,4 +432,33 @@ class Project(ToStringMixin):
             failure_messages = "\n".join([f"{lang.value}: {e}" for lang, e in exceptions.items()])
             raise Exception(f"Failed to start language servers:\n{failure_messages}")
 
-        return LanguageServerManager(language_servers, language_server_factories)
+        self.language_server_manager = LanguageServerManager(language_servers, language_server_factories, language_server_factory_params)
+        return self.language_server_manager
+
+    def add_language(self, language: Language) -> None:
+        """
+        Adds a new programming language to the project configuration, starting the corresponding
+        language server instance if the LS manager is active.
+        The project configuration is saved to disk after adding the language.
+
+        :param language: the programming language to add
+        """
+        if language in self.project_config.languages:
+            log.info(f"Language {language.value} is already present in the project configuration.")
+            return
+
+        # start the language server (if the LS manager is active)
+        if self.language_server_manager is None:
+            log.info("Language server manager is not active; skipping language server startup for the new language.")
+        else:
+            log.info("Adding and starting the language server for new language %s ...", language.value)
+            self.language_server_manager.add_language_server(language)
+
+        # update the project configuration
+        self.project_config.languages.append(language)
+        self.save_config()
+
+    def shutdown(self) -> None:
+        if self.language_server_manager is not None:
+            self.language_server_manager.stop_all(save_cache=True)
+            self.language_server_manager = None