Improve configurability of modes

Modes can now be configured in three places (in reverse order of precedence)
  * global config
  * project config
  * command line

Furthermore, we differentiate base modes from default modes,
where command line parameters override only default modes,
adding more flexibility

Resolves #926

docs/02-usage/040_workflow.md

@@ -53,6 +53,7 @@ The file allows you to configure ...
   * write access
   * an initial prompt that shall be passed to the LLM whenever the project is activated 
   * the name by which you want to refer to the project (relevant when telling the LLM to dynamically activate the project)
+  * the set of tools and modes to use by default
 
 For detailed information on the parameters and possible settings, see the 
 [template file](https://github.com/oraios/serena/blob/main/src/serena/resources/project.template.yml). 

docs/02-usage/050_configuration.md

@@ -25,6 +25,7 @@ Some of the configurable settings include:
   * the language backend to use by default (i.e., the JetBrains plugin or language servers)
   * UI settings affecting the [Serena Dashboard and GUI tool](060_dashboard.md)
   * the set of tools to enable/disable by default
+  * the set of modes to use by default
   * tool execution parameters (timeout, max. answer length)
   * global ignore rules
   * logging settings

src/serena/agent.py

@@ -6,7 +6,7 @@ import os
 import platform
 import subprocess
 import sys
-from collections.abc import Callable
+from collections.abc import Callable, Sequence
 from logging import Logger
 from typing import TYPE_CHECKING, Optional, TypeVar
 
@@ -17,7 +17,7 @@ from interprompt.jinja_template import JinjaTemplate
 from serena import serena_version
 from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats
 from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
-from serena.config.serena_config import LanguageBackend, SerenaConfig, ToolInclusionDefinition
+from serena.config.serena_config import LanguageBackend, ModeSelectionDefinition, SerenaConfig, ToolInclusionDefinition
 from serena.dashboard import SerenaDashboardAPI
 from serena.ls_manager import LanguageServerManager
 from serena.project import Project
@@ -167,6 +167,47 @@ class ToolSet:
         return tool_name in self._tool_names
 
 
+class ActiveModes:
+    def __init__(self) -> None:
+        self._base_modes: Sequence[str] | None = None
+        self._default_modes: Sequence[str] | None = None
+        self._active_mode_names: Sequence[str] | None = []
+        self._active_modes: Sequence[SerenaAgentMode] | None = []
+
+    def apply(self, mode_selection: ModeSelectionDefinition) -> None:
+        # invalidate active modes
+        self._active_mode_names = None
+        self._active_modes = None
+
+        # apply overrides
+        log.debug("Applying mode selection: default_modes=%s, base_modes=%s", mode_selection.default_modes, mode_selection.base_modes)
+        if mode_selection.base_modes is not None:
+            self._base_modes = mode_selection.base_modes
+        if mode_selection.default_modes is not None:
+            self._default_modes = mode_selection.default_modes
+        log.debug("Current mode selection: base_modes=%s, default_modes=%s", self._base_modes, self._default_modes)
+
+    def get_mode_names(self) -> Sequence[str]:
+        if self._active_mode_names is not None:
+            return self._active_mode_names
+        self._active_mode_names = []
+        if self._base_modes is not None:
+            self._active_mode_names.extend(self._base_modes)
+        if self._default_modes is not None:
+            self._active_mode_names.extend(self._default_modes)
+        log.info("Active modes: %s", self._active_mode_names)
+        return self._active_mode_names
+
+    def get_modes(self) -> Sequence[SerenaAgentMode]:
+        if self._active_modes is not None:
+            return self._active_modes
+        self._active_modes = []
+        for mode_name in self.get_mode_names():
+            mode = SerenaAgentMode.load(mode_name)
+            self._active_modes.append(mode)
+        return self._active_modes
+
+
 class SerenaAgent:
     def __init__(
         self,
@@ -174,7 +215,7 @@ class SerenaAgent:
         project_activation_callback: Callable[[], None] | None = None,
         serena_config: SerenaConfig | None = None,
         context: SerenaAgentContext | None = None,
-        modes: list[SerenaAgentMode] | None = None,
+        modes: ModeSelectionDefinition | None = None,
         memory_log_handler: MemoryLogHandler | None = None,
     ):
         """
@@ -290,22 +331,27 @@ class SerenaAgent:
         self.prompt_factory = SerenaPromptFactory()
         self._project_activation_callback = project_activation_callback
 
-        # set the active modes
-        if modes is None:
-            modes = SerenaAgentMode.load_default_modes()
-        self._modes = modes
-
-        # determine the subset of active tools (depending on active modes and the active project)
-        self._active_tools: AvailableTools = self._exposed_tools
-        self._update_active_tools()
+        # initialize active modes based on the global configuration
+        self._active_modes = ActiveModes()
+        self._active_modes.apply(self.serena_config)
 
         # activate a project configuration (if provided or if there is only a single project available)
         if project is not None:
             try:
-                self.activate_project_from_path_or_name(project)
+                # Note: This will apply the project's mode selections (if any).
+                # We do not yet update the tool set, as we need to apply mode overrides first (see below).
+                self.activate_project_from_path_or_name(project, update_tools=False)
             except Exception as e:
                 log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e)
 
+        # apply modes from the provided mode selection overrides (precedence over project configuration)
+        if modes is not None:
+            self._active_modes.apply(modes)
+
+        # determine the subset of active tools (depending on active modes and the active project)
+        self._active_tools: AvailableTools = self._exposed_tools
+        self._update_active_tools()
+
         # start the dashboard (web frontend), registering its log handler
         # should be the last thing to happen in the initialization since the dashboard
         # may access various parts of the agent
@@ -476,22 +522,22 @@ class SerenaAgent:
             raise ValueError("No active project. Please activate a project first.")
         return project
 
-    def set_modes(self, modes: list[SerenaAgentMode]) -> None:
+    def set_modes(self, mode_names: list[str]) -> None:
         """
         Set the current mode configurations.
 
-        :param modes: List of mode names or paths to use
+        :param mode_names: List of mode names or paths to use
         """
-        self._modes = modes
+        self._active_modes.apply(ModeSelectionDefinition(default_modes=mode_names))
         self._update_active_tools()
 
-        log.info(f"Set modes to {[mode.name for mode in modes]}")
+        log.info(f"Set modes to {[mode.name for mode in self.get_active_modes()]}")
 
     def get_active_modes(self) -> list[SerenaAgentMode]:
         """
         :return: the list of active modes
         """
-        return list(self._modes)
+        return list(self._active_modes.get_modes())
 
     def _format_prompt(self, prompt_template: str) -> str:
         template = JinjaTemplate(prompt_template)
@@ -503,7 +549,7 @@ class SerenaAgent:
         log.info("Generating system prompt with available_tools=(see active tools), available_markers=%s", available_markers)
         system_prompt = self.prompt_factory.create_system_prompt(
             context_system_prompt=self._format_prompt(self._context.prompt),
-            mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self._modes],
+            mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self.get_active_modes()],
             available_tools=available_tools.tool_names,
             available_markers=available_markers,
         )
@@ -522,7 +568,7 @@ class SerenaAgent:
         (as well as any internal modes that are not handled dynamically, such as JetBrains mode).
         """
         # apply modes
-        tool_set = self._base_tool_set.apply(*self._modes)
+        tool_set = self._base_tool_set.apply(*self._active_modes.get_modes())
 
         # apply active project configuration (if any)
         if self._active_project is not None:
@@ -569,10 +615,15 @@ class SerenaAgent:
         """
         return self.serena_config.language_backend == LanguageBackend.LSP
 
-    def _activate_project(self, project: Project) -> None:
+    def _activate_project(self, project: Project, update_tools: bool = True) -> None:
         log.info(f"Activating {project.project_name} at {project.project_root}")
         self._active_project = project
-        self._update_active_tools()
+
+        # update the active modes based on the project configuration
+        self._active_modes.apply(project.project_config)
+
+        if update_tools:
+            self._update_active_tools()
 
         def init_language_server_manager() -> None:
             # start the language server
@@ -603,7 +654,7 @@ class SerenaAgent:
             log.info(f"Added new project {project_instance.project_name} for path {project_instance.project_root}")
         return project_instance
 
-    def activate_project_from_path_or_name(self, project_root_or_name: str) -> Project:
+    def activate_project_from_path_or_name(self, project_root_or_name: str, update_tools: bool = True) -> Project:
         """
         Activate a project from a path or a name.
         If the project was already registered, it will just be activated.
@@ -619,7 +670,7 @@ class SerenaAgent:
                 f"Project '{project_root_or_name}' not found: Not a valid project name or directory. "
                 f"Existing project names: {self.serena_config.project_names}"
             )
-        self._activate_project(project_instance)
+        self._activate_project(project_instance, update_tools=update_tools)
         return project_instance
 
     def get_active_tool_names(self) -> list[str]:

src/serena/cli.py

@@ -5,7 +5,7 @@ import os
 import shutil
 import subprocess
 import sys
-from collections.abc import Iterator
+from collections.abc import Iterator, Sequence
 from logging import Logger
 from pathlib import Path
 from typing import Any, Literal
@@ -18,10 +18,16 @@ from tqdm import tqdm
 
 from serena.agent import SerenaAgent
 from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
-from serena.config.serena_config import LanguageBackend, ProjectConfig, RegisteredProject, SerenaConfig, SerenaPaths
+from serena.config.serena_config import (
+    LanguageBackend,
+    ModeSelectionDefinition,
+    ProjectConfig,
+    RegisteredProject,
+    SerenaConfig,
+    SerenaPaths,
+)
 from serena.constants import (
     DEFAULT_CONTEXT,
-    DEFAULT_MODES,
     PROMPT_TEMPLATES_DIR_INTERNAL,
     SERENA_LOG_FORMAT,
     SERENAS_OWN_CONTEXT_YAMLS_DIR,
@@ -152,7 +158,7 @@ class TopLevelCommands(AutoRegisteringGroup):
         "modes",
         type=str,
         multiple=True,
-        default=DEFAULT_MODES,
+        default=(),
         show_default=True,
         help="Built-in mode names or paths to custom mode YAMLs.",
     )
@@ -221,7 +227,7 @@ class TopLevelCommands(AutoRegisteringGroup):
         project_file_arg: str | None,
         project_from_cwd: bool | None,
         context: str,
-        modes: tuple[str, ...],
+        modes: Sequence[str],
         language_backend: str | None,
         transport: Literal["stdio", "sse", "streamable-http"],
         host: str,
@@ -302,11 +308,13 @@ class TopLevelCommands(AutoRegisteringGroup):
         "modes",
         type=str,
         multiple=True,
-        default=DEFAULT_MODES,
+        default=(),
         show_default=True,
         help="Built-in mode names or paths to custom mode YAMLs.",
     )
-    def print_system_prompt(project: str, log_level: str, only_instructions: bool, context: str, modes: tuple[str, ...]) -> None:
+    def print_system_prompt(
+        project: str, log_level: str, only_instructions: bool, context: str, modes: Sequence[str] | None = None
+    ) -> None:
         prefix = "You will receive access to Serena's symbolic tools. Below are instructions for using them, take them into account."
         postfix = "You begin by acknowledging that you understood the above instructions and are ready to receive tasks."
         from serena.tools.workflow_tools import InitialInstructionsTool
@@ -314,12 +322,14 @@ class TopLevelCommands(AutoRegisteringGroup):
         lvl = logging.getLevelNamesMapping()[log_level.upper()]
         logging.configure(level=lvl)
         context_instance = SerenaAgentContext.load(context)
-        mode_instances = [SerenaAgentMode.load(mode) for mode in modes]
+        modes_selection_def: ModeSelectionDefinition | None = None
+        if modes:
+            modes_selection_def = ModeSelectionDefinition(default_modes=modes)
         agent = SerenaAgent(
             project=os.path.abspath(project),
             serena_config=SerenaConfig(web_dashboard=False, log_level=lvl),
             context=context_instance,
-            modes=mode_instances,
+            modes=modes_selection_def,
         )
         tool = agent.get_tool(InitialInstructionsTool)
         instr = tool.apply()

src/serena/config/context_mode.py

@@ -14,7 +14,6 @@ from sensai.util.string import ToStringMixin
 from serena.config.serena_config import SerenaPaths, ToolInclusionDefinition
 from serena.constants import (
     DEFAULT_CONTEXT,
-    DEFAULT_MODES,
     INTERNAL_MODE_YAMLS_DIR,
     SERENA_FILE_ENCODING,
     SERENAS_OWN_CONTEXT_YAMLS_DIR,
@@ -117,11 +116,6 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin):
         """Names of all custom modes defined by the user."""
         return [f.stem for f in Path(SerenaPaths().user_modes_dir).glob("*.yml")]
 
-    @classmethod
-    def load_default_modes(cls) -> list[Self]:
-        """Load the default modes (interactive and editing)."""
-        return [cls.from_name(mode) for mode in DEFAULT_MODES]
-
     @classmethod
     def load(cls, name_or_path: str | Path) -> Self:
         # Check if it's a file path that exists

src/serena/config/serena_config.py

@@ -127,6 +127,12 @@ class ToolInclusionDefinition:
         return num_fixed > 0
 
 
+@dataclass
+class ModeSelectionDefinition:
+    base_modes: Sequence[str] | None = None
+    default_modes: Sequence[str] | None = None
+
+
 class SerenaConfigError(Exception):
     pass
 
@@ -156,7 +162,7 @@ class LanguageBackend(Enum):
 
 
 @dataclass(kw_only=True)
-class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
+class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMixin):
     project_name: str
     languages: list[Language]
     ignored_paths: list[str] = field(default_factory=list)
@@ -166,6 +172,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
     encoding: str = DEFAULT_SOURCE_FILE_ENCODING
 
     SERENA_DEFAULT_PROJECT_FILE = "project.yml"
+    FIELDS_WITHOUT_DEFAULTS = {"project_name", "languages"}
 
     def _tostring_includes(self) -> list[str]:
         return ["project_name"]
@@ -258,20 +265,18 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
 
     @classmethod
     def _apply_defaults_to_dict(cls, data: TDict) -> TDict:
-        # apply defaults for new fields
-        data["languages"] = data.get("languages", [])
-        data["ignored_paths"] = data.get("ignored_paths", [])
-        data["excluded_tools"] = data.get("excluded_tools", [])
-        data["included_optional_tools"] = data.get("included_optional_tools", [])
-        data["read_only"] = data.get("read_only", False)
-        data["ignore_all_files_in_gitignore"] = data.get("ignore_all_files_in_gitignore", True)
-        data["initial_prompt"] = data.get("initial_prompt", "")
-        data["encoding"] = data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING)
+        # apply defaults
+        for field_info in dataclasses.fields(cls):
+            key = field_info.name
+            if key in cls.FIELDS_WITHOUT_DEFAULTS:
+                continue
+            if key not in data:
+                default_value = get_dataclass_default(cls, key)
+                data.setdefault(key, default_value)
 
         # backward compatibility: handle single "language" field
-        if len(data["languages"]) == 0 and "language" in data:
+        if "languages" not in data and "language" in data:
             data["languages"] = [data["language"]]
-        if "language" in data:
             del data["language"]
 
         return data
@@ -314,11 +319,14 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
             languages=languages,
             ignored_paths=data["ignored_paths"],
             excluded_tools=data["excluded_tools"],
+            fixed_tools=data["fixed_tools"],
             included_optional_tools=data["included_optional_tools"],
             read_only=data["read_only"],
             ignore_all_files_in_gitignore=data["ignore_all_files_in_gitignore"],
             initial_prompt=data["initial_prompt"],
             encoding=data["encoding"],
+            base_modes=data["base_modes"],
+            default_modes=data["default_modes"],
         )
 
     def to_yaml_dict(self) -> dict:
@@ -404,7 +412,7 @@ class RegisteredProject(ToStringMixin):
 
 
 @dataclass(kw_only=True)
-class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
+class SerenaConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMixin):
     """
     Holds the Serena agent configuration, which is typically loaded from a YAML configuration file
     (when instantiated via :method:`from_config_file`), which is updated when projects are added or removed.
@@ -444,6 +452,9 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
     ls_specific_settings: dict = field(default_factory=dict)
     """Advanced configuration option allowing to configure language server implementation specific options, see SolidLSPSettings for more info."""
 
+    # settings with overridden defaults
+    default_modes: Sequence[str] | None = ("interactive", "editing")
+
     # *** fields that are NOT mapped to/from the configuration file ***
 
     _loaded_commented_yaml: CommentedMap | None = None

src/serena/constants.py

@@ -20,7 +20,6 @@ SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons")
 DEFAULT_SOURCE_FILE_ENCODING = "utf-8"
 """The default encoding assumed for project source files."""
 DEFAULT_CONTEXT = "desktop-app"
-DEFAULT_MODES = ("interactive", "editing")
 
 SERENA_FILE_ENCODING = "utf-8"
 """The encoding used for Serena's own files, such as configuration files and memories."""

src/serena/mcp.py

@@ -21,9 +21,9 @@ from serena.agent import (
     SerenaAgent,
     SerenaConfig,
 )
-from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
-from serena.config.serena_config import LanguageBackend
-from serena.constants import DEFAULT_CONTEXT, DEFAULT_MODES, SERENA_LOG_FORMAT
+from serena.config.context_mode import SerenaAgentContext
+from serena.config.serena_config import LanguageBackend, ModeSelectionDefinition
+from serena.constants import DEFAULT_CONTEXT, SERENA_LOG_FORMAT
 from serena.tools import Tool
 from serena.util.exception import show_fatal_exception_safe
 from serena.util.logging import MemoryLogHandler
@@ -260,7 +260,7 @@ class SerenaMCPFactory:
                 mcp._tool_manager._tools[tool.get_name()] = mcp_tool
             log.info(f"Starting MCP server with {len(mcp._tool_manager._tools)} tools: {list(mcp._tool_manager._tools.keys())}")
 
-    def _create_serena_agent(self, serena_config: SerenaConfig, modes: list[SerenaAgentMode]) -> SerenaAgent:
+    def _create_serena_agent(self, serena_config: SerenaConfig, modes: ModeSelectionDefinition | None = None) -> SerenaAgent:
         return SerenaAgent(
             project=self.project, serena_config=serena_config, context=self.context, modes=modes, memory_log_handler=self.memory_log_handler
         )
@@ -272,7 +272,7 @@ class SerenaMCPFactory:
         self,
         host: str = "0.0.0.0",
         port: int = 8000,
-        modes: Sequence[str] = DEFAULT_MODES,
+        modes: Sequence[str] = (),
         language_backend: LanguageBackend | None = None,
         enable_web_dashboard: bool | None = None,
         enable_gui_log_window: bool | None = None,
@@ -318,8 +318,10 @@ class SerenaMCPFactory:
             if language_backend is not None:
                 config.language_backend = language_backend
 
-            modes_instances = [SerenaAgentMode.load(mode) for mode in modes]
-            self.agent = self._create_serena_agent(config, modes_instances)
+            mode_selection_def: ModeSelectionDefinition | None = None
+            if modes:
+                mode_selection_def = ModeSelectionDefinition(default_modes=modes)
+            self.agent = self._create_serena_agent(config, mode_selection_def)
 
         except Exception as e:
             show_fatal_exception_safe(e)

src/serena/resources/project.template.yml

@@ -80,6 +80,21 @@ read_only: false
 #  * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
 excluded_tools: []
 
+# list of mode names to that are always to be included in the set of active modes
+# The full set of modes to be activated is base_modes + default_modes.
+# If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this setting overrides the global configuration.
+# Set this to [] to disable base modes for this project.
+# Set this to a list of mode names to always include the respective modes for this project.
+base_modes:
+
+# list of mode names that are to be activated by default.
+# The full set of modes to be activated is base_modes + default_modes.
+# If the setting is undefined, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+default_modes:
+
 # initial prompt for the project. It will always be given to the LLM upon activating the project
 # (contrary to the memories, which are loaded on demand).
 initial_prompt: ""

src/serena/resources/serena_config.template.yml

@@ -77,6 +77,19 @@ fixed_tools: []
 # fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
 # This cannot be combined with non-empty excluded_tools or included_optional_tools.
 
+base_modes:
+# list of mode names to that are always to be included in the set of active modes
+# The full set of modes to be activated is base_modes + default_modes.
+# If this is undefined, no base modes are included.
+# The project configuration (project.yml) may override this setting.
+
+default_modes:
+  - interactive
+  - editing
+# list of mode names that are to be activated by default.
+# The full set of modes to be activated is base_modes + default_modes.
+# These modes can be overridden by the project configuration (project.yml) or through the CLI (--mode).
+
 default_max_tool_answer_chars: 150000
 # Used as default for tools where the apply method has a default maximal answer length.
 # Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default

src/serena/tools/config_tools.py

@@ -1,4 +1,3 @@
-from serena.config.context_mode import SerenaAgentMode
 from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional
 
 
@@ -61,11 +60,11 @@ class SwitchModesTool(Tool, ToolMarkerOptional):
 
         :param modes: the names of the modes to activate
         """
-        mode_instances = [SerenaAgentMode.load(mode) for mode in modes]
-        self.agent.set_modes(mode_instances)
+        self.agent.set_modes(modes)
 
         # Inform the Agent about the activated modes and the currently active tools
-        result_str = f"Successfully activated modes: {', '.join([mode.name for mode in mode_instances])}" + "\n"
+        mode_instances = self.agent.get_active_modes()
+        result_str = f"Active modes: {', '.join([mode.name for mode in mode_instances])}" + "\n"
         result_str += "\n".join([mode_instance.prompt for mode_instance in mode_instances]) + "\n"
         result_str += f"Currently active tools: {', '.join(self.agent.get_active_tool_names())}"
         return result_str

test/serena/test_tool_parameter_types.py

@@ -18,7 +18,7 @@ def test_all_tool_parameters_have_type(context):
     cfg.included_optional_tools = tuple(registry.get_tool_names_optional())
     factory = SerenaMCPFactory(context=context)
     # Initialize the agent so that the tools are available
-    factory.agent = factory._create_serena_agent(cfg, [])
+    factory.agent = factory._create_serena_agent(cfg)
     tools = list(factory._iter_tools())
 
     for tool in tools: