From 5b103ee281afb9c20800df8bded86e8636c86b05 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 26 Jan 2026 17:03:12 +0100 Subject: [PATCH 01/19] Improve configurability of modes Modes can now be configured in three places (in reverse order of precedence) * global config * project config * command line Furthermore, we differentiate base modes from default modes, where command line parameters override only default modes, adding more flexibility Resolves #926 --- docs/02-usage/040_workflow.md | 1 + docs/02-usage/050_configuration.md | 1 + src/serena/agent.py | 97 ++++++++++++++----- src/serena/cli.py | 28 ++++-- src/serena/config/context_mode.py | 6 -- src/serena/config/serena_config.py | 37 ++++--- src/serena/constants.py | 1 - src/serena/mcp.py | 16 +-- src/serena/resources/project.template.yml | 15 +++ .../resources/serena_config.template.yml | 13 +++ src/serena/tools/config_tools.py | 7 +- test/serena/test_tool_parameter_types.py | 2 +- 12 files changed, 160 insertions(+), 64 deletions(-) diff --git a/docs/02-usage/040_workflow.md b/docs/02-usage/040_workflow.md index 91d27d3d..8f90fe68 100644 --- a/docs/02-usage/040_workflow.md +++ b/docs/02-usage/040_workflow.md @@ -53,6 +53,7 @@ The file allows you to configure ... * write access * an initial prompt that shall be passed to the LLM whenever the project is activated * the name by which you want to refer to the project (relevant when telling the LLM to dynamically activate the project) + * the set of tools and modes to use by default For detailed information on the parameters and possible settings, see the [template file](https://github.com/oraios/serena/blob/main/src/serena/resources/project.template.yml). diff --git a/docs/02-usage/050_configuration.md b/docs/02-usage/050_configuration.md index 716d541a..c86a7e72 100644 --- a/docs/02-usage/050_configuration.md +++ b/docs/02-usage/050_configuration.md @@ -25,6 +25,7 @@ Some of the configurable settings include: * the language backend to use by default (i.e., the JetBrains plugin or language servers) * UI settings affecting the [Serena Dashboard and GUI tool](060_dashboard.md) * the set of tools to enable/disable by default + * the set of modes to use by default * tool execution parameters (timeout, max. answer length) * global ignore rules * logging settings diff --git a/src/serena/agent.py b/src/serena/agent.py index dbeb27e8..a5156e1b 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -6,7 +6,7 @@ import os import platform import subprocess import sys -from collections.abc import Callable +from collections.abc import Callable, Sequence from logging import Logger from typing import TYPE_CHECKING, Optional, TypeVar @@ -17,7 +17,7 @@ from interprompt.jinja_template import JinjaTemplate from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import LanguageBackend, SerenaConfig, ToolInclusionDefinition +from serena.config.serena_config import LanguageBackend, ModeSelectionDefinition, SerenaConfig, ToolInclusionDefinition from serena.dashboard import SerenaDashboardAPI from serena.ls_manager import LanguageServerManager from serena.project import Project @@ -167,6 +167,47 @@ class ToolSet: return tool_name in self._tool_names +class ActiveModes: + def __init__(self) -> None: + self._base_modes: Sequence[str] | None = None + self._default_modes: Sequence[str] | None = None + self._active_mode_names: Sequence[str] | None = [] + self._active_modes: Sequence[SerenaAgentMode] | None = [] + + def apply(self, mode_selection: ModeSelectionDefinition) -> None: + # invalidate active modes + self._active_mode_names = None + self._active_modes = None + + # apply overrides + log.debug("Applying mode selection: default_modes=%s, base_modes=%s", mode_selection.default_modes, mode_selection.base_modes) + if mode_selection.base_modes is not None: + self._base_modes = mode_selection.base_modes + if mode_selection.default_modes is not None: + self._default_modes = mode_selection.default_modes + log.debug("Current mode selection: base_modes=%s, default_modes=%s", self._base_modes, self._default_modes) + + def get_mode_names(self) -> Sequence[str]: + if self._active_mode_names is not None: + return self._active_mode_names + self._active_mode_names = [] + if self._base_modes is not None: + self._active_mode_names.extend(self._base_modes) + if self._default_modes is not None: + self._active_mode_names.extend(self._default_modes) + log.info("Active modes: %s", self._active_mode_names) + return self._active_mode_names + + def get_modes(self) -> Sequence[SerenaAgentMode]: + if self._active_modes is not None: + return self._active_modes + self._active_modes = [] + for mode_name in self.get_mode_names(): + mode = SerenaAgentMode.load(mode_name) + self._active_modes.append(mode) + return self._active_modes + + class SerenaAgent: def __init__( self, @@ -174,7 +215,7 @@ class SerenaAgent: project_activation_callback: Callable[[], None] | None = None, serena_config: SerenaConfig | None = None, context: SerenaAgentContext | None = None, - modes: list[SerenaAgentMode] | None = None, + modes: ModeSelectionDefinition | None = None, memory_log_handler: MemoryLogHandler | None = None, ): """ @@ -290,22 +331,27 @@ class SerenaAgent: self.prompt_factory = SerenaPromptFactory() self._project_activation_callback = project_activation_callback - # set the active modes - if modes is None: - modes = SerenaAgentMode.load_default_modes() - self._modes = modes - - # determine the subset of active tools (depending on active modes and the active project) - self._active_tools: AvailableTools = self._exposed_tools - self._update_active_tools() + # initialize active modes based on the global configuration + self._active_modes = ActiveModes() + self._active_modes.apply(self.serena_config) # activate a project configuration (if provided or if there is only a single project available) if project is not None: try: - self.activate_project_from_path_or_name(project) + # Note: This will apply the project's mode selections (if any). + # We do not yet update the tool set, as we need to apply mode overrides first (see below). + self.activate_project_from_path_or_name(project, update_tools=False) except Exception as e: log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e) + # apply modes from the provided mode selection overrides (precedence over project configuration) + if modes is not None: + self._active_modes.apply(modes) + + # determine the subset of active tools (depending on active modes and the active project) + self._active_tools: AvailableTools = self._exposed_tools + self._update_active_tools() + # start the dashboard (web frontend), registering its log handler # should be the last thing to happen in the initialization since the dashboard # may access various parts of the agent @@ -476,22 +522,22 @@ class SerenaAgent: raise ValueError("No active project. Please activate a project first.") return project - def set_modes(self, modes: list[SerenaAgentMode]) -> None: + def set_modes(self, mode_names: list[str]) -> None: """ Set the current mode configurations. - :param modes: List of mode names or paths to use + :param mode_names: List of mode names or paths to use """ - self._modes = modes + self._active_modes.apply(ModeSelectionDefinition(default_modes=mode_names)) self._update_active_tools() - log.info(f"Set modes to {[mode.name for mode in modes]}") + log.info(f"Set modes to {[mode.name for mode in self.get_active_modes()]}") def get_active_modes(self) -> list[SerenaAgentMode]: """ :return: the list of active modes """ - return list(self._modes) + return list(self._active_modes.get_modes()) def _format_prompt(self, prompt_template: str) -> str: template = JinjaTemplate(prompt_template) @@ -503,7 +549,7 @@ class SerenaAgent: log.info("Generating system prompt with available_tools=(see active tools), available_markers=%s", available_markers) system_prompt = self.prompt_factory.create_system_prompt( context_system_prompt=self._format_prompt(self._context.prompt), - mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self._modes], + mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self.get_active_modes()], available_tools=available_tools.tool_names, available_markers=available_markers, ) @@ -522,7 +568,7 @@ class SerenaAgent: (as well as any internal modes that are not handled dynamically, such as JetBrains mode). """ # apply modes - tool_set = self._base_tool_set.apply(*self._modes) + tool_set = self._base_tool_set.apply(*self._active_modes.get_modes()) # apply active project configuration (if any) if self._active_project is not None: @@ -569,10 +615,15 @@ class SerenaAgent: """ return self.serena_config.language_backend == LanguageBackend.LSP - def _activate_project(self, project: Project) -> None: + def _activate_project(self, project: Project, update_tools: bool = True) -> None: log.info(f"Activating {project.project_name} at {project.project_root}") self._active_project = project - self._update_active_tools() + + # update the active modes based on the project configuration + self._active_modes.apply(project.project_config) + + if update_tools: + self._update_active_tools() def init_language_server_manager() -> None: # start the language server @@ -603,7 +654,7 @@ class SerenaAgent: log.info(f"Added new project {project_instance.project_name} for path {project_instance.project_root}") return project_instance - def activate_project_from_path_or_name(self, project_root_or_name: str) -> Project: + def activate_project_from_path_or_name(self, project_root_or_name: str, update_tools: bool = True) -> Project: """ Activate a project from a path or a name. If the project was already registered, it will just be activated. @@ -619,7 +670,7 @@ class SerenaAgent: f"Project '{project_root_or_name}' not found: Not a valid project name or directory. " f"Existing project names: {self.serena_config.project_names}" ) - self._activate_project(project_instance) + self._activate_project(project_instance, update_tools=update_tools) return project_instance def get_active_tool_names(self) -> list[str]: diff --git a/src/serena/cli.py b/src/serena/cli.py index fba9ccdc..f5dc51a4 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -5,7 +5,7 @@ import os import shutil import subprocess import sys -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from logging import Logger from pathlib import Path from typing import Any, Literal @@ -18,10 +18,16 @@ from tqdm import tqdm from serena.agent import SerenaAgent from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import LanguageBackend, ProjectConfig, RegisteredProject, SerenaConfig, SerenaPaths +from serena.config.serena_config import ( + LanguageBackend, + ModeSelectionDefinition, + ProjectConfig, + RegisteredProject, + SerenaConfig, + SerenaPaths, +) from serena.constants import ( DEFAULT_CONTEXT, - DEFAULT_MODES, PROMPT_TEMPLATES_DIR_INTERNAL, SERENA_LOG_FORMAT, SERENAS_OWN_CONTEXT_YAMLS_DIR, @@ -152,7 +158,7 @@ class TopLevelCommands(AutoRegisteringGroup): "modes", type=str, multiple=True, - default=DEFAULT_MODES, + default=(), show_default=True, help="Built-in mode names or paths to custom mode YAMLs.", ) @@ -221,7 +227,7 @@ class TopLevelCommands(AutoRegisteringGroup): project_file_arg: str | None, project_from_cwd: bool | None, context: str, - modes: tuple[str, ...], + modes: Sequence[str], language_backend: str | None, transport: Literal["stdio", "sse", "streamable-http"], host: str, @@ -302,11 +308,13 @@ class TopLevelCommands(AutoRegisteringGroup): "modes", type=str, multiple=True, - default=DEFAULT_MODES, + default=(), show_default=True, help="Built-in mode names or paths to custom mode YAMLs.", ) - def print_system_prompt(project: str, log_level: str, only_instructions: bool, context: str, modes: tuple[str, ...]) -> None: + def print_system_prompt( + project: str, log_level: str, only_instructions: bool, context: str, modes: Sequence[str] | None = None + ) -> None: prefix = "You will receive access to Serena's symbolic tools. Below are instructions for using them, take them into account." postfix = "You begin by acknowledging that you understood the above instructions and are ready to receive tasks." from serena.tools.workflow_tools import InitialInstructionsTool @@ -314,12 +322,14 @@ class TopLevelCommands(AutoRegisteringGroup): lvl = logging.getLevelNamesMapping()[log_level.upper()] logging.configure(level=lvl) context_instance = SerenaAgentContext.load(context) - mode_instances = [SerenaAgentMode.load(mode) for mode in modes] + modes_selection_def: ModeSelectionDefinition | None = None + if modes: + modes_selection_def = ModeSelectionDefinition(default_modes=modes) agent = SerenaAgent( project=os.path.abspath(project), serena_config=SerenaConfig(web_dashboard=False, log_level=lvl), context=context_instance, - modes=mode_instances, + modes=modes_selection_def, ) tool = agent.get_tool(InitialInstructionsTool) instr = tool.apply() diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index 9238d806..e417fe32 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -14,7 +14,6 @@ from sensai.util.string import ToStringMixin from serena.config.serena_config import SerenaPaths, ToolInclusionDefinition from serena.constants import ( DEFAULT_CONTEXT, - DEFAULT_MODES, INTERNAL_MODE_YAMLS_DIR, SERENA_FILE_ENCODING, SERENAS_OWN_CONTEXT_YAMLS_DIR, @@ -117,11 +116,6 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): """Names of all custom modes defined by the user.""" return [f.stem for f in Path(SerenaPaths().user_modes_dir).glob("*.yml")] - @classmethod - def load_default_modes(cls) -> list[Self]: - """Load the default modes (interactive and editing).""" - return [cls.from_name(mode) for mode in DEFAULT_MODES] - @classmethod def load(cls, name_or_path: str | Path) -> Self: # Check if it's a file path that exists diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index bde964b9..43fc3f0a 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -127,6 +127,12 @@ class ToolInclusionDefinition: return num_fixed > 0 +@dataclass +class ModeSelectionDefinition: + base_modes: Sequence[str] | None = None + default_modes: Sequence[str] | None = None + + class SerenaConfigError(Exception): pass @@ -156,7 +162,7 @@ class LanguageBackend(Enum): @dataclass(kw_only=True) -class ProjectConfig(ToolInclusionDefinition, ToStringMixin): +class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMixin): project_name: str languages: list[Language] ignored_paths: list[str] = field(default_factory=list) @@ -166,6 +172,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): encoding: str = DEFAULT_SOURCE_FILE_ENCODING SERENA_DEFAULT_PROJECT_FILE = "project.yml" + FIELDS_WITHOUT_DEFAULTS = {"project_name", "languages"} def _tostring_includes(self) -> list[str]: return ["project_name"] @@ -258,20 +265,18 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): @classmethod def _apply_defaults_to_dict(cls, data: TDict) -> TDict: - # apply defaults for new fields - data["languages"] = data.get("languages", []) - data["ignored_paths"] = data.get("ignored_paths", []) - data["excluded_tools"] = data.get("excluded_tools", []) - data["included_optional_tools"] = data.get("included_optional_tools", []) - data["read_only"] = data.get("read_only", False) - data["ignore_all_files_in_gitignore"] = data.get("ignore_all_files_in_gitignore", True) - data["initial_prompt"] = data.get("initial_prompt", "") - data["encoding"] = data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING) + # apply defaults + for field_info in dataclasses.fields(cls): + key = field_info.name + if key in cls.FIELDS_WITHOUT_DEFAULTS: + continue + if key not in data: + default_value = get_dataclass_default(cls, key) + data.setdefault(key, default_value) # backward compatibility: handle single "language" field - if len(data["languages"]) == 0 and "language" in data: + if "languages" not in data and "language" in data: data["languages"] = [data["language"]] - if "language" in data: del data["language"] return data @@ -314,11 +319,14 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): languages=languages, ignored_paths=data["ignored_paths"], excluded_tools=data["excluded_tools"], + fixed_tools=data["fixed_tools"], included_optional_tools=data["included_optional_tools"], read_only=data["read_only"], ignore_all_files_in_gitignore=data["ignore_all_files_in_gitignore"], initial_prompt=data["initial_prompt"], encoding=data["encoding"], + base_modes=data["base_modes"], + default_modes=data["default_modes"], ) def to_yaml_dict(self) -> dict: @@ -404,7 +412,7 @@ class RegisteredProject(ToStringMixin): @dataclass(kw_only=True) -class SerenaConfig(ToolInclusionDefinition, ToStringMixin): +class SerenaConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMixin): """ Holds the Serena agent configuration, which is typically loaded from a YAML configuration file (when instantiated via :method:`from_config_file`), which is updated when projects are added or removed. @@ -444,6 +452,9 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): ls_specific_settings: dict = field(default_factory=dict) """Advanced configuration option allowing to configure language server implementation specific options, see SolidLSPSettings for more info.""" + # settings with overridden defaults + default_modes: Sequence[str] | None = ("interactive", "editing") + # *** fields that are NOT mapped to/from the configuration file *** _loaded_commented_yaml: CommentedMap | None = None diff --git a/src/serena/constants.py b/src/serena/constants.py index f5ccd660..3a9e2bf5 100644 --- a/src/serena/constants.py +++ b/src/serena/constants.py @@ -20,7 +20,6 @@ SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons") DEFAULT_SOURCE_FILE_ENCODING = "utf-8" """The default encoding assumed for project source files.""" DEFAULT_CONTEXT = "desktop-app" -DEFAULT_MODES = ("interactive", "editing") SERENA_FILE_ENCODING = "utf-8" """The encoding used for Serena's own files, such as configuration files and memories.""" diff --git a/src/serena/mcp.py b/src/serena/mcp.py index b0367717..d7ff379b 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -21,9 +21,9 @@ from serena.agent import ( SerenaAgent, SerenaConfig, ) -from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import LanguageBackend -from serena.constants import DEFAULT_CONTEXT, DEFAULT_MODES, SERENA_LOG_FORMAT +from serena.config.context_mode import SerenaAgentContext +from serena.config.serena_config import LanguageBackend, ModeSelectionDefinition +from serena.constants import DEFAULT_CONTEXT, SERENA_LOG_FORMAT from serena.tools import Tool from serena.util.exception import show_fatal_exception_safe from serena.util.logging import MemoryLogHandler @@ -260,7 +260,7 @@ class SerenaMCPFactory: mcp._tool_manager._tools[tool.get_name()] = mcp_tool log.info(f"Starting MCP server with {len(mcp._tool_manager._tools)} tools: {list(mcp._tool_manager._tools.keys())}") - def _create_serena_agent(self, serena_config: SerenaConfig, modes: list[SerenaAgentMode]) -> SerenaAgent: + def _create_serena_agent(self, serena_config: SerenaConfig, modes: ModeSelectionDefinition | None = None) -> SerenaAgent: return SerenaAgent( project=self.project, serena_config=serena_config, context=self.context, modes=modes, memory_log_handler=self.memory_log_handler ) @@ -272,7 +272,7 @@ class SerenaMCPFactory: self, host: str = "0.0.0.0", port: int = 8000, - modes: Sequence[str] = DEFAULT_MODES, + modes: Sequence[str] = (), language_backend: LanguageBackend | None = None, enable_web_dashboard: bool | None = None, enable_gui_log_window: bool | None = None, @@ -318,8 +318,10 @@ class SerenaMCPFactory: if language_backend is not None: config.language_backend = language_backend - modes_instances = [SerenaAgentMode.load(mode) for mode in modes] - self.agent = self._create_serena_agent(config, modes_instances) + mode_selection_def: ModeSelectionDefinition | None = None + if modes: + mode_selection_def = ModeSelectionDefinition(default_modes=modes) + self.agent = self._create_serena_agent(config, mode_selection_def) except Exception as e: show_fatal_exception_safe(e) diff --git a/src/serena/resources/project.template.yml b/src/serena/resources/project.template.yml index a0f79106..a05555d7 100644 --- a/src/serena/resources/project.template.yml +++ b/src/serena/resources/project.template.yml @@ -80,6 +80,21 @@ read_only: false # * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. excluded_tools: [] +# list of mode names to that are always to be included in the set of active modes +# The full set of modes to be activated is base_modes + default_modes. +# If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this setting overrides the global configuration. +# Set this to [] to disable base modes for this project. +# Set this to a list of mode names to always include the respective modes for this project. +base_modes: + +# list of mode names that are to be activated by default. +# The full set of modes to be activated is base_modes + default_modes. +# If the setting is undefined, the default_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this overrides the setting from the global configuration (serena_config.yml). +# This setting can, in turn, be overridden by CLI parameters (--mode). +default_modes: + # initial prompt for the project. It will always be given to the LLM upon activating the project # (contrary to the memories, which are loaded on demand). initial_prompt: "" diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index ddb41e35..f36f5895 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -77,6 +77,19 @@ fixed_tools: [] # fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. # This cannot be combined with non-empty excluded_tools or included_optional_tools. +base_modes: +# list of mode names to that are always to be included in the set of active modes +# The full set of modes to be activated is base_modes + default_modes. +# If this is undefined, no base modes are included. +# The project configuration (project.yml) may override this setting. + +default_modes: + - interactive + - editing +# list of mode names that are to be activated by default. +# The full set of modes to be activated is base_modes + default_modes. +# These modes can be overridden by the project configuration (project.yml) or through the CLI (--mode). + default_max_tool_answer_chars: 150000 # Used as default for tools where the apply method has a default maximal answer length. # Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default diff --git a/src/serena/tools/config_tools.py b/src/serena/tools/config_tools.py index 7de51e59..7a0472cf 100644 --- a/src/serena/tools/config_tools.py +++ b/src/serena/tools/config_tools.py @@ -1,4 +1,3 @@ -from serena.config.context_mode import SerenaAgentMode from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional @@ -61,11 +60,11 @@ class SwitchModesTool(Tool, ToolMarkerOptional): :param modes: the names of the modes to activate """ - mode_instances = [SerenaAgentMode.load(mode) for mode in modes] - self.agent.set_modes(mode_instances) + self.agent.set_modes(modes) # Inform the Agent about the activated modes and the currently active tools - result_str = f"Successfully activated modes: {', '.join([mode.name for mode in mode_instances])}" + "\n" + mode_instances = self.agent.get_active_modes() + result_str = f"Active modes: {', '.join([mode.name for mode in mode_instances])}" + "\n" result_str += "\n".join([mode_instance.prompt for mode_instance in mode_instances]) + "\n" result_str += f"Currently active tools: {', '.join(self.agent.get_active_tool_names())}" return result_str diff --git a/test/serena/test_tool_parameter_types.py b/test/serena/test_tool_parameter_types.py index 9899a5bb..83a0981c 100644 --- a/test/serena/test_tool_parameter_types.py +++ b/test/serena/test_tool_parameter_types.py @@ -18,7 +18,7 @@ def test_all_tool_parameters_have_type(context): cfg.included_optional_tools = tuple(registry.get_tool_names_optional()) factory = SerenaMCPFactory(context=context) # Initialize the agent so that the tools are available - factory.agent = factory._create_serena_agent(cfg, []) + factory.agent = factory._create_serena_agent(cfg) tools = list(factory._iter_tools()) for tool in tools: From 40a3075cc0fca0d005ad250a8d8da7413fbc85a1 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 26 Jan 2026 20:18:37 +0100 Subject: [PATCH 02/19] Ensure consistent mode configuration for project Enabled modes must not depend on project activation order. Always apply, in order: SerenaConfig, ProjectConfig, current CLI overrides --- src/serena/agent.py | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index a5156e1b..35ca3042 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -331,26 +331,20 @@ class SerenaAgent: self.prompt_factory = SerenaPromptFactory() self._project_activation_callback = project_activation_callback - # initialize active modes based on the global configuration + # initialize members for modes and tools + self._mode_overrides = modes self._active_modes = ActiveModes() - self._active_modes.apply(self.serena_config) + self._active_tools: AvailableTools = self._exposed_tools # activate a project configuration (if provided or if there is only a single project available) if project is not None: try: - # Note: This will apply the project's mode selections (if any). - # We do not yet update the tool set, as we need to apply mode overrides first (see below). - self.activate_project_from_path_or_name(project, update_tools=False) + self.activate_project_from_path_or_name(project, update_modes_and_tools=False) except Exception as e: log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e) - # apply modes from the provided mode selection overrides (precedence over project configuration) - if modes is not None: - self._active_modes.apply(modes) - - # determine the subset of active tools (depending on active modes and the active project) - self._active_tools: AvailableTools = self._exposed_tools - self._update_active_tools() + # update active modes and active tools (considering the active project, if any) + self._update_active_modes_and_tools() # start the dashboard (web frontend), registering its log handler # should be the last thing to happen in the initialization since the dashboard @@ -529,7 +523,7 @@ class SerenaAgent: :param mode_names: List of mode names or paths to use """ self._active_modes.apply(ModeSelectionDefinition(default_modes=mode_names)) - self._update_active_tools() + self._update_active_modes_and_tools() log.info(f"Set modes to {[mode.name for mode in self.get_active_modes()]}") @@ -561,12 +555,20 @@ class SerenaAgent: log.info("System prompt:\n%s", system_prompt) return system_prompt - def _update_active_tools(self) -> None: + def _update_active_modes_and_tools(self) -> None: """ - Update the active tools based on enabled modes and the active project. + Updates the active modes and, subsequently, the active tools based on the modes and the active project. The base tool set already takes the Serena configuration and the context into account (as well as any internal modes that are not handled dynamically, such as JetBrains mode). """ + # determine active modes from serena config, active project, and mode overrides + self._active_modes = ActiveModes() + self._active_modes.apply(self.serena_config) + if self._active_project: + self._active_modes.apply(self._active_project.project_config) + if self._mode_overrides: + self._active_modes.apply(self._mode_overrides) + # apply modes tool_set = self._base_tool_set.apply(*self._active_modes.get_modes()) @@ -615,15 +617,12 @@ class SerenaAgent: """ return self.serena_config.language_backend == LanguageBackend.LSP - def _activate_project(self, project: Project, update_tools: bool = True) -> None: + def _activate_project(self, project: Project, update_modes_and_tools: bool = True) -> None: log.info(f"Activating {project.project_name} at {project.project_root}") self._active_project = project - # update the active modes based on the project configuration - self._active_modes.apply(project.project_config) - - if update_tools: - self._update_active_tools() + if update_modes_and_tools: + self._update_active_modes_and_tools() def init_language_server_manager() -> None: # start the language server @@ -654,7 +653,7 @@ class SerenaAgent: log.info(f"Added new project {project_instance.project_name} for path {project_instance.project_root}") return project_instance - def activate_project_from_path_or_name(self, project_root_or_name: str, update_tools: bool = True) -> Project: + def activate_project_from_path_or_name(self, project_root_or_name: str, update_modes_and_tools: bool = True) -> Project: """ Activate a project from a path or a name. If the project was already registered, it will just be activated. @@ -670,7 +669,7 @@ class SerenaAgent: f"Project '{project_root_or_name}' not found: Not a valid project name or directory. " f"Existing project names: {self.serena_config.project_names}" ) - self._activate_project(project_instance, update_tools=update_tools) + self._activate_project(project_instance, update_modes_and_tools=update_modes_and_tools) return project_instance def get_active_tool_names(self) -> list[str]: From 7ad00d8e516f429d22bffbc4dd456de5d399fb28 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 26 Jan 2026 20:48:57 +0100 Subject: [PATCH 03/19] Ensure that project.yml files are complete, re-saving incomplete ones with defaults --- .serena/project.yml | 70 ++++++++++++++++++++--- src/serena/config/serena_config.py | 64 +++++++++++++++------ src/serena/project.py | 7 +-- src/serena/resources/project.template.yml | 7 +++ test/serena/config/test_serena_config.py | 7 +++ 5 files changed, 121 insertions(+), 34 deletions(-) diff --git a/.serena/project.yml b/.serena/project.yml index edba1eeb..d7c316a4 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -1,14 +1,45 @@ +# the name by which the project can be referenced within Serena +project_name: "serena" + + +# list of languages for which language servers are started; choose from: +# al bash clojure cpp csharp +# csharp_omnisharp dart elixir elm erlang +# fortran fsharp go groovy haskell +# java julia kotlin lua markdown +# matlab nix pascal perl php +# powershell python python_jedi r rego +# ruby ruby_solargraph rust scala swift +# terraform toml typescript typescript_vts vue +# yaml zig +# (This list may be outdated. For the current list, see values of Language enum here: +# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py +# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) +# Note: +# - For C, use cpp +# - For JavaScript, use typescript +# - For Free Pascal/Lazarus, use pascal +# Special requirements: +# - csharp: Requires the presence of a .sln file in the project folder. +# - pascal: Requires Free Pascal Compiler (fpc) and optionally Lazarus. +# When using multiple languages, the first language server that supports a given file will be used for that file. +# The first language is the default language and the respective language server will be used as a fallback. +# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. +languages: +- python +- typescript + + +# whether to use project's .gitignore files to ignore files ignore_all_files_in_gitignore: true -# list of additional paths to ignore + +# list of additional paths to ignore in all projects # same syntax as gitignore, so you can use * and ** -# Was previously called `ignored_dirs`, please update your config if you are using that. -# Added (renamed)on 2025-04-07 ignored_paths: [] # whether the project is in read-only mode # If set to true, all editing tools will be disabled and attempts to use them will result in an error -# Added on 2025-04-18 read_only: false @@ -53,13 +84,34 @@ read_only: false # * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. excluded_tools: [] +# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default) +included_optional_tools: [] + +# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. +# This cannot be combined with non-empty excluded_tools or included_optional_tools. # initial prompt for the project. It will always be given to the LLM upon activating the project # (contrary to the memories, which are loaded on demand). initial_prompt: "" -project_name: "serena" -languages: -- python -- typescript -included_optional_tools: [] + encoding: utf-8 + + +# list of mode names to that are always to be included in the set of active modes +# The full set of modes to be activated is base_modes + default_modes. +# If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this setting overrides the global configuration. +# Set this to [] to disable base modes for this project. +# Set this to a list of mode names to always include the respective modes for this project. +base_modes: + +# list of mode names that are to be activated by default. +# The full set of modes to be activated is base_modes + default_modes. +# If the setting is undefined, the default_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this overrides the setting from the global configuration (serena_config.yml). +# This setting can, in turn, be overridden by CLI parameters (--mode). +default_modes: + +# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. +# This cannot be combined with non-empty excluded_tools or included_optional_tools. +fixed_tools: [] diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 43fc3f0a..91c109e4 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -246,7 +246,7 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi log.info("Using languages: %s", languages_to_use) else: languages_to_use = [lang.value for lang in languages] - config_with_comments = cls.load_commented_map(PROJECT_TEMPLATE_FILE) + config_with_comments, _ = cls._load_yaml(PROJECT_TEMPLATE_FILE) config_with_comments["project_name"] = project_name config_with_comments["languages"] = languages_to_use if save_to_disk: @@ -264,13 +264,27 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi return os.path.join(SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE) @classmethod - def _apply_defaults_to_dict(cls, data: TDict) -> TDict: + def _load_yaml(cls, yml_path: str) -> tuple[CommentedMap, bool]: + """ + Load the project configuration as a CommentedMap, preserving comments and ensuring + completeness of the configuration by applying default values for missing fields + and backward compatibility adjustments. + + :param yml_path: the path to the project.yml file + :return: a tuple `(dict, was_complete)` where dict is a CommentedMap representing a + full project configuration and `was_complete` indicates whether the loaded configuration + was complete (i.e., did not require any default values to be applied) + """ + data = load_yaml(yml_path, preserve_comments=True) + # apply defaults + was_complete = True for field_info in dataclasses.fields(cls): key = field_info.name if key in cls.FIELDS_WITHOUT_DEFAULTS: continue if key not in data: + was_complete = False default_value = get_dataclass_default(cls, key) data.setdefault(key, default_value) @@ -279,20 +293,7 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi data["languages"] = [data["language"]] del data["language"] - return data - - @classmethod - def load_commented_map(cls, yml_path: str) -> CommentedMap: - """ - Load the project configuration as a CommentedMap, preserving comments and ensuring - completeness of the configuration by applying default values for missing fields - and backward compatibility adjustments. - - :param yml_path: the path to the project.yml file - :return: a CommentedMap representing a full project configuration - """ - data = load_yaml(yml_path, preserve_comments=True) - return cls._apply_defaults_to_dict(data) + return data, was_complete @classmethod def _from_dict(cls, data: dict[str, Any]) -> Self: @@ -329,7 +330,7 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi default_modes=data["default_modes"], ) - def to_yaml_dict(self) -> dict: + def _to_yaml_dict(self) -> dict: """ :return: a yaml-serializable dictionary representation of this configuration """ @@ -344,15 +345,40 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi """ project_root = Path(project_root) yaml_path = project_root / cls.rel_path_to_project_yml() + + # auto-generate if necessary if not yaml_path.exists(): if autogenerate: return cls.autogenerate(project_root) else: raise FileNotFoundError(f"Project configuration file not found: {yaml_path}") - yaml_data = cls.load_commented_map(str(yaml_path)) + + # load the configuration dictionary + yaml_data, was_complete = cls._load_yaml(str(yaml_path)) if "project_name" not in yaml_data: yaml_data["project_name"] = project_root.name - return cls._from_dict(yaml_data) + + # instantiate the ProjectConfig + project_config = cls._from_dict(yaml_data) + + # if the configuration was incomplete, re-save it to disk + if not was_complete: + log.info("Project configuration in %s was incomplete, re-saving with default values for missing fields", yaml_path) + project_config.save(project_root) + + return project_config + + def save(self, project_root: Path | str) -> None: + """ + Saves the project configuration to disk. + + :param project_root: the root directory of the project + """ + config_path = self.path_to_project_yml(project_root) + log.info("Saving updated project configuration to %s", config_path) + config_with_comments, _ = self._load_yaml(config_path) + config_with_comments.update(self._to_yaml_dict()) + save_yaml(config_path, config_with_comments, preserve_comments=True) class RegisteredProject(ToStringMixin): diff --git a/src/serena/project.py b/src/serena/project.py index 90a2a33e..06f39954 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -12,7 +12,6 @@ from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME from serena.ls_manager import LanguageServerFactory, LanguageServerManager from serena.text_utils import MatchedConsecutiveLines, search_files from serena.util.file_system import GitignoreParser, match_path -from serena.util.general import save_yaml from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import FileUtils @@ -114,11 +113,7 @@ class Project(ToStringMixin): """ Saves the current project configuration to disk. """ - config_path = os.path.join(self.project_root, self.project_config.rel_path_to_project_yml()) - log.info("Saving updated project configuration to %s", config_path) - config_with_comments = ProjectConfig.load_commented_map(config_path) - config_with_comments.update(self.project_config.to_yaml_dict()) - save_yaml(config_path, config_with_comments, preserve_comments=True) + self.project_config.save(self.project_root) def path_to_serena_data_folder(self) -> str: return os.path.join(self.project_root, SERENA_MANAGED_DIR_NAME) diff --git a/src/serena/resources/project.template.yml b/src/serena/resources/project.template.yml index a05555d7..02ba1b26 100644 --- a/src/serena/resources/project.template.yml +++ b/src/serena/resources/project.template.yml @@ -80,6 +80,13 @@ read_only: false # * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. excluded_tools: [] +# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default) +included_optional_tools: [] + +# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. +# This cannot be combined with non-empty excluded_tools or included_optional_tools. +fixed_tools: [] + # list of mode names to that are always to be included in the set of active modes # The full set of modes to be activated is base_modes + default_modes. # If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply. diff --git a/test/serena/config/test_serena_config.py b/test/serena/config/test_serena_config.py index 048b3dbc..93abd5a6 100644 --- a/test/serena/config/test_serena_config.py +++ b/test/serena/config/test_serena_config.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest from serena.config.serena_config import ProjectConfig +from serena.constants import PROJECT_TEMPLATE_FILE from solidlsp.ls_config import Language @@ -117,3 +118,9 @@ class TestProjectConfigAutogenerate: assert config.project_name == custom_name assert config.languages == [Language.TYPESCRIPT] + + +class TestProjectConfig: + def test_template_is_complete(self): + _, is_complete = ProjectConfig._load_yaml(PROJECT_TEMPLATE_FILE) + assert is_complete, "Project template YAML is incomplete; all fields must be present (with descriptions)." From c6b00c39ebfe3558d147c4eea9747fbd47e42625 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 26 Jan 2026 22:20:37 +0100 Subject: [PATCH 04/19] Separate YAML and dataclass utils --- src/serena/util/dataclass.py | 21 +++++++++++++++++++++ src/serena/util/{general.py => yaml.py} | 22 +--------------------- 2 files changed, 22 insertions(+), 21 deletions(-) create mode 100644 src/serena/util/dataclass.py rename src/serena/util/{general.py => yaml.py} (60%) diff --git a/src/serena/util/dataclass.py b/src/serena/util/dataclass.py new file mode 100644 index 00000000..c5518bf4 --- /dev/null +++ b/src/serena/util/dataclass.py @@ -0,0 +1,21 @@ +from dataclasses import MISSING, Field +from typing import Any, cast + + +def get_dataclass_default(cls: type, field_name: str) -> Any: + """ + Gets the default value of a dataclass field. + + :param cls: The dataclass type. + :param field_name: The name of the field. + :return: The default value of the field (either from default or default_factory). + """ + field = cast(Field, cls.__dataclass_fields__[field_name]) # type: ignore[attr-defined] + + if field.default is not MISSING: + return field.default + + if field.default_factory is not MISSING: # default_factory is a function + return field.default_factory() + + raise AttributeError(f"{field_name} has no default") diff --git a/src/serena/util/general.py b/src/serena/util/yaml.py similarity index 60% rename from src/serena/util/general.py rename to src/serena/util/yaml.py index f380c699..4a1397e2 100644 --- a/src/serena/util/general.py +++ b/src/serena/util/yaml.py @@ -1,6 +1,5 @@ import os -from dataclasses import MISSING, Field -from typing import Any, Literal, cast, overload +from typing import Literal, overload from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -33,22 +32,3 @@ def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = Fa os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding=SERENA_FILE_ENCODING) as f: yaml.dump(data, f) - - -def get_dataclass_default(cls: type, field_name: str) -> Any: - """ - Gets the default value of a dataclass field. - - :param cls: The dataclass type. - :param field_name: The name of the field. - :return: The default value of the field (either from default or default_factory). - """ - field = cast(Field, cls.__dataclass_fields__[field_name]) # type: ignore[attr-defined] - - if field.default is not MISSING: - return field.default - - if field.default_factory is not MISSING: # default_factory is a function - return field.default_factory() - - raise AttributeError(f"{field_name} has no default") From 71fca6c56714c0ab80fa6c9b9d972000a66e77c3 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 26 Jan 2026 22:30:26 +0100 Subject: [PATCH 05/19] Remove unnecessary overload (always preserve comments) --- src/serena/config/serena_config.py | 20 ++++++++++++-------- src/serena/util/yaml.py | 15 +++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 91c109e4..7e085ec1 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -28,13 +28,14 @@ from serena.constants import ( SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME, ) -from serena.util.general import get_dataclass_default, load_yaml, save_yaml from serena.util.inspection import determine_programming_language_composition +from serena.util.yaml import load_yaml, save_yaml from solidlsp.ls_config import Language from ..analytics import RegisteredTokenCountEstimator from ..util.class_decorators import singleton from ..util.cli_util import ask_yes_no +from ..util.dataclass import get_dataclass_default if TYPE_CHECKING: from ..project import Project @@ -252,7 +253,7 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi if save_to_disk: project_yml_path = cls.path_to_project_yml(project_root) log.info("Saving project configuration to %s", project_yml_path) - save_yaml(project_yml_path, config_with_comments, preserve_comments=True) + save_yaml(project_yml_path, config_with_comments) return cls._from_dict(config_with_comments) @classmethod @@ -275,7 +276,7 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi full project configuration and `was_complete` indicates whether the loaded configuration was complete (i.e., did not require any default values to be applied) """ - data = load_yaml(yml_path, preserve_comments=True) + data = load_yaml(yml_path) # apply defaults was_complete = True @@ -376,9 +377,12 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi """ config_path = self.path_to_project_yml(project_root) log.info("Saving updated project configuration to %s", config_path) + + # load original commented map and update it with current values config_with_comments, _ = self._load_yaml(config_path) config_with_comments.update(self._to_yaml_dict()) - save_yaml(config_path, config_with_comments, preserve_comments=True) + + save_yaml(config_path, config_with_comments) class RegisteredProject(ToStringMixin): @@ -521,8 +525,8 @@ class SerenaConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMix :param config_file_path: the path where the configuration file should be generated """ log.info(f"Auto-generating Serena configuration file in {config_file_path}") - loaded_commented_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE, preserve_comments=True) - save_yaml(config_file_path, loaded_commented_yaml, preserve_comments=True) + loaded_commented_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE) + save_yaml(config_file_path, loaded_commented_yaml) @classmethod def _determine_config_file_path(cls) -> str: @@ -558,7 +562,7 @@ class SerenaConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMix # load the configuration log.info(f"Loading Serena configuration from {config_file_path}") try: - loaded_commented_yaml = load_yaml(config_file_path, preserve_comments=True) + loaded_commented_yaml = load_yaml(config_file_path) except Exception as e: raise ValueError(f"Error loading Serena configuration from {config_file_path}: {e}") from e @@ -766,7 +770,7 @@ class SerenaConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMix # convert language backend to string commented_yaml["language_backend"] = self.language_backend.value - save_yaml(self.config_file_path, commented_yaml, preserve_comments=True) + save_yaml(self.config_file_path, commented_yaml) def propagate_settings(self) -> None: """ diff --git a/src/serena/util/yaml.py b/src/serena/util/yaml.py index 4a1397e2..ea9bece6 100644 --- a/src/serena/util/yaml.py +++ b/src/serena/util/yaml.py @@ -1,5 +1,4 @@ import os -from typing import Literal, overload from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -7,7 +6,7 @@ from ruamel.yaml.comments import CommentedMap from serena.constants import SERENA_FILE_ENCODING -def _create_YAML(preserve_comments: bool = False) -> YAML: +def _create_yaml(preserve_comments: bool = False) -> YAML: """ Creates a YAML that can load/save with comments if preserve_comments is True. """ @@ -17,18 +16,14 @@ def _create_YAML(preserve_comments: bool = False) -> YAML: return result -@overload -def load_yaml(path: str, preserve_comments: Literal[False]) -> dict: ... -@overload -def load_yaml(path: str, preserve_comments: Literal[True]) -> CommentedMap: ... -def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap: +def load_yaml(path: str) -> CommentedMap: with open(path, encoding=SERENA_FILE_ENCODING) as f: - yaml = _create_YAML(preserve_comments) + yaml = _create_yaml(preserve_comments=True) return yaml.load(f) -def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = False) -> None: - yaml = _create_YAML(preserve_comments) +def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = True) -> None: + yaml = _create_yaml(preserve_comments) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding=SERENA_FILE_ENCODING) as f: yaml.dump(data, f) From 57423142fe36e65cd2209cbc77d21e44c79d3e77 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 26 Jan 2026 23:13:56 +0100 Subject: [PATCH 06/19] Add YAML comment normalisation to ensure that comments are properly associated with the respective keys (given a file that uniformly uses pre-key comments) --- src/serena/resources/project.template.yml | 6 ++- src/serena/util/yaml.py | 58 ++++++++++++++++++++++- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/serena/resources/project.template.yml b/src/serena/resources/project.template.yml index 02ba1b26..584cec7d 100644 --- a/src/serena/resources/project.template.yml +++ b/src/serena/resources/project.template.yml @@ -1,3 +1,7 @@ +# the name by which the project can be referenced within Serena +project_name: "project_name" + + # list of languages for which language servers are started; choose from: # al bash clojure cpp csharp # csharp_omnisharp dart elixir elm erlang @@ -105,5 +109,3 @@ default_modes: # initial prompt for the project. It will always be given to the LLM upon activating the project # (contrary to the memories, which are loaded on demand). initial_prompt: "" - -project_name: "project_name" diff --git a/src/serena/util/yaml.py b/src/serena/util/yaml.py index ea9bece6..e9679d02 100644 --- a/src/serena/util/yaml.py +++ b/src/serena/util/yaml.py @@ -1,4 +1,6 @@ import os +from enum import Enum +from typing import Any from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -16,10 +18,62 @@ def _create_yaml(preserve_comments: bool = False) -> YAML: return result -def load_yaml(path: str) -> CommentedMap: +class YamlCommentNormalisation(Enum): + NONE = "none" + LEADING = "leading" + """ + Document is assumed to have leading comments only, i.e. comments before keys. + This normalisation achieves that comments are properly associated with keys as leading comments. + """ + + +DOC_COMMENT_INDEX_POST = 0 +DOC_COMMENT_INDEX_PRE = 1 + +# item comment indices: (post key, pre key, post value, pre value) +ITEM_COMMENT_INDEX_BEFORE = 1 # Must be a list at this index +ITEM_COMMENT_INDEX_AFTER = 2 # Not a list at this index + + +def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE) -> CommentedMap: with open(path, encoding=SERENA_FILE_ENCODING) as f: yaml = _create_yaml(preserve_comments=True) - return yaml.load(f) + commented_map: CommentedMap = yaml.load(f) + + def make_list(comment_entry: Any) -> list: + if not isinstance(comment_entry, list): + return [comment_entry] + return comment_entry + + match comment_normalisation: + case YamlCommentNormalisation.NONE: + pass + case YamlCommentNormalisation.LEADING: + # Comments are supposed to be leading comments (i.e., before a key and associated with the key). + # When ruamel parses a YAML, however, comments belonging to a key may be stored as trailing + # comments of the previous key or as a document-level comment. + # Move them accordingly. + keys = list(commented_map.keys()) + comment_items = commented_map.ca.items + doc_comment = commented_map.ca.comment + preceding_comment = None + for i, key in enumerate(keys): + current_comment = comment_items.get(key, [None] * 4) + comment_items[key] = current_comment + if current_comment[ITEM_COMMENT_INDEX_BEFORE] is None: + if i == 0 and doc_comment is not None and doc_comment[DOC_COMMENT_INDEX_PRE] is not None: + # move document pre-comment to leading comment of first key + current_comment[ITEM_COMMENT_INDEX_BEFORE] = make_list(doc_comment[DOC_COMMENT_INDEX_PRE]) + doc_comment[DOC_COMMENT_INDEX_PRE] = None + elif preceding_comment is not None and preceding_comment[ITEM_COMMENT_INDEX_AFTER] is not None: + # move trailing comment of preceding key to leading comment of current key + current_comment[ITEM_COMMENT_INDEX_BEFORE] = make_list(preceding_comment[ITEM_COMMENT_INDEX_AFTER]) + preceding_comment[ITEM_COMMENT_INDEX_AFTER] = None + preceding_comment = current_comment + case _: + raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}") + + return commented_map def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = True) -> None: From a58a1c2f16fc05fbeee45f1e35b0b5a3d0335cf9 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 26 Jan 2026 23:38:21 +0100 Subject: [PATCH 07/19] Transfer comments from template when re-saving project.yml files --- src/serena/config/serena_config.py | 19 ++++++++++++--- src/serena/util/yaml.py | 39 ++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 7e085ec1..6adf9e8a 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -29,7 +29,7 @@ from serena.constants import ( SERENA_MANAGED_DIR_NAME, ) from serena.util.inspection import determine_programming_language_composition -from serena.util.yaml import load_yaml, save_yaml +from serena.util.yaml import YamlCommentNormalisation, load_yaml, save_yaml, transfer_missing_yaml_comments from solidlsp.ls_config import Language from ..analytics import RegisteredTokenCountEstimator @@ -174,6 +174,11 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi SERENA_DEFAULT_PROJECT_FILE = "project.yml" FIELDS_WITHOUT_DEFAULTS = {"project_name", "languages"} + YAML_COMMENT_NORMALISATION = YamlCommentNormalisation.LEADING + """ + the comment normalisation strategy to use when loading/saving project configuration files. + The template file must match this configuration (i.e. it must use leading comments if this is set to LEADING). + """ def _tostring_includes(self) -> list[str]: return ["project_name"] @@ -265,7 +270,9 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi return os.path.join(SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE) @classmethod - def _load_yaml(cls, yml_path: str) -> tuple[CommentedMap, bool]: + def _load_yaml( + cls, yml_path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE + ) -> tuple[CommentedMap, bool]: """ Load the project configuration as a CommentedMap, preserving comments and ensuring completeness of the configuration by applying default values for missing fields @@ -276,7 +283,7 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi full project configuration and `was_complete` indicates whether the loaded configuration was complete (i.e., did not require any default values to be applied) """ - data = load_yaml(yml_path) + data = load_yaml(yml_path, comment_normalisation=comment_normalisation) # apply defaults was_complete = True @@ -379,9 +386,13 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi log.info("Saving updated project configuration to %s", config_path) # load original commented map and update it with current values - config_with_comments, _ = self._load_yaml(config_path) + config_with_comments, _ = self._load_yaml(config_path, self.YAML_COMMENT_NORMALISATION) config_with_comments.update(self._to_yaml_dict()) + # transfer missing comments from the template file + template_config, _ = self._load_yaml(PROJECT_TEMPLATE_FILE, self.YAML_COMMENT_NORMALISATION) + transfer_missing_yaml_comments(template_config, config_with_comments, self.YAML_COMMENT_NORMALISATION) + save_yaml(config_path, config_with_comments) diff --git a/src/serena/util/yaml.py b/src/serena/util/yaml.py index e9679d02..22928200 100644 --- a/src/serena/util/yaml.py +++ b/src/serena/util/yaml.py @@ -31,8 +31,8 @@ DOC_COMMENT_INDEX_POST = 0 DOC_COMMENT_INDEX_PRE = 1 # item comment indices: (post key, pre key, post value, pre value) -ITEM_COMMENT_INDEX_BEFORE = 1 # Must be a list at this index -ITEM_COMMENT_INDEX_AFTER = 2 # Not a list at this index +ITEM_COMMENT_INDEX_BEFORE = 1 # (pre-key; must be a list at this index) +ITEM_COMMENT_INDEX_AFTER = 2 # (post-value; typically no list at this index) def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE) -> CommentedMap: @@ -81,3 +81,38 @@ def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = Tr os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding=SERENA_FILE_ENCODING) as f: yaml.dump(data, f) + + +def transfer_missing_yaml_comments_by_index(source: CommentedMap, target: CommentedMap, indices: list[int]) -> None: + """ + :param source: the source, from which to transfer missing comments + :param target: the target map, whose comments will be updated + :param indices: list of comment indices to transfer + """ + for key in target.keys(): + if key in source: + source_comment = source.ca.items.get(key) + if source_comment is None: + continue + target_comment = target.ca.items.get(key, [None, None, None, None]) + target.ca.items[key] = target_comment + for index in indices: + if target_comment[index] is None: + target_comment[index] = source_comment[index] + + +def transfer_missing_yaml_comments(source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation) -> None: + """ + Transfers missing comments from source to target YAML. + + :param source: the source, from which to transfer missing comments + :param target: the target map, whose comments will be updated. + :param comment_normalisation: the comment normalisation to assume; if NONE, no comments are transferred + """ + match comment_normalisation: + case YamlCommentNormalisation.NONE: + pass + case YamlCommentNormalisation.LEADING: + transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE]) + case _: + raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}") From 7c7f100d754727f4e40815731884c4afef7e0a9e Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 27 Jan 2026 11:37:48 +0100 Subject: [PATCH 08/19] transfer_missing_yaml_comments_by_index: Also transfer source comment if target comment is effectively empty --- src/serena/util/yaml.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/serena/util/yaml.py b/src/serena/util/yaml.py index 22928200..8bc1fc99 100644 --- a/src/serena/util/yaml.py +++ b/src/serena/util/yaml.py @@ -2,7 +2,7 @@ import os from enum import Enum from typing import Any -from ruamel.yaml import YAML +from ruamel.yaml import YAML, CommentToken from ruamel.yaml.comments import CommentedMap from serena.constants import SERENA_FILE_ENCODING @@ -89,6 +89,23 @@ def transfer_missing_yaml_comments_by_index(source: CommentedMap, target: Commen :param target: the target map, whose comments will be updated :param indices: list of comment indices to transfer """ + + def is_empty(comment_entry: Any) -> bool: + if comment_entry is None: + return True + elif isinstance(comment_entry, list): + for item in comment_entry: + if isinstance(item, CommentToken): + if item.value.strip() != "": + return False + else: + return False + return True + elif isinstance(comment_entry, CommentToken): + return comment_entry.value.strip() == "" + else: + return False + for key in target.keys(): if key in source: source_comment = source.ca.items.get(key) @@ -97,7 +114,7 @@ def transfer_missing_yaml_comments_by_index(source: CommentedMap, target: Commen target_comment = target.ca.items.get(key, [None, None, None, None]) target.ca.items[key] = target_comment for index in indices: - if target_comment[index] is None: + if is_empty(target_comment[index]): target_comment[index] = source_comment[index] From 5125831c3a18882d9ded3efa52373c851893a39b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 27 Jan 2026 11:44:56 +0100 Subject: [PATCH 09/19] Improve comment on special requirements for language servers, linking to docs --- src/serena/resources/project.template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/serena/resources/project.template.yml b/src/serena/resources/project.template.yml index 584cec7d..9472ba95 100644 --- a/src/serena/resources/project.template.yml +++ b/src/serena/resources/project.template.yml @@ -20,8 +20,8 @@ project_name: "project_name" # - For JavaScript, use typescript # - For Free Pascal/Lazarus, use pascal # Special requirements: -# - csharp: Requires the presence of a .sln file in the project folder. -# - pascal: Requires Free Pascal Compiler (fpc) and optionally Lazarus. +# Some languages require additional setup/installations. +# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers # When using multiple languages, the first language server that supports a given file will be used for that file. # The first language is the default language and the respective language server will be used as a fallback. # Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. From a8753d9618db17111f6b77c4aa836fa55059098b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 27 Jan 2026 13:02:31 +0100 Subject: [PATCH 10/19] Apply YAML comment transfer to serena_config.yml when re-saving * Use leading comment convention in template, because trailing comments are much more difficult to deal with and very hard to normalise * Apply a conversion to the user's yaml content (translating from trailing comments where necessary) before applying the comment transfer --- src/serena/config/serena_config.py | 9 +- .../resources/serena_config.template.yml | 43 +++--- src/serena/util/yaml.py | 137 +++++++++++++----- 3 files changed, 133 insertions(+), 56 deletions(-) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 6adf9e8a..03dd6abf 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -29,7 +29,7 @@ from serena.constants import ( SERENA_MANAGED_DIR_NAME, ) from serena.util.inspection import determine_programming_language_composition -from serena.util.yaml import YamlCommentNormalisation, load_yaml, save_yaml, transfer_missing_yaml_comments +from serena.util.yaml import YamlCommentNormalisation, load_yaml, normalise_yaml_comments, save_yaml, transfer_missing_yaml_comments from solidlsp.ls_config import Language from ..analytics import RegisteredTokenCountEstimator @@ -781,6 +781,13 @@ class SerenaConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMix # convert language backend to string commented_yaml["language_backend"] = self.language_backend.value + # transfer comments from the template file + # NOTE: The template file now uses leading comments, but we previously used trailing comments, + # so we apply a conversion, which detects the old style and transforms it. + normalise_yaml_comments(commented_yaml, YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING) + template_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE, comment_normalisation=YamlCommentNormalisation.LEADING) + transfer_missing_yaml_comments(template_yaml, commented_yaml, YamlCommentNormalisation.LEADING, fully_missing_only=False) + save_yaml(self.config_file_path, commented_yaml) def propagate_settings(self) -> None: diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index f36f5895..11842b53 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -1,4 +1,3 @@ -language_backend: LSP # the language backend to use for code understanding and manipulation. # Possible values are: # * LSP: Use the language server protocol (LSP), spawning freely available language servers @@ -6,8 +5,8 @@ language_backend: LSP # * JetBrains: Use the Serena plugin in your JetBrains IDE. # (requires the plugin to be installed and the project being worked on to be open # in your IDE). +language_backend: LSP -gui_log_window: False # whether to open a graphical window with Serena's logs. # This is mainly supported on Windows and (partly) on Linux; not available on macOS. # If you prefer a browser-based tool, use the `web_dashboard` option instead. @@ -22,8 +21,8 @@ gui_log_window: False # previous instances. This can lead to multiple log windows being opened, and only the last # window being updated. Since we can't control how agno or Claude Desktop start Serena, # we have to live with this limitation for now. +gui_log_window: False -web_dashboard: True # whether to open the Serena web dashboard (which will be accessible through your web browser) that # provides access to Serena's configuration and state as well as the current session logs. # The web dashboard is supported on all platforms. @@ -34,11 +33,11 @@ web_dashboard: True # You can then open the dashboard by asking your agent to do so (e.g., by saying "open the dashboard"), Serena provides # a tool for this. # Further information: https://oraios.github.io/serena/02-usage/060_dashboard.html +web_dashboard: True -web_dashboard_listen_address: 127.0.0.1 # The address where the web dashboard will listen on +web_dashboard_listen_address: 127.0.0.1 -web_dashboard_open_on_launch: True # whether to open a browser window with the web dashboard when Serena starts (provided that web_dashboard # is enabled). See also the web_dashboard option. # If set to false, you can still open the dashboard manually by @@ -46,56 +45,55 @@ web_dashboard_open_on_launch: True # b) manually navigating to http://localhost:24282/dashboard/ in your web browser (actual port # may be higher if you have multiple instances running; try ports 24283, 24284, etc.) # See also: https://oraios.github.io/serena/02-usage/060_dashboard.html +web_dashboard_open_on_launch: True -jetbrains_plugin_server_address: 127.0.0.1 # address where JetBrains plugin servers are running (only relevant when using the JetBrains language backend) +jetbrains_plugin_server_address: 127.0.0.1 -log_level: 20 # the minimum log level for the GUI log window and the dashboard (10 = debug, 20 = info, 30 = warning, 40 = error) +log_level: 20 -trace_lsp_communication: False # whether to trace the communication between Serena and the language servers. # This is useful for debugging language server issues. +trace_lsp_communication: False -ls_specific_settings: {} -# Added on 23.08.2025 -# Advanced configuration option allowing to configure language server implementation specific options. Maps the language -# (same entry as in project.yml) to the options. +# Advanced configuration option allowing to configure language server-specific options. +# Maps the language key to the options. # Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. # No documentation on options means no options are available. +ls_specific_settings: {} -tool_timeout: 240 # timeout, in seconds, after which tool executions are terminated +tool_timeout: 240 -excluded_tools: [] # list of tools to be globally excluded +excluded_tools: [] -included_optional_tools: [] # list of optional tools (which are disabled by default) to be included +included_optional_tools: [] -fixed_tools: [] # fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. # This cannot be combined with non-empty excluded_tools or included_optional_tools. +fixed_tools: [] -base_modes: # list of mode names to that are always to be included in the set of active modes # The full set of modes to be activated is base_modes + default_modes. # If this is undefined, no base modes are included. # The project configuration (project.yml) may override this setting. +base_modes: -default_modes: - - interactive - - editing # list of mode names that are to be activated by default. # The full set of modes to be activated is base_modes + default_modes. # These modes can be overridden by the project configuration (project.yml) or through the CLI (--mode). +default_modes: + - interactive + - editing -default_max_tool_answer_chars: 150000 # Used as default for tools where the apply method has a default maximal answer length. # Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default # through the global configuration. +default_max_tool_answer_chars: 150000 -token_count_estimator: CHAR_COUNT # the name of the token count estimator to use for tool usage statistics. # See the `RegisteredTokenCountEstimator` enum for available options. # @@ -103,6 +101,7 @@ token_count_estimator: CHAR_COUNT # You can configure this to TIKTOKEN_GPT4 to use a local tiktoken-based estimator for GPT-4 (will download tiktoken # data files on first run), or ANTHROPIC_CLAUDE_SONNET_4 which will use the (free of cost) Anthropic API to # estimate the token count using the Claude Sonnet 4 tokenizer. +token_count_estimator: CHAR_COUNT # The list of registered project paths (updated automatically). projects: [] diff --git a/src/serena/util/yaml.py b/src/serena/util/yaml.py index 8bc1fc99..20bddc07 100644 --- a/src/serena/util/yaml.py +++ b/src/serena/util/yaml.py @@ -1,12 +1,15 @@ +import logging import os from enum import Enum from typing import Any -from ruamel.yaml import YAML, CommentToken +from ruamel.yaml import YAML, CommentToken, StreamMark from ruamel.yaml.comments import CommentedMap from serena.constants import SERENA_FILE_ENCODING +log = logging.getLogger(__name__) + def _create_yaml(preserve_comments: bool = False) -> YAML: """ @@ -20,35 +23,69 @@ def _create_yaml(preserve_comments: bool = False) -> YAML: class YamlCommentNormalisation(Enum): NONE = "none" + """ + No comment normalisation is performed. + Comments are kept as parsed by ruamel.yaml. + """ LEADING = "leading" """ - Document is assumed to have leading comments only, i.e. comments before keys. + Document is assumed to have leading comments only, i.e. comments before keys, only full-line comments. This normalisation achieves that comments are properly associated with keys as leading comments. """ + LEADING_WITH_CONVERSION_FROM_TRAILING = "leading_with_conversion_from_trailing" + # NOTE: Normalisation for trailing comments was attempted but is extremely hard, because + # it is difficult to position the comments properly after values, especially for complex values. DOC_COMMENT_INDEX_POST = 0 DOC_COMMENT_INDEX_PRE = 1 # item comment indices: (post key, pre key, post value, pre value) -ITEM_COMMENT_INDEX_BEFORE = 1 # (pre-key; must be a list at this index) -ITEM_COMMENT_INDEX_AFTER = 2 # (post-value; typically no list at this index) +ITEM_COMMENT_INDEX_BEFORE = 1 # (pre-key; must be a list of CommentToken at this index) +ITEM_COMMENT_INDEX_AFTER = 2 # (post-value; must be an instance of CommentToken at this index) def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE) -> CommentedMap: with open(path, encoding=SERENA_FILE_ENCODING) as f: yaml = _create_yaml(preserve_comments=True) commented_map: CommentedMap = yaml.load(f) + normalise_yaml_comments(commented_map, comment_normalisation) + return commented_map + +def normalise_yaml_comments(commented_map: CommentedMap, comment_normalisation: YamlCommentNormalisation) -> None: def make_list(comment_entry: Any) -> list: if not isinstance(comment_entry, list): return [comment_entry] return comment_entry + def make_unit(comment_entry: Any) -> Any: + """ + Converts a list-valued comment entry into a single comment entry. + """ + if isinstance(comment_entry, list): + if len(comment_entry) == 0: + return None + elif len(comment_entry) == 1: + return comment_entry[0] + else: + if all(isinstance(item, CommentToken) for item in comment_entry): + start_mark = StreamMark(name="", index=0, line=0, column=0) + comment_str = "".join(item.value for item in comment_entry) + if not comment_str.startswith("\n"): + comment_str = "\n" + comment_str + return CommentToken(value=comment_str, start_mark=start_mark, end_mark=None) + else: + types = set(type(item) for item in comment_entry) + log.warning("Unhandled types in list-valued comment entry: %s; not updating entry", types) + return None + else: + return comment_entry + match comment_normalisation: case YamlCommentNormalisation.NONE: pass - case YamlCommentNormalisation.LEADING: + case YamlCommentNormalisation.LEADING | YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING: # Comments are supposed to be leading comments (i.e., before a key and associated with the key). # When ruamel parses a YAML, however, comments belonging to a key may be stored as trailing # comments of the previous key or as a document-level comment. @@ -70,11 +107,31 @@ def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlC current_comment[ITEM_COMMENT_INDEX_BEFORE] = make_list(preceding_comment[ITEM_COMMENT_INDEX_AFTER]) preceding_comment[ITEM_COMMENT_INDEX_AFTER] = None preceding_comment = current_comment + + if comment_normalisation == YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING: + # Second pass: conversion of trailing comments + # If a leading comment ends with "\n\n", i.e. it has an empty line between the comment and the key, + # it was actually intended as a trailing comment for the preceding key, so we associate it with + # the preceding key instead. + preceding_comment = None + for key in keys: + current_comment = comment_items.get(key, [None] * 4) + if current_comment[ITEM_COMMENT_INDEX_BEFORE] is not None: + token_list = make_list(current_comment[ITEM_COMMENT_INDEX_BEFORE]) + if len(token_list) > 0: + last_token = token_list[-1] + if isinstance(last_token, CommentToken) and last_token.value.endswith("\n\n"): + # move comment to preceding key, removing the empty line + if preceding_comment is not None and yaml_comment_entry_is_empty( + preceding_comment[ITEM_COMMENT_INDEX_BEFORE] + ): + last_token.value = last_token.value[:-1] + preceding_comment[ITEM_COMMENT_INDEX_BEFORE] = token_list + current_comment[ITEM_COMMENT_INDEX_BEFORE] = None + preceding_comment = current_comment case _: raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}") - return commented_map - def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = True) -> None: yaml = _create_yaml(preserve_comments) @@ -83,53 +140,67 @@ def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = Tr yaml.dump(data, f) -def transfer_missing_yaml_comments_by_index(source: CommentedMap, target: CommentedMap, indices: list[int]) -> None: +def yaml_comment_entry_is_empty(comment_entry: Any) -> bool: + if comment_entry is None: + return True + elif isinstance(comment_entry, list): + for item in comment_entry: + if isinstance(item, CommentToken): + if item.value.strip() != "": + return False + else: + return False + return True + elif isinstance(comment_entry, CommentToken): + return comment_entry.value.strip() == "" + else: + return False + + +def transfer_missing_yaml_comments_by_index( + source: CommentedMap, target: CommentedMap, indices: list[int], fully_missing_only: bool = False +) -> None: """ :param source: the source, from which to transfer missing comments :param target: the target map, whose comments will be updated :param indices: list of comment indices to transfer + :param fully_missing_only: if True, only transfer comments for keys that have no comment definition at all (newly added) """ - - def is_empty(comment_entry: Any) -> bool: - if comment_entry is None: - return True - elif isinstance(comment_entry, list): - for item in comment_entry: - if isinstance(item, CommentToken): - if item.value.strip() != "": - return False - else: - return False - return True - elif isinstance(comment_entry, CommentToken): - return comment_entry.value.strip() == "" - else: - return False - for key in target.keys(): if key in source: source_comment = source.ca.items.get(key) if source_comment is None: continue - target_comment = target.ca.items.get(key, [None, None, None, None]) - target.ca.items[key] = target_comment - for index in indices: - if is_empty(target_comment[index]): - target_comment[index] = source_comment[index] + target_comment = target.ca.items.get(key) + if target_comment is None and fully_missing_only: + # transfer full comment and we're done + target.ca.items[key] = source_comment + else: + # initialise target comment if needed + if target_comment is None: + target_comment = [None] * 4 + target.ca.items[key] = target_comment + # transfer missing comments at specified indices + for index in indices: + if yaml_comment_entry_is_empty(target_comment[index]): + target_comment[index] = source_comment[index] -def transfer_missing_yaml_comments(source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation) -> None: +def transfer_missing_yaml_comments( + source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation, fully_missing_only: bool = False +) -> None: """ Transfers missing comments from source to target YAML. :param source: the source, from which to transfer missing comments :param target: the target map, whose comments will be updated. :param comment_normalisation: the comment normalisation to assume; if NONE, no comments are transferred + :param fully_missing_only: if True, only transfer comments for keys that have no comment definition at all (newly added) """ match comment_normalisation: case YamlCommentNormalisation.NONE: pass - case YamlCommentNormalisation.LEADING: - transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE]) + case YamlCommentNormalisation.LEADING | YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING: + transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE], fully_missing_only=fully_missing_only) case _: raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}") From 712c493360445cb643b8147be6aa966a82aa6ce5 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 27 Jan 2026 14:11:34 +0100 Subject: [PATCH 11/19] Handle newline differences between leading and trailing comments --- src/serena/util/yaml.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/serena/util/yaml.py b/src/serena/util/yaml.py index 20bddc07..9f221389 100644 --- a/src/serena/util/yaml.py +++ b/src/serena/util/yaml.py @@ -82,6 +82,17 @@ def normalise_yaml_comments(commented_map: CommentedMap, comment_normalisation: else: return comment_entry + def trailing_to_leading(comment_entry: Any) -> Any: + if comment_entry is None: + return None + token_list = make_list(comment_entry) + first_token = token_list[0] + if isinstance(first_token, CommentToken): + # remove leading newline if present + if first_token.value.startswith("\n"): + first_token.value = first_token.value[1:] + return token_list + match comment_normalisation: case YamlCommentNormalisation.NONE: pass @@ -104,7 +115,7 @@ def normalise_yaml_comments(commented_map: CommentedMap, comment_normalisation: doc_comment[DOC_COMMENT_INDEX_PRE] = None elif preceding_comment is not None and preceding_comment[ITEM_COMMENT_INDEX_AFTER] is not None: # move trailing comment of preceding key to leading comment of current key - current_comment[ITEM_COMMENT_INDEX_BEFORE] = make_list(preceding_comment[ITEM_COMMENT_INDEX_AFTER]) + current_comment[ITEM_COMMENT_INDEX_BEFORE] = trailing_to_leading(preceding_comment[ITEM_COMMENT_INDEX_AFTER]) preceding_comment[ITEM_COMMENT_INDEX_AFTER] = None preceding_comment = current_comment @@ -112,7 +123,7 @@ def normalise_yaml_comments(commented_map: CommentedMap, comment_normalisation: # Second pass: conversion of trailing comments # If a leading comment ends with "\n\n", i.e. it has an empty line between the comment and the key, # it was actually intended as a trailing comment for the preceding key, so we associate it with - # the preceding key instead. + # the preceding key instead (if the preceding key has no leading comment already). preceding_comment = None for key in keys: current_comment = comment_items.get(key, [None] * 4) @@ -121,11 +132,18 @@ def normalise_yaml_comments(commented_map: CommentedMap, comment_normalisation: if len(token_list) > 0: last_token = token_list[-1] if isinstance(last_token, CommentToken) and last_token.value.endswith("\n\n"): - # move comment to preceding key, removing the empty line + # move comment to preceding key, removing the empty line, + # and adding an empty line at the beginning instead if preceding_comment is not None and yaml_comment_entry_is_empty( preceding_comment[ITEM_COMMENT_INDEX_BEFORE] ): last_token.value = last_token.value[:-1] + + first_token = token_list[0] + if isinstance(first_token, CommentToken): + if not first_token.value.startswith("\n"): + first_token.value = "\n" + first_token.value + preceding_comment[ITEM_COMMENT_INDEX_BEFORE] = token_list current_comment[ITEM_COMMENT_INDEX_BEFORE] = None preceding_comment = current_comment From 3c29a8fbe816a8713eb4119fb562665a1df80998 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 28 Jan 2026 16:18:18 +0100 Subject: [PATCH 12/19] Deduplicate active modes --- src/serena/agent.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 35ca3042..18e71d5c 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -190,11 +190,12 @@ class ActiveModes: def get_mode_names(self) -> Sequence[str]: if self._active_mode_names is not None: return self._active_mode_names - self._active_mode_names = [] + active_mode_names: set[str] = set() if self._base_modes is not None: - self._active_mode_names.extend(self._base_modes) + active_mode_names.update(self._base_modes) if self._default_modes is not None: - self._active_mode_names.extend(self._default_modes) + active_mode_names.update(self._default_modes) + self._active_mode_names = sorted(active_mode_names) log.info("Active modes: %s", self._active_mode_names) return self._active_mode_names From f0cb06eb4bfd07fd5524813ed613e34ae8cb0e1b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 28 Jan 2026 16:19:35 +0100 Subject: [PATCH 13/19] Add docstrings --- src/serena/util/yaml.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/serena/util/yaml.py b/src/serena/util/yaml.py index 9f221389..a9e6ca7c 100644 --- a/src/serena/util/yaml.py +++ b/src/serena/util/yaml.py @@ -22,6 +22,16 @@ def _create_yaml(preserve_comments: bool = False) -> YAML: class YamlCommentNormalisation(Enum): + """ + Defines a normalisation to be applied to the comment representation in a ruamel CommentedMap. + + Note that even though a YAML document may seem to consistently contain, for example, leading comments + before a key only, ruamel may still parse some comments as trailing comments of the previous key + or as document-level comments. + The normalisations define ways to adjust the comment representation accordingly, clearly associating + comments with the keys they belong to. + """ + NONE = "none" """ No comment normalisation is performed. @@ -33,6 +43,10 @@ class YamlCommentNormalisation(Enum): This normalisation achieves that comments are properly associated with keys as leading comments. """ LEADING_WITH_CONVERSION_FROM_TRAILING = "leading_with_conversion_from_trailing" + """ + Document is assumed to have a mixture of leading comments (before keys) and trailing comments (after values), only full-line comments. + This normalisation achieves that all comments are converted to leading comments and properly associated with keys. + """ # NOTE: Normalisation for trailing comments was attempted but is extremely hard, because # it is difficult to position the comments properly after values, especially for complex values. @@ -46,6 +60,11 @@ ITEM_COMMENT_INDEX_AFTER = 2 # (post-value; must be an instance of CommentToken def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE) -> CommentedMap: + """ + :param path: the path to the YAML file to load + :param comment_normalisation: the comment normalisation to apply after loading + :return: the loaded commented map + """ with open(path, encoding=SERENA_FILE_ENCODING) as f: yaml = _create_yaml(preserve_comments=True) commented_map: CommentedMap = yaml.load(f) @@ -54,6 +73,13 @@ def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlC def normalise_yaml_comments(commented_map: CommentedMap, comment_normalisation: YamlCommentNormalisation) -> None: + """ + Applies the given comment normalisation to the given commented map in-place. + + :param commented_map: the commented map whose comments are to be normalised + :param comment_normalisation: the comment normalisation to apply + """ + def make_list(comment_entry: Any) -> list: if not isinstance(comment_entry, list): return [comment_entry] From 754d7e51081f625ca0604eddc3211a4cd4b7b03b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 28 Jan 2026 16:38:48 +0100 Subject: [PATCH 14/19] transfer_missing_yaml_comments: Remove obsolete parameter fully_missing_only --- src/serena/config/serena_config.py | 2 +- src/serena/util/yaml.py | 32 ++++++++++-------------------- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 03dd6abf..e746cafb 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -786,7 +786,7 @@ class SerenaConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMix # so we apply a conversion, which detects the old style and transforms it. normalise_yaml_comments(commented_yaml, YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING) template_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE, comment_normalisation=YamlCommentNormalisation.LEADING) - transfer_missing_yaml_comments(template_yaml, commented_yaml, YamlCommentNormalisation.LEADING, fully_missing_only=False) + transfer_missing_yaml_comments(template_yaml, commented_yaml, YamlCommentNormalisation.LEADING) save_yaml(self.config_file_path, commented_yaml) diff --git a/src/serena/util/yaml.py b/src/serena/util/yaml.py index a9e6ca7c..bb6bd290 100644 --- a/src/serena/util/yaml.py +++ b/src/serena/util/yaml.py @@ -201,14 +201,11 @@ def yaml_comment_entry_is_empty(comment_entry: Any) -> bool: return False -def transfer_missing_yaml_comments_by_index( - source: CommentedMap, target: CommentedMap, indices: list[int], fully_missing_only: bool = False -) -> None: +def transfer_missing_yaml_comments_by_index(source: CommentedMap, target: CommentedMap, indices: list[int]) -> None: """ :param source: the source, from which to transfer missing comments :param target: the target map, whose comments will be updated :param indices: list of comment indices to transfer - :param fully_missing_only: if True, only transfer comments for keys that have no comment definition at all (newly added) """ for key in target.keys(): if key in source: @@ -216,35 +213,28 @@ def transfer_missing_yaml_comments_by_index( if source_comment is None: continue target_comment = target.ca.items.get(key) - if target_comment is None and fully_missing_only: - # transfer full comment and we're done - target.ca.items[key] = source_comment - else: - # initialise target comment if needed - if target_comment is None: - target_comment = [None] * 4 - target.ca.items[key] = target_comment - # transfer missing comments at specified indices - for index in indices: - if yaml_comment_entry_is_empty(target_comment[index]): - target_comment[index] = source_comment[index] + # initialise target comment if needed + if target_comment is None: + target_comment = [None] * 4 + target.ca.items[key] = target_comment + # transfer missing comments at specified indices + for index in indices: + if yaml_comment_entry_is_empty(target_comment[index]): + target_comment[index] = source_comment[index] -def transfer_missing_yaml_comments( - source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation, fully_missing_only: bool = False -) -> None: +def transfer_missing_yaml_comments(source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation) -> None: """ Transfers missing comments from source to target YAML. :param source: the source, from which to transfer missing comments :param target: the target map, whose comments will be updated. :param comment_normalisation: the comment normalisation to assume; if NONE, no comments are transferred - :param fully_missing_only: if True, only transfer comments for keys that have no comment definition at all (newly added) """ match comment_normalisation: case YamlCommentNormalisation.NONE: pass case YamlCommentNormalisation.LEADING | YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING: - transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE], fully_missing_only=fully_missing_only) + transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE]) case _: raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}") From df0c10fa1af51b20cb810b7132874a27bb8871f6 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Thu, 29 Jan 2026 12:50:56 +0700 Subject: [PATCH 15/19] Minor: declare mode-related fields instead of setting them The fields _active_modes and _active_tools were overwritten directly after having been set, this is no longer the case --- src/serena/agent.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 18e71d5c..a4b21978 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -332,11 +332,6 @@ class SerenaAgent: self.prompt_factory = SerenaPromptFactory() self._project_activation_callback = project_activation_callback - # initialize members for modes and tools - self._mode_overrides = modes - self._active_modes = ActiveModes() - self._active_tools: AvailableTools = self._exposed_tools - # activate a project configuration (if provided or if there is only a single project available) if project is not None: try: @@ -345,6 +340,10 @@ class SerenaAgent: log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e) # update active modes and active tools (considering the active project, if any) + # declared attributes are set in the call to _update_active_modes_and_tools() + self._mode_overrides = modes + self._active_modes: ActiveModes + self._active_tools: AvailableTools self._update_active_modes_and_tools() # start the dashboard (web frontend), registering its log handler From c41810a227bc78f3b64b9584172b3378acd0e530 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Thu, 29 Jan 2026 13:47:38 +0700 Subject: [PATCH 16/19] Better documentation of modes in cli --- src/serena/cli.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index f5dc51a4..e63957d1 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -43,6 +43,9 @@ from solidlsp.util.subprocess_util import subprocess_kwargs log = logging.getLogger(__name__) _MAX_CONTENT_WIDTH = 100 +_MODES_EXPLANATION = """Built-in mode names or paths to custom mode YAMLs. +By default, the `default_modes` entry from your serena config (at `~/.serena/serena_config.yml`) will be used. +If not configured otherwise, this will result in ('interactive', 'editing').""" def find_project_root(root: str | Path | None = None) -> str: @@ -159,8 +162,8 @@ class TopLevelCommands(AutoRegisteringGroup): type=str, multiple=True, default=(), - show_default=True, - help="Built-in mode names or paths to custom mode YAMLs.", + show_default=False, + help=_MODES_EXPLANATION, ) @click.option( "--language-backend", @@ -309,8 +312,8 @@ class TopLevelCommands(AutoRegisteringGroup): type=str, multiple=True, default=(), - show_default=True, - help="Built-in mode names or paths to custom mode YAMLs.", + show_default=False, + help=_MODES_EXPLANATION, ) def print_system_prompt( project: str, log_level: str, only_instructions: bool, context: str, modes: Sequence[str] | None = None From 153ccb5238bc5b68f4f721ec2757b085bacdcfd9 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 29 Jan 2026 12:02:24 +0100 Subject: [PATCH 17/19] Improve --mode help text --- src/serena/cli.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index e63957d1..ecb5dcca 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -36,6 +36,7 @@ from serena.constants import ( from serena.mcp import SerenaMCPFactory from serena.project import Project from serena.tools import FindReferencingSymbolsTool, FindSymbolTool, GetSymbolsOverviewTool, SearchForPatternTool, ToolRegistry +from serena.util.dataclass import get_dataclass_default from serena.util.logging import MemoryLogHandler from solidlsp.ls_config import Language from solidlsp.util.subprocess_util import subprocess_kwargs @@ -43,9 +44,15 @@ from solidlsp.util.subprocess_util import subprocess_kwargs log = logging.getLogger(__name__) _MAX_CONTENT_WIDTH = 100 -_MODES_EXPLANATION = """Built-in mode names or paths to custom mode YAMLs. -By default, the `default_modes` entry from your serena config (at `~/.serena/serena_config.yml`) will be used. -If not configured otherwise, this will result in ('interactive', 'editing').""" +_MODES_EXPLANATION = f"""\b\nBuilt-in mode names or paths to custom mode YAMLs with which to +override the default modes defined in the global Serena configuration or +the active project. +For details on mode configuration, see + https://oraios.github.io/serena/02-usage/050_configuration.html#modes. +If no configuration changes were made, the base defaults are: + {get_dataclass_default(SerenaConfig, "default_modes")}. +Overriding them means that they no longer apply, so you will need to +re-specify them in addition to further modes if you want to keep them.""" def find_project_root(root: str | Path | None = None) -> str: From ad47570c4f51e7817b5629c512115a6cd3a5d382 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 29 Jan 2026 19:01:06 +0100 Subject: [PATCH 18/19] Minor changes to improve uniformity of comments --- src/serena/resources/serena_config.template.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index 11842b53..7a660e64 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -35,7 +35,7 @@ gui_log_window: False # Further information: https://oraios.github.io/serena/02-usage/060_dashboard.html web_dashboard: True -# The address where the web dashboard will listen on +# the address the web dashboard will listen on (bind address). web_dashboard_listen_address: 127.0.0.1 # whether to open a browser window with the web dashboard when Serena starts (provided that web_dashboard @@ -57,7 +57,7 @@ log_level: 20 # This is useful for debugging language server issues. trace_lsp_communication: False -# Advanced configuration option allowing to configure language server-specific options. +# advanced configuration option allowing to configure language server-specific options. # Maps the language key to the options. # Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. # No documentation on options means no options are available. @@ -103,5 +103,5 @@ default_max_tool_answer_chars: 150000 # estimate the token count using the Claude Sonnet 4 tokenizer. token_count_estimator: CHAR_COUNT -# The list of registered project paths (updated automatically). +# the list of registered project paths (updated automatically). projects: [] From 80f3586186f12ebcd69a1f39ba4052c74ece10bf Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 29 Jan 2026 19:02:07 +0100 Subject: [PATCH 19/19] Support forced comment updates when transferring comments Apply forced update for 'projects' key (old comment is misleading) --- src/serena/config/serena_config.py | 3 ++- src/serena/util/yaml.py | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index e746cafb..0f108bac 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -784,9 +784,10 @@ class SerenaConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMix # transfer comments from the template file # NOTE: The template file now uses leading comments, but we previously used trailing comments, # so we apply a conversion, which detects the old style and transforms it. + # For some keys, we force updates, because old comments are problematic/misleading. normalise_yaml_comments(commented_yaml, YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING) template_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE, comment_normalisation=YamlCommentNormalisation.LEADING) - transfer_missing_yaml_comments(template_yaml, commented_yaml, YamlCommentNormalisation.LEADING) + transfer_missing_yaml_comments(template_yaml, commented_yaml, YamlCommentNormalisation.LEADING, forced_update_keys=["projects"]) save_yaml(self.config_file_path, commented_yaml) diff --git a/src/serena/util/yaml.py b/src/serena/util/yaml.py index bb6bd290..32a8f364 100644 --- a/src/serena/util/yaml.py +++ b/src/serena/util/yaml.py @@ -1,5 +1,6 @@ import logging import os +from collections.abc import Sequence from enum import Enum from typing import Any @@ -201,11 +202,14 @@ def yaml_comment_entry_is_empty(comment_entry: Any) -> bool: return False -def transfer_missing_yaml_comments_by_index(source: CommentedMap, target: CommentedMap, indices: list[int]) -> None: +def transfer_missing_yaml_comments_by_index( + source: CommentedMap, target: CommentedMap, indices: list[int], forced_update_keys: Sequence[str] = () +) -> None: """ :param source: the source, from which to transfer missing comments :param target: the target map, whose comments will be updated :param indices: list of comment indices to transfer + :param forced_update_keys: keys for which comments are always transferred, even if present in target """ for key in target.keys(): if key in source: @@ -217,24 +221,28 @@ def transfer_missing_yaml_comments_by_index(source: CommentedMap, target: Commen if target_comment is None: target_comment = [None] * 4 target.ca.items[key] = target_comment - # transfer missing comments at specified indices + # transfer comments at specified indices for index in indices: - if yaml_comment_entry_is_empty(target_comment[index]): + is_forced_update = key in forced_update_keys + if is_forced_update or yaml_comment_entry_is_empty(target_comment[index]): target_comment[index] = source_comment[index] -def transfer_missing_yaml_comments(source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation) -> None: +def transfer_missing_yaml_comments( + source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation, forced_update_keys: Sequence[str] = () +) -> None: """ Transfers missing comments from source to target YAML. :param source: the source, from which to transfer missing comments :param target: the target map, whose comments will be updated. :param comment_normalisation: the comment normalisation to assume; if NONE, no comments are transferred + :param forced_update_keys: keys for which comments are always transferred, even if present in target """ match comment_normalisation: case YamlCommentNormalisation.NONE: pass case YamlCommentNormalisation.LEADING | YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING: - transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE]) + transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE], forced_update_keys=forced_update_keys) case _: raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}")