diff --git a/.serena/project.yml b/.serena/project.yml index edba1eeb..d7c316a4 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -1,14 +1,45 @@ +# the name by which the project can be referenced within Serena +project_name: "serena" + + +# list of languages for which language servers are started; choose from: +# al bash clojure cpp csharp +# csharp_omnisharp dart elixir elm erlang +# fortran fsharp go groovy haskell +# java julia kotlin lua markdown +# matlab nix pascal perl php +# powershell python python_jedi r rego +# ruby ruby_solargraph rust scala swift +# terraform toml typescript typescript_vts vue +# yaml zig +# (This list may be outdated. For the current list, see values of Language enum here: +# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py +# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) +# Note: +# - For C, use cpp +# - For JavaScript, use typescript +# - For Free Pascal/Lazarus, use pascal +# Special requirements: +# - csharp: Requires the presence of a .sln file in the project folder. +# - pascal: Requires Free Pascal Compiler (fpc) and optionally Lazarus. +# When using multiple languages, the first language server that supports a given file will be used for that file. +# The first language is the default language and the respective language server will be used as a fallback. +# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. +languages: +- python +- typescript + + +# whether to use project's .gitignore files to ignore files ignore_all_files_in_gitignore: true -# list of additional paths to ignore + +# list of additional paths to ignore in all projects # same syntax as gitignore, so you can use * and ** -# Was previously called `ignored_dirs`, please update your config if you are using that. -# Added (renamed)on 2025-04-07 ignored_paths: [] # whether the project is in read-only mode # If set to true, all editing tools will be disabled and attempts to use them will result in an error -# Added on 2025-04-18 read_only: false @@ -53,13 +84,34 @@ read_only: false # * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. excluded_tools: [] +# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default) +included_optional_tools: [] + +# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. +# This cannot be combined with non-empty excluded_tools or included_optional_tools. # initial prompt for the project. It will always be given to the LLM upon activating the project # (contrary to the memories, which are loaded on demand). initial_prompt: "" -project_name: "serena" -languages: -- python -- typescript -included_optional_tools: [] + encoding: utf-8 + + +# list of mode names to that are always to be included in the set of active modes +# The full set of modes to be activated is base_modes + default_modes. +# If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this setting overrides the global configuration. +# Set this to [] to disable base modes for this project. +# Set this to a list of mode names to always include the respective modes for this project. +base_modes: + +# list of mode names that are to be activated by default. +# The full set of modes to be activated is base_modes + default_modes. +# If the setting is undefined, the default_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this overrides the setting from the global configuration (serena_config.yml). +# This setting can, in turn, be overridden by CLI parameters (--mode). +default_modes: + +# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. +# This cannot be combined with non-empty excluded_tools or included_optional_tools. +fixed_tools: [] diff --git a/docs/02-usage/040_workflow.md b/docs/02-usage/040_workflow.md index 91d27d3d..8f90fe68 100644 --- a/docs/02-usage/040_workflow.md +++ b/docs/02-usage/040_workflow.md @@ -53,6 +53,7 @@ The file allows you to configure ... * write access * an initial prompt that shall be passed to the LLM whenever the project is activated * the name by which you want to refer to the project (relevant when telling the LLM to dynamically activate the project) + * the set of tools and modes to use by default For detailed information on the parameters and possible settings, see the [template file](https://github.com/oraios/serena/blob/main/src/serena/resources/project.template.yml). diff --git a/docs/02-usage/050_configuration.md b/docs/02-usage/050_configuration.md index 716d541a..c86a7e72 100644 --- a/docs/02-usage/050_configuration.md +++ b/docs/02-usage/050_configuration.md @@ -25,6 +25,7 @@ Some of the configurable settings include: * the language backend to use by default (i.e., the JetBrains plugin or language servers) * UI settings affecting the [Serena Dashboard and GUI tool](060_dashboard.md) * the set of tools to enable/disable by default + * the set of modes to use by default * tool execution parameters (timeout, max. answer length) * global ignore rules * logging settings diff --git a/src/serena/agent.py b/src/serena/agent.py index dbeb27e8..a4b21978 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -6,7 +6,7 @@ import os import platform import subprocess import sys -from collections.abc import Callable +from collections.abc import Callable, Sequence from logging import Logger from typing import TYPE_CHECKING, Optional, TypeVar @@ -17,7 +17,7 @@ from interprompt.jinja_template import JinjaTemplate from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import LanguageBackend, SerenaConfig, ToolInclusionDefinition +from serena.config.serena_config import LanguageBackend, ModeSelectionDefinition, SerenaConfig, ToolInclusionDefinition from serena.dashboard import SerenaDashboardAPI from serena.ls_manager import LanguageServerManager from serena.project import Project @@ -167,6 +167,48 @@ class ToolSet: return tool_name in self._tool_names +class ActiveModes: + def __init__(self) -> None: + self._base_modes: Sequence[str] | None = None + self._default_modes: Sequence[str] | None = None + self._active_mode_names: Sequence[str] | None = [] + self._active_modes: Sequence[SerenaAgentMode] | None = [] + + def apply(self, mode_selection: ModeSelectionDefinition) -> None: + # invalidate active modes + self._active_mode_names = None + self._active_modes = None + + # apply overrides + log.debug("Applying mode selection: default_modes=%s, base_modes=%s", mode_selection.default_modes, mode_selection.base_modes) + if mode_selection.base_modes is not None: + self._base_modes = mode_selection.base_modes + if mode_selection.default_modes is not None: + self._default_modes = mode_selection.default_modes + log.debug("Current mode selection: base_modes=%s, default_modes=%s", self._base_modes, self._default_modes) + + def get_mode_names(self) -> Sequence[str]: + if self._active_mode_names is not None: + return self._active_mode_names + active_mode_names: set[str] = set() + if self._base_modes is not None: + active_mode_names.update(self._base_modes) + if self._default_modes is not None: + active_mode_names.update(self._default_modes) + self._active_mode_names = sorted(active_mode_names) + log.info("Active modes: %s", self._active_mode_names) + return self._active_mode_names + + def get_modes(self) -> Sequence[SerenaAgentMode]: + if self._active_modes is not None: + return self._active_modes + self._active_modes = [] + for mode_name in self.get_mode_names(): + mode = SerenaAgentMode.load(mode_name) + self._active_modes.append(mode) + return self._active_modes + + class SerenaAgent: def __init__( self, @@ -174,7 +216,7 @@ class SerenaAgent: project_activation_callback: Callable[[], None] | None = None, serena_config: SerenaConfig | None = None, context: SerenaAgentContext | None = None, - modes: list[SerenaAgentMode] | None = None, + modes: ModeSelectionDefinition | None = None, memory_log_handler: MemoryLogHandler | None = None, ): """ @@ -290,22 +332,20 @@ class SerenaAgent: self.prompt_factory = SerenaPromptFactory() self._project_activation_callback = project_activation_callback - # set the active modes - if modes is None: - modes = SerenaAgentMode.load_default_modes() - self._modes = modes - - # determine the subset of active tools (depending on active modes and the active project) - self._active_tools: AvailableTools = self._exposed_tools - self._update_active_tools() - # activate a project configuration (if provided or if there is only a single project available) if project is not None: try: - self.activate_project_from_path_or_name(project) + self.activate_project_from_path_or_name(project, update_modes_and_tools=False) except Exception as e: log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e) + # update active modes and active tools (considering the active project, if any) + # declared attributes are set in the call to _update_active_modes_and_tools() + self._mode_overrides = modes + self._active_modes: ActiveModes + self._active_tools: AvailableTools + self._update_active_modes_and_tools() + # start the dashboard (web frontend), registering its log handler # should be the last thing to happen in the initialization since the dashboard # may access various parts of the agent @@ -476,22 +516,22 @@ class SerenaAgent: raise ValueError("No active project. Please activate a project first.") return project - def set_modes(self, modes: list[SerenaAgentMode]) -> None: + def set_modes(self, mode_names: list[str]) -> None: """ Set the current mode configurations. - :param modes: List of mode names or paths to use + :param mode_names: List of mode names or paths to use """ - self._modes = modes - self._update_active_tools() + self._active_modes.apply(ModeSelectionDefinition(default_modes=mode_names)) + self._update_active_modes_and_tools() - log.info(f"Set modes to {[mode.name for mode in modes]}") + log.info(f"Set modes to {[mode.name for mode in self.get_active_modes()]}") def get_active_modes(self) -> list[SerenaAgentMode]: """ :return: the list of active modes """ - return list(self._modes) + return list(self._active_modes.get_modes()) def _format_prompt(self, prompt_template: str) -> str: template = JinjaTemplate(prompt_template) @@ -503,7 +543,7 @@ class SerenaAgent: log.info("Generating system prompt with available_tools=(see active tools), available_markers=%s", available_markers) system_prompt = self.prompt_factory.create_system_prompt( context_system_prompt=self._format_prompt(self._context.prompt), - mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self._modes], + mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self.get_active_modes()], available_tools=available_tools.tool_names, available_markers=available_markers, ) @@ -515,14 +555,22 @@ class SerenaAgent: log.info("System prompt:\n%s", system_prompt) return system_prompt - def _update_active_tools(self) -> None: + def _update_active_modes_and_tools(self) -> None: """ - Update the active tools based on enabled modes and the active project. + Updates the active modes and, subsequently, the active tools based on the modes and the active project. The base tool set already takes the Serena configuration and the context into account (as well as any internal modes that are not handled dynamically, such as JetBrains mode). """ + # determine active modes from serena config, active project, and mode overrides + self._active_modes = ActiveModes() + self._active_modes.apply(self.serena_config) + if self._active_project: + self._active_modes.apply(self._active_project.project_config) + if self._mode_overrides: + self._active_modes.apply(self._mode_overrides) + # apply modes - tool_set = self._base_tool_set.apply(*self._modes) + tool_set = self._base_tool_set.apply(*self._active_modes.get_modes()) # apply active project configuration (if any) if self._active_project is not None: @@ -569,10 +617,12 @@ class SerenaAgent: """ return self.serena_config.language_backend == LanguageBackend.LSP - def _activate_project(self, project: Project) -> None: + def _activate_project(self, project: Project, update_modes_and_tools: bool = True) -> None: log.info(f"Activating {project.project_name} at {project.project_root}") self._active_project = project - self._update_active_tools() + + if update_modes_and_tools: + self._update_active_modes_and_tools() def init_language_server_manager() -> None: # start the language server @@ -603,7 +653,7 @@ class SerenaAgent: log.info(f"Added new project {project_instance.project_name} for path {project_instance.project_root}") return project_instance - def activate_project_from_path_or_name(self, project_root_or_name: str) -> Project: + def activate_project_from_path_or_name(self, project_root_or_name: str, update_modes_and_tools: bool = True) -> Project: """ Activate a project from a path or a name. If the project was already registered, it will just be activated. @@ -619,7 +669,7 @@ class SerenaAgent: f"Project '{project_root_or_name}' not found: Not a valid project name or directory. " f"Existing project names: {self.serena_config.project_names}" ) - self._activate_project(project_instance) + self._activate_project(project_instance, update_modes_and_tools=update_modes_and_tools) return project_instance def get_active_tool_names(self) -> list[str]: diff --git a/src/serena/cli.py b/src/serena/cli.py index fba9ccdc..ecb5dcca 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -5,7 +5,7 @@ import os import shutil import subprocess import sys -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from logging import Logger from pathlib import Path from typing import Any, Literal @@ -18,10 +18,16 @@ from tqdm import tqdm from serena.agent import SerenaAgent from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import LanguageBackend, ProjectConfig, RegisteredProject, SerenaConfig, SerenaPaths +from serena.config.serena_config import ( + LanguageBackend, + ModeSelectionDefinition, + ProjectConfig, + RegisteredProject, + SerenaConfig, + SerenaPaths, +) from serena.constants import ( DEFAULT_CONTEXT, - DEFAULT_MODES, PROMPT_TEMPLATES_DIR_INTERNAL, SERENA_LOG_FORMAT, SERENAS_OWN_CONTEXT_YAMLS_DIR, @@ -30,6 +36,7 @@ from serena.constants import ( from serena.mcp import SerenaMCPFactory from serena.project import Project from serena.tools import FindReferencingSymbolsTool, FindSymbolTool, GetSymbolsOverviewTool, SearchForPatternTool, ToolRegistry +from serena.util.dataclass import get_dataclass_default from serena.util.logging import MemoryLogHandler from solidlsp.ls_config import Language from solidlsp.util.subprocess_util import subprocess_kwargs @@ -37,6 +44,15 @@ from solidlsp.util.subprocess_util import subprocess_kwargs log = logging.getLogger(__name__) _MAX_CONTENT_WIDTH = 100 +_MODES_EXPLANATION = f"""\b\nBuilt-in mode names or paths to custom mode YAMLs with which to +override the default modes defined in the global Serena configuration or +the active project. +For details on mode configuration, see + https://oraios.github.io/serena/02-usage/050_configuration.html#modes. +If no configuration changes were made, the base defaults are: + {get_dataclass_default(SerenaConfig, "default_modes")}. +Overriding them means that they no longer apply, so you will need to +re-specify them in addition to further modes if you want to keep them.""" def find_project_root(root: str | Path | None = None) -> str: @@ -152,9 +168,9 @@ class TopLevelCommands(AutoRegisteringGroup): "modes", type=str, multiple=True, - default=DEFAULT_MODES, - show_default=True, - help="Built-in mode names or paths to custom mode YAMLs.", + default=(), + show_default=False, + help=_MODES_EXPLANATION, ) @click.option( "--language-backend", @@ -221,7 +237,7 @@ class TopLevelCommands(AutoRegisteringGroup): project_file_arg: str | None, project_from_cwd: bool | None, context: str, - modes: tuple[str, ...], + modes: Sequence[str], language_backend: str | None, transport: Literal["stdio", "sse", "streamable-http"], host: str, @@ -302,11 +318,13 @@ class TopLevelCommands(AutoRegisteringGroup): "modes", type=str, multiple=True, - default=DEFAULT_MODES, - show_default=True, - help="Built-in mode names or paths to custom mode YAMLs.", + default=(), + show_default=False, + help=_MODES_EXPLANATION, ) - def print_system_prompt(project: str, log_level: str, only_instructions: bool, context: str, modes: tuple[str, ...]) -> None: + def print_system_prompt( + project: str, log_level: str, only_instructions: bool, context: str, modes: Sequence[str] | None = None + ) -> None: prefix = "You will receive access to Serena's symbolic tools. Below are instructions for using them, take them into account." postfix = "You begin by acknowledging that you understood the above instructions and are ready to receive tasks." from serena.tools.workflow_tools import InitialInstructionsTool @@ -314,12 +332,14 @@ class TopLevelCommands(AutoRegisteringGroup): lvl = logging.getLevelNamesMapping()[log_level.upper()] logging.configure(level=lvl) context_instance = SerenaAgentContext.load(context) - mode_instances = [SerenaAgentMode.load(mode) for mode in modes] + modes_selection_def: ModeSelectionDefinition | None = None + if modes: + modes_selection_def = ModeSelectionDefinition(default_modes=modes) agent = SerenaAgent( project=os.path.abspath(project), serena_config=SerenaConfig(web_dashboard=False, log_level=lvl), context=context_instance, - modes=mode_instances, + modes=modes_selection_def, ) tool = agent.get_tool(InitialInstructionsTool) instr = tool.apply() diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index 9238d806..e417fe32 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -14,7 +14,6 @@ from sensai.util.string import ToStringMixin from serena.config.serena_config import SerenaPaths, ToolInclusionDefinition from serena.constants import ( DEFAULT_CONTEXT, - DEFAULT_MODES, INTERNAL_MODE_YAMLS_DIR, SERENA_FILE_ENCODING, SERENAS_OWN_CONTEXT_YAMLS_DIR, @@ -117,11 +116,6 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): """Names of all custom modes defined by the user.""" return [f.stem for f in Path(SerenaPaths().user_modes_dir).glob("*.yml")] - @classmethod - def load_default_modes(cls) -> list[Self]: - """Load the default modes (interactive and editing).""" - return [cls.from_name(mode) for mode in DEFAULT_MODES] - @classmethod def load(cls, name_or_path: str | Path) -> Self: # Check if it's a file path that exists diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index bde964b9..0f108bac 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -28,13 +28,14 @@ from serena.constants import ( SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME, ) -from serena.util.general import get_dataclass_default, load_yaml, save_yaml from serena.util.inspection import determine_programming_language_composition +from serena.util.yaml import YamlCommentNormalisation, load_yaml, normalise_yaml_comments, save_yaml, transfer_missing_yaml_comments from solidlsp.ls_config import Language from ..analytics import RegisteredTokenCountEstimator from ..util.class_decorators import singleton from ..util.cli_util import ask_yes_no +from ..util.dataclass import get_dataclass_default if TYPE_CHECKING: from ..project import Project @@ -127,6 +128,12 @@ class ToolInclusionDefinition: return num_fixed > 0 +@dataclass +class ModeSelectionDefinition: + base_modes: Sequence[str] | None = None + default_modes: Sequence[str] | None = None + + class SerenaConfigError(Exception): pass @@ -156,7 +163,7 @@ class LanguageBackend(Enum): @dataclass(kw_only=True) -class ProjectConfig(ToolInclusionDefinition, ToStringMixin): +class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMixin): project_name: str languages: list[Language] ignored_paths: list[str] = field(default_factory=list) @@ -166,6 +173,12 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): encoding: str = DEFAULT_SOURCE_FILE_ENCODING SERENA_DEFAULT_PROJECT_FILE = "project.yml" + FIELDS_WITHOUT_DEFAULTS = {"project_name", "languages"} + YAML_COMMENT_NORMALISATION = YamlCommentNormalisation.LEADING + """ + the comment normalisation strategy to use when loading/saving project configuration files. + The template file must match this configuration (i.e. it must use leading comments if this is set to LEADING). + """ def _tostring_includes(self) -> list[str]: return ["project_name"] @@ -239,13 +252,13 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): log.info("Using languages: %s", languages_to_use) else: languages_to_use = [lang.value for lang in languages] - config_with_comments = cls.load_commented_map(PROJECT_TEMPLATE_FILE) + config_with_comments, _ = cls._load_yaml(PROJECT_TEMPLATE_FILE) config_with_comments["project_name"] = project_name config_with_comments["languages"] = languages_to_use if save_to_disk: project_yml_path = cls.path_to_project_yml(project_root) log.info("Saving project configuration to %s", project_yml_path) - save_yaml(project_yml_path, config_with_comments, preserve_comments=True) + save_yaml(project_yml_path, config_with_comments) return cls._from_dict(config_with_comments) @classmethod @@ -257,37 +270,38 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): return os.path.join(SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE) @classmethod - def _apply_defaults_to_dict(cls, data: TDict) -> TDict: - # apply defaults for new fields - data["languages"] = data.get("languages", []) - data["ignored_paths"] = data.get("ignored_paths", []) - data["excluded_tools"] = data.get("excluded_tools", []) - data["included_optional_tools"] = data.get("included_optional_tools", []) - data["read_only"] = data.get("read_only", False) - data["ignore_all_files_in_gitignore"] = data.get("ignore_all_files_in_gitignore", True) - data["initial_prompt"] = data.get("initial_prompt", "") - data["encoding"] = data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING) - - # backward compatibility: handle single "language" field - if len(data["languages"]) == 0 and "language" in data: - data["languages"] = [data["language"]] - if "language" in data: - del data["language"] - - return data - - @classmethod - def load_commented_map(cls, yml_path: str) -> CommentedMap: + def _load_yaml( + cls, yml_path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE + ) -> tuple[CommentedMap, bool]: """ Load the project configuration as a CommentedMap, preserving comments and ensuring completeness of the configuration by applying default values for missing fields and backward compatibility adjustments. :param yml_path: the path to the project.yml file - :return: a CommentedMap representing a full project configuration + :return: a tuple `(dict, was_complete)` where dict is a CommentedMap representing a + full project configuration and `was_complete` indicates whether the loaded configuration + was complete (i.e., did not require any default values to be applied) """ - data = load_yaml(yml_path, preserve_comments=True) - return cls._apply_defaults_to_dict(data) + data = load_yaml(yml_path, comment_normalisation=comment_normalisation) + + # apply defaults + was_complete = True + for field_info in dataclasses.fields(cls): + key = field_info.name + if key in cls.FIELDS_WITHOUT_DEFAULTS: + continue + if key not in data: + was_complete = False + default_value = get_dataclass_default(cls, key) + data.setdefault(key, default_value) + + # backward compatibility: handle single "language" field + if "languages" not in data and "language" in data: + data["languages"] = [data["language"]] + del data["language"] + + return data, was_complete @classmethod def _from_dict(cls, data: dict[str, Any]) -> Self: @@ -314,14 +328,17 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): languages=languages, ignored_paths=data["ignored_paths"], excluded_tools=data["excluded_tools"], + fixed_tools=data["fixed_tools"], included_optional_tools=data["included_optional_tools"], read_only=data["read_only"], ignore_all_files_in_gitignore=data["ignore_all_files_in_gitignore"], initial_prompt=data["initial_prompt"], encoding=data["encoding"], + base_modes=data["base_modes"], + default_modes=data["default_modes"], ) - def to_yaml_dict(self) -> dict: + def _to_yaml_dict(self) -> dict: """ :return: a yaml-serializable dictionary representation of this configuration """ @@ -336,15 +353,47 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): """ project_root = Path(project_root) yaml_path = project_root / cls.rel_path_to_project_yml() + + # auto-generate if necessary if not yaml_path.exists(): if autogenerate: return cls.autogenerate(project_root) else: raise FileNotFoundError(f"Project configuration file not found: {yaml_path}") - yaml_data = cls.load_commented_map(str(yaml_path)) + + # load the configuration dictionary + yaml_data, was_complete = cls._load_yaml(str(yaml_path)) if "project_name" not in yaml_data: yaml_data["project_name"] = project_root.name - return cls._from_dict(yaml_data) + + # instantiate the ProjectConfig + project_config = cls._from_dict(yaml_data) + + # if the configuration was incomplete, re-save it to disk + if not was_complete: + log.info("Project configuration in %s was incomplete, re-saving with default values for missing fields", yaml_path) + project_config.save(project_root) + + return project_config + + def save(self, project_root: Path | str) -> None: + """ + Saves the project configuration to disk. + + :param project_root: the root directory of the project + """ + config_path = self.path_to_project_yml(project_root) + log.info("Saving updated project configuration to %s", config_path) + + # load original commented map and update it with current values + config_with_comments, _ = self._load_yaml(config_path, self.YAML_COMMENT_NORMALISATION) + config_with_comments.update(self._to_yaml_dict()) + + # transfer missing comments from the template file + template_config, _ = self._load_yaml(PROJECT_TEMPLATE_FILE, self.YAML_COMMENT_NORMALISATION) + transfer_missing_yaml_comments(template_config, config_with_comments, self.YAML_COMMENT_NORMALISATION) + + save_yaml(config_path, config_with_comments) class RegisteredProject(ToStringMixin): @@ -404,7 +453,7 @@ class RegisteredProject(ToStringMixin): @dataclass(kw_only=True) -class SerenaConfig(ToolInclusionDefinition, ToStringMixin): +class SerenaConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMixin): """ Holds the Serena agent configuration, which is typically loaded from a YAML configuration file (when instantiated via :method:`from_config_file`), which is updated when projects are added or removed. @@ -444,6 +493,9 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): ls_specific_settings: dict = field(default_factory=dict) """Advanced configuration option allowing to configure language server implementation specific options, see SolidLSPSettings for more info.""" + # settings with overridden defaults + default_modes: Sequence[str] | None = ("interactive", "editing") + # *** fields that are NOT mapped to/from the configuration file *** _loaded_commented_yaml: CommentedMap | None = None @@ -484,8 +536,8 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): :param config_file_path: the path where the configuration file should be generated """ log.info(f"Auto-generating Serena configuration file in {config_file_path}") - loaded_commented_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE, preserve_comments=True) - save_yaml(config_file_path, loaded_commented_yaml, preserve_comments=True) + loaded_commented_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE) + save_yaml(config_file_path, loaded_commented_yaml) @classmethod def _determine_config_file_path(cls) -> str: @@ -521,7 +573,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): # load the configuration log.info(f"Loading Serena configuration from {config_file_path}") try: - loaded_commented_yaml = load_yaml(config_file_path, preserve_comments=True) + loaded_commented_yaml = load_yaml(config_file_path) except Exception as e: raise ValueError(f"Error loading Serena configuration from {config_file_path}: {e}") from e @@ -729,7 +781,15 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): # convert language backend to string commented_yaml["language_backend"] = self.language_backend.value - save_yaml(self.config_file_path, commented_yaml, preserve_comments=True) + # transfer comments from the template file + # NOTE: The template file now uses leading comments, but we previously used trailing comments, + # so we apply a conversion, which detects the old style and transforms it. + # For some keys, we force updates, because old comments are problematic/misleading. + normalise_yaml_comments(commented_yaml, YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING) + template_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE, comment_normalisation=YamlCommentNormalisation.LEADING) + transfer_missing_yaml_comments(template_yaml, commented_yaml, YamlCommentNormalisation.LEADING, forced_update_keys=["projects"]) + + save_yaml(self.config_file_path, commented_yaml) def propagate_settings(self) -> None: """ diff --git a/src/serena/constants.py b/src/serena/constants.py index f5ccd660..3a9e2bf5 100644 --- a/src/serena/constants.py +++ b/src/serena/constants.py @@ -20,7 +20,6 @@ SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons") DEFAULT_SOURCE_FILE_ENCODING = "utf-8" """The default encoding assumed for project source files.""" DEFAULT_CONTEXT = "desktop-app" -DEFAULT_MODES = ("interactive", "editing") SERENA_FILE_ENCODING = "utf-8" """The encoding used for Serena's own files, such as configuration files and memories.""" diff --git a/src/serena/mcp.py b/src/serena/mcp.py index b0367717..d7ff379b 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -21,9 +21,9 @@ from serena.agent import ( SerenaAgent, SerenaConfig, ) -from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import LanguageBackend -from serena.constants import DEFAULT_CONTEXT, DEFAULT_MODES, SERENA_LOG_FORMAT +from serena.config.context_mode import SerenaAgentContext +from serena.config.serena_config import LanguageBackend, ModeSelectionDefinition +from serena.constants import DEFAULT_CONTEXT, SERENA_LOG_FORMAT from serena.tools import Tool from serena.util.exception import show_fatal_exception_safe from serena.util.logging import MemoryLogHandler @@ -260,7 +260,7 @@ class SerenaMCPFactory: mcp._tool_manager._tools[tool.get_name()] = mcp_tool log.info(f"Starting MCP server with {len(mcp._tool_manager._tools)} tools: {list(mcp._tool_manager._tools.keys())}") - def _create_serena_agent(self, serena_config: SerenaConfig, modes: list[SerenaAgentMode]) -> SerenaAgent: + def _create_serena_agent(self, serena_config: SerenaConfig, modes: ModeSelectionDefinition | None = None) -> SerenaAgent: return SerenaAgent( project=self.project, serena_config=serena_config, context=self.context, modes=modes, memory_log_handler=self.memory_log_handler ) @@ -272,7 +272,7 @@ class SerenaMCPFactory: self, host: str = "0.0.0.0", port: int = 8000, - modes: Sequence[str] = DEFAULT_MODES, + modes: Sequence[str] = (), language_backend: LanguageBackend | None = None, enable_web_dashboard: bool | None = None, enable_gui_log_window: bool | None = None, @@ -318,8 +318,10 @@ class SerenaMCPFactory: if language_backend is not None: config.language_backend = language_backend - modes_instances = [SerenaAgentMode.load(mode) for mode in modes] - self.agent = self._create_serena_agent(config, modes_instances) + mode_selection_def: ModeSelectionDefinition | None = None + if modes: + mode_selection_def = ModeSelectionDefinition(default_modes=modes) + self.agent = self._create_serena_agent(config, mode_selection_def) except Exception as e: show_fatal_exception_safe(e) diff --git a/src/serena/project.py b/src/serena/project.py index 90a2a33e..06f39954 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -12,7 +12,6 @@ from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME from serena.ls_manager import LanguageServerFactory, LanguageServerManager from serena.text_utils import MatchedConsecutiveLines, search_files from serena.util.file_system import GitignoreParser, match_path -from serena.util.general import save_yaml from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import FileUtils @@ -114,11 +113,7 @@ class Project(ToStringMixin): """ Saves the current project configuration to disk. """ - config_path = os.path.join(self.project_root, self.project_config.rel_path_to_project_yml()) - log.info("Saving updated project configuration to %s", config_path) - config_with_comments = ProjectConfig.load_commented_map(config_path) - config_with_comments.update(self.project_config.to_yaml_dict()) - save_yaml(config_path, config_with_comments, preserve_comments=True) + self.project_config.save(self.project_root) def path_to_serena_data_folder(self) -> str: return os.path.join(self.project_root, SERENA_MANAGED_DIR_NAME) diff --git a/src/serena/resources/project.template.yml b/src/serena/resources/project.template.yml index a0f79106..9472ba95 100644 --- a/src/serena/resources/project.template.yml +++ b/src/serena/resources/project.template.yml @@ -1,3 +1,7 @@ +# the name by which the project can be referenced within Serena +project_name: "project_name" + + # list of languages for which language servers are started; choose from: # al bash clojure cpp csharp # csharp_omnisharp dart elixir elm erlang @@ -16,8 +20,8 @@ # - For JavaScript, use typescript # - For Free Pascal/Lazarus, use pascal # Special requirements: -# - csharp: Requires the presence of a .sln file in the project folder. -# - pascal: Requires Free Pascal Compiler (fpc) and optionally Lazarus. +# Some languages require additional setup/installations. +# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers # When using multiple languages, the first language server that supports a given file will be used for that file. # The first language is the default language and the respective language server will be used as a fallback. # Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. @@ -80,8 +84,28 @@ read_only: false # * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. excluded_tools: [] +# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default) +included_optional_tools: [] + +# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. +# This cannot be combined with non-empty excluded_tools or included_optional_tools. +fixed_tools: [] + +# list of mode names to that are always to be included in the set of active modes +# The full set of modes to be activated is base_modes + default_modes. +# If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this setting overrides the global configuration. +# Set this to [] to disable base modes for this project. +# Set this to a list of mode names to always include the respective modes for this project. +base_modes: + +# list of mode names that are to be activated by default. +# The full set of modes to be activated is base_modes + default_modes. +# If the setting is undefined, the default_modes from the global configuration (serena_config.yml) apply. +# Otherwise, this overrides the setting from the global configuration (serena_config.yml). +# This setting can, in turn, be overridden by CLI parameters (--mode). +default_modes: + # initial prompt for the project. It will always be given to the LLM upon activating the project # (contrary to the memories, which are loaded on demand). initial_prompt: "" - -project_name: "project_name" diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index ddb41e35..7a660e64 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -1,4 +1,3 @@ -language_backend: LSP # the language backend to use for code understanding and manipulation. # Possible values are: # * LSP: Use the language server protocol (LSP), spawning freely available language servers @@ -6,8 +5,8 @@ language_backend: LSP # * JetBrains: Use the Serena plugin in your JetBrains IDE. # (requires the plugin to be installed and the project being worked on to be open # in your IDE). +language_backend: LSP -gui_log_window: False # whether to open a graphical window with Serena's logs. # This is mainly supported on Windows and (partly) on Linux; not available on macOS. # If you prefer a browser-based tool, use the `web_dashboard` option instead. @@ -22,8 +21,8 @@ gui_log_window: False # previous instances. This can lead to multiple log windows being opened, and only the last # window being updated. Since we can't control how agno or Claude Desktop start Serena, # we have to live with this limitation for now. +gui_log_window: False -web_dashboard: True # whether to open the Serena web dashboard (which will be accessible through your web browser) that # provides access to Serena's configuration and state as well as the current session logs. # The web dashboard is supported on all platforms. @@ -34,11 +33,11 @@ web_dashboard: True # You can then open the dashboard by asking your agent to do so (e.g., by saying "open the dashboard"), Serena provides # a tool for this. # Further information: https://oraios.github.io/serena/02-usage/060_dashboard.html +web_dashboard: True +# the address the web dashboard will listen on (bind address). web_dashboard_listen_address: 127.0.0.1 -# The address where the web dashboard will listen on -web_dashboard_open_on_launch: True # whether to open a browser window with the web dashboard when Serena starts (provided that web_dashboard # is enabled). See also the web_dashboard option. # If set to false, you can still open the dashboard manually by @@ -46,43 +45,55 @@ web_dashboard_open_on_launch: True # b) manually navigating to http://localhost:24282/dashboard/ in your web browser (actual port # may be higher if you have multiple instances running; try ports 24283, 24284, etc.) # See also: https://oraios.github.io/serena/02-usage/060_dashboard.html +web_dashboard_open_on_launch: True -jetbrains_plugin_server_address: 127.0.0.1 # address where JetBrains plugin servers are running (only relevant when using the JetBrains language backend) +jetbrains_plugin_server_address: 127.0.0.1 -log_level: 20 # the minimum log level for the GUI log window and the dashboard (10 = debug, 20 = info, 30 = warning, 40 = error) +log_level: 20 -trace_lsp_communication: False # whether to trace the communication between Serena and the language servers. # This is useful for debugging language server issues. +trace_lsp_communication: False -ls_specific_settings: {} -# Added on 23.08.2025 -# Advanced configuration option allowing to configure language server implementation specific options. Maps the language -# (same entry as in project.yml) to the options. +# advanced configuration option allowing to configure language server-specific options. +# Maps the language key to the options. # Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. # No documentation on options means no options are available. +ls_specific_settings: {} -tool_timeout: 240 # timeout, in seconds, after which tool executions are terminated +tool_timeout: 240 -excluded_tools: [] # list of tools to be globally excluded +excluded_tools: [] -included_optional_tools: [] # list of optional tools (which are disabled by default) to be included +included_optional_tools: [] -fixed_tools: [] # fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. # This cannot be combined with non-empty excluded_tools or included_optional_tools. +fixed_tools: [] + +# list of mode names to that are always to be included in the set of active modes +# The full set of modes to be activated is base_modes + default_modes. +# If this is undefined, no base modes are included. +# The project configuration (project.yml) may override this setting. +base_modes: + +# list of mode names that are to be activated by default. +# The full set of modes to be activated is base_modes + default_modes. +# These modes can be overridden by the project configuration (project.yml) or through the CLI (--mode). +default_modes: + - interactive + - editing -default_max_tool_answer_chars: 150000 # Used as default for tools where the apply method has a default maximal answer length. # Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default # through the global configuration. +default_max_tool_answer_chars: 150000 -token_count_estimator: CHAR_COUNT # the name of the token count estimator to use for tool usage statistics. # See the `RegisteredTokenCountEstimator` enum for available options. # @@ -90,6 +101,7 @@ token_count_estimator: CHAR_COUNT # You can configure this to TIKTOKEN_GPT4 to use a local tiktoken-based estimator for GPT-4 (will download tiktoken # data files on first run), or ANTHROPIC_CLAUDE_SONNET_4 which will use the (free of cost) Anthropic API to # estimate the token count using the Claude Sonnet 4 tokenizer. +token_count_estimator: CHAR_COUNT -# The list of registered project paths (updated automatically). +# the list of registered project paths (updated automatically). projects: [] diff --git a/src/serena/tools/config_tools.py b/src/serena/tools/config_tools.py index 7de51e59..7a0472cf 100644 --- a/src/serena/tools/config_tools.py +++ b/src/serena/tools/config_tools.py @@ -1,4 +1,3 @@ -from serena.config.context_mode import SerenaAgentMode from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional @@ -61,11 +60,11 @@ class SwitchModesTool(Tool, ToolMarkerOptional): :param modes: the names of the modes to activate """ - mode_instances = [SerenaAgentMode.load(mode) for mode in modes] - self.agent.set_modes(mode_instances) + self.agent.set_modes(modes) # Inform the Agent about the activated modes and the currently active tools - result_str = f"Successfully activated modes: {', '.join([mode.name for mode in mode_instances])}" + "\n" + mode_instances = self.agent.get_active_modes() + result_str = f"Active modes: {', '.join([mode.name for mode in mode_instances])}" + "\n" result_str += "\n".join([mode_instance.prompt for mode_instance in mode_instances]) + "\n" result_str += f"Currently active tools: {', '.join(self.agent.get_active_tool_names())}" return result_str diff --git a/src/serena/util/dataclass.py b/src/serena/util/dataclass.py new file mode 100644 index 00000000..c5518bf4 --- /dev/null +++ b/src/serena/util/dataclass.py @@ -0,0 +1,21 @@ +from dataclasses import MISSING, Field +from typing import Any, cast + + +def get_dataclass_default(cls: type, field_name: str) -> Any: + """ + Gets the default value of a dataclass field. + + :param cls: The dataclass type. + :param field_name: The name of the field. + :return: The default value of the field (either from default or default_factory). + """ + field = cast(Field, cls.__dataclass_fields__[field_name]) # type: ignore[attr-defined] + + if field.default is not MISSING: + return field.default + + if field.default_factory is not MISSING: # default_factory is a function + return field.default_factory() + + raise AttributeError(f"{field_name} has no default") diff --git a/src/serena/util/general.py b/src/serena/util/general.py deleted file mode 100644 index f380c699..00000000 --- a/src/serena/util/general.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -from dataclasses import MISSING, Field -from typing import Any, Literal, cast, overload - -from ruamel.yaml import YAML -from ruamel.yaml.comments import CommentedMap - -from serena.constants import SERENA_FILE_ENCODING - - -def _create_YAML(preserve_comments: bool = False) -> YAML: - """ - Creates a YAML that can load/save with comments if preserve_comments is True. - """ - typ = None if preserve_comments else "safe" - result = YAML(typ=typ) - result.preserve_quotes = preserve_comments - return result - - -@overload -def load_yaml(path: str, preserve_comments: Literal[False]) -> dict: ... -@overload -def load_yaml(path: str, preserve_comments: Literal[True]) -> CommentedMap: ... -def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap: - with open(path, encoding=SERENA_FILE_ENCODING) as f: - yaml = _create_YAML(preserve_comments) - return yaml.load(f) - - -def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = False) -> None: - yaml = _create_YAML(preserve_comments) - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w", encoding=SERENA_FILE_ENCODING) as f: - yaml.dump(data, f) - - -def get_dataclass_default(cls: type, field_name: str) -> Any: - """ - Gets the default value of a dataclass field. - - :param cls: The dataclass type. - :param field_name: The name of the field. - :return: The default value of the field (either from default or default_factory). - """ - field = cast(Field, cls.__dataclass_fields__[field_name]) # type: ignore[attr-defined] - - if field.default is not MISSING: - return field.default - - if field.default_factory is not MISSING: # default_factory is a function - return field.default_factory() - - raise AttributeError(f"{field_name} has no default") diff --git a/src/serena/util/yaml.py b/src/serena/util/yaml.py new file mode 100644 index 00000000..32a8f364 --- /dev/null +++ b/src/serena/util/yaml.py @@ -0,0 +1,248 @@ +import logging +import os +from collections.abc import Sequence +from enum import Enum +from typing import Any + +from ruamel.yaml import YAML, CommentToken, StreamMark +from ruamel.yaml.comments import CommentedMap + +from serena.constants import SERENA_FILE_ENCODING + +log = logging.getLogger(__name__) + + +def _create_yaml(preserve_comments: bool = False) -> YAML: + """ + Creates a YAML that can load/save with comments if preserve_comments is True. + """ + typ = None if preserve_comments else "safe" + result = YAML(typ=typ) + result.preserve_quotes = preserve_comments + return result + + +class YamlCommentNormalisation(Enum): + """ + Defines a normalisation to be applied to the comment representation in a ruamel CommentedMap. + + Note that even though a YAML document may seem to consistently contain, for example, leading comments + before a key only, ruamel may still parse some comments as trailing comments of the previous key + or as document-level comments. + The normalisations define ways to adjust the comment representation accordingly, clearly associating + comments with the keys they belong to. + """ + + NONE = "none" + """ + No comment normalisation is performed. + Comments are kept as parsed by ruamel.yaml. + """ + LEADING = "leading" + """ + Document is assumed to have leading comments only, i.e. comments before keys, only full-line comments. + This normalisation achieves that comments are properly associated with keys as leading comments. + """ + LEADING_WITH_CONVERSION_FROM_TRAILING = "leading_with_conversion_from_trailing" + """ + Document is assumed to have a mixture of leading comments (before keys) and trailing comments (after values), only full-line comments. + This normalisation achieves that all comments are converted to leading comments and properly associated with keys. + """ + # NOTE: Normalisation for trailing comments was attempted but is extremely hard, because + # it is difficult to position the comments properly after values, especially for complex values. + + +DOC_COMMENT_INDEX_POST = 0 +DOC_COMMENT_INDEX_PRE = 1 + +# item comment indices: (post key, pre key, post value, pre value) +ITEM_COMMENT_INDEX_BEFORE = 1 # (pre-key; must be a list of CommentToken at this index) +ITEM_COMMENT_INDEX_AFTER = 2 # (post-value; must be an instance of CommentToken at this index) + + +def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE) -> CommentedMap: + """ + :param path: the path to the YAML file to load + :param comment_normalisation: the comment normalisation to apply after loading + :return: the loaded commented map + """ + with open(path, encoding=SERENA_FILE_ENCODING) as f: + yaml = _create_yaml(preserve_comments=True) + commented_map: CommentedMap = yaml.load(f) + normalise_yaml_comments(commented_map, comment_normalisation) + return commented_map + + +def normalise_yaml_comments(commented_map: CommentedMap, comment_normalisation: YamlCommentNormalisation) -> None: + """ + Applies the given comment normalisation to the given commented map in-place. + + :param commented_map: the commented map whose comments are to be normalised + :param comment_normalisation: the comment normalisation to apply + """ + + def make_list(comment_entry: Any) -> list: + if not isinstance(comment_entry, list): + return [comment_entry] + return comment_entry + + def make_unit(comment_entry: Any) -> Any: + """ + Converts a list-valued comment entry into a single comment entry. + """ + if isinstance(comment_entry, list): + if len(comment_entry) == 0: + return None + elif len(comment_entry) == 1: + return comment_entry[0] + else: + if all(isinstance(item, CommentToken) for item in comment_entry): + start_mark = StreamMark(name="", index=0, line=0, column=0) + comment_str = "".join(item.value for item in comment_entry) + if not comment_str.startswith("\n"): + comment_str = "\n" + comment_str + return CommentToken(value=comment_str, start_mark=start_mark, end_mark=None) + else: + types = set(type(item) for item in comment_entry) + log.warning("Unhandled types in list-valued comment entry: %s; not updating entry", types) + return None + else: + return comment_entry + + def trailing_to_leading(comment_entry: Any) -> Any: + if comment_entry is None: + return None + token_list = make_list(comment_entry) + first_token = token_list[0] + if isinstance(first_token, CommentToken): + # remove leading newline if present + if first_token.value.startswith("\n"): + first_token.value = first_token.value[1:] + return token_list + + match comment_normalisation: + case YamlCommentNormalisation.NONE: + pass + case YamlCommentNormalisation.LEADING | YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING: + # Comments are supposed to be leading comments (i.e., before a key and associated with the key). + # When ruamel parses a YAML, however, comments belonging to a key may be stored as trailing + # comments of the previous key or as a document-level comment. + # Move them accordingly. + keys = list(commented_map.keys()) + comment_items = commented_map.ca.items + doc_comment = commented_map.ca.comment + preceding_comment = None + for i, key in enumerate(keys): + current_comment = comment_items.get(key, [None] * 4) + comment_items[key] = current_comment + if current_comment[ITEM_COMMENT_INDEX_BEFORE] is None: + if i == 0 and doc_comment is not None and doc_comment[DOC_COMMENT_INDEX_PRE] is not None: + # move document pre-comment to leading comment of first key + current_comment[ITEM_COMMENT_INDEX_BEFORE] = make_list(doc_comment[DOC_COMMENT_INDEX_PRE]) + doc_comment[DOC_COMMENT_INDEX_PRE] = None + elif preceding_comment is not None and preceding_comment[ITEM_COMMENT_INDEX_AFTER] is not None: + # move trailing comment of preceding key to leading comment of current key + current_comment[ITEM_COMMENT_INDEX_BEFORE] = trailing_to_leading(preceding_comment[ITEM_COMMENT_INDEX_AFTER]) + preceding_comment[ITEM_COMMENT_INDEX_AFTER] = None + preceding_comment = current_comment + + if comment_normalisation == YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING: + # Second pass: conversion of trailing comments + # If a leading comment ends with "\n\n", i.e. it has an empty line between the comment and the key, + # it was actually intended as a trailing comment for the preceding key, so we associate it with + # the preceding key instead (if the preceding key has no leading comment already). + preceding_comment = None + for key in keys: + current_comment = comment_items.get(key, [None] * 4) + if current_comment[ITEM_COMMENT_INDEX_BEFORE] is not None: + token_list = make_list(current_comment[ITEM_COMMENT_INDEX_BEFORE]) + if len(token_list) > 0: + last_token = token_list[-1] + if isinstance(last_token, CommentToken) and last_token.value.endswith("\n\n"): + # move comment to preceding key, removing the empty line, + # and adding an empty line at the beginning instead + if preceding_comment is not None and yaml_comment_entry_is_empty( + preceding_comment[ITEM_COMMENT_INDEX_BEFORE] + ): + last_token.value = last_token.value[:-1] + + first_token = token_list[0] + if isinstance(first_token, CommentToken): + if not first_token.value.startswith("\n"): + first_token.value = "\n" + first_token.value + + preceding_comment[ITEM_COMMENT_INDEX_BEFORE] = token_list + current_comment[ITEM_COMMENT_INDEX_BEFORE] = None + preceding_comment = current_comment + case _: + raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}") + + +def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = True) -> None: + yaml = _create_yaml(preserve_comments) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding=SERENA_FILE_ENCODING) as f: + yaml.dump(data, f) + + +def yaml_comment_entry_is_empty(comment_entry: Any) -> bool: + if comment_entry is None: + return True + elif isinstance(comment_entry, list): + for item in comment_entry: + if isinstance(item, CommentToken): + if item.value.strip() != "": + return False + else: + return False + return True + elif isinstance(comment_entry, CommentToken): + return comment_entry.value.strip() == "" + else: + return False + + +def transfer_missing_yaml_comments_by_index( + source: CommentedMap, target: CommentedMap, indices: list[int], forced_update_keys: Sequence[str] = () +) -> None: + """ + :param source: the source, from which to transfer missing comments + :param target: the target map, whose comments will be updated + :param indices: list of comment indices to transfer + :param forced_update_keys: keys for which comments are always transferred, even if present in target + """ + for key in target.keys(): + if key in source: + source_comment = source.ca.items.get(key) + if source_comment is None: + continue + target_comment = target.ca.items.get(key) + # initialise target comment if needed + if target_comment is None: + target_comment = [None] * 4 + target.ca.items[key] = target_comment + # transfer comments at specified indices + for index in indices: + is_forced_update = key in forced_update_keys + if is_forced_update or yaml_comment_entry_is_empty(target_comment[index]): + target_comment[index] = source_comment[index] + + +def transfer_missing_yaml_comments( + source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation, forced_update_keys: Sequence[str] = () +) -> None: + """ + Transfers missing comments from source to target YAML. + + :param source: the source, from which to transfer missing comments + :param target: the target map, whose comments will be updated. + :param comment_normalisation: the comment normalisation to assume; if NONE, no comments are transferred + :param forced_update_keys: keys for which comments are always transferred, even if present in target + """ + match comment_normalisation: + case YamlCommentNormalisation.NONE: + pass + case YamlCommentNormalisation.LEADING | YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING: + transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE], forced_update_keys=forced_update_keys) + case _: + raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}") diff --git a/test/serena/config/test_serena_config.py b/test/serena/config/test_serena_config.py index 048b3dbc..93abd5a6 100644 --- a/test/serena/config/test_serena_config.py +++ b/test/serena/config/test_serena_config.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest from serena.config.serena_config import ProjectConfig +from serena.constants import PROJECT_TEMPLATE_FILE from solidlsp.ls_config import Language @@ -117,3 +118,9 @@ class TestProjectConfigAutogenerate: assert config.project_name == custom_name assert config.languages == [Language.TYPESCRIPT] + + +class TestProjectConfig: + def test_template_is_complete(self): + _, is_complete = ProjectConfig._load_yaml(PROJECT_TEMPLATE_FILE) + assert is_complete, "Project template YAML is incomplete; all fields must be present (with descriptions)." diff --git a/test/serena/test_tool_parameter_types.py b/test/serena/test_tool_parameter_types.py index 9899a5bb..83a0981c 100644 --- a/test/serena/test_tool_parameter_types.py +++ b/test/serena/test_tool_parameter_types.py @@ -18,7 +18,7 @@ def test_all_tool_parameters_have_type(context): cfg.included_optional_tools = tuple(registry.get_tool_names_optional()) factory = SerenaMCPFactory(context=context) # Initialize the agent so that the tools are available - factory.agent = factory._create_serena_agent(cfg, []) + factory.agent = factory._create_serena_agent(cfg) tools = list(factory._iter_tools()) for tool in tools: