From a8753d9618db17111f6b77c4aa836fa55059098b Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Tue, 27 Jan 2026 13:02:31 +0100 Subject: [PATCH] Apply YAML comment transfer to serena_config.yml when re-saving * Use leading comment convention in template, because trailing comments are much more difficult to deal with and very hard to normalise * Apply a conversion to the user's yaml content (translating from trailing comments where necessary) before applying the comment transfer --- src/serena/config/serena_config.py | 9 +- .../resources/serena_config.template.yml | 43 +++--- src/serena/util/yaml.py | 137 +++++++++++++----- 3 files changed, 133 insertions(+), 56 deletions(-) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 6adf9e8a..03dd6abf 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -29,7 +29,7 @@ from serena.constants import ( SERENA_MANAGED_DIR_NAME, ) from serena.util.inspection import determine_programming_language_composition -from serena.util.yaml import YamlCommentNormalisation, load_yaml, save_yaml, transfer_missing_yaml_comments +from serena.util.yaml import YamlCommentNormalisation, load_yaml, normalise_yaml_comments, save_yaml, transfer_missing_yaml_comments from solidlsp.ls_config import Language from ..analytics import RegisteredTokenCountEstimator @@ -781,6 +781,13 @@ class SerenaConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMix # convert language backend to string commented_yaml["language_backend"] = self.language_backend.value + # transfer comments from the template file + # NOTE: The template file now uses leading comments, but we previously used trailing comments, + # so we apply a conversion, which detects the old style and transforms it. + normalise_yaml_comments(commented_yaml, YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING) + template_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE, comment_normalisation=YamlCommentNormalisation.LEADING) + transfer_missing_yaml_comments(template_yaml, commented_yaml, YamlCommentNormalisation.LEADING, fully_missing_only=False) + save_yaml(self.config_file_path, commented_yaml) def propagate_settings(self) -> None: diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index f36f5895..11842b53 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -1,4 +1,3 @@ -language_backend: LSP # the language backend to use for code understanding and manipulation. # Possible values are: # * LSP: Use the language server protocol (LSP), spawning freely available language servers @@ -6,8 +5,8 @@ language_backend: LSP # * JetBrains: Use the Serena plugin in your JetBrains IDE. # (requires the plugin to be installed and the project being worked on to be open # in your IDE). +language_backend: LSP -gui_log_window: False # whether to open a graphical window with Serena's logs. # This is mainly supported on Windows and (partly) on Linux; not available on macOS. # If you prefer a browser-based tool, use the `web_dashboard` option instead. @@ -22,8 +21,8 @@ gui_log_window: False # previous instances. This can lead to multiple log windows being opened, and only the last # window being updated. Since we can't control how agno or Claude Desktop start Serena, # we have to live with this limitation for now. +gui_log_window: False -web_dashboard: True # whether to open the Serena web dashboard (which will be accessible through your web browser) that # provides access to Serena's configuration and state as well as the current session logs. # The web dashboard is supported on all platforms. @@ -34,11 +33,11 @@ web_dashboard: True # You can then open the dashboard by asking your agent to do so (e.g., by saying "open the dashboard"), Serena provides # a tool for this. # Further information: https://oraios.github.io/serena/02-usage/060_dashboard.html +web_dashboard: True -web_dashboard_listen_address: 127.0.0.1 # The address where the web dashboard will listen on +web_dashboard_listen_address: 127.0.0.1 -web_dashboard_open_on_launch: True # whether to open a browser window with the web dashboard when Serena starts (provided that web_dashboard # is enabled). See also the web_dashboard option. # If set to false, you can still open the dashboard manually by @@ -46,56 +45,55 @@ web_dashboard_open_on_launch: True # b) manually navigating to http://localhost:24282/dashboard/ in your web browser (actual port # may be higher if you have multiple instances running; try ports 24283, 24284, etc.) # See also: https://oraios.github.io/serena/02-usage/060_dashboard.html +web_dashboard_open_on_launch: True -jetbrains_plugin_server_address: 127.0.0.1 # address where JetBrains plugin servers are running (only relevant when using the JetBrains language backend) +jetbrains_plugin_server_address: 127.0.0.1 -log_level: 20 # the minimum log level for the GUI log window and the dashboard (10 = debug, 20 = info, 30 = warning, 40 = error) +log_level: 20 -trace_lsp_communication: False # whether to trace the communication between Serena and the language servers. # This is useful for debugging language server issues. +trace_lsp_communication: False -ls_specific_settings: {} -# Added on 23.08.2025 -# Advanced configuration option allowing to configure language server implementation specific options. Maps the language -# (same entry as in project.yml) to the options. +# Advanced configuration option allowing to configure language server-specific options. +# Maps the language key to the options. # Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. # No documentation on options means no options are available. +ls_specific_settings: {} -tool_timeout: 240 # timeout, in seconds, after which tool executions are terminated +tool_timeout: 240 -excluded_tools: [] # list of tools to be globally excluded +excluded_tools: [] -included_optional_tools: [] # list of optional tools (which are disabled by default) to be included +included_optional_tools: [] -fixed_tools: [] # fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. # This cannot be combined with non-empty excluded_tools or included_optional_tools. +fixed_tools: [] -base_modes: # list of mode names to that are always to be included in the set of active modes # The full set of modes to be activated is base_modes + default_modes. # If this is undefined, no base modes are included. # The project configuration (project.yml) may override this setting. +base_modes: -default_modes: - - interactive - - editing # list of mode names that are to be activated by default. # The full set of modes to be activated is base_modes + default_modes. # These modes can be overridden by the project configuration (project.yml) or through the CLI (--mode). +default_modes: + - interactive + - editing -default_max_tool_answer_chars: 150000 # Used as default for tools where the apply method has a default maximal answer length. # Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default # through the global configuration. +default_max_tool_answer_chars: 150000 -token_count_estimator: CHAR_COUNT # the name of the token count estimator to use for tool usage statistics. # See the `RegisteredTokenCountEstimator` enum for available options. # @@ -103,6 +101,7 @@ token_count_estimator: CHAR_COUNT # You can configure this to TIKTOKEN_GPT4 to use a local tiktoken-based estimator for GPT-4 (will download tiktoken # data files on first run), or ANTHROPIC_CLAUDE_SONNET_4 which will use the (free of cost) Anthropic API to # estimate the token count using the Claude Sonnet 4 tokenizer. +token_count_estimator: CHAR_COUNT # The list of registered project paths (updated automatically). projects: [] diff --git a/src/serena/util/yaml.py b/src/serena/util/yaml.py index 8bc1fc99..20bddc07 100644 --- a/src/serena/util/yaml.py +++ b/src/serena/util/yaml.py @@ -1,12 +1,15 @@ +import logging import os from enum import Enum from typing import Any -from ruamel.yaml import YAML, CommentToken +from ruamel.yaml import YAML, CommentToken, StreamMark from ruamel.yaml.comments import CommentedMap from serena.constants import SERENA_FILE_ENCODING +log = logging.getLogger(__name__) + def _create_yaml(preserve_comments: bool = False) -> YAML: """ @@ -20,35 +23,69 @@ def _create_yaml(preserve_comments: bool = False) -> YAML: class YamlCommentNormalisation(Enum): NONE = "none" + """ + No comment normalisation is performed. + Comments are kept as parsed by ruamel.yaml. + """ LEADING = "leading" """ - Document is assumed to have leading comments only, i.e. comments before keys. + Document is assumed to have leading comments only, i.e. comments before keys, only full-line comments. This normalisation achieves that comments are properly associated with keys as leading comments. """ + LEADING_WITH_CONVERSION_FROM_TRAILING = "leading_with_conversion_from_trailing" + # NOTE: Normalisation for trailing comments was attempted but is extremely hard, because + # it is difficult to position the comments properly after values, especially for complex values. DOC_COMMENT_INDEX_POST = 0 DOC_COMMENT_INDEX_PRE = 1 # item comment indices: (post key, pre key, post value, pre value) -ITEM_COMMENT_INDEX_BEFORE = 1 # (pre-key; must be a list at this index) -ITEM_COMMENT_INDEX_AFTER = 2 # (post-value; typically no list at this index) +ITEM_COMMENT_INDEX_BEFORE = 1 # (pre-key; must be a list of CommentToken at this index) +ITEM_COMMENT_INDEX_AFTER = 2 # (post-value; must be an instance of CommentToken at this index) def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE) -> CommentedMap: with open(path, encoding=SERENA_FILE_ENCODING) as f: yaml = _create_yaml(preserve_comments=True) commented_map: CommentedMap = yaml.load(f) + normalise_yaml_comments(commented_map, comment_normalisation) + return commented_map + +def normalise_yaml_comments(commented_map: CommentedMap, comment_normalisation: YamlCommentNormalisation) -> None: def make_list(comment_entry: Any) -> list: if not isinstance(comment_entry, list): return [comment_entry] return comment_entry + def make_unit(comment_entry: Any) -> Any: + """ + Converts a list-valued comment entry into a single comment entry. + """ + if isinstance(comment_entry, list): + if len(comment_entry) == 0: + return None + elif len(comment_entry) == 1: + return comment_entry[0] + else: + if all(isinstance(item, CommentToken) for item in comment_entry): + start_mark = StreamMark(name="", index=0, line=0, column=0) + comment_str = "".join(item.value for item in comment_entry) + if not comment_str.startswith("\n"): + comment_str = "\n" + comment_str + return CommentToken(value=comment_str, start_mark=start_mark, end_mark=None) + else: + types = set(type(item) for item in comment_entry) + log.warning("Unhandled types in list-valued comment entry: %s; not updating entry", types) + return None + else: + return comment_entry + match comment_normalisation: case YamlCommentNormalisation.NONE: pass - case YamlCommentNormalisation.LEADING: + case YamlCommentNormalisation.LEADING | YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING: # Comments are supposed to be leading comments (i.e., before a key and associated with the key). # When ruamel parses a YAML, however, comments belonging to a key may be stored as trailing # comments of the previous key or as a document-level comment. @@ -70,11 +107,31 @@ def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlC current_comment[ITEM_COMMENT_INDEX_BEFORE] = make_list(preceding_comment[ITEM_COMMENT_INDEX_AFTER]) preceding_comment[ITEM_COMMENT_INDEX_AFTER] = None preceding_comment = current_comment + + if comment_normalisation == YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING: + # Second pass: conversion of trailing comments + # If a leading comment ends with "\n\n", i.e. it has an empty line between the comment and the key, + # it was actually intended as a trailing comment for the preceding key, so we associate it with + # the preceding key instead. + preceding_comment = None + for key in keys: + current_comment = comment_items.get(key, [None] * 4) + if current_comment[ITEM_COMMENT_INDEX_BEFORE] is not None: + token_list = make_list(current_comment[ITEM_COMMENT_INDEX_BEFORE]) + if len(token_list) > 0: + last_token = token_list[-1] + if isinstance(last_token, CommentToken) and last_token.value.endswith("\n\n"): + # move comment to preceding key, removing the empty line + if preceding_comment is not None and yaml_comment_entry_is_empty( + preceding_comment[ITEM_COMMENT_INDEX_BEFORE] + ): + last_token.value = last_token.value[:-1] + preceding_comment[ITEM_COMMENT_INDEX_BEFORE] = token_list + current_comment[ITEM_COMMENT_INDEX_BEFORE] = None + preceding_comment = current_comment case _: raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}") - return commented_map - def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = True) -> None: yaml = _create_yaml(preserve_comments) @@ -83,53 +140,67 @@ def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = Tr yaml.dump(data, f) -def transfer_missing_yaml_comments_by_index(source: CommentedMap, target: CommentedMap, indices: list[int]) -> None: +def yaml_comment_entry_is_empty(comment_entry: Any) -> bool: + if comment_entry is None: + return True + elif isinstance(comment_entry, list): + for item in comment_entry: + if isinstance(item, CommentToken): + if item.value.strip() != "": + return False + else: + return False + return True + elif isinstance(comment_entry, CommentToken): + return comment_entry.value.strip() == "" + else: + return False + + +def transfer_missing_yaml_comments_by_index( + source: CommentedMap, target: CommentedMap, indices: list[int], fully_missing_only: bool = False +) -> None: """ :param source: the source, from which to transfer missing comments :param target: the target map, whose comments will be updated :param indices: list of comment indices to transfer + :param fully_missing_only: if True, only transfer comments for keys that have no comment definition at all (newly added) """ - - def is_empty(comment_entry: Any) -> bool: - if comment_entry is None: - return True - elif isinstance(comment_entry, list): - for item in comment_entry: - if isinstance(item, CommentToken): - if item.value.strip() != "": - return False - else: - return False - return True - elif isinstance(comment_entry, CommentToken): - return comment_entry.value.strip() == "" - else: - return False - for key in target.keys(): if key in source: source_comment = source.ca.items.get(key) if source_comment is None: continue - target_comment = target.ca.items.get(key, [None, None, None, None]) - target.ca.items[key] = target_comment - for index in indices: - if is_empty(target_comment[index]): - target_comment[index] = source_comment[index] + target_comment = target.ca.items.get(key) + if target_comment is None and fully_missing_only: + # transfer full comment and we're done + target.ca.items[key] = source_comment + else: + # initialise target comment if needed + if target_comment is None: + target_comment = [None] * 4 + target.ca.items[key] = target_comment + # transfer missing comments at specified indices + for index in indices: + if yaml_comment_entry_is_empty(target_comment[index]): + target_comment[index] = source_comment[index] -def transfer_missing_yaml_comments(source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation) -> None: +def transfer_missing_yaml_comments( + source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation, fully_missing_only: bool = False +) -> None: """ Transfers missing comments from source to target YAML. :param source: the source, from which to transfer missing comments :param target: the target map, whose comments will be updated. :param comment_normalisation: the comment normalisation to assume; if NONE, no comments are transferred + :param fully_missing_only: if True, only transfer comments for keys that have no comment definition at all (newly added) """ match comment_normalisation: case YamlCommentNormalisation.NONE: pass - case YamlCommentNormalisation.LEADING: - transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE]) + case YamlCommentNormalisation.LEADING | YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING: + transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE], fully_missing_only=fully_missing_only) case _: raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}")