Apply YAML comment transfer to serena_config.yml when re-saving

* Use leading comment convention in template, because trailing
   comments are much more difficult to deal with and very hard to normalise
 * Apply a conversion to the user's yaml content (translating from trailing
   comments where necessary) before applying the comment transfer

src/serena/config/serena_config.py

@@ -29,7 +29,7 @@ from serena.constants import (
     SERENA_MANAGED_DIR_NAME,
 )
 from serena.util.inspection import determine_programming_language_composition
-from serena.util.yaml import YamlCommentNormalisation, load_yaml, save_yaml, transfer_missing_yaml_comments
+from serena.util.yaml import YamlCommentNormalisation, load_yaml, normalise_yaml_comments, save_yaml, transfer_missing_yaml_comments
 from solidlsp.ls_config import Language
 
 from ..analytics import RegisteredTokenCountEstimator
@@ -781,6 +781,13 @@ class SerenaConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMix
         # convert language backend to string
         commented_yaml["language_backend"] = self.language_backend.value
 
+        # transfer comments from the template file
+        # NOTE: The template file now uses leading comments, but we previously used trailing comments,
+        #       so we apply a conversion, which detects the old style and transforms it.
+        normalise_yaml_comments(commented_yaml, YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING)
+        template_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE, comment_normalisation=YamlCommentNormalisation.LEADING)
+        transfer_missing_yaml_comments(template_yaml, commented_yaml, YamlCommentNormalisation.LEADING, fully_missing_only=False)
+
         save_yaml(self.config_file_path, commented_yaml)
 
     def propagate_settings(self) -> None:

src/serena/resources/serena_config.template.yml

@@ -1,4 +1,3 @@
-language_backend: LSP
 # the language backend to use for code understanding and manipulation.
 # Possible values are:
 #  * LSP: Use the language server protocol (LSP), spawning freely available language servers
@@ -6,8 +5,8 @@ language_backend: LSP
 #  * JetBrains: Use the Serena plugin in your JetBrains IDE.
 #      (requires the plugin to be installed and the project being worked on to be open
 #      in your IDE).
+language_backend: LSP
 
-gui_log_window: False
 # whether to open a graphical window with Serena's logs.
 # This is mainly supported on Windows and (partly) on Linux; not available on macOS.
 # If you prefer a browser-based tool, use the `web_dashboard` option instead.
@@ -22,8 +21,8 @@ gui_log_window: False
 # previous instances. This can lead to multiple log windows being opened, and only the last
 # window being updated. Since we can't control how agno or Claude Desktop start Serena,
 # we have to live with this limitation for now.
+gui_log_window: False
 
-web_dashboard: True
 # whether to open the Serena web dashboard (which will be accessible through your web browser) that
 # provides access to Serena's configuration and state as well as the current session logs.
 # The web dashboard is supported on all platforms.
@@ -34,11 +33,11 @@ web_dashboard: True
 # You can then open the dashboard by asking your agent to do so (e.g., by saying "open the dashboard"), Serena provides
 # a tool for this.
 # Further information: https://oraios.github.io/serena/02-usage/060_dashboard.html
+web_dashboard: True
 
-web_dashboard_listen_address: 127.0.0.1
 # The address where the web dashboard will listen on
+web_dashboard_listen_address: 127.0.0.1
 
-web_dashboard_open_on_launch: True
 # whether to open a browser window with the web dashboard when Serena starts (provided that web_dashboard
 # is enabled). See also the web_dashboard option.
 # If set to false, you can still open the dashboard manually by
@@ -46,56 +45,55 @@ web_dashboard_open_on_launch: True
 # b) manually navigating to http://localhost:24282/dashboard/ in your web browser (actual port
 #    may be higher if you have multiple instances running; try ports 24283, 24284, etc.)
 # See also: https://oraios.github.io/serena/02-usage/060_dashboard.html
+web_dashboard_open_on_launch: True
 
-jetbrains_plugin_server_address: 127.0.0.1
 # address where JetBrains plugin servers are running (only relevant when using the JetBrains language backend)
+jetbrains_plugin_server_address: 127.0.0.1
 
-log_level: 20
 # the minimum log level for the GUI log window and the dashboard (10 = debug, 20 = info, 30 = warning, 40 = error)
+log_level: 20
 
-trace_lsp_communication: False
 # whether to trace the communication between Serena and the language servers.
 # This is useful for debugging language server issues.
+trace_lsp_communication: False
 
-ls_specific_settings: {}
-# Added on 23.08.2025
-# Advanced configuration option allowing to configure language server implementation specific options. Maps the language
-# (same entry as in project.yml) to the options.
+# Advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
 # Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available.
 # No documentation on options means no options are available.
+ls_specific_settings: {}
 
-tool_timeout: 240
 # timeout, in seconds, after which tool executions are terminated
+tool_timeout: 240
 
-excluded_tools: []
 # list of tools to be globally excluded
+excluded_tools: []
 
-included_optional_tools: []
 # list of optional tools (which are disabled by default) to be included
+included_optional_tools: []
 
-fixed_tools: []
 # fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
 # This cannot be combined with non-empty excluded_tools or included_optional_tools.
+fixed_tools: []
 
-base_modes:
 # list of mode names to that are always to be included in the set of active modes
 # The full set of modes to be activated is base_modes + default_modes.
 # If this is undefined, no base modes are included.
 # The project configuration (project.yml) may override this setting.
+base_modes:
 
-default_modes:
-  - interactive
-  - editing
 # list of mode names that are to be activated by default.
 # The full set of modes to be activated is base_modes + default_modes.
 # These modes can be overridden by the project configuration (project.yml) or through the CLI (--mode).
+default_modes:
+  - interactive
+  - editing
 
-default_max_tool_answer_chars: 150000
 # Used as default for tools where the apply method has a default maximal answer length.
 # Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default
 # through the global configuration.
+default_max_tool_answer_chars: 150000
 
-token_count_estimator: CHAR_COUNT
 # the name of the token count estimator to use for tool usage statistics.
 # See the `RegisteredTokenCountEstimator` enum for available options.
 #
@@ -103,6 +101,7 @@ token_count_estimator: CHAR_COUNT
 # You can configure this to TIKTOKEN_GPT4 to use a local tiktoken-based estimator for GPT-4 (will download tiktoken
 # data files on first run), or ANTHROPIC_CLAUDE_SONNET_4 which will use the (free of cost) Anthropic API to
 # estimate the token count using the Claude Sonnet 4 tokenizer.
+token_count_estimator: CHAR_COUNT
 
 # The list of registered project paths (updated automatically).
 projects: []

src/serena/util/yaml.py

@@ -1,12 +1,15 @@
+import logging
 import os
 from enum import Enum
 from typing import Any
 
-from ruamel.yaml import YAML, CommentToken
+from ruamel.yaml import YAML, CommentToken, StreamMark
 from ruamel.yaml.comments import CommentedMap
 
 from serena.constants import SERENA_FILE_ENCODING
 
+log = logging.getLogger(__name__)
+
 
 def _create_yaml(preserve_comments: bool = False) -> YAML:
     """
@@ -20,35 +23,69 @@ def _create_yaml(preserve_comments: bool = False) -> YAML:
 
 class YamlCommentNormalisation(Enum):
     NONE = "none"
+    """
+    No comment normalisation is performed.
+    Comments are kept as parsed by ruamel.yaml.
+    """
     LEADING = "leading"
     """
-    Document is assumed to have leading comments only, i.e. comments before keys.
+    Document is assumed to have leading comments only, i.e. comments before keys, only full-line comments.
     This normalisation achieves that comments are properly associated with keys as leading comments.
     """
+    LEADING_WITH_CONVERSION_FROM_TRAILING = "leading_with_conversion_from_trailing"
+    # NOTE: Normalisation for trailing comments was attempted but is extremely hard, because
+    #  it is difficult to position the comments properly after values, especially for complex values.
 
 
 DOC_COMMENT_INDEX_POST = 0
 DOC_COMMENT_INDEX_PRE = 1
 
 # item comment indices: (post key, pre key, post value, pre value)
-ITEM_COMMENT_INDEX_BEFORE = 1  # (pre-key; must be a list at this index)
-ITEM_COMMENT_INDEX_AFTER = 2  # (post-value; typically no list at this index)
+ITEM_COMMENT_INDEX_BEFORE = 1  # (pre-key; must be a list of CommentToken at this index)
+ITEM_COMMENT_INDEX_AFTER = 2  # (post-value; must be an instance of CommentToken at this index)
 
 
 def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE) -> CommentedMap:
     with open(path, encoding=SERENA_FILE_ENCODING) as f:
         yaml = _create_yaml(preserve_comments=True)
         commented_map: CommentedMap = yaml.load(f)
+    normalise_yaml_comments(commented_map, comment_normalisation)
+    return commented_map
 
+
+def normalise_yaml_comments(commented_map: CommentedMap, comment_normalisation: YamlCommentNormalisation) -> None:
     def make_list(comment_entry: Any) -> list:
         if not isinstance(comment_entry, list):
             return [comment_entry]
         return comment_entry
 
+    def make_unit(comment_entry: Any) -> Any:
+        """
+        Converts a list-valued comment entry into a single comment entry.
+        """
+        if isinstance(comment_entry, list):
+            if len(comment_entry) == 0:
+                return None
+            elif len(comment_entry) == 1:
+                return comment_entry[0]
+            else:
+                if all(isinstance(item, CommentToken) for item in comment_entry):
+                    start_mark = StreamMark(name="", index=0, line=0, column=0)
+                    comment_str = "".join(item.value for item in comment_entry)
+                    if not comment_str.startswith("\n"):
+                        comment_str = "\n" + comment_str
+                    return CommentToken(value=comment_str, start_mark=start_mark, end_mark=None)
+                else:
+                    types = set(type(item) for item in comment_entry)
+                    log.warning("Unhandled types in list-valued comment entry: %s; not updating entry", types)
+                    return None
+        else:
+            return comment_entry
+
     match comment_normalisation:
         case YamlCommentNormalisation.NONE:
             pass
-        case YamlCommentNormalisation.LEADING:
+        case YamlCommentNormalisation.LEADING | YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING:
             # Comments are supposed to be leading comments (i.e., before a key and associated with the key).
             # When ruamel parses a YAML, however, comments belonging to a key may be stored as trailing
             # comments of the previous key or as a document-level comment.
@@ -70,11 +107,31 @@ def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlC
                         current_comment[ITEM_COMMENT_INDEX_BEFORE] = make_list(preceding_comment[ITEM_COMMENT_INDEX_AFTER])
                         preceding_comment[ITEM_COMMENT_INDEX_AFTER] = None
                 preceding_comment = current_comment
+
+            if comment_normalisation == YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING:
+                # Second pass: conversion of trailing comments
+                # If a leading comment ends with "\n\n", i.e. it has an empty line between the comment and the key,
+                # it was actually intended as a trailing comment for the preceding key, so we associate it with
+                # the preceding key instead.
+                preceding_comment = None
+                for key in keys:
+                    current_comment = comment_items.get(key, [None] * 4)
+                    if current_comment[ITEM_COMMENT_INDEX_BEFORE] is not None:
+                        token_list = make_list(current_comment[ITEM_COMMENT_INDEX_BEFORE])
+                        if len(token_list) > 0:
+                            last_token = token_list[-1]
+                            if isinstance(last_token, CommentToken) and last_token.value.endswith("\n\n"):
+                                # move comment to preceding key, removing the empty line
+                                if preceding_comment is not None and yaml_comment_entry_is_empty(
+                                    preceding_comment[ITEM_COMMENT_INDEX_BEFORE]
+                                ):
+                                    last_token.value = last_token.value[:-1]
+                                    preceding_comment[ITEM_COMMENT_INDEX_BEFORE] = token_list
+                                    current_comment[ITEM_COMMENT_INDEX_BEFORE] = None
+                    preceding_comment = current_comment
         case _:
             raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}")
 
-    return commented_map
-
 
 def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = True) -> None:
     yaml = _create_yaml(preserve_comments)
@@ -83,53 +140,67 @@ def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = Tr
         yaml.dump(data, f)
 
 
-def transfer_missing_yaml_comments_by_index(source: CommentedMap, target: CommentedMap, indices: list[int]) -> None:
+def yaml_comment_entry_is_empty(comment_entry: Any) -> bool:
+    if comment_entry is None:
+        return True
+    elif isinstance(comment_entry, list):
+        for item in comment_entry:
+            if isinstance(item, CommentToken):
+                if item.value.strip() != "":
+                    return False
+            else:
+                return False
+        return True
+    elif isinstance(comment_entry, CommentToken):
+        return comment_entry.value.strip() == ""
+    else:
+        return False
+
+
+def transfer_missing_yaml_comments_by_index(
+    source: CommentedMap, target: CommentedMap, indices: list[int], fully_missing_only: bool = False
+) -> None:
     """
     :param source: the source, from which to transfer missing comments
     :param target: the target map, whose comments will be updated
     :param indices: list of comment indices to transfer
+    :param fully_missing_only: if True, only transfer comments for keys that have no comment definition at all (newly added)
     """
-
-    def is_empty(comment_entry: Any) -> bool:
-        if comment_entry is None:
-            return True
-        elif isinstance(comment_entry, list):
-            for item in comment_entry:
-                if isinstance(item, CommentToken):
-                    if item.value.strip() != "":
-                        return False
-                else:
-                    return False
-            return True
-        elif isinstance(comment_entry, CommentToken):
-            return comment_entry.value.strip() == ""
-        else:
-            return False
-
     for key in target.keys():
         if key in source:
             source_comment = source.ca.items.get(key)
             if source_comment is None:
                 continue
-            target_comment = target.ca.items.get(key, [None, None, None, None])
-            target.ca.items[key] = target_comment
-            for index in indices:
-                if is_empty(target_comment[index]):
-                    target_comment[index] = source_comment[index]
+            target_comment = target.ca.items.get(key)
+            if target_comment is None and fully_missing_only:
+                # transfer full comment and we're done
+                target.ca.items[key] = source_comment
+            else:
+                # initialise target comment if needed
+                if target_comment is None:
+                    target_comment = [None] * 4
+                    target.ca.items[key] = target_comment
+                # transfer missing comments at specified indices
+                for index in indices:
+                    if yaml_comment_entry_is_empty(target_comment[index]):
+                        target_comment[index] = source_comment[index]
 
 
-def transfer_missing_yaml_comments(source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation) -> None:
+def transfer_missing_yaml_comments(
+    source: CommentedMap, target: CommentedMap, comment_normalisation: YamlCommentNormalisation, fully_missing_only: bool = False
+) -> None:
     """
     Transfers missing comments from source to target YAML.
 
     :param source: the source, from which to transfer missing comments
     :param target: the target map, whose comments will be updated.
     :param comment_normalisation: the comment normalisation to assume; if NONE, no comments are transferred
+    :param fully_missing_only: if True, only transfer comments for keys that have no comment definition at all (newly added)
     """
     match comment_normalisation:
         case YamlCommentNormalisation.NONE:
             pass
-        case YamlCommentNormalisation.LEADING:
-            transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE])
+        case YamlCommentNormalisation.LEADING | YamlCommentNormalisation.LEADING_WITH_CONVERSION_FROM_TRAILING:
+            transfer_missing_yaml_comments_by_index(source, target, [ITEM_COMMENT_INDEX_BEFORE], fully_missing_only=fully_missing_only)
         case _:
             raise ValueError(f"Unhandled comment normalisation: {comment_normalisation}")