Ensure that project.yml files are complete, re-saving incomplete ones with defaults

.serena/project.yml

@@ -1,14 +1,45 @@
+# the name by which the project can be referenced within Serena
+project_name: "serena"
+
+
+# list of languages for which language servers are started; choose from:
+#   al                  bash                clojure             cpp                 csharp
+#   csharp_omnisharp    dart                elixir              elm                 erlang
+#   fortran             fsharp              go                  groovy              haskell
+#   java                julia               kotlin              lua                 markdown
+#   matlab              nix                 pascal              perl                php
+#   powershell          python              python_jedi         r                   rego
+#   ruby                ruby_solargraph     rust                scala               swift
+#   terraform           toml                typescript          typescript_vts      vue
+#   yaml                zig
+#   (This list may be outdated. For the current list, see values of Language enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py
+#   For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Free Pascal/Lazarus, use pascal
+# Special requirements:
+#   - csharp: Requires the presence of a .sln file in the project folder.
+#   - pascal: Requires Free Pascal Compiler (fpc) and optionally Lazarus.
+# When using multiple languages, the first language server that supports a given file will be used for that file.
+# The first language is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+languages:
+- python
+- typescript
+
+
+# whether to use project's .gitignore files to ignore files
 ignore_all_files_in_gitignore: true
 
-# list of additional paths to ignore
+
+# list of additional paths to ignore in all projects
 # same syntax as gitignore, so you can use * and **
-# Was previously called `ignored_dirs`, please update your config if you are using that.
-# Added (renamed)on 2025-04-07
 ignored_paths: []
 
 # whether the project is in read-only mode
 # If set to true, all editing tools will be disabled and attempts to use them will result in an error
-# Added on 2025-04-18
 read_only: false
 
 
@@ -53,13 +84,34 @@ read_only: false
 #  * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
 excluded_tools: []
 
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default)
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
 # initial prompt for the project. It will always be given to the LLM upon activating the project
 # (contrary to the memories, which are loaded on demand).
 initial_prompt: ""
 
-project_name: "serena"
-languages:
-- python
-- typescript
-included_optional_tools: []
+
 encoding: utf-8
+
+
+# list of mode names to that are always to be included in the set of active modes
+# The full set of modes to be activated is base_modes + default_modes.
+# If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this setting overrides the global configuration.
+# Set this to [] to disable base modes for this project.
+# Set this to a list of mode names to always include the respective modes for this project.
+base_modes:
+
+# list of mode names that are to be activated by default.
+# The full set of modes to be activated is base_modes + default_modes.
+# If the setting is undefined, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+default_modes:
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+fixed_tools: []

src/serena/config/serena_config.py

@@ -246,7 +246,7 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi
                 log.info("Using languages: %s", languages_to_use)
             else:
                 languages_to_use = [lang.value for lang in languages]
-            config_with_comments = cls.load_commented_map(PROJECT_TEMPLATE_FILE)
+            config_with_comments, _ = cls._load_yaml(PROJECT_TEMPLATE_FILE)
             config_with_comments["project_name"] = project_name
             config_with_comments["languages"] = languages_to_use
             if save_to_disk:
@@ -264,13 +264,27 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi
         return os.path.join(SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE)
 
     @classmethod
-    def _apply_defaults_to_dict(cls, data: TDict) -> TDict:
+    def _load_yaml(cls, yml_path: str) -> tuple[CommentedMap, bool]:
+        """
+        Load the project configuration as a CommentedMap, preserving comments and ensuring
+        completeness of the configuration by applying default values for missing fields
+        and backward compatibility adjustments.
+
+        :param yml_path: the path to the project.yml file
+        :return: a tuple `(dict, was_complete)` where dict is a CommentedMap representing a
+          full project configuration and `was_complete` indicates whether the loaded configuration
+          was complete (i.e., did not require any default values to be applied)
+        """
+        data = load_yaml(yml_path, preserve_comments=True)
+
         # apply defaults
+        was_complete = True
         for field_info in dataclasses.fields(cls):
             key = field_info.name
             if key in cls.FIELDS_WITHOUT_DEFAULTS:
                 continue
             if key not in data:
+                was_complete = False
                 default_value = get_dataclass_default(cls, key)
                 data.setdefault(key, default_value)
 
@@ -279,20 +293,7 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi
             data["languages"] = [data["language"]]
             del data["language"]
 
-        return data
-
-    @classmethod
-    def load_commented_map(cls, yml_path: str) -> CommentedMap:
-        """
-        Load the project configuration as a CommentedMap, preserving comments and ensuring
-        completeness of the configuration by applying default values for missing fields
-        and backward compatibility adjustments.
-
-        :param yml_path: the path to the project.yml file
-        :return: a CommentedMap representing a full project configuration
-        """
-        data = load_yaml(yml_path, preserve_comments=True)
-        return cls._apply_defaults_to_dict(data)
+        return data, was_complete
 
     @classmethod
     def _from_dict(cls, data: dict[str, Any]) -> Self:
@@ -329,7 +330,7 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi
             default_modes=data["default_modes"],
         )
 
-    def to_yaml_dict(self) -> dict:
+    def _to_yaml_dict(self) -> dict:
         """
         :return: a yaml-serializable dictionary representation of this configuration
         """
@@ -344,15 +345,40 @@ class ProjectConfig(ToolInclusionDefinition, ModeSelectionDefinition, ToStringMi
         """
         project_root = Path(project_root)
         yaml_path = project_root / cls.rel_path_to_project_yml()
+
+        # auto-generate if necessary
         if not yaml_path.exists():
             if autogenerate:
                 return cls.autogenerate(project_root)
             else:
                 raise FileNotFoundError(f"Project configuration file not found: {yaml_path}")
-        yaml_data = cls.load_commented_map(str(yaml_path))
+
+        # load the configuration dictionary
+        yaml_data, was_complete = cls._load_yaml(str(yaml_path))
         if "project_name" not in yaml_data:
             yaml_data["project_name"] = project_root.name
-        return cls._from_dict(yaml_data)
+
+        # instantiate the ProjectConfig
+        project_config = cls._from_dict(yaml_data)
+
+        # if the configuration was incomplete, re-save it to disk
+        if not was_complete:
+            log.info("Project configuration in %s was incomplete, re-saving with default values for missing fields", yaml_path)
+            project_config.save(project_root)
+
+        return project_config
+
+    def save(self, project_root: Path | str) -> None:
+        """
+        Saves the project configuration to disk.
+
+        :param project_root: the root directory of the project
+        """
+        config_path = self.path_to_project_yml(project_root)
+        log.info("Saving updated project configuration to %s", config_path)
+        config_with_comments, _ = self._load_yaml(config_path)
+        config_with_comments.update(self._to_yaml_dict())
+        save_yaml(config_path, config_with_comments, preserve_comments=True)
 
 
 class RegisteredProject(ToStringMixin):

src/serena/project.py

@@ -12,7 +12,6 @@ from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME
 from serena.ls_manager import LanguageServerFactory, LanguageServerManager
 from serena.text_utils import MatchedConsecutiveLines, search_files
 from serena.util.file_system import GitignoreParser, match_path
-from serena.util.general import save_yaml
 from solidlsp import SolidLanguageServer
 from solidlsp.ls_config import Language
 from solidlsp.ls_utils import FileUtils
@@ -114,11 +113,7 @@ class Project(ToStringMixin):
         """
         Saves the current project configuration to disk.
         """
-        config_path = os.path.join(self.project_root, self.project_config.rel_path_to_project_yml())
-        log.info("Saving updated project configuration to %s", config_path)
-        config_with_comments = ProjectConfig.load_commented_map(config_path)
-        config_with_comments.update(self.project_config.to_yaml_dict())
-        save_yaml(config_path, config_with_comments, preserve_comments=True)
+        self.project_config.save(self.project_root)
 
     def path_to_serena_data_folder(self) -> str:
         return os.path.join(self.project_root, SERENA_MANAGED_DIR_NAME)

src/serena/resources/project.template.yml

@@ -80,6 +80,13 @@ read_only: false
 #  * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
 excluded_tools: []
 
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default)
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+fixed_tools: []
+
 # list of mode names to that are always to be included in the set of active modes
 # The full set of modes to be activated is base_modes + default_modes.
 # If the setting is undefined, the base_modes from the global configuration (serena_config.yml) apply.

test/serena/config/test_serena_config.py

@@ -5,6 +5,7 @@ from pathlib import Path
 import pytest
 
 from serena.config.serena_config import ProjectConfig
+from serena.constants import PROJECT_TEMPLATE_FILE
 from solidlsp.ls_config import Language
 
 
@@ -117,3 +118,9 @@ class TestProjectConfigAutogenerate:
 
         assert config.project_name == custom_name
         assert config.languages == [Language.TYPESCRIPT]
+
+
+class TestProjectConfig:
+    def test_template_is_complete(self):
+        _, is_complete = ProjectConfig._load_yaml(PROJECT_TEMPLATE_FILE)
+        assert is_complete, "Project template YAML is incomplete; all fields must be present (with descriptions)."