Merge pull request #1134 from oraios/project-local-yml

Add support for project.local.yml

CHANGELOG.md

@@ -1,6 +1,6 @@
-# latest
-Status of the `main` branch. Changes prior to the next official version change will appear here.
+# Latest
 
+Status of the `main` branch. Changes prior to the next official version change will appear here.
 
 * General:
     * Add monorepo/multi-language support
@@ -11,7 +11,6 @@ Status of the `main` branch. Changes prior to the next official version change w
       The `QueryProjectTool` allows Serena tools to be called on other projects.
         * For the LSP backend, calling symbolic tools require a project server to be spawned that will launch the respective language servers
         * For the JetBrains backend, all projects for which IDE instances are open can directly be queried
-    * The current project can be activated automatically even when the MCP configuration is global (through the --project-from-cwd flag)
     * Support overloaded symbols in `FindSymbolTool` and related tools
         * Name paths of overloaded symbols now include an index (e.g., `myOverloadedFunction[2]`)
         * Responses of the Java language server, which handled this in its own way, are now adapted accordingly,
@@ -23,23 +22,30 @@ Status of the `main` branch. Changes prior to the next official version change w
         * View tool usage statistics
         * View and create memories and edit the serena configuration file
         * Log page now has save (downloads a snapshot) and clear (resets log view) buttons alongside the existing copy button
-    * New two-tier caching of language server document symbols and considerable performance improvements surrounding symbol retrieval/indexing
+    * Language server backend:
+        * New two-tier caching of language server document symbols and considerable performance improvements surrounding symbol retrieval/indexing
+        * Allow passing language server specific settings through `ls_specific_settings` field (in `serena_config.yml`)
+    * Add the JetBrains language backend as an alternative to language servers
+    * Improve management of the Serena projects
+        * Facilitate project activation based on the current directory (through the `--project-from-cwd` parameter)
+        * Add notion of a "single-project context" (flag `single_project`), allowing user-defined contexts to behave
+          like the built-in `ide-assistant` context (where the available tools are restricted to ones required by the active
+          project and project changes are disabled)
+        * The location of Serena's project-specific data folder can now be flexibly configured, allowing, in particular,
+          locations outside of the project folder, thus improving support for read-only projects.
+        * Add support for `project.local.yml` for local overrides that should not be versioned 
     * Various fixes related to indexing, special paths and determination of ignored paths
-    * Decreased `TOOL_DEFAULT_MAX_ANSWER_LENGTH` to be in accordance with (below) typical max-tokens configurations
-    * Allow passing language server specific settings through `ls_specific_settings` field (in `serena_config.yml`)
-    * Add notion of a "single-project context" (flag `single_project`), allowing user-defined contexts to behave 
-      like the built-in `ide-assistant` context (where the available tools are restricted to ones required by the active 
-      project and project changes are disabled)
 
 * Client support:
     * New mode `oaicompat-agent` and extensions enhancing OpenAI tool compatibility, permitting Serena to work with llama.cpp
 
 * Tools:
-  * *New tool*: `jet_brains_type_hierarchy`
   * Symbol information (hover, docstring, quick-info) is now provided as part of `find_symbol` and related tool responses.
+  * Added `QueryProjectTool` and `ListQueryableProjectTool` (see above)
   * Added `RenameSymbolTool` for renaming symbols across the codebase (if LS supports this operation).
   * Replaced `ReplaceRegexTool` with `ReplaceContentTool`, which supports both plain text and regex-based replacements
     (and which requires no escaping in the replacement text, making it more robust) 
+  * Decreased `TOOL_DEFAULT_MAX_ANSWER_LENGTH` to be in accordance with (below) typical max-tokens configurations
 
 * Language support:
 

docs/02-usage/040_workflow.md

@@ -59,6 +59,11 @@ The file allows you to configure ...
 For detailed information on the parameters and possible settings, see the 
 [template file](https://github.com/oraios/serena/blob/main/src/serena/resources/project.template.yml). 
 
+**Local Overrides**. The project.yml file is intended to be versioned together with the project.
+You can specify local overrides for the settings in a `project.local.yml` file in the same directory
+(which, by default, is ignored by git). 
+Any keys defined therein will override the respective key in `project.yml`.
+
 (indexing)=
 ### Indexing
 

src/serena/config/serena_config.py

@@ -23,6 +23,7 @@ from sensai.util.string import ToStringMixin
 
 from serena.constants import (
     DEFAULT_SOURCE_FILE_ENCODING,
+    PROJECT_LOCAL_TEMPLATE_FILE,
     PROJECT_TEMPLATE_FILE,
     REPO_ROOT,
     SERENA_CONFIG_TEMPLATE_FILE,
@@ -213,7 +214,8 @@ class ProjectConfig(SharedConfig):
     initial_prompt: str = ""
     encoding: str = DEFAULT_SOURCE_FILE_ENCODING
 
-    SERENA_DEFAULT_PROJECT_FILE = "project.yml"
+    SERENA_PROJECT_FILE = "project.yml"
+    SERENA_LOCAL_PROJECT_FILE = "project.local.yml"
     FIELDS_WITHOUT_DEFAULTS = {"project_name", "languages"}
     YAML_COMMENT_NORMALISATION = YamlCommentNormalisation.LEADING
     """
@@ -296,13 +298,16 @@ class ProjectConfig(SharedConfig):
                 log.info("Using languages: %s", languages_to_use)
             else:
                 languages_to_use = [lang.value for lang in languages]
-            config_with_comments, _ = cls._load_yaml(PROJECT_TEMPLATE_FILE)
+            config_with_comments, _ = cls._load_yaml_dict(PROJECT_TEMPLATE_FILE)
             config_with_comments["project_name"] = project_name
             config_with_comments["languages"] = languages_to_use
+
             if save_to_disk:
                 project_yml_path = serena_config.get_project_yml_location(str(project_root))
                 log.info("Saving project configuration to %s", project_yml_path)
                 save_yaml(project_yml_path, config_with_comments)
+                project_local_yml_path = os.path.join(os.path.dirname(project_yml_path), cls.SERENA_LOCAL_PROJECT_FILE)
+                shutil.copy(PROJECT_LOCAL_TEMPLATE_FILE, project_local_yml_path)
             return cls._from_dict(config_with_comments)
 
     @classmethod
@@ -312,10 +317,10 @@ class ProjectConfig(SharedConfig):
             This is suitable as a fallback when no ``SerenaConfig`` is available to resolve
             a potentially customised location.
         """
-        return os.path.join(str(project_root), SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE)
+        return os.path.join(str(project_root), SERENA_MANAGED_DIR_NAME, cls.SERENA_PROJECT_FILE)
 
     @classmethod
-    def _load_yaml(
+    def _load_yaml_dict(
         cls, yml_path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE
     ) -> tuple[CommentedMap, bool]:
         """
@@ -429,7 +434,7 @@ class ProjectConfig(SharedConfig):
                 raise FileNotFoundError(f"Project configuration file not found: {yaml_path}")
 
         # load the configuration dictionary
-        yaml_data, was_complete = cls._load_yaml(str(yaml_path))
+        yaml_data, was_complete = cls._load_yaml_dict(str(yaml_path))
         if "project_name" not in yaml_data:
             yaml_data["project_name"] = project_folder_name
 
@@ -453,11 +458,11 @@ class ProjectConfig(SharedConfig):
         log.info("Saving updated project configuration to %s", config_path)
 
         # load original commented map and update it with current values
-        config_with_comments, _ = self._load_yaml(config_path, self.YAML_COMMENT_NORMALISATION)
+        config_with_comments, _ = self._load_yaml_dict(config_path, self.YAML_COMMENT_NORMALISATION)
         config_with_comments.update(self._to_yaml_dict())
 
         # transfer missing comments from the template file
-        template_config, _ = self._load_yaml(PROJECT_TEMPLATE_FILE, self.YAML_COMMENT_NORMALISATION)
+        template_config, _ = self._load_yaml_dict(PROJECT_TEMPLATE_FILE, self.YAML_COMMENT_NORMALISATION)
         transfer_missing_yaml_comments(template_config, config_with_comments, self.YAML_COMMENT_NORMALISATION)
 
         save_yaml(config_path, config_with_comments)
@@ -981,7 +986,7 @@ class SerenaConfig(SharedConfig):
         :return: the resolved absolute path to the project's project.yml file
         """
         serena_folder = self.get_project_serena_folder(project_root)
-        return os.path.join(serena_folder, ProjectConfig.SERENA_DEFAULT_PROJECT_FILE)
+        return os.path.join(serena_folder, ProjectConfig.SERENA_PROJECT_FILE)
 
     def propagate_settings(self) -> None:
         """

src/serena/constants.py

@@ -25,6 +25,7 @@ SERENA_FILE_ENCODING = "utf-8"
 """The encoding used for Serena's own files, such as configuration files and memories."""
 
 PROJECT_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.template.yml")
+PROJECT_LOCAL_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.local.template.yml")
 SERENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml")
 
 SERENA_LOG_FORMAT = "%(levelname)-5s %(asctime)-15s [%(threadName)s] %(name)s:%(funcName)s:%(lineno)d - %(message)s"

src/serena/project.py

@@ -219,6 +219,7 @@ class Project(ToStringMixin):
             log.info(f"Creating .gitignore file in {serena_data_gitignore_path}")
             with open(serena_data_gitignore_path, "w", encoding="utf-8") as f:
                 f.write(f"/{SolidLanguageServer.CACHE_FOLDER_NAME}\n")
+                f.write(f"/{ProjectConfig.SERENA_LOCAL_PROJECT_FILE}\n")
 
         # prepare ignore spec asynchronously, ensuring immediate project activation.
         self.__ignored_patterns: list[str]

src/serena/resources/project.local.template.yml

@@ -0,0 +1,5 @@
+# This file allows you to locally override settings in project.yml for development purposes.
+#
+# Use the same keys as in project.yml here. Any setting you specify will override the corresponding
+# setting in project.yml, allowing you to customise the configuration for your local development environment
+# without affecting the project configuration in project.yml (which is intended to be versioned).

test/serena/config/test_serena_config.py

@@ -135,7 +135,7 @@ class TestProjectConfigAutogenerate:
 
 class TestProjectConfig:
     def test_template_is_complete(self):
-        _, is_complete = ProjectConfig._load_yaml(PROJECT_TEMPLATE_FILE)
+        _, is_complete = ProjectConfig._load_yaml_dict(PROJECT_TEMPLATE_FILE)
         assert is_complete, "Project template YAML is incomplete; all fields must be present (with descriptions)."
 
 
@@ -177,7 +177,7 @@ class TestProjectConfigLanguageBackend:
     def test_language_backend_parsed_from_dict(self):
         """Test that _from_dict parses language_backend correctly."""
         template_path = PROJECT_TEMPLATE_FILE
-        data, _ = ProjectConfig._load_yaml(template_path)
+        data, _ = ProjectConfig._load_yaml_dict(template_path)
         data["project_name"] = "test"
         data["languages"] = ["python"]
         data["language_backend"] = "JetBrains"
@@ -187,7 +187,7 @@ class TestProjectConfigLanguageBackend:
     def test_language_backend_none_when_missing_from_dict(self):
         """Test that _from_dict handles missing language_backend gracefully."""
         template_path = PROJECT_TEMPLATE_FILE
-        data, _ = ProjectConfig._load_yaml(template_path)
+        data, _ = ProjectConfig._load_yaml_dict(template_path)
         data["project_name"] = "test"
         data["languages"] = ["python"]
         data.pop("language_backend", None)

test/solidlsp/fsharp/test_fsharp_basic.py

@@ -109,6 +109,7 @@ class TestFSharpLanguageServer:
         # The subtract function should be referenced in Program.fs
         assert any("Program.fs" in ref.get("relativePath", "") for ref in refs), "Program.fs should reference subtract function"
 
+    @pytest.mark.xfail(is_ci, reason="Test is flaky")  # TODO: Re-enable if the LS can be made more reliable #1040
     @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True)
     def test_go_to_definition(self, language_server: SolidLanguageServer) -> None:
         """Test go-to-definition functionality."""