Allow language server priorities to be configured in serena_config.yml (#1736)

CHANGELOG.md

@@ -5,8 +5,10 @@ Status of the `main` branch. Changes prior to the next official version change w
 * General:
   - Add Grok Build support (context `grok`, setup CLI, hooks)
 
-* Language Servers:
-    - Add `python_basedpyright` as an alternative Python language server
+* Language Servers: 
+  - Allow language server priorities to be configured in `serena_config.yml` (for auto-detection during 
+    project creation) 
+  - Add `python_basedpyright` as an alternative Python language server
 
 * Hooks:
   - Add `serena-hooks --client=grok`, including Grok-native PreToolUse allow/deny output.

docs/02-usage/050_configuration.md

@@ -33,6 +33,7 @@ Some of the configurable settings include:
   * logging settings
   * the set of trusted project paths
   * advanced settings specific to individual language servers (see [below](ls-specific-settings))
+  * priorities of language servers, which affect auto-detection 
 
 The global configuration settings apply to all projects.
 Some of the settings it contains can, however, be *extended* or *overridden* in project-specific settings, contexts and modes.

src/serena/config/serena_config.py

@@ -356,11 +356,20 @@ class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes):
         return ["project_name"]
 
     @classmethod
-    def _determine_project_languages(cls, project_root: str, interactive: bool) -> list[Language]:
-        # determine languages automatically
+    def _determine_project_languages(cls, project_root: str, interactive: bool, serena_config: "SerenaConfig") -> list[Language]:
         log.info("Determining programming languages used in the project")
-        language_composition = determine_programming_language_composition(project_root)
-        log.info("Language composition: %s", language_composition)
+
+        # determine languages to be considered and their priorities
+        language_priorities = {}
+        for language in Language:
+            priority = serena_config.get_ls_priority(language)
+            if priority > 0:
+                language_priorities[language] = priority
+
+        log.debug("Language priorities: %s", language_priorities)
+        language_composition = determine_programming_language_composition(project_root, list(language_priorities.keys()))
+        log.info("Project language composition: %s", language_composition)
+
         if len(language_composition) == 0:
             log.warning(
                 "No source files for supported language servers were found in %s. "
@@ -374,7 +383,7 @@ class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes):
         else:
             # sort languages by number of files found
             languages_and_percentages = sorted(
-                language_composition.items(), key=lambda item: (item[1], item[0].get_priority()), reverse=True
+                language_composition.items(), key=lambda item: (item[1], language_priorities[item[0]]), reverse=True
             )
             # find the language with the highest percentage and enable it
             top_language_pair = languages_and_percentages[0]
@@ -395,6 +404,7 @@ class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes):
                     if enable:
                         languages_to_use.append(lang)
                 print()
+
         log.info("Using languages: %s", languages_to_use)
         return languages_to_use
 
@@ -438,7 +448,9 @@ class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes):
                     use_asynchronous_language_determination = True
                     languages_to_use = []  # temporarily empty, will be determined in background thread
                 else:
-                    determined_languages = cls._determine_project_languages(str(project_root), interactive=interactive)
+                    determined_languages = cls._determine_project_languages(
+                        str(project_root), interactive=interactive, serena_config=serena_config
+                    )
                     languages_to_use = [l.value for l in determined_languages]
             else:
                 languages_to_use = [lang.value for lang in languages]
@@ -465,7 +477,9 @@ class ProjectConfig(SharedConfig, ModeSelectionDefinitionWithAddedModes):
                 def async_language_determination():
                     try:
                         with LogTime("Asynchronous language determination", logger=log):
-                            project_config.languages = cls._determine_project_languages(str(project_root), interactive=False)
+                            project_config.languages = cls._determine_project_languages(
+                                str(project_root), interactive=False, serena_config=serena_config
+                            )
                             if save_to_disk:
                                 project_config.save(project_yml_path)
                     finally:
@@ -914,6 +928,11 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes):
     defined in the configuration template file. 
     """
 
+    ls_priorities: dict[str, int] | None = None
+    """
+    mapping from language server keys to their priority (higher number = higher priority).
+    """
+
     # settings with overridden defaults
 
     language_backend: LanguageBackend = LanguageBackend.LSP
@@ -1431,3 +1450,19 @@ class SerenaConfig(SharedConfig, ModeSelectionDefinitionWithBaseModes):
             if log_choice:
                 log.info(f"Using language backend from global configuration: {language_backend.name}")
         return language_backend
+
+    def get_ls_priority(self, language: Language) -> int:
+        """
+        Gets the priority value associated with a language server
+
+        :param language: identifies the language server
+        :return: the integer priority
+        """
+        if self.ls_priorities is not None:
+            try:
+                configured_value = self.ls_priorities.get(language.value)
+                if configured_value is not None:
+                    return int(configured_value)
+            except Exception as e:
+                log.error("Error reading language priority for %s: %s. Using default priority.", language.value, e)
+        return language.get_priority()

src/serena/resources/serena_config.template.yml

@@ -100,6 +100,24 @@ trace_lsp_communication: False
 # No documentation on options means no options are available.
 ls_specific_settings: {}
 
+# mapping from language server keys to integer priority values (higher = more preferred), which determine
+# language server selection during automatic project creation and are used to break ties when multiple
+# language servers match an equal number of source files.
+# The priority values override Serena's default priorities:
+#   * 0 = experimental or secondary language servers, which are never auto-detected
+#         (e.g. `python_*` and `php_*` are "secondary"; also applies to non-programming languages like `json`)
+#   * 1 = supersets of other languages, which shall be preferred only if they match more files
+#         (e.g. `vue` and `svelte`, which are supersets of `typescript`)
+#   * 2 = default priority
+# Example:
+#   ls_priorities:
+#     python: 0
+#     python_basedpyright: 3
+# This example would achieve that `python_basedpyright` becomes the preferred language server for Python,
+# giving it a high priority of 3, while the default Python language server is fully excluded from
+# auto-detection with priority 0.
+ls_priorities:
+
 # list of paths to ignore across all projects.
 # Same syntax as gitignore, so you can use * and **.
 # These patterns are merged additively with each project's own ignored_paths.

src/serena/util/inspection.py

@@ -27,7 +27,7 @@ def iter_subclasses(
             yield from iter_subclasses(subclass, recursive, inclusion_predicate)
 
 
-def determine_programming_language_composition(repo_path: str) -> dict[Language, float]:
+def determine_programming_language_composition(repo_path: str, languages: list[Language] | None = None) -> dict[Language, float]:
     """
     Determine the programming language composition of a repository.
 
@@ -36,18 +36,20 @@ def determine_programming_language_composition(repo_path: str) -> dict[Language,
     belong to no supported language (images, plain text, licenses, lock files, etc.)
     from diluting language percentages in repositories where such files dominate.
 
-    :param repo_path: Path to the repository to analyze
-    :return: Dictionary mapping languages to percentages of recognised source files
-        matching each language (denominator = files matched by at least one language)
+    :param repo_path: path to the repository to analyze
+    :param languages: the list of languages to consider; if None, use default languages
+    :return: dictionary mapping languages to percentages of recognised source files
+        matching the respective language (denominator = files matched by at least one language)
     """
+    if languages is None:
+        languages = list(Language.iter_all(include_experimental=False))
+
     all_files = find_all_non_ignored_files(repo_path)
 
     if not all_files:
         return {}
 
-    # collect all language matchers once
-    all_languages = list(Language.iter_all(include_experimental=False))
-    matchers = {lang: lang.get_source_fn_matcher() for lang in all_languages}
+    matchers = {lang: lang.get_source_fn_matcher() for lang in languages}
 
     # count files per language in a single pass over the files
     language_counts: dict[Language, int] = {}

src/solidlsp/ls_config.py

@@ -288,7 +288,7 @@ class Language(str, Enum):
     """
 
     @classmethod
-    def iter_all(cls, include_experimental: bool = False, include_non_programming_languages: bool = True) -> Iterable[Self]:
+    def iter_all(cls, include_experimental: bool = True, include_non_programming_languages: bool = True) -> Iterable[Self]:
         for lang in cls:
             if include_experimental or not lang.is_experimental():
                 if include_non_programming_languages or lang.is_programming_language():
@@ -296,11 +296,8 @@ class Language(str, Enum):
 
     def is_experimental(self) -> bool:
         """
-        Check if the language server is experimental or deprecated.
-
-        Note for serena users/developers:
-        Experimental languages are not autodetected and must be explicitly specified
-        in the project.yml configuration.
+        Check if the language server is experimental (potentially not robust),
+        secondary (not default for respective language) or deprecated.
         """
         return self in {
             self.ANSIBLE,

test/serena/config/test_serena_config.py

@@ -2,6 +2,7 @@ import logging
 import os
 import shutil
 import tempfile
+from copy import deepcopy
 from pathlib import Path
 
 import pytest
@@ -63,6 +64,22 @@ class TestProjectConfigAutogenerate:
         assert config.project_name == self.project_path.name
         assert config.languages == [Language.PYTHON]
 
+    def test_autogenerate_with_python_files_and_custom_ls_priorities(self):
+        """Test successful autogeneration with Python source files, using custom language server priorities."""
+        # Create a Python file
+        python_file = self.project_path / "main.py"
+        python_file.write_text("def hello():\n    print('Hello, world!')\n")
+
+        serena_config = deepcopy(self.serena_config)
+        serena_config.ls_priorities = {Language.PYTHON_TY.value: 3}
+
+        # Run autogenerate
+        config = ProjectConfig.autogenerate(self.project_path, serena_config, save_to_disk=False)
+
+        # Verify the configuration
+        assert config.project_name == self.project_path.name
+        assert config.languages == [Language.PYTHON_TY]
+
     def test_autogenerate_with_js_files(self):
         """Test successful autogeneration with JavaScript source files."""
         # Create files for multiple languages