Merge branch 'main' of github.com:oraios/serena

Conflicts: src/serena/agent.py src/serena/resources/dashboard/dashboard.css src/serena/resources/dashboard/dashboard.js src/serena/resources/dashboard/index.html
2026-10-13 14:16:21 +00:00 · 2025-10-30 13:26:40 +01:00
commit 14f12c4cb0
@@ -1,11 +1,5 @@
-# language of the project (csharp, python, rust, java, typescript, javascript, go, cpp, or ruby)
-# Special requirements:
-#  * csharp: Requires the presence of a .sln file in the project folder.
-language: python
-
-# whether to use the project's gitignore file to ignore files
-# Added on 2025-04-07
 ignore_all_files_in_gitignore: true
+
 # list of additional paths to ignore
 # same syntax as gitignore, so you can use * and **
 # Was previously called `ignored_dirs`, please update your config if you are using that.
@@ -64,3 +58,8 @@ excluded_tools: []
 initial_prompt: ""

 project_name: "serena"
+languages:
+- python
+- typescript
+included_optional_tools: []
+encoding: utf-8
@@ -2,6 +2,7 @@
 The Serena Model Context Protocol (MCP) Server
 """

+import concurrent.futures
 import multiprocessing
 import os
 import platform
@@ -9,12 +10,15 @@ import sys
 import threading
 import webbrowser
 from collections.abc import Callable
-from concurrent.futures import Future, ThreadPoolExecutor
+from concurrent.futures import Future
 from logging import Logger
+from queue import Queue
+from threading import Thread
 from typing import TYPE_CHECKING, Any, Optional, TypeVar

 from sensai.util import logging
 from sensai.util.logging import LogTime
+from sensai.util.string import ToStringMixin

 from interprompt.jinja_template import JinjaTemplate
 from serena import serena_version
@@ -153,7 +157,9 @@ class SerenaAgent:

        # create executor for starting the language server and running tools in another thread
        # This executor is used to achieve linear task execution, so it is important to use a single-threaded executor.
-        self._task_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="SerenaAgentExecutor")
+        self._task_executor_queue = Queue()
+        self._task_executor_thread = Thread(target=self._process_task_queue, name="SerenaAgentTaskExecutor", daemon=True)
+        self._task_executor_thread.start()
        self._task_executor_lock = threading.Lock()
        self._task_executor_task_index = 1

@@ -192,6 +198,29 @@ class SerenaAgent:
                process.start()
                process.join(timeout=1)

+    class Task(ToStringMixin):
+        def __init__(self, function: Callable[[], Any], name: str):
+            self.name = name
+            self.future = concurrent.futures.Future()
+            self._function = function
+
+        def _tostring_includes(self) -> list[str]:
+            return ["name"]
+
+        def execute(self):
+            try:
+                with LogTime(self.name, logger=log):
+                    result = self._function()
+                    self.future.set_result(result)
+            except Exception as e:
+                log.error(f"Error during execution of {self.name}: {e}", exc_info=e)
+                self.future.set_exception(e)
+
+    def _process_task_queue(self):
+        while True:
+            task: SerenaAgent.Task = self._task_executor_queue.get()
+            task.execute()
+
    def get_language_server_manager(self) -> LanguageServerManager | None:
        if self._active_project is not None:
            return self._active_project.language_server_manager
@@ -376,16 +405,10 @@ class SerenaAgent:
        with self._task_executor_lock:
            task_name = f"Task-{self._task_executor_task_index}[{name or task.__name__}]"
            self._task_executor_task_index += 1
-
-            def task_execution_wrapper() -> Any:
-                try:
-                    with LogTime(task_name, logger=log):
-                        return task()
-                except Exception as e:
-                    log.error(f"Error during execution of {task_name}: {e}", exc_info=e)
-
            log.info(f"Scheduling {task_name}")
-            return self._task_executor.submit(task_execution_wrapper)
+            task = SerenaAgent.Task(function=task, name=task_name)
+            self._task_executor_queue.put(task)
+            return task.future

    def execute_task(self, task: Callable[[], T], name: str | None = None) -> T:
        """
@@ -543,22 +566,15 @@ class SerenaAgent:
            ls_specific_settings=self.serena_config.ls_specific_settings,
        )

-    def add_language(self, language: Language, wait_for_ls_startup: bool = False) -> None:
+    def add_language(self, language: Language) -> None:
        """
        Adds a new language to the active project, spawning the respective language server and updating the project configuration.
        The addition is scheduled via the agent's task executor and executed synchronously, i.e. the method returns
        when the addition is complete.

        :param language: the language to add
-        :param wait_for_ls_startup: whether to wait for the language server to start up before returning (synchronous execution)
-            or to just issue the addition task and return immediately (asynchronous execution)
        """
-        task = lambda: self.get_active_project_or_raise().add_language(language)
-        task_name = f"AddLanguage:{language.value}"
-        if wait_for_ls_startup:
-            self.execute_task(task, name=task_name)
-        else:
-            self.issue_task(task, name=task_name)
+        self.execute_task(lambda: self.get_active_project_or_raise().add_language(language), name=f"AddLanguage:{language.value}")

    def remove_language(self, language: Language) -> None:
        """
@@ -41,6 +41,8 @@ if TYPE_CHECKING:
 log = logging.getLogger(__name__)
 T = TypeVar("T")
 DEFAULT_TOOL_TIMEOUT: float = 240
+DictType = dict | CommentedMap
+TDict = TypeVar("TDict", bound=DictType)


@singleton
@@ -212,7 +214,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
                languages_to_use = [dominant_language]
            else:
                languages_to_use = [lang.value for lang in languages]
-            config_with_comments = load_yaml(PROJECT_TEMPLATE_FILE, preserve_comments=True)
+            config_with_comments = cls.load_commented_map(PROJECT_TEMPLATE_FILE)
            config_with_comments["project_name"] = project_name
            config_with_comments["languages"] = languages_to_use
            if save_to_disk:
@@ -223,21 +225,47 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
    def rel_path_to_project_yml(cls) -> str:
        return os.path.join(SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE)

+    @classmethod
+    def _apply_defaults_to_dict(cls, data: TDict) -> TDict:
+        # apply defaults for new fields
+        data["languages"] = data.get("languages", [])
+        data["ignored_paths"] = data.get("ignored_paths", [])
+        data["excluded_tools"] = data.get("excluded_tools", [])
+        data["included_optional_tools"] = data.get("included_optional_tools", [])
+        data["read_only"] = data.get("read_only", False)
+        data["ignore_all_files_in_gitignore"] = data.get("ignore_all_files_in_gitignore", True)
+        data["initial_prompt"] = data.get("initial_prompt", "")
+        data["encoding"] = data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING)
+
+        # backward compatibility: handle single "language" field
+        if len(data["languages"]) == 0 and "language" in data:
+            data["languages"] = [data["language"]]
+        if "language" in data:
+            del data["language"]
+
+        return data
+
+    @classmethod
+    def load_commented_map(cls, yml_path: str) -> CommentedMap:
+        """
+        Load the project configuration as a CommentedMap, preserving comments and ensuring
+        completeness of the configuration by applying default values for missing fields
+        and backward compatibility adjustments.
+
+        :param yml_path: the path to the project.yml file
+        :return: a CommentedMap representing a full project configuration
+        """
+        data = load_yaml(yml_path, preserve_comments=True)
+        return cls._apply_defaults_to_dict(data)
+
    @classmethod
    def _from_dict(cls, data: dict[str, Any]) -> Self:
        """
-        Create a ProjectConfig instance from a configuration dictionary
+        Create a ProjectConfig instance from a (full) configuration dictionary
        """
-        project_name = data["project_name"]
-        language_strings = data.get("languages", [])
-
-        # backwards compatibility
        lang_name_mapping = {"javascript": "typescript"}
-        if len(language_strings) == 0 and "language" in data:
-            language_strings = [data["language"]]
-
        languages: list[Language] = []
-        for language_str in language_strings:
+        for language_str in data["languages"]:
            try:
                language_str = language_str.lower()
                if language_str in lang_name_mapping:
@@ -248,15 +276,15 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
                raise ValueError(f"Invalid language: {data['language']}.\nValid language_strings are: {[l.value for l in Language]}") from e

        return cls(
-            project_name=project_name,
+            project_name=data["project_name"],
            languages=languages,
-            ignored_paths=data.get("ignored_paths", []),
-            excluded_tools=data.get("excluded_tools", []),
-            included_optional_tools=data.get("included_optional_tools", []),
-            read_only=data.get("read_only", False),
-            ignore_all_files_in_gitignore=data.get("ignore_all_files_in_gitignore", True),
-            initial_prompt=data.get("initial_prompt", ""),
-            encoding=data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING),
+            ignored_paths=data["ignored_paths"],
+            excluded_tools=data["excluded_tools"],
+            included_optional_tools=data["included_optional_tools"],
+            read_only=data["read_only"],
+            ignore_all_files_in_gitignore=data["ignore_all_files_in_gitignore"],
+            initial_prompt=data["initial_prompt"],
+            encoding=data["encoding"],
        )

    def to_yaml_dict(self) -> dict:
@@ -279,8 +307,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
                return cls.autogenerate(project_root)
            else:
                raise FileNotFoundError(f"Project configuration file not found: {yaml_path}")
-        with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f:
-            yaml_data = yaml.safe_load(f)
+        yaml_data = cls.load_commented_map(str(yaml_path))
        if "project_name" not in yaml_data:
            yaml_data["project_name"] = project_root.name
        return cls._from_dict(yaml_data)
@@ -414,7 +414,7 @@ class SerenaDashboardAPI:
            raise ValueError(f"Invalid language: {request_add_language.language}")

        # Add the language to the active project
-        self._agent.add_language(language, wait_for_ls_startup=False)
+        self._agent.add_language(language)

    def _remove_language(self, request_remove_language: RequestRemoveLanguage) -> None:
        from solidlsp.ls_config import Language
@@ -12,7 +12,7 @@ from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME
 from serena.ls_manager import LanguageServerFactory, LanguageServerManager
 from serena.text_utils import MatchedConsecutiveLines, search_files
 from serena.util.file_system import GitignoreParser, match_path
-from serena.util.general import load_yaml, save_yaml
+from serena.util.general import save_yaml
 from solidlsp import SolidLanguageServer
 from solidlsp.ls_config import Language
 from solidlsp.ls_utils import FileUtils
@@ -116,7 +116,7 @@ class Project(ToStringMixin):
        """
        config_path = os.path.join(self.project_root, self.project_config.rel_path_to_project_yml())
        log.info("Saving updated project configuration to %s", config_path)
-        config_with_comments = load_yaml(config_path, preserve_comments=True)
+        config_with_comments = ProjectConfig.load_commented_map(config_path)
        config_with_comments.update(self.project_config.to_yaml_dict())
        save_yaml(config_path, config_with_comments, preserve_comments=True)

@@ -405,9 +405,6 @@ class Project(ToStringMixin):
        if language in self.project_config.languages:
            log.info(f"Language {language.value} is already present in the project configuration.")
            return
-        # update the project configuration
-        self.project_config.languages.append(language)
-        self.save_config()

        # start the language server (if the LS manager is active)
        if self.language_server_manager is None:
@@ -416,6 +413,10 @@ class Project(ToStringMixin):
            log.info("Adding and starting the language server for new language %s ...", language.value)
            self.language_server_manager.add_language_server(language)

+        # update the project configuration
+        self.project_config.languages.append(language)
+        self.save_config()
+
    def remove_language(self, language: Language) -> None:
        """
        Removes a programming language from the project configuration, stopping the corresponding
@@ -1,3 +1,7 @@
+html {
+    scrollbar-gutter: stable; /* Prevent layout shift when scrollbar appears */
+}
+
 :root {
    /* Light theme variables */
    --bg-primary: #f5f5f5;
@@ -43,15 +47,16 @@
 body {
    font-family: 'Consolas', 'Monaco', 'Courier New', monospace;
    margin: 0;
-    padding: 20px;
-    padding-top: 130px;
    background-color: var(--bg-primary);
    color: var(--text-primary);
    transition: background-color 0.3s ease, color 0.3s ease;
 }

+.main {
+    padding: 20px;
+}
+
 .header {
-    position: fixed;
    top: 0;
    left: 0;
    right: 0;
@@ -72,7 +77,7 @@ body {
 }

 .logo-container img {
-    max-width: 300px;
+    max-width: 250px;
    height: auto;
    display: block;
 }
@@ -712,6 +717,46 @@ body {
    transform: scale(1.1);
 }

+.language-add-btn {
+    padding: 6px 12px;
+    font-size: 13px;
+    font-weight: 500;
+    border-radius: 6px;
+    border: 1px dashed var(--border-color);
+    background: var(--bg-secondary);
+    color: var(--text-primary);
+    cursor: pointer;
+    transition: all 0.2s;
+}
+
+.language-add-btn:hover {
+    background: var(--bg-tertiary);
+    border-color: var(--btn-primary);
+    color: var(--btn-primary);
+}
+
+.language-spinner {
+    display: inline-flex;
+    align-items: center;
+    justify-content: center;
+    padding: 6px 12px;
+}
+
+.spinner {
+    width: 16px;
+    height: 16px;
+    border: 2px solid var(--border-color);
+    border-top-color: var(--btn-primary);
+    border-radius: 50%;
+    animation: spin 0.8s linear infinite;
+}
+
+@keyframes spin {
+    to {
+        transform: rotate(360deg);
+    }
+}
+
 /* Memory Editor Styles */
 .modal-content-large {
    max-width: 800px;
@@ -87,7 +87,6 @@ class Dashboard {
        this.$availableToolsDisplay = $('#available-tools-display');
        this.$availableModesDisplay = $('#available-modes-display');
        this.$availableContextsDisplay = $('#available-contexts-display');
-        this.$addLanguageBtn = $('#add-language-btn');
        this.$addLanguageModal = $('#add-language-modal');
        this.$modalLanguageSelect = $('#modal-language-select');
        this.$modalProjectName = $('#modal-project-name');
@@ -127,7 +126,6 @@ class Dashboard {
        this.$themeToggle.click(this.toggleTheme.bind(this));
        this.$refreshStats.click(this.loadStats.bind(this));
        this.$clearStats.click(this.clearStats.bind(this));
-        this.$addLanguageBtn.click(this.openLanguageModal.bind(this));
        this.$modalAddBtn.click(this.addLanguageFromModal.bind(this));
        this.$modalCancelBtn.click(this.closeLanguageModal.bind(this));
        this.$modalClose.click(this.closeLanguageModal.bind(this));
@@ -340,6 +338,13 @@ class Dashboard {
                    }
                    html += '</div>';
                });
+                // Add the "Add Language" button inline with language badges (only if active project exists)
+                if (config.active_project && config.active_project.name) {
+                    html += '<button id="add-language-btn" class="btn language-add-btn">+ Add Language</button>';
+                    html += '<div id="add-language-spinner" class="language-spinner" style="display:none;">';
+                    html += '<div class="spinner"></div>';
+                    html += '</div>';
+                }
                html += '</div>';
            } else {
                html += 'N/A';
@@ -370,12 +375,6 @@ class Dashboard {

        html += '</div>';

-        // Configuration help link
-        html += '<div style="margin-top: 15px; padding: 10px; background: var(--bg-secondary); border-radius: 4px; font-size: 13px; border: 1px solid var(--border-color);">';
-        html += '<span style="color: var(--text-muted);">📖</span>';
-        html += '<a href="https://github.com/oraios/serena#configuration" target="_blank" rel="noopener noreferrer" style="color: var(--btn-primary); text-decoration: none; font-weight: 500;">View Configuration Guide</a>';
-        html += '</div>';
-
        // Active tools - collapsible
        html += '<div style="margin-top: 20px;">';
        html += '<h3 class="collapsible-header" id="tools-header" style="font-size: 16px; margin: 0;">';
@@ -407,14 +406,16 @@ class Dashboard {
            html += '</div>';
        }

+        // Configuration help link
+        html += '<div style="margin-top: 15px; padding: 10px; background: var(--bg-secondary); border-radius: 4px; font-size: 13px; border: 1px solid var(--border-color);">';
+        html += '<span style="color: var(--text-muted);">📖</span> ';
+        html += '<a href="https://github.com/oraios/serena#configuration" target="_blank" rel="noopener noreferrer" style="color: var(--btn-primary); text-decoration: none; font-weight: 500;">View Configuration Guide</a>';
+        html += '</div>';
+
        this.$configDisplay.html(html);

-        // Show/hide add language button based on jetbrains mode
-        if (this.jetbrainsMode || this.activeProjectName === null) {
-            this.$addLanguageBtn.hide();
-        } else {
-            this.$addLanguageBtn.show();
-        }
+        // Attach event handlers for the dynamically created add language button
+        $('#add-language-btn').click(this.openLanguageModal.bind(this));

        // Attach event handlers for language remove buttons
        const self = this;
@@ -1052,9 +1053,9 @@ class Dashboard {
        const logoElement = document.getElementById('serena-logo');
        if (logoElement) {
            if (theme === 'dark') {
-                logoElement.src = 'serena-logs-dark-mode.png';
+                logoElement.src = 'serena-logo-dark-mode.svg';
            } else {
-                logoElement.src = 'serena-logs.png';
+                logoElement.src = 'serena-logo.svg';
            }
        }
    }
@@ -1205,8 +1206,13 @@ class Dashboard {
        }

        const self = this;
-        // Disable button during request
-        self.$modalAddBtn.prop('disabled', true).text('Adding...');
+
+        // Close modal immediately
+        self.closeLanguageModal();
+
+        // Hide the inline add language button and show spinner
+        $('#add-language-btn').hide();
+        $('#add-language-spinner').show();

        $.ajax({
            url: '/add_language',
@@ -1217,21 +1223,21 @@ class Dashboard {
            }),
            success: function(response) {
                if (response.status === 'success') {
-                    // Close modal
-                    self.closeLanguageModal();
-                    // Reload config to show updated language
+                    // Reload config to show updated language (this will restore the button)
                    self.loadConfigOverview();
                } else {
                    alert('Error: ' + response.message);
+                    // Restore button visibility on error
+                    $('#add-language-btn').show();
+                    $('#add-language-spinner').hide();
                }
            },
            error: function(xhr, status, error) {
                console.error('Error adding language:', error);
                alert('Error adding language: ' + (xhr.responseJSON ? xhr.responseJSON.message : error));
-            },
-            complete: function() {
-                // Re-enable button
-                self.$modalAddBtn.prop('disabled', false).text('Add Language');
+                // Restore button visibility on error
+                $('#add-language-btn').show();
+                $('#add-language-spinner').hide();
            }
        });
    }
@@ -38,7 +38,7 @@
            </div>
        </nav>
    </header>
-
+    <div class="main">
    <!-- Overview Page (Landing) -->
    <div id="page-overview" class="page-view">
        <div class="overview-container">
@@ -46,7 +46,6 @@
                <section class="config-section">
                    <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 15px;">
                        <h2 style="margin: 0;">Current Configuration</h2>
-                        <button id="add-language-btn" class="btn" style="padding: 6px 12px; font-size: 14px;">Add Language</button>
                    </div>
                    <div id="config-display">
                        <div class="loading">Loading configuration...</div>
@@ -210,7 +209,7 @@
                <button id="delete-memory-ok-btn" class="btn" style="padding: 8px 16px;">OK</button>
            </div>
        </div>
-    </div>
+    </div> <!-- End of Main -->

    <script>
        $(document).ready(function () {
@@ -51,6 +51,8 @@ class ClangdLanguageServer(SolidLanguageServer):
        """
        Setup runtime dependencies for ClangdLanguageServer and return the command to start the server.
        """
+        import shutil
+
        deps = RuntimeDependencyCollection(
            [
                RuntimeDependency(
@@ -89,21 +91,40 @@ class ClangdLanguageServer(SolidLanguageServer):
        )

        clangd_ls_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "clangd")
-        dep = deps.get_single_dep_for_current_platform()
-        clangd_executable_path = deps.binary_path(clangd_ls_dir)
-        if not os.path.exists(clangd_executable_path):
-            logger.log(
-                f"Clangd executable not found at {clangd_executable_path}. Downloading from {dep.url}",
-                logging.INFO,
-            )
-            deps.install(logger, clangd_ls_dir)
-        if not os.path.exists(clangd_executable_path):
-            raise FileNotFoundError(
-                f"Clangd executable not found at {clangd_executable_path}.\n"
-                "Make sure you have installed clangd. See https://clangd.llvm.org/installation"
-            )
-        os.chmod(clangd_executable_path, 0o755)

+        try:
+            dep = deps.get_single_dep_for_current_platform()
+        except RuntimeError:
+            dep = None
+
+        if dep is None:
+            # No prebuilt binary available, look for system-installed clangd
+            clangd_executable_path = shutil.which("clangd")
+            if not clangd_executable_path:
+                raise FileNotFoundError(
+                    "Clangd is not installed on your system.\n"
+                    + "Please install clangd using your system package manager:\n"
+                    + "  Ubuntu/Debian: sudo apt-get install clangd\n"
+                    + "  Fedora/RHEL: sudo dnf install clang-tools-extra\n"
+                    + "  Arch Linux: sudo pacman -S clang\n"
+                    + "See https://clangd.llvm.org/installation for more details."
+                )
+            logger.log(f"Using system-installed clangd at {clangd_executable_path}", logging.INFO)
+        else:
+            # Standard download and install for platforms with prebuilt binaries
+            clangd_executable_path = deps.binary_path(clangd_ls_dir)
+            if not os.path.exists(clangd_executable_path):
+                logger.log(
+                    f"Clangd executable not found at {clangd_executable_path}. Downloading from {dep.url}",
+                    logging.INFO,
+                )
+                _ = deps.install(logger, clangd_ls_dir)
+            if not os.path.exists(clangd_executable_path):
+                raise FileNotFoundError(
+                    f"Clangd executable not found at {clangd_executable_path}.\n"
+                    + "Make sure you have installed clangd. See https://clangd.llvm.org/installation"
+                )
+            os.chmod(clangd_executable_path, 0o755)
        return clangd_executable_path

    @staticmethod