Merge branch 'feature/web-server-logs' of github.com:oraios/serena into feature/web-server-logs

Conflicts:
  src/serena/agent.py

.gitignore

@@ -1,4 +1,64 @@
-#
+# macOS specific files
+.DS_Store
+.AppleDouble
+.LSOverride
+._*
+.Spotlight-V100
+.Trashes
+Icon
+.fseventsd
+.DocumentRevisions-V100
+.TemporaryItems
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Windows specific files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+*.stackdump
+[Dd]esktop.ini
+$RECYCLE.BIN/
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+*.lnk
+
+# Linux specific files
+*~
+.fuse_hidden*
+.directory
+.Trash-*
+.nfs*
+
+# IDE/Text Editors
+# VS Code
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+.history/
+
+# JetBrains IDEs (beyond .idea/)
+*.iml
+*.ipr
+*.iws
+out/
+.idea_modules/
+
+# Sublime Text
+*.tmlanguage.cache
+*.tmPreferences.cache
+*.stTheme.cache
+*.sublime-workspace
+*.sublime-project
+
+# Project specific ignore
 .idea
 temp
 

pyproject.toml

@@ -36,6 +36,7 @@ dependencies = [
   "psutil>=7.0.0",
   "agno>=1.2.15",
   "docstring_parser>=0.16",
+  "joblib>=1.5.1",
 ]
 
 [project.scripts]
@@ -257,4 +258,4 @@ markers = [
     "rust: language server running for Rust",
     "typescript: language server running for TypeScript",
     "php: language server running for PHP",
-]
+]

serena_config.template.yml

@@ -24,14 +24,12 @@ gui_log_level: 20
 # minimum log level for the GUI log window (10 = debug, 20 = info, 30 = warning, 40 = error)
 
 
-# Add your list of projects here (which you can switch between using "activate_project").
-# Every list item must be a path to the project's root directory (e.g. "/path/to/myproject"), with the
-# configuration file being located in "/path/to/myproject/.serena/project.yml", or a dict mapping
-# project_name to project_path (to avoid name collisions).
-# You can activate a project by its name using the "activate_project" command.
-# If the name is not explicitly set, the name of the project will be the name of the directory 
-# ("myproject" in the example above).
-# Note: comments in the block below are not preserved when saving the config file,
-# so we recommend to not have any comments directly below the "projects" key.
+# The list of registered projects (which you can switch between using "activate_project").
+# MANAGED BY SERENA, KEEP AT THE BOTTOM OF THE YAML AND DON'T ADJUST WITHOUT NEED (you may add paths though)
+# Whenever a project is activated (either by using the tool or by starting the server with --project),
+# the project's path will be added here and you will be able to activate it by its name in the future.
+# By default, the project's name will be the name of the directory containing the project, but you may change it
+# by changing the entry in the (autogenerated) `.serena/project.yml` file.
+# Note: make sure there are no name collisions in the names of registered projects
 projects:
   - /path/to/myproject

src/multilspy/language_server.py

@@ -136,11 +136,11 @@ class LanguageServer:
             return PyrightServer(config, logger, repository_root_path)
             # It used to be jedi, but pyright is a bit faster, and also more actively maintained
             # Keeping the previous code for reference
-            from multilspy.language_servers.jedi_language_server.jedi_server import (
-                JediServer,
-            )
+            # from multilspy.language_servers.jedi_language_server.jedi_server import (
+            #     JediServer,
+            # )
 
-            return JediServer(config, logger, repository_root_path)
+            # return JediServer(config, logger, repository_root_path)
         elif config.code_language == Language.JAVA:
             from multilspy.language_servers.eclipse_jdtls.eclipse_jdtls import (
                 EclipseJDTLS,
@@ -1177,42 +1177,25 @@ class LanguageServer:
 
 
     async def request_parsed_files(self) -> list[str]:
-        """
-        Retrieves relative paths of all files analyzed by the Language Server.
-
-        This is slow, as it finds all files by finding all symbols.
-
-        This seems to be the only way, the LSP does not provide any endpoints for listing project files."""
+        """Retrieves relative paths of all files analyzed by the Language Server."""
         if not self.server_started:
             self.logger.log(
                 "request_parsed_files called before Language Server started",
                 logging.ERROR,
             )
             raise MultilspyException("Language Server not started")
-        # TODO: this worked in jedi, but pyright and basedpyright return nothing...
-        # I don't know why
-        # params = LSPTypes.WorkspaceSymbolParams(query="")  # Empty query returns all symbols
-        # symbols = await self.server.send.workspace_symbol(params) or []
-
-        # Thus, instead of calling all symbols, we hack this and use the symbol tree instead, which
-        # seems to work in all these language servers
-        # walk through all children recursively, find all symbols of type Module and collect their relative paths
-        roots = await self.request_full_symbol_tree()
-        paths = []
-        def collect_module_files(symbol):
-            if symbol["kind"] == multilspy_types.SymbolKind.File:
-                assert "location" in symbol
-                paths.append(symbol["location"]["relativePath"])
-
-            elif symbol["kind"] == multilspy_types.SymbolKind.Package:
-                for child in symbol["children"]:
-                    collect_module_files(child)
-
-        for root in roots:
-            collect_module_files(root)
-
-        return paths
-
+        rel_file_paths = []
+        for root, dirs, files in os.walk(self.repository_root_path):
+            # Don't go into directories that are ignored by modifying dirs inplace
+            # Explanation for the  + "/" part:
+            # pathspec can't handle the matching of directories if they don't end with a slash!
+            # see https://github.com/cpburnz/python-pathspec/issues/89
+            dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))]
+            for file in files:
+                rel_file_path = os.path.join(root, file)
+                if not self.is_ignored_path(rel_file_path):
+                    rel_file_paths.append(rel_file_path)
+        return rel_file_paths
 
     async def search_files_for_pattern(
         self,
@@ -1926,9 +1909,7 @@ class SyncLanguageServer:
         return self.language_server.retrieve_symbol_body(symbol)
 
     def request_parsed_files(self) -> list[str]:
-        """This is slow, as it finds all files by finding all symbols.
-
-        This seems to be the only way, the LSP does not provide any endpoints for listing project files."""
+        """Retrieves relative paths of all files analyzed by the Language Server."""
         assert self.loop
         result = asyncio.run_coroutine_threadsafe(
             self.language_server.request_parsed_files(), self.loop

src/serena/agent.py

@@ -21,6 +21,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, Self, TypeVar, Union
 
 import yaml
+from overrides import override
 from ruamel.yaml.comments import CommentedMap
 from sensai.util import logging
 from sensai.util.logging import FallbackHandler
@@ -108,6 +109,12 @@ class ProjectConfig(ToStringMixin):
             raise FileNotFoundError(f"Project root not found: {project_root}")
         project_name = project_name or project_root.name
         language_composition = determine_programming_language_composition(str(project_root))
+        if len(language_composition) == 0:
+            raise ValueError(
+                f"Failed to autogenerate project.yaml: no programming language detected in project {project_root}. "
+                f"You can either add some files that correspond to one of the supported programming languages, "
+                f"or create the file {os.path.join(project_root, cls.rel_path_to_project_yml())} manually and specify the language there."
+            )
         # find the language with the highest percentage
         dominant_language = max(language_composition.keys(), key=lambda lang: language_composition[lang])
         result = cls(
@@ -197,19 +204,16 @@ class Project:
         return os.path.join(self.project_root, self.project_config.rel_path_to_project_yml())
 
 
-@dataclass
-class SerenaConfig:
+@dataclass(kw_only=True)
+class SerenaConfigBase(ABC):
     """
-    Handles user-defined Serena configuration based on the configuration file
+    Abstract base class for Serena configuration handling
     """
 
     projects: list[Project] = field(default_factory=list)
     gui_log_window_enabled: bool = False
     gui_log_window_level: int = logging.INFO
     web_dashboard: bool = True
-    loaded_original_yaml: dict | CommentedMap = field(default_factory=dict)
-
-    CONFIG_FILE = "serena_config.yml"
 
     @cached_property
     def project_paths(self) -> list[str]:
@@ -219,87 +223,7 @@ class SerenaConfig:
     def project_names(self) -> list[str]:
         return sorted(project.project_config.project_name for project in self.projects)
 
-    @classmethod
-    def get_config_file_path(cls) -> str:
-        return os.path.join(serena_root_path(), cls.CONFIG_FILE)
-
-    @classmethod
-    def from_config_file(cls, preserve_comments: bool = False) -> "SerenaConfig":
-        """
-        Static constructor to create SerenaConfig from the configuration file
-        """
-        config_file = cls.get_config_file_path()
-        if not os.path.exists(config_file):
-            raise FileNotFoundError(f"Serena configuration file not found: {config_file}")
-
-        log.info(f"Loading Serena configuration from {config_file}")
-        try:
-            loaded_original_yaml = load_yaml(config_file, preserve_comments)
-        except Exception as e:
-            raise ValueError(f"Error loading Serena configuration from {config_file}: {e}") from e
-
-        # Create instance
-        instance = cls()
-
-        # read projects
-        if "projects" not in loaded_original_yaml:
-            raise SerenaConfigError("`projects` key not found in Serena configuration. Please update your `serena_config.yml` file.")
-
-        # load list of known projects
-        instance.projects = []
-        num_project_migrations = 0
-        for path in loaded_original_yaml["projects"]:
-            path = Path(path).resolve()
-            if not path.exists():
-                log.warning(f"Project path {path} does not exist, skipping.")
-                continue
-            if path.is_file():
-                path = cls._migrate_out_of_project_config_file(path)
-                if path is None:
-                    continue
-                num_project_migrations += 1
-            project = Project.load(path)
-            instance.projects.append(project)
-
-        instance.gui_log_window_enabled = loaded_original_yaml.get("gui_log_window", False)
-        instance.gui_log_window_level = loaded_original_yaml.get("gui_log_level", logging.INFO)
-        instance.web_dashboard = loaded_original_yaml.get("web_dashboard", True)
-        instance.loaded_original_yaml = loaded_original_yaml
-
-        # re-save the configuration file if any migrations were performed
-        if num_project_migrations > 0:
-            log.info(
-                f"Migrated {num_project_migrations} project configurations from legacy format to in-project configuration; re-saving configuration"
-            )
-            instance.save()
-
-        return instance
-
-    @classmethod
-    def _migrate_out_of_project_config_file(self, path: Path) -> Path | None:
-        """
-        Migrates a legacy project configuration file (which is a YAML file containing the project root) to the
-        in-project configuration file (project.yml) inside the project root directory.
-
-        :param path: the path to the legacy project configuration file
-        :return: the project root path if the migration was successful, None otherwise.
-        """
-        log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.")
-        try:
-            with open(path, encoding="utf-8") as f:
-                project_config_data = yaml.safe_load(f)
-            if "project_name" not in project_config_data:
-                project_name = path.stem
-                with open(path, "a", encoding="utf-8") as f:
-                    f.write(f"\nproject_name: {project_name}")
-            project_root = project_config_data["project_root"]
-            shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml()))
-            return Path(project_root).resolve()
-        except Exception as e:
-            log.error(f"Error migrating configuration file: {e}")
-            return None
-
-    def find_project(self, project_root_or_name: str) -> Project | None:
+    def get_project(self, project_root_or_name: str) -> Project | None:
         for project in self.projects:
             if project.project_config.project_name == project_root_or_name:
                 return project
@@ -310,22 +234,14 @@ class SerenaConfig:
                     return project
         return None
 
-    def save(self, preserve_comments: bool = True) -> None:
-        loaded_original_yaml = deepcopy(self.loaded_original_yaml)
-        loaded_original_yaml["projects"] = [project.project_root for project in self.projects]
-        save_yaml(self.get_config_file_path(), loaded_original_yaml, preserve_comments)
-
-    def add_project_from_path(
-        self, project_root: Path | str, project_name: str | None = None, update_config_file: bool = True
-    ) -> tuple[Project, bool]:
+    def add_project_from_path(self, project_root: Path | str, project_name: str | None = None) -> tuple[Project, bool]:
         """
         Add a project to the Serena configuration from a given path. Will raise a FileExistsError if the
         name or path is already registered.
 
-        :param path: the path to the project to add
+        :param project_root: the path to the project to add
         :param project_name: the name of the project to add; if None, the name of the project will be the name of the directory
             containing the project
-        :param update_config_file: whether to update the Serena configuration file on disk
         :return: the project that was added and a boolean indicating whether a new project configuration was generated and
             saved to disk. It may be that no new project configuration was generated if the project configuration already
             exists on disk but the project itself was not added yet to the Serena configuration.
@@ -356,16 +272,18 @@ class SerenaConfig:
             new_project_config_generated = True
 
         new_project = Project(project_root=str(project_root), project_config=project_config)
-        self.projects.append(new_project)
+        self._add_new_project(new_project)
 
-        # Update the config file on disk and preserve comments
-        if update_config_file:
-            config_with_comments = SerenaConfig.from_config_file(preserve_comments=True)
-            config_with_comments.projects.append(new_project)
-            config_with_comments.save(preserve_comments=True)
         return new_project, new_project_config_generated
 
-    def remove_project(self, project_name: str, update_config_file: bool = True) -> None:
+    def _add_new_project(self, project: Project) -> None:
+        """
+        Adds a new project to the Serena configuration. No checks are performed here,
+        this method is intended to be overridden by subclasses.
+        """
+        self.projects.append(project)
+
+    def remove_project(self, project_name: str) -> None:
         # find the index of the project with the desired name and remove it
         for i, project in enumerate(self.projects):
             if project.project_name == project_name:
@@ -374,14 +292,114 @@ class SerenaConfig:
         else:
             raise ValueError(f"Project '{project_name}' not found in Serena configuration; valid project names: {self.project_names}")
 
-        # Update the config file on disk and preserve comments
-        if update_config_file:
-            config_with_comments = SerenaConfig.from_config_file(preserve_comments=True)
-            for i, project in enumerate(config_with_comments.projects):
-                if project.project_name == project_name:
-                    del config_with_comments.projects[i]
-                    break
-            config_with_comments.save(preserve_comments=True)
+
+@dataclass(kw_only=True)
+class SerenaConfig(SerenaConfigBase):
+    """
+    Handles user-defined Serena configuration based on the (fixed) Serena configuration file.
+    Updates to the instance will be automatically saved to the configuration file.
+    Usually, there should be only one instance of this class in the application.
+    """
+
+    loaded_commented_yaml: CommentedMap
+
+    CONFIG_FILE = "serena_config.yml"
+
+    @classmethod
+    def get_config_file_path(cls) -> str:
+        return os.path.join(serena_root_path(), cls.CONFIG_FILE)
+
+    @classmethod
+    def from_config_file(cls) -> "SerenaConfig":
+        """
+        Static constructor to create SerenaConfig from the configuration file
+        """
+        config_file = cls.get_config_file_path()
+        if not os.path.exists(config_file):
+            raise FileNotFoundError(f"Serena configuration file not found: {config_file}")
+
+        log.info(f"Loading Serena configuration from {config_file}")
+        try:
+            loaded_commented_yaml = load_yaml(config_file, preserve_comments=True)
+        except Exception as e:
+            raise ValueError(f"Error loading Serena configuration from {config_file}: {e}") from e
+
+        # Create instance
+        instance = cls(loaded_commented_yaml=loaded_commented_yaml)
+
+        # read projects
+        if "projects" not in loaded_commented_yaml:
+            raise SerenaConfigError("`projects` key not found in Serena configuration. Please update your `serena_config.yml` file.")
+
+        # load list of known projects
+        instance.projects = []
+        num_project_migrations = 0
+        for path in loaded_commented_yaml["projects"]:
+            path = Path(path).resolve()
+            if not path.exists():
+                log.warning(f"Project path {path} does not exist, skipping.")
+                continue
+            if path.is_file():
+                path = cls._migrate_out_of_project_config_file(path)
+                if path is None:
+                    continue
+                num_project_migrations += 1
+            project = Project.load(path)
+            instance.projects.append(project)
+
+        instance.gui_log_window_enabled = loaded_commented_yaml.get("gui_log_window", False)
+        instance.gui_log_window_level = loaded_commented_yaml.get("gui_log_level", logging.INFO)
+        instance.web_dashboard = loaded_commented_yaml.get("web_dashboard", True)
+
+        # re-save the configuration file if any migrations were performed
+        if num_project_migrations > 0:
+            log.info(
+                f"Migrated {num_project_migrations} project configurations from legacy format to in-project configuration; re-saving configuration"
+            )
+            instance.save()
+
+        return instance
+
+    @classmethod
+    def _migrate_out_of_project_config_file(cls, path: Path) -> Path | None:
+        """
+        Migrates a legacy project configuration file (which is a YAML file containing the project root) to the
+        in-project configuration file (project.yml) inside the project root directory.
+
+        :param path: the path to the legacy project configuration file
+        :return: the project root path if the migration was successful, None otherwise.
+        """
+        log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.")
+        try:
+            with open(path, encoding="utf-8") as f:
+                project_config_data = yaml.safe_load(f)
+            if "project_name" not in project_config_data:
+                project_name = path.stem
+                with open(path, "a", encoding="utf-8") as f:
+                    f.write(f"\nproject_name: {project_name}")
+            project_root = project_config_data["project_root"]
+            shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml()))
+            return Path(project_root).resolve()
+        except Exception as e:
+            log.error(f"Error migrating configuration file: {e}")
+            return None
+
+    def save(self) -> None:
+        loaded_original_yaml = deepcopy(self.loaded_commented_yaml)
+        # projects are unique absolute paths
+        # we also canonicalize them before saving
+        loaded_original_yaml["projects"] = sorted({str(Path(project.project_root).resolve()) for project in self.projects})
+        save_yaml(self.get_config_file_path(), loaded_original_yaml, preserve_comments=True)
+
+    @override
+    def _add_new_project(self, project: Project) -> None:
+        super()._add_new_project(project)
+        self.save()
+
+    @override
+    def remove_project(self, project_name: str) -> None:
+        super().remove_project(project_name)
+        self.save()
 
 
 class LinesRead:
@@ -454,7 +472,7 @@ class SerenaAgent:
         self,
         project: str | None = None,
         project_activation_callback: Callable[[], None] | None = None,
-        serena_config: SerenaConfig | None = None,
+        serena_config: SerenaConfigBase | None = None,
         context: SerenaAgentContext | None = None,
         modes: list[SerenaAgentMode] | None = None,
     ):
@@ -626,7 +644,7 @@ class SerenaAgent:
         """
         new_project_generated = False
         new_project_config_generated = False
-        project_instance: Project | None = self.serena_config.find_project(project_root_or_name)
+        project_instance: Project | None = self.serena_config.get_project(project_root_or_name)
         if project_instance is not None:
             log.info(f"Found registered project {project_instance.project_name} at path {project_instance.project_root}.")
         else:
@@ -635,9 +653,7 @@ class SerenaAgent:
                     f"Project '{project_root_or_name}' not found: Not a valid project name or directory. "
                     f"Existing project names: {self.serena_config.project_names}"
                 )
-            project_instance, new_project_config_generated = self.serena_config.add_project_from_path(
-                project_root_or_name, update_config_file=True
-            )
+            project_instance, new_project_config_generated = self.serena_config.add_project_from_path(project_root_or_name)
             new_project_generated = True
             log.info(f"Added new project {project_instance.project_name} for path {project_instance.project_root}.")
             if new_project_config_generated:
@@ -1749,7 +1765,7 @@ class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject):
 
         :param project_name: Name of the project to remove
         """
-        self.agent.serena_config.remove_project(project_name, update_config_file=True)
+        self.agent.serena_config.remove_project(project_name)
         return f"Successfully removed project '{project_name}' from configuration."
 
 

src/serena/agno.py

@@ -70,20 +70,32 @@ class SerenaAgnoAgentProvider:
             load_dotenv()
 
             parser = argparse.ArgumentParser(description="Serena coding assistant")
-            parser.add_argument(
+
+            # Create a mutually exclusive group
+            group = parser.add_mutually_exclusive_group()
+
+            # Add arguments to the group, both pointing to the same destination
+            group.add_argument(
+                "--project-file",
+                required=False,
+                help="Path to the project (or project.yml file).",
+            )
+            group.add_argument(
                 "--project",
                 required=False,
                 help="Path to the project (or project.yml file).",
             )
             args = parser.parse_args()
 
-            if args.project_file:
-                project_file = Path(args.project_file).resolve()
+            args_project_file = args.project or args.project_file
+
+            if args_project_file:
+                project_file = Path(args_project_file).resolve()
                 # If project file path is relative, make it absolute by joining with project root
                 if not project_file.is_absolute():
                     # Get the project root directory (parent of scripts directory)
                     project_root = Path(serena_root_path())
-                    project_file = project_root / args.project_file
+                    project_file = project_root / args_project_file
 
                 # Ensure the path is normalized and absolute
                 project_file = str(project_file.resolve())

src/serena/dashboard.py

@@ -16,22 +16,22 @@ log = logging.getLogger(__name__)
 
 
 class MemoryLogHandler(logging.Handler):
-    def __init__(self, level=logging.NOTSET):
+    def __init__(self, level: int = logging.NOTSET) -> None:
         super().__init__(level=level)
         self.setFormatter(logging.Formatter(logging.LOG_DEFAULT_FORMAT))
         self._log_buffer = LogBuffer()
-        self._log_queue = queue.Queue()
+        self._log_queue: queue.Queue[str] = queue.Queue()
         self._stop_event = threading.Event()
 
         # start background thread to process logs
         self.worker_thread = threading.Thread(target=self._process_queue, daemon=True)
         self.worker_thread.start()
 
-    def emit(self, record):
+    def emit(self, record: logging.LogRecord) -> None:
         msg = self.format(record)
         self._log_queue.put_nowait(msg)
 
-    def _process_queue(self):
+    def _process_queue(self) -> None:
         while not self._stop_event.is_set():
             try:
                 msg = self._log_queue.get(timeout=1)
@@ -45,10 +45,10 @@ class MemoryLogHandler(logging.Handler):
 
 
 class LogBuffer:
-    def __init__(self):
-        self.logs = []
+    def __init__(self) -> None:
+        self.logs: list[str] = []
 
-    def append(self, msg: str):
+    def append(self, msg: str) -> None:
         self.logs.append(msg)
 
 
@@ -68,13 +68,13 @@ class ResponseToolNames(BaseModel):
 class SerenaDashboardAPI:
     log = logging.getLogger(__qualname__)
 
-    def __init__(self, memory_log_handler: MemoryLogHandler, tool_names: list[str]):
+    def __init__(self, memory_log_handler: MemoryLogHandler, tool_names: list[str]) -> None:
         self._memory_log_handler = memory_log_handler
         self._tool_names = tool_names
         self._app = FastAPI(title="Serena Dashboard")
         self._setup_routes()
 
-    def _setup_routes(self):
+    def _setup_routes(self) -> None:
         static_dir = os.path.join(serena_root_path(), "dashboard")
         self._app.mount("/dashboard", StaticFiles(directory=static_dir), name="dashboard")
 
@@ -110,7 +110,10 @@ class SerenaDashboardAPI:
 
         raise RuntimeError(f"No free ports found starting from {start_port}")
 
-    def run(self, host="127.0.0.1", port=0x5EDA):
+    def run(self, host: str = "127.0.0.1", port: int = 0x5EDA) -> int:
+        """
+        Runs the dashboard on the given host and port and returns the port number.
+        """
         uvicorn.run(self._app, host=host, port=port, workers=1, log_config=None, log_level="critical")
         return port
 

src/serena/mcp.py

@@ -7,6 +7,7 @@ from collections.abc import AsyncIterator, Sequence
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from logging import Formatter, Logger, StreamHandler
+from pathlib import Path
 from typing import Any, Literal
 
 import click  # Add click import
@@ -105,7 +106,9 @@ def create_mcp_server_and_agent(
     """
     Create an MCP server.
 
-    :param project: The path to the project directory or the `project.yml` file therein, or None.
+    :param project: "Either an absolute path to the project directory or a name of an already registered project. "
+        "If the project passed here hasn't been registered yet, it will be registered automatically and can be activated by its name "
+        "afterwards.
     :param host: The host to bind to
     :param port: The port to bind to
     :param context: The context name or path to context file
@@ -157,33 +160,43 @@ def create_mcp_server_and_agent(
     return mcp, agent
 
 
+class ProjectType(click.ParamType):
+    name = "[PROJECT_NAME|PROJECT_PATH]"
+
+    def convert(self, value, param, ctx):
+        path = Path(value).resolve()
+        if path.exists() and path.is_dir():
+            return str(path)  # Valid path
+        return value  # Assume it's a project name
+
+
+PROJECT_TYPE = ProjectType()
+
+
 @click.command()
-# Add --project option as the primary, more intuitive interface
 @click.option(
     "--project",
-    "project_file_opt",  # Use same destination variable to avoid conflicts
-    type=click.Path(exists=True, dir_okay=True, resolve_path=True),
+    "project_file_opt",
+    type=PROJECT_TYPE,
     default=None,
-    help="Path to the .yml project file. "
-    "Does not need to be provided at startup since you can activate a project later by simply asking the agent to do so "
-    "(there is a dedicated tool for this purpose).",
+    help="Either an absolute path to the project directory or a name of an already registered project. "
+    "If the project passed here hasn't been registered yet, it will be registered automatically and can be activated by its name afterwards.",
 )
 # Keep --project-file for backwards compatibility
 @click.option(
     "--project-file",
     "project_file_opt",  # Use same destination variable to avoid conflicts
-    type=click.Path(exists=True, dir_okay=True, resolve_path=True),
+    type=PROJECT_TYPE,
     default=None,
-    help="[DEPRECATED] Use --project instead. Optional path to the .yml project file via option."
-    "Does not need to be provided at startup since you can activate a project later by simply asking the agent to do so "
-    "(there is a dedicated tool for this purpose).",
+    help="[DEPRECATED] Use --project instead.",
 )
 # Positional argument for backwards compatibility
 @click.argument(
     "project_file_arg",
-    type=click.Path(exists=True, dir_okay=True, resolve_path=True),
+    type=PROJECT_TYPE,
     required=False,
     default=None,
+    metavar="",  # don't display anything since it's deprecated
 )
 @click.option(
     "--context",
@@ -208,7 +221,7 @@ def create_mcp_server_and_agent(
     type=click.Choice(["stdio", "sse"]),
     default="stdio",
     show_default=True,
-    help="Transport protocol.",
+    help="Transport protocol. If you start the server yourself (as opposed to an MCP Client starting the server), sse is recommended.",
 )
 @click.option(
     "--host",
@@ -233,11 +246,11 @@ def start_mcp_server(
     host: str = "0.0.0.0",
     port: int = 8000,
 ) -> None:
-    """Starts the Serena MCP server.
-
-    Accepts a path to the project directory or the `project.yml` file therein via the --project option.
+    """Starts the Serena MCP server. By default, will not activate any project at startup.
+    If you want to start with an already active project, use --project to pass the project name or path.
 
     Use --context to specify the execution environment and --mode to specify behavior mode(s).
+    The modes may be adjusted after startup (via the corresponding tool), but the context cannot be changed.
     """
     # Prioritize the positional argument if provided
     # This is for backward compatibility with the old CLI, should be removed in the future!

src/serena/symbol.py

@@ -20,7 +20,8 @@ log = logging.getLogger(__name__)
 @dataclass
 class SymbolLocation:
     """
-    Represents the (start) location of a symbol identifier
+    Represents the location of a symbol, including the line where the identifier
+    is defined and the end line of the symbol's body
     """
 
     relative_path: str | None
@@ -37,6 +38,16 @@ class SymbolLocation:
     the column number in which the symbol identifier is defined (if the symbol is a function, class, etc.);
     may be None for some types of symbols (e.g. SymbolKind.File)
     """
+    end_line: int | None = None
+    """
+    the line in which the symbol's body ends. For methods that search based on SymbolLocation,
+    the end line is not needed, as it is unused by the language server in the search.
+    However, the end_line will typically be included in the results of search methods,
+    and thus be provided to the user in the response to such requests.
+    This is useful for the LLMs, especially the less intelligent ones, as they often fail
+    to perform symbolic operations and when falling back to line-editing tools, will just make up
+    an end line.
+    """
 
     def __post_init__(self) -> None:
         if self.relative_path is not None:
@@ -118,10 +129,7 @@ class Symbol(ToStringMixin):
 
     @property
     def location(self) -> SymbolLocation:
-        """
-        :return: the start location of the actual symbol identifier
-        """
-        return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column)
+        return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column, end_line=self.end_line)
 
     @property
     def body_start_position(self) -> Position | None:
@@ -147,12 +155,21 @@ class Symbol(ToStringMixin):
 
     @property
     def line(self) -> int | None:
+        """The line in which the symbol identifier is defined (start line)."""
         if "selectionRange" in self.symbol_root:
             return self.symbol_root["selectionRange"]["start"]["line"]
         else:
             # line is expected to be undefined for some types of symbols (e.g. SymbolKind.File)
             return None
 
+    @property
+    def end_line(self) -> int | None:
+        """The end line of the symbol body, also contained in the `body_end_position`."""
+        body_end_position = self.body_end_position
+        if body_end_position is not None:
+            return body_end_position["line"]
+        return None
+
     @property
     def column(self) -> int | None:
         if "selectionRange" in self.symbol_root:
@@ -365,7 +382,8 @@ class SymbolManager:
         """
         Find all symbols that reference the symbol at the given location.
 
-        :param symbol_location: the location of the symbol for which to find references
+        :param symbol_location: the location of the symbol for which to find references.
+            Does not need to include an end_line, as it is unused in the search.
         :param include_body: whether to include the body of all symbols in the result.
             Note: you can filter out the bodies of the children if you set include_children_body=False
             in the to_dict method.
@@ -420,7 +438,7 @@ class SymbolManager:
         """
         Replace the body of the symbol at the given location with the given body
 
-        :param location: the location of the symbol to replace
+        :param location: the location of the symbol to replace.
         :param body: the new body
         """
         # make sure body always ends with at least one newline

src/serena/text_utils.py

@@ -3,7 +3,9 @@ import re
 from collections.abc import Callable
 from dataclasses import dataclass, field
 from enum import StrEnum
+from typing import Any
 
+from joblib import Parallel, delayed
 from pathspec import PathSpec
 from pathspec.patterns.gitwildmatch import GitWildMatchPattern
 
@@ -103,7 +105,8 @@ def search_text(
         content: The text content to search. May be None if source_file_path is provided.
         source_file_path: Optional path to the source file. If content is None,
             this has to be passed and the file will be read.
-        allow_multiline_match: Whether to search across multiple lines
+        allow_multiline_match: Whether to search across multiple lines. Currently, the default
+            option (False) is very inefficient, so it is recommended to set this to True.
         context_lines_before: Number of context lines to include before matches
         context_lines_after: Number of context lines to include after matches
         is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py")
@@ -188,6 +191,8 @@ def search_text(
 
             matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path))
     else:
+        # TODO: extremely inefficient! Since we currently don't use this option in SerenaAgent or LanguageServer,
+        #   it is not urgent to fix, but should be either improved or the option should be removed.
         # Search line by line
         for i, line in enumerate(lines):
             line_num = i + 1
@@ -242,10 +247,11 @@ def search_files(
     :param paths_exclude_glob: Optional glob pattern to exclude files from the list
     :return: List of MatchedConsecutiveLines objects
     """
-    matches = []
+    # Pre-filter paths (done sequentially to avoid overhead)
     include_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_include_glob]) if paths_include_glob else None
     exclude_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_exclude_glob]) if paths_exclude_glob else None
-    skipped_file_error_tuples: list[tuple[str, str]] = []
+
+    filtered_paths = []
     for path in file_paths:
         if include_spec and not include_spec.match_file(path):
             log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}")
@@ -253,26 +259,47 @@ def search_files(
         if exclude_spec and exclude_spec.match_file(path):
             log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}")
             continue
+        filtered_paths.append(path)
+
+    log.info(f"Processing {len(filtered_paths)} files.")
+
+    def process_single_file(path: str) -> dict[str, Any]:
+        """Process a single file - this function will be parallelized."""
         try:
             file_content = file_reader(path)
+            search_results = search_text(
+                pattern,
+                content=file_content,
+                source_file_path=path,
+                allow_multiline_match=True,
+                context_lines_before=context_lines_before,
+                context_lines_after=context_lines_after,
+            )
+            if len(search_results) > 0:
+                log.debug(f"Found {len(search_results)} matches in {path}")
+            return {"path": path, "results": search_results, "error": None}
         except Exception as e:
-            skipped_file_error_tuples.append((path, str(e)))
-            continue
+            log.debug(f"Error processing {path}: {e}")
+            return {"path": path, "results": [], "error": str(e)}
+
+    # Execute in parallel using joblib
+    results = Parallel(
+        n_jobs=-1,
+        backend="threading",
+    )(delayed(process_single_file)(path) for path in filtered_paths)
+
+    # Collect results and errors
+    matches = []
+    skipped_file_error_tuples = []
+
+    for result in results:
+        if result["error"]:
+            skipped_file_error_tuples.append((result["path"], result["error"]))
+        else:
+            matches.extend(result["results"])
 
-        search_results = search_text(
-            pattern,
-            file_content,
-            source_file_path=path,
-            allow_multiline_match=True,
-            context_lines_before=context_lines_before,
-            context_lines_after=context_lines_after,
-        )
-        if len(search_results) > 0:
-            log.debug(f"Found {len(search_results)} matches in {path}")
-            matches.extend(search_results)
     if skipped_file_error_tuples:
-        log.debug(
-            f"Failed to read {len(skipped_file_error_tuples)} files. Here the full list of files and errors:\n{skipped_file_error_tuples}"
-        )
+        log.debug(f"Failed to read {len(skipped_file_error_tuples)} files: {skipped_file_error_tuples}")
 
+    log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files")
     return matches

src/serena/util/general.py

@@ -1,8 +1,10 @@
+from typing import Literal, overload
+
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
 
 
-def create_YAML(preserve_comments: bool = False) -> YAML:
+def _create_YAML(preserve_comments: bool = False) -> YAML:
     """
     Creates a YAML that can load/save with comments if preserve_comments is True.
     """
@@ -12,13 +14,17 @@ def create_YAML(preserve_comments: bool = False) -> YAML:
     return result
 
 
-def load_yaml(path: str, preserve_comments: bool = False) -> dict:
+@overload
+def load_yaml(path: str, preserve_comments: Literal[False]) -> dict: ...
+@overload
+def load_yaml(path: str, preserve_comments: Literal[True]) -> CommentedMap: ...
+def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap:
     with open(path, encoding="utf-8") as f:
-        yaml = create_YAML(preserve_comments)
+        yaml = _create_YAML(preserve_comments)
         return yaml.load(f)
 
 
 def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = False) -> None:
-    yaml = create_YAML(preserve_comments)
+    yaml = _create_YAML(preserve_comments)
     with open(path, "w", encoding="utf-8") as f:
         yaml.dump(data, f)

src/serena/util/shell.py

@@ -1,4 +1,5 @@
 import os
+import platform
 import subprocess
 
 from pydantic import BaseModel
@@ -25,7 +26,7 @@ def execute_shell_command(command: str, cwd: str | None = None, capture_stderr:
 
     process = subprocess.Popen(
         command,
-        shell=True,
+        shell=platform.system() != "Windows",
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE if capture_stderr else None,
         text=True,

test/serena/test_serena_agent.py

@@ -1,17 +1,28 @@
 import json
 import os
 import time
+from dataclasses import dataclass
 
 import pytest
 
 from multilspy.multilspy_config import Language
-from serena.agent import FindReferencingSymbolsTool, FindSymbolTool, SerenaAgent, SerenaConfig
+from serena.agent import FindReferencingSymbolsTool, FindSymbolTool, SerenaAgent, SerenaConfigBase
 from test.conftest import LanguageParamRequest, get_repo_path
 
 
+@dataclass
+class SerenaConfigForTests(SerenaConfigBase):
+    """
+    In-memory implementation of Serena configuration with the GUI disabled.
+    """
+
+    gui_log_window_enabled: bool = False
+    web_dashboard: bool = False
+
+
 @pytest.fixture
 def serena_config():
-    return SerenaConfig()
+    return SerenaConfigForTests()
 
 
 @pytest.fixture

uv.lock

@@ -455,6 +455,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/4c/47/5e0b94c603d8e54dd1faab439b40b832c277d3b90743e7835879ab663757/jiter-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:923b54afdd697dfd00d368b7ccad008cccfeb1efb4e621f32860c75e9f25edbd", size = 210119, upload-time = "2025-03-10T21:35:43.46Z" },
 ]
 
+[[package]]
+name = "joblib"
+version = "1.5.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/dc/fe/0f5a938c54105553436dbff7a61dc4fed4b1b2c98852f8833beaf4d5968f/joblib-1.5.1.tar.gz", hash = "sha256:f4f86e351f39fe3d0d32a9f2c3d8af1ee4cec285aafcb27003dda5205576b444", size = 330475, upload-time = "2025-05-23T12:04:37.097Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7d/4f/1195bbac8e0c2acc5f740661631d8d750dc38d4a32b23ee5df3cde6f4e0d/joblib-1.5.1-py3-none-any.whl", hash = "sha256:4719a31f054c7d766948dcd83e9613686b27114f190f717cec7eaa2084f8a74a", size = 307746, upload-time = "2025-05-23T12:04:35.124Z" },
+]
+
 [[package]]
 name = "markdown-it-py"
 version = "3.0.0"
@@ -967,6 +976,7 @@ dependencies = [
     { name = "fastapi" },
     { name = "fastmcp" },
     { name = "jinja2" },
+    { name = "joblib" },
     { name = "mcp" },
     { name = "overrides" },
     { name = "pathspec" },
@@ -1016,6 +1026,7 @@ requires-dist = [
     { name = "google-genai", marker = "extra == 'google'", specifier = ">=1.8.0" },
     { name = "jinja2", specifier = ">=3.1.6" },
     { name = "jinja2", marker = "extra == 'dev'" },
+    { name = "joblib", specifier = ">=1.5.1" },
     { name = "mcp", specifier = ">=1.5.0" },
     { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.4.1" },
     { name = "overrides", specifier = ">=7.7.0,<8" },