diff --git a/.gitignore b/.gitignore index 9fdcc92a..ce9c54f5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,64 @@ -# +# macOS specific files +.DS_Store +.AppleDouble +.LSOverride +._* +.Spotlight-V100 +.Trashes +Icon +.fseventsd +.DocumentRevisions-V100 +.TemporaryItems +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Windows specific files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db +*.stackdump +[Dd]esktop.ini +$RECYCLE.BIN/ +*.cab +*.msi +*.msix +*.msm +*.msp +*.lnk + +# Linux specific files +*~ +.fuse_hidden* +.directory +.Trash-* +.nfs* + +# IDE/Text Editors +# VS Code +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace +.history/ + +# JetBrains IDEs (beyond .idea/) +*.iml +*.ipr +*.iws +out/ +.idea_modules/ + +# Sublime Text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache +*.sublime-workspace +*.sublime-project + +# Project specific ignore .idea temp diff --git a/pyproject.toml b/pyproject.toml index 16ebc869..5ef68fe4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "psutil>=7.0.0", "agno>=1.2.15", "docstring_parser>=0.16", + "joblib>=1.5.1", ] [project.scripts] @@ -257,4 +258,4 @@ markers = [ "rust: language server running for Rust", "typescript: language server running for TypeScript", "php: language server running for PHP", -] \ No newline at end of file +] diff --git a/serena_config.template.yml b/serena_config.template.yml index f5a66f78..db921f96 100644 --- a/serena_config.template.yml +++ b/serena_config.template.yml @@ -24,14 +24,12 @@ gui_log_level: 20 # minimum log level for the GUI log window (10 = debug, 20 = info, 30 = warning, 40 = error) -# Add your list of projects here (which you can switch between using "activate_project"). -# Every list item must be a path to the project's root directory (e.g. "/path/to/myproject"), with the -# configuration file being located in "/path/to/myproject/.serena/project.yml", or a dict mapping -# project_name to project_path (to avoid name collisions). -# You can activate a project by its name using the "activate_project" command. -# If the name is not explicitly set, the name of the project will be the name of the directory -# ("myproject" in the example above). -# Note: comments in the block below are not preserved when saving the config file, -# so we recommend to not have any comments directly below the "projects" key. +# The list of registered projects (which you can switch between using "activate_project"). +# MANAGED BY SERENA, KEEP AT THE BOTTOM OF THE YAML AND DON'T ADJUST WITHOUT NEED (you may add paths though) +# Whenever a project is activated (either by using the tool or by starting the server with --project), +# the project's path will be added here and you will be able to activate it by its name in the future. +# By default, the project's name will be the name of the directory containing the project, but you may change it +# by changing the entry in the (autogenerated) `.serena/project.yml` file. +# Note: make sure there are no name collisions in the names of registered projects projects: - /path/to/myproject diff --git a/src/multilspy/language_server.py b/src/multilspy/language_server.py index 6ca98b9a..fab22e52 100644 --- a/src/multilspy/language_server.py +++ b/src/multilspy/language_server.py @@ -136,11 +136,11 @@ class LanguageServer: return PyrightServer(config, logger, repository_root_path) # It used to be jedi, but pyright is a bit faster, and also more actively maintained # Keeping the previous code for reference - from multilspy.language_servers.jedi_language_server.jedi_server import ( - JediServer, - ) + # from multilspy.language_servers.jedi_language_server.jedi_server import ( + # JediServer, + # ) - return JediServer(config, logger, repository_root_path) + # return JediServer(config, logger, repository_root_path) elif config.code_language == Language.JAVA: from multilspy.language_servers.eclipse_jdtls.eclipse_jdtls import ( EclipseJDTLS, @@ -1177,42 +1177,25 @@ class LanguageServer: async def request_parsed_files(self) -> list[str]: - """ - Retrieves relative paths of all files analyzed by the Language Server. - - This is slow, as it finds all files by finding all symbols. - - This seems to be the only way, the LSP does not provide any endpoints for listing project files.""" + """Retrieves relative paths of all files analyzed by the Language Server.""" if not self.server_started: self.logger.log( "request_parsed_files called before Language Server started", logging.ERROR, ) raise MultilspyException("Language Server not started") - # TODO: this worked in jedi, but pyright and basedpyright return nothing... - # I don't know why - # params = LSPTypes.WorkspaceSymbolParams(query="") # Empty query returns all symbols - # symbols = await self.server.send.workspace_symbol(params) or [] - - # Thus, instead of calling all symbols, we hack this and use the symbol tree instead, which - # seems to work in all these language servers - # walk through all children recursively, find all symbols of type Module and collect their relative paths - roots = await self.request_full_symbol_tree() - paths = [] - def collect_module_files(symbol): - if symbol["kind"] == multilspy_types.SymbolKind.File: - assert "location" in symbol - paths.append(symbol["location"]["relativePath"]) - - elif symbol["kind"] == multilspy_types.SymbolKind.Package: - for child in symbol["children"]: - collect_module_files(child) - - for root in roots: - collect_module_files(root) - - return paths - + rel_file_paths = [] + for root, dirs, files in os.walk(self.repository_root_path): + # Don't go into directories that are ignored by modifying dirs inplace + # Explanation for the + "/" part: + # pathspec can't handle the matching of directories if they don't end with a slash! + # see https://github.com/cpburnz/python-pathspec/issues/89 + dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))] + for file in files: + rel_file_path = os.path.join(root, file) + if not self.is_ignored_path(rel_file_path): + rel_file_paths.append(rel_file_path) + return rel_file_paths async def search_files_for_pattern( self, @@ -1926,9 +1909,7 @@ class SyncLanguageServer: return self.language_server.retrieve_symbol_body(symbol) def request_parsed_files(self) -> list[str]: - """This is slow, as it finds all files by finding all symbols. - - This seems to be the only way, the LSP does not provide any endpoints for listing project files.""" + """Retrieves relative paths of all files analyzed by the Language Server.""" assert self.loop result = asyncio.run_coroutine_threadsafe( self.language_server.request_parsed_files(), self.loop diff --git a/src/serena/agent.py b/src/serena/agent.py index 155bfcd7..3313a469 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -21,6 +21,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Self, TypeVar, Union import yaml +from overrides import override from ruamel.yaml.comments import CommentedMap from sensai.util import logging from sensai.util.logging import FallbackHandler @@ -108,6 +109,12 @@ class ProjectConfig(ToStringMixin): raise FileNotFoundError(f"Project root not found: {project_root}") project_name = project_name or project_root.name language_composition = determine_programming_language_composition(str(project_root)) + if len(language_composition) == 0: + raise ValueError( + f"Failed to autogenerate project.yaml: no programming language detected in project {project_root}. " + f"You can either add some files that correspond to one of the supported programming languages, " + f"or create the file {os.path.join(project_root, cls.rel_path_to_project_yml())} manually and specify the language there." + ) # find the language with the highest percentage dominant_language = max(language_composition.keys(), key=lambda lang: language_composition[lang]) result = cls( @@ -197,19 +204,16 @@ class Project: return os.path.join(self.project_root, self.project_config.rel_path_to_project_yml()) -@dataclass -class SerenaConfig: +@dataclass(kw_only=True) +class SerenaConfigBase(ABC): """ - Handles user-defined Serena configuration based on the configuration file + Abstract base class for Serena configuration handling """ projects: list[Project] = field(default_factory=list) gui_log_window_enabled: bool = False gui_log_window_level: int = logging.INFO web_dashboard: bool = True - loaded_original_yaml: dict | CommentedMap = field(default_factory=dict) - - CONFIG_FILE = "serena_config.yml" @cached_property def project_paths(self) -> list[str]: @@ -219,87 +223,7 @@ class SerenaConfig: def project_names(self) -> list[str]: return sorted(project.project_config.project_name for project in self.projects) - @classmethod - def get_config_file_path(cls) -> str: - return os.path.join(serena_root_path(), cls.CONFIG_FILE) - - @classmethod - def from_config_file(cls, preserve_comments: bool = False) -> "SerenaConfig": - """ - Static constructor to create SerenaConfig from the configuration file - """ - config_file = cls.get_config_file_path() - if not os.path.exists(config_file): - raise FileNotFoundError(f"Serena configuration file not found: {config_file}") - - log.info(f"Loading Serena configuration from {config_file}") - try: - loaded_original_yaml = load_yaml(config_file, preserve_comments) - except Exception as e: - raise ValueError(f"Error loading Serena configuration from {config_file}: {e}") from e - - # Create instance - instance = cls() - - # read projects - if "projects" not in loaded_original_yaml: - raise SerenaConfigError("`projects` key not found in Serena configuration. Please update your `serena_config.yml` file.") - - # load list of known projects - instance.projects = [] - num_project_migrations = 0 - for path in loaded_original_yaml["projects"]: - path = Path(path).resolve() - if not path.exists(): - log.warning(f"Project path {path} does not exist, skipping.") - continue - if path.is_file(): - path = cls._migrate_out_of_project_config_file(path) - if path is None: - continue - num_project_migrations += 1 - project = Project.load(path) - instance.projects.append(project) - - instance.gui_log_window_enabled = loaded_original_yaml.get("gui_log_window", False) - instance.gui_log_window_level = loaded_original_yaml.get("gui_log_level", logging.INFO) - instance.web_dashboard = loaded_original_yaml.get("web_dashboard", True) - instance.loaded_original_yaml = loaded_original_yaml - - # re-save the configuration file if any migrations were performed - if num_project_migrations > 0: - log.info( - f"Migrated {num_project_migrations} project configurations from legacy format to in-project configuration; re-saving configuration" - ) - instance.save() - - return instance - - @classmethod - def _migrate_out_of_project_config_file(self, path: Path) -> Path | None: - """ - Migrates a legacy project configuration file (which is a YAML file containing the project root) to the - in-project configuration file (project.yml) inside the project root directory. - - :param path: the path to the legacy project configuration file - :return: the project root path if the migration was successful, None otherwise. - """ - log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.") - try: - with open(path, encoding="utf-8") as f: - project_config_data = yaml.safe_load(f) - if "project_name" not in project_config_data: - project_name = path.stem - with open(path, "a", encoding="utf-8") as f: - f.write(f"\nproject_name: {project_name}") - project_root = project_config_data["project_root"] - shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml())) - return Path(project_root).resolve() - except Exception as e: - log.error(f"Error migrating configuration file: {e}") - return None - - def find_project(self, project_root_or_name: str) -> Project | None: + def get_project(self, project_root_or_name: str) -> Project | None: for project in self.projects: if project.project_config.project_name == project_root_or_name: return project @@ -310,22 +234,14 @@ class SerenaConfig: return project return None - def save(self, preserve_comments: bool = True) -> None: - loaded_original_yaml = deepcopy(self.loaded_original_yaml) - loaded_original_yaml["projects"] = [project.project_root for project in self.projects] - save_yaml(self.get_config_file_path(), loaded_original_yaml, preserve_comments) - - def add_project_from_path( - self, project_root: Path | str, project_name: str | None = None, update_config_file: bool = True - ) -> tuple[Project, bool]: + def add_project_from_path(self, project_root: Path | str, project_name: str | None = None) -> tuple[Project, bool]: """ Add a project to the Serena configuration from a given path. Will raise a FileExistsError if the name or path is already registered. - :param path: the path to the project to add + :param project_root: the path to the project to add :param project_name: the name of the project to add; if None, the name of the project will be the name of the directory containing the project - :param update_config_file: whether to update the Serena configuration file on disk :return: the project that was added and a boolean indicating whether a new project configuration was generated and saved to disk. It may be that no new project configuration was generated if the project configuration already exists on disk but the project itself was not added yet to the Serena configuration. @@ -356,16 +272,18 @@ class SerenaConfig: new_project_config_generated = True new_project = Project(project_root=str(project_root), project_config=project_config) - self.projects.append(new_project) + self._add_new_project(new_project) - # Update the config file on disk and preserve comments - if update_config_file: - config_with_comments = SerenaConfig.from_config_file(preserve_comments=True) - config_with_comments.projects.append(new_project) - config_with_comments.save(preserve_comments=True) return new_project, new_project_config_generated - def remove_project(self, project_name: str, update_config_file: bool = True) -> None: + def _add_new_project(self, project: Project) -> None: + """ + Adds a new project to the Serena configuration. No checks are performed here, + this method is intended to be overridden by subclasses. + """ + self.projects.append(project) + + def remove_project(self, project_name: str) -> None: # find the index of the project with the desired name and remove it for i, project in enumerate(self.projects): if project.project_name == project_name: @@ -374,14 +292,114 @@ class SerenaConfig: else: raise ValueError(f"Project '{project_name}' not found in Serena configuration; valid project names: {self.project_names}") - # Update the config file on disk and preserve comments - if update_config_file: - config_with_comments = SerenaConfig.from_config_file(preserve_comments=True) - for i, project in enumerate(config_with_comments.projects): - if project.project_name == project_name: - del config_with_comments.projects[i] - break - config_with_comments.save(preserve_comments=True) + +@dataclass(kw_only=True) +class SerenaConfig(SerenaConfigBase): + """ + Handles user-defined Serena configuration based on the (fixed) Serena configuration file. + Updates to the instance will be automatically saved to the configuration file. + Usually, there should be only one instance of this class in the application. + """ + + loaded_commented_yaml: CommentedMap + + CONFIG_FILE = "serena_config.yml" + + @classmethod + def get_config_file_path(cls) -> str: + return os.path.join(serena_root_path(), cls.CONFIG_FILE) + + @classmethod + def from_config_file(cls) -> "SerenaConfig": + """ + Static constructor to create SerenaConfig from the configuration file + """ + config_file = cls.get_config_file_path() + if not os.path.exists(config_file): + raise FileNotFoundError(f"Serena configuration file not found: {config_file}") + + log.info(f"Loading Serena configuration from {config_file}") + try: + loaded_commented_yaml = load_yaml(config_file, preserve_comments=True) + except Exception as e: + raise ValueError(f"Error loading Serena configuration from {config_file}: {e}") from e + + # Create instance + instance = cls(loaded_commented_yaml=loaded_commented_yaml) + + # read projects + if "projects" not in loaded_commented_yaml: + raise SerenaConfigError("`projects` key not found in Serena configuration. Please update your `serena_config.yml` file.") + + # load list of known projects + instance.projects = [] + num_project_migrations = 0 + for path in loaded_commented_yaml["projects"]: + path = Path(path).resolve() + if not path.exists(): + log.warning(f"Project path {path} does not exist, skipping.") + continue + if path.is_file(): + path = cls._migrate_out_of_project_config_file(path) + if path is None: + continue + num_project_migrations += 1 + project = Project.load(path) + instance.projects.append(project) + + instance.gui_log_window_enabled = loaded_commented_yaml.get("gui_log_window", False) + instance.gui_log_window_level = loaded_commented_yaml.get("gui_log_level", logging.INFO) + instance.web_dashboard = loaded_commented_yaml.get("web_dashboard", True) + + # re-save the configuration file if any migrations were performed + if num_project_migrations > 0: + log.info( + f"Migrated {num_project_migrations} project configurations from legacy format to in-project configuration; re-saving configuration" + ) + instance.save() + + return instance + + @classmethod + def _migrate_out_of_project_config_file(cls, path: Path) -> Path | None: + """ + Migrates a legacy project configuration file (which is a YAML file containing the project root) to the + in-project configuration file (project.yml) inside the project root directory. + + :param path: the path to the legacy project configuration file + :return: the project root path if the migration was successful, None otherwise. + """ + log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.") + try: + with open(path, encoding="utf-8") as f: + project_config_data = yaml.safe_load(f) + if "project_name" not in project_config_data: + project_name = path.stem + with open(path, "a", encoding="utf-8") as f: + f.write(f"\nproject_name: {project_name}") + project_root = project_config_data["project_root"] + shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml())) + return Path(project_root).resolve() + except Exception as e: + log.error(f"Error migrating configuration file: {e}") + return None + + def save(self) -> None: + loaded_original_yaml = deepcopy(self.loaded_commented_yaml) + # projects are unique absolute paths + # we also canonicalize them before saving + loaded_original_yaml["projects"] = sorted({str(Path(project.project_root).resolve()) for project in self.projects}) + save_yaml(self.get_config_file_path(), loaded_original_yaml, preserve_comments=True) + + @override + def _add_new_project(self, project: Project) -> None: + super()._add_new_project(project) + self.save() + + @override + def remove_project(self, project_name: str) -> None: + super().remove_project(project_name) + self.save() class LinesRead: @@ -454,7 +472,7 @@ class SerenaAgent: self, project: str | None = None, project_activation_callback: Callable[[], None] | None = None, - serena_config: SerenaConfig | None = None, + serena_config: SerenaConfigBase | None = None, context: SerenaAgentContext | None = None, modes: list[SerenaAgentMode] | None = None, ): @@ -626,7 +644,7 @@ class SerenaAgent: """ new_project_generated = False new_project_config_generated = False - project_instance: Project | None = self.serena_config.find_project(project_root_or_name) + project_instance: Project | None = self.serena_config.get_project(project_root_or_name) if project_instance is not None: log.info(f"Found registered project {project_instance.project_name} at path {project_instance.project_root}.") else: @@ -635,9 +653,7 @@ class SerenaAgent: f"Project '{project_root_or_name}' not found: Not a valid project name or directory. " f"Existing project names: {self.serena_config.project_names}" ) - project_instance, new_project_config_generated = self.serena_config.add_project_from_path( - project_root_or_name, update_config_file=True - ) + project_instance, new_project_config_generated = self.serena_config.add_project_from_path(project_root_or_name) new_project_generated = True log.info(f"Added new project {project_instance.project_name} for path {project_instance.project_root}.") if new_project_config_generated: @@ -1749,7 +1765,7 @@ class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): :param project_name: Name of the project to remove """ - self.agent.serena_config.remove_project(project_name, update_config_file=True) + self.agent.serena_config.remove_project(project_name) return f"Successfully removed project '{project_name}' from configuration." diff --git a/src/serena/agno.py b/src/serena/agno.py index 3e3fec49..f6684eca 100644 --- a/src/serena/agno.py +++ b/src/serena/agno.py @@ -70,20 +70,32 @@ class SerenaAgnoAgentProvider: load_dotenv() parser = argparse.ArgumentParser(description="Serena coding assistant") - parser.add_argument( + + # Create a mutually exclusive group + group = parser.add_mutually_exclusive_group() + + # Add arguments to the group, both pointing to the same destination + group.add_argument( + "--project-file", + required=False, + help="Path to the project (or project.yml file).", + ) + group.add_argument( "--project", required=False, help="Path to the project (or project.yml file).", ) args = parser.parse_args() - if args.project_file: - project_file = Path(args.project_file).resolve() + args_project_file = args.project or args.project_file + + if args_project_file: + project_file = Path(args_project_file).resolve() # If project file path is relative, make it absolute by joining with project root if not project_file.is_absolute(): # Get the project root directory (parent of scripts directory) project_root = Path(serena_root_path()) - project_file = project_root / args.project_file + project_file = project_root / args_project_file # Ensure the path is normalized and absolute project_file = str(project_file.resolve()) diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index f6f129a4..d52040b5 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -16,22 +16,22 @@ log = logging.getLogger(__name__) class MemoryLogHandler(logging.Handler): - def __init__(self, level=logging.NOTSET): + def __init__(self, level: int = logging.NOTSET) -> None: super().__init__(level=level) self.setFormatter(logging.Formatter(logging.LOG_DEFAULT_FORMAT)) self._log_buffer = LogBuffer() - self._log_queue = queue.Queue() + self._log_queue: queue.Queue[str] = queue.Queue() self._stop_event = threading.Event() # start background thread to process logs self.worker_thread = threading.Thread(target=self._process_queue, daemon=True) self.worker_thread.start() - def emit(self, record): + def emit(self, record: logging.LogRecord) -> None: msg = self.format(record) self._log_queue.put_nowait(msg) - def _process_queue(self): + def _process_queue(self) -> None: while not self._stop_event.is_set(): try: msg = self._log_queue.get(timeout=1) @@ -45,10 +45,10 @@ class MemoryLogHandler(logging.Handler): class LogBuffer: - def __init__(self): - self.logs = [] + def __init__(self) -> None: + self.logs: list[str] = [] - def append(self, msg: str): + def append(self, msg: str) -> None: self.logs.append(msg) @@ -68,13 +68,13 @@ class ResponseToolNames(BaseModel): class SerenaDashboardAPI: log = logging.getLogger(__qualname__) - def __init__(self, memory_log_handler: MemoryLogHandler, tool_names: list[str]): + def __init__(self, memory_log_handler: MemoryLogHandler, tool_names: list[str]) -> None: self._memory_log_handler = memory_log_handler self._tool_names = tool_names self._app = FastAPI(title="Serena Dashboard") self._setup_routes() - def _setup_routes(self): + def _setup_routes(self) -> None: static_dir = os.path.join(serena_root_path(), "dashboard") self._app.mount("/dashboard", StaticFiles(directory=static_dir), name="dashboard") @@ -110,7 +110,10 @@ class SerenaDashboardAPI: raise RuntimeError(f"No free ports found starting from {start_port}") - def run(self, host="127.0.0.1", port=0x5EDA): + def run(self, host: str = "127.0.0.1", port: int = 0x5EDA) -> int: + """ + Runs the dashboard on the given host and port and returns the port number. + """ uvicorn.run(self._app, host=host, port=port, workers=1, log_config=None, log_level="critical") return port diff --git a/src/serena/mcp.py b/src/serena/mcp.py index b99762f8..9508a33f 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -7,6 +7,7 @@ from collections.abc import AsyncIterator, Sequence from contextlib import asynccontextmanager from dataclasses import dataclass from logging import Formatter, Logger, StreamHandler +from pathlib import Path from typing import Any, Literal import click # Add click import @@ -105,7 +106,9 @@ def create_mcp_server_and_agent( """ Create an MCP server. - :param project: The path to the project directory or the `project.yml` file therein, or None. + :param project: "Either an absolute path to the project directory or a name of an already registered project. " + "If the project passed here hasn't been registered yet, it will be registered automatically and can be activated by its name " + "afterwards. :param host: The host to bind to :param port: The port to bind to :param context: The context name or path to context file @@ -157,33 +160,43 @@ def create_mcp_server_and_agent( return mcp, agent +class ProjectType(click.ParamType): + name = "[PROJECT_NAME|PROJECT_PATH]" + + def convert(self, value, param, ctx): + path = Path(value).resolve() + if path.exists() and path.is_dir(): + return str(path) # Valid path + return value # Assume it's a project name + + +PROJECT_TYPE = ProjectType() + + @click.command() -# Add --project option as the primary, more intuitive interface @click.option( "--project", - "project_file_opt", # Use same destination variable to avoid conflicts - type=click.Path(exists=True, dir_okay=True, resolve_path=True), + "project_file_opt", + type=PROJECT_TYPE, default=None, - help="Path to the .yml project file. " - "Does not need to be provided at startup since you can activate a project later by simply asking the agent to do so " - "(there is a dedicated tool for this purpose).", + help="Either an absolute path to the project directory or a name of an already registered project. " + "If the project passed here hasn't been registered yet, it will be registered automatically and can be activated by its name afterwards.", ) # Keep --project-file for backwards compatibility @click.option( "--project-file", "project_file_opt", # Use same destination variable to avoid conflicts - type=click.Path(exists=True, dir_okay=True, resolve_path=True), + type=PROJECT_TYPE, default=None, - help="[DEPRECATED] Use --project instead. Optional path to the .yml project file via option." - "Does not need to be provided at startup since you can activate a project later by simply asking the agent to do so " - "(there is a dedicated tool for this purpose).", + help="[DEPRECATED] Use --project instead.", ) # Positional argument for backwards compatibility @click.argument( "project_file_arg", - type=click.Path(exists=True, dir_okay=True, resolve_path=True), + type=PROJECT_TYPE, required=False, default=None, + metavar="", # don't display anything since it's deprecated ) @click.option( "--context", @@ -208,7 +221,7 @@ def create_mcp_server_and_agent( type=click.Choice(["stdio", "sse"]), default="stdio", show_default=True, - help="Transport protocol.", + help="Transport protocol. If you start the server yourself (as opposed to an MCP Client starting the server), sse is recommended.", ) @click.option( "--host", @@ -233,11 +246,11 @@ def start_mcp_server( host: str = "0.0.0.0", port: int = 8000, ) -> None: - """Starts the Serena MCP server. - - Accepts a path to the project directory or the `project.yml` file therein via the --project option. + """Starts the Serena MCP server. By default, will not activate any project at startup. + If you want to start with an already active project, use --project to pass the project name or path. Use --context to specify the execution environment and --mode to specify behavior mode(s). + The modes may be adjusted after startup (via the corresponding tool), but the context cannot be changed. """ # Prioritize the positional argument if provided # This is for backward compatibility with the old CLI, should be removed in the future! diff --git a/src/serena/symbol.py b/src/serena/symbol.py index 171e209a..3178d553 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -20,7 +20,8 @@ log = logging.getLogger(__name__) @dataclass class SymbolLocation: """ - Represents the (start) location of a symbol identifier + Represents the location of a symbol, including the line where the identifier + is defined and the end line of the symbol's body """ relative_path: str | None @@ -37,6 +38,16 @@ class SymbolLocation: the column number in which the symbol identifier is defined (if the symbol is a function, class, etc.); may be None for some types of symbols (e.g. SymbolKind.File) """ + end_line: int | None = None + """ + the line in which the symbol's body ends. For methods that search based on SymbolLocation, + the end line is not needed, as it is unused by the language server in the search. + However, the end_line will typically be included in the results of search methods, + and thus be provided to the user in the response to such requests. + This is useful for the LLMs, especially the less intelligent ones, as they often fail + to perform symbolic operations and when falling back to line-editing tools, will just make up + an end line. + """ def __post_init__(self) -> None: if self.relative_path is not None: @@ -118,10 +129,7 @@ class Symbol(ToStringMixin): @property def location(self) -> SymbolLocation: - """ - :return: the start location of the actual symbol identifier - """ - return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column) + return SymbolLocation(relative_path=self.relative_path, line=self.line, column=self.column, end_line=self.end_line) @property def body_start_position(self) -> Position | None: @@ -147,12 +155,21 @@ class Symbol(ToStringMixin): @property def line(self) -> int | None: + """The line in which the symbol identifier is defined (start line).""" if "selectionRange" in self.symbol_root: return self.symbol_root["selectionRange"]["start"]["line"] else: # line is expected to be undefined for some types of symbols (e.g. SymbolKind.File) return None + @property + def end_line(self) -> int | None: + """The end line of the symbol body, also contained in the `body_end_position`.""" + body_end_position = self.body_end_position + if body_end_position is not None: + return body_end_position["line"] + return None + @property def column(self) -> int | None: if "selectionRange" in self.symbol_root: @@ -365,7 +382,8 @@ class SymbolManager: """ Find all symbols that reference the symbol at the given location. - :param symbol_location: the location of the symbol for which to find references + :param symbol_location: the location of the symbol for which to find references. + Does not need to include an end_line, as it is unused in the search. :param include_body: whether to include the body of all symbols in the result. Note: you can filter out the bodies of the children if you set include_children_body=False in the to_dict method. @@ -420,7 +438,7 @@ class SymbolManager: """ Replace the body of the symbol at the given location with the given body - :param location: the location of the symbol to replace + :param location: the location of the symbol to replace. :param body: the new body """ # make sure body always ends with at least one newline diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 1bae75a8..6cab02a8 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -3,7 +3,9 @@ import re from collections.abc import Callable from dataclasses import dataclass, field from enum import StrEnum +from typing import Any +from joblib import Parallel, delayed from pathspec import PathSpec from pathspec.patterns.gitwildmatch import GitWildMatchPattern @@ -103,7 +105,8 @@ def search_text( content: The text content to search. May be None if source_file_path is provided. source_file_path: Optional path to the source file. If content is None, this has to be passed and the file will be read. - allow_multiline_match: Whether to search across multiple lines + allow_multiline_match: Whether to search across multiple lines. Currently, the default + option (False) is very inefficient, so it is recommended to set this to True. context_lines_before: Number of context lines to include before matches context_lines_after: Number of context lines to include after matches is_glob: If True, pattern is treated as a glob-like pattern (e.g., "*.py", "test_??.py") @@ -188,6 +191,8 @@ def search_text( matches.append(MatchedConsecutiveLines(lines=context_lines, source_file_path=source_file_path)) else: + # TODO: extremely inefficient! Since we currently don't use this option in SerenaAgent or LanguageServer, + # it is not urgent to fix, but should be either improved or the option should be removed. # Search line by line for i, line in enumerate(lines): line_num = i + 1 @@ -242,10 +247,11 @@ def search_files( :param paths_exclude_glob: Optional glob pattern to exclude files from the list :return: List of MatchedConsecutiveLines objects """ - matches = [] + # Pre-filter paths (done sequentially to avoid overhead) include_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_include_glob]) if paths_include_glob else None exclude_spec = PathSpec.from_lines(GitWildMatchPattern, [paths_exclude_glob]) if paths_exclude_glob else None - skipped_file_error_tuples: list[tuple[str, str]] = [] + + filtered_paths = [] for path in file_paths: if include_spec and not include_spec.match_file(path): log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}") @@ -253,26 +259,47 @@ def search_files( if exclude_spec and exclude_spec.match_file(path): log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}") continue + filtered_paths.append(path) + + log.info(f"Processing {len(filtered_paths)} files.") + + def process_single_file(path: str) -> dict[str, Any]: + """Process a single file - this function will be parallelized.""" try: file_content = file_reader(path) + search_results = search_text( + pattern, + content=file_content, + source_file_path=path, + allow_multiline_match=True, + context_lines_before=context_lines_before, + context_lines_after=context_lines_after, + ) + if len(search_results) > 0: + log.debug(f"Found {len(search_results)} matches in {path}") + return {"path": path, "results": search_results, "error": None} except Exception as e: - skipped_file_error_tuples.append((path, str(e))) - continue + log.debug(f"Error processing {path}: {e}") + return {"path": path, "results": [], "error": str(e)} + + # Execute in parallel using joblib + results = Parallel( + n_jobs=-1, + backend="threading", + )(delayed(process_single_file)(path) for path in filtered_paths) + + # Collect results and errors + matches = [] + skipped_file_error_tuples = [] + + for result in results: + if result["error"]: + skipped_file_error_tuples.append((result["path"], result["error"])) + else: + matches.extend(result["results"]) - search_results = search_text( - pattern, - file_content, - source_file_path=path, - allow_multiline_match=True, - context_lines_before=context_lines_before, - context_lines_after=context_lines_after, - ) - if len(search_results) > 0: - log.debug(f"Found {len(search_results)} matches in {path}") - matches.extend(search_results) if skipped_file_error_tuples: - log.debug( - f"Failed to read {len(skipped_file_error_tuples)} files. Here the full list of files and errors:\n{skipped_file_error_tuples}" - ) + log.debug(f"Failed to read {len(skipped_file_error_tuples)} files: {skipped_file_error_tuples}") + log.info(f"Found {len(matches)} total matches across {len(filtered_paths)} files") return matches diff --git a/src/serena/util/general.py b/src/serena/util/general.py index 8cabcd82..bfddaf47 100644 --- a/src/serena/util/general.py +++ b/src/serena/util/general.py @@ -1,8 +1,10 @@ +from typing import Literal, overload + from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap -def create_YAML(preserve_comments: bool = False) -> YAML: +def _create_YAML(preserve_comments: bool = False) -> YAML: """ Creates a YAML that can load/save with comments if preserve_comments is True. """ @@ -12,13 +14,17 @@ def create_YAML(preserve_comments: bool = False) -> YAML: return result -def load_yaml(path: str, preserve_comments: bool = False) -> dict: +@overload +def load_yaml(path: str, preserve_comments: Literal[False]) -> dict: ... +@overload +def load_yaml(path: str, preserve_comments: Literal[True]) -> CommentedMap: ... +def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap: with open(path, encoding="utf-8") as f: - yaml = create_YAML(preserve_comments) + yaml = _create_YAML(preserve_comments) return yaml.load(f) def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = False) -> None: - yaml = create_YAML(preserve_comments) + yaml = _create_YAML(preserve_comments) with open(path, "w", encoding="utf-8") as f: yaml.dump(data, f) diff --git a/src/serena/util/shell.py b/src/serena/util/shell.py index 3d9e66eb..d940384d 100644 --- a/src/serena/util/shell.py +++ b/src/serena/util/shell.py @@ -1,4 +1,5 @@ import os +import platform import subprocess from pydantic import BaseModel @@ -25,7 +26,7 @@ def execute_shell_command(command: str, cwd: str | None = None, capture_stderr: process = subprocess.Popen( command, - shell=True, + shell=platform.system() != "Windows", stdout=subprocess.PIPE, stderr=subprocess.PIPE if capture_stderr else None, text=True, diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 604b9021..691c75d7 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -1,17 +1,28 @@ import json import os import time +from dataclasses import dataclass import pytest from multilspy.multilspy_config import Language -from serena.agent import FindReferencingSymbolsTool, FindSymbolTool, SerenaAgent, SerenaConfig +from serena.agent import FindReferencingSymbolsTool, FindSymbolTool, SerenaAgent, SerenaConfigBase from test.conftest import LanguageParamRequest, get_repo_path +@dataclass +class SerenaConfigForTests(SerenaConfigBase): + """ + In-memory implementation of Serena configuration with the GUI disabled. + """ + + gui_log_window_enabled: bool = False + web_dashboard: bool = False + + @pytest.fixture def serena_config(): - return SerenaConfig() + return SerenaConfigForTests() @pytest.fixture diff --git a/uv.lock b/uv.lock index 5390942a..b9d538e1 100644 --- a/uv.lock +++ b/uv.lock @@ -455,6 +455,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/47/5e0b94c603d8e54dd1faab439b40b832c277d3b90743e7835879ab663757/jiter-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:923b54afdd697dfd00d368b7ccad008cccfeb1efb4e621f32860c75e9f25edbd", size = 210119, upload-time = "2025-03-10T21:35:43.46Z" }, ] +[[package]] +name = "joblib" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/fe/0f5a938c54105553436dbff7a61dc4fed4b1b2c98852f8833beaf4d5968f/joblib-1.5.1.tar.gz", hash = "sha256:f4f86e351f39fe3d0d32a9f2c3d8af1ee4cec285aafcb27003dda5205576b444", size = 330475, upload-time = "2025-05-23T12:04:37.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/4f/1195bbac8e0c2acc5f740661631d8d750dc38d4a32b23ee5df3cde6f4e0d/joblib-1.5.1-py3-none-any.whl", hash = "sha256:4719a31f054c7d766948dcd83e9613686b27114f190f717cec7eaa2084f8a74a", size = 307746, upload-time = "2025-05-23T12:04:35.124Z" }, +] + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -967,6 +976,7 @@ dependencies = [ { name = "fastapi" }, { name = "fastmcp" }, { name = "jinja2" }, + { name = "joblib" }, { name = "mcp" }, { name = "overrides" }, { name = "pathspec" }, @@ -1016,6 +1026,7 @@ requires-dist = [ { name = "google-genai", marker = "extra == 'google'", specifier = ">=1.8.0" }, { name = "jinja2", specifier = ">=3.1.6" }, { name = "jinja2", marker = "extra == 'dev'" }, + { name = "joblib", specifier = ">=1.5.1" }, { name = "mcp", specifier = ">=1.5.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.4.1" }, { name = "overrides", specifier = ">=7.7.0,<8" },