diff --git a/.github/workflows/junie.yml b/.github/workflows/junie.yml new file mode 100644 index 00000000..9860e79e --- /dev/null +++ b/.github/workflows/junie.yml @@ -0,0 +1,21 @@ +name: Junie +run-name: Junie run ${{ inputs.run_id }} + +permissions: + contents: write + +on: + workflow_dispatch: + inputs: + run_id: + description: "id of workflow process" + required: true + workflow_params: + description: "stringified params" + required: true + +jobs: + call-workflow-passing-data: + uses: jetbrains-junie/junie-workflows/.github/workflows/ej-issue.yml@main + with: + workflow_params: ${{ inputs.workflow_params }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index dc4be450..49d8e857 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -28,6 +28,12 @@ jobs: - uses: actions/setup-go@v5 with: go-version: ">=1.17.0" + - name: Ensure cached directory exist before calling cache-related actions + shell: bash + run: | + mkdir -p $HOME/.serena/language_servers/static + mkdir -p $HOME/.cache/go-build + mkdir -p $HOME/go/bin # Add Go bin directory to PATH for this workflow # GITHUB_PATH is a special file that GitHub Actions uses to modify PATH # Writing to this file adds the directory to the PATH for subsequent steps @@ -92,6 +98,13 @@ jobs: echo "=== Adding Swift toolchain to PATH ===" echo "$HOME/.local/share/swiftly/bin" >> $GITHUB_PATH echo "Swiftly installed successfully!" + - name: Install Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.4' + - name: Install Ruby language server + shell: bash + run: gem install solargraph - name: Install uv shell: bash run: curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/CHANGELOG.md b/CHANGELOG.md index dfbf52c4..a32f46cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Status of the `main` branch. Changes prior to the next official version change w * Reliably detect language server termination and propagate the respective error all the way back to the tool application, where an unexpected termination is handled by restarting the language server and subsequently retrying the tool application. + * Add comprehensive bash language support with bash-language-server integration + * Support for .sh and .bash file extensions + * Hybrid function detection (LSP + regex fallback) for reliable symbolic editing + * Automatic Node.js/npm dependency management + * Cross-platform compatibility including Windows Fixes: * Ignore `.git` in check for ignored paths and improve performance of `find_all_non_ignored_files` diff --git a/DOCKER.md b/DOCKER.md index 5ee8829c..e2647503 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -68,6 +68,8 @@ SERENA_DASHBOARD_PORT=8080 docker-compose up serena docker-compose up serena-dev ``` +Note: Edit the `compose.yaml` file to customize volume mounts for your projects. + ### Using Docker directly ```bash @@ -83,6 +85,24 @@ docker run -it --rm \ serena ``` +### Using Docker Compose with Merge Compose files + +To use Docker Compose with merge files, you can create a `compose.override.yml` file to customize the configuration: + +```yaml +services: + serena: + # To work with projects, you must mount them as volumes: + volumes: + - ./my-project:/workspace/my-project + - /path/to/another/project:/workspace/another-project + # Add the context for the IDE assistant option: + command: + - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide-assistant" +``` + +See the [Docker Merge Compose files documentation](https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/) for more details on using merge files. + ## Accessing the Dashboard Once running, access the web dashboard at: diff --git a/Dockerfile b/Dockerfile index d88d4e26..205f4ee6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,8 @@ FROM python:3.11-slim AS base SHELL ["/bin/bash", "-c"] # Set environment variables to make Python print directly to the terminal and avoid .pyc files. -ENV PYTHONUNBUFFERED 1 -ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 # Install system dependencies required for package manager and build tools. RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -18,6 +18,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN python3 -m pip install --no-cache-dir pipx \ && pipx ensurepath +# Install nodejs +ENV NVM_VERSION=0.40.3 +ENV NODE_VERSION=22.18.0 +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh | bash +# standard location +ENV NVM_DIR=/root/.nvm +RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} +ENV PATH="${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH}" + # Add local bin to the path ENV PATH="${PATH}:/root/.local/bin" diff --git a/README.md b/README.md index 14a7d839..f37dc3e9 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ With Serena, we provide * Java (_Note_: startup is slow, initial startup especially so. There may be issues with java on macos and linux, we are working on it.) * Elixir (Requires NextLS and Elixir install; **Windows not supported** - Next LS does not provide Windows binaries) * Clojure + * Bash (Requires Node.js and npm for bash-language-server installation) * C/C++ (You may experience issues with finding references, we are working on it) * Swift (requires Swift and sourcekit-lsp to be installed) * indirect support (may require some code changes/manual installation) for: diff --git a/compose.yaml b/compose.yaml index bb8f683e..48923e27 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,10 @@ services: serena: image: serena:latest + # To work with projects, you must mount them as volumes: + # volumes: + # - ./my-project:/workspace/my-project + # - /path/to/another/project:/workspace/another-project build: context: ./ dockerfile: Dockerfile @@ -12,6 +16,8 @@ services: - SERENA_DOCKER=1 command: - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0" + # Add the context for the IDE assistant + # - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide-assistant" serena-dev: image: serena:dev diff --git a/docs/serena_on_chatgpt.md b/docs/serena_on_chatgpt.md new file mode 100644 index 00000000..ec872696 --- /dev/null +++ b/docs/serena_on_chatgpt.md @@ -0,0 +1,102 @@ + +# Connecting Serena MCP Server to ChatGPT via MCPO & Cloudflare Tunnel + +This guide explains how to expose a **locally running Serena MCP server** (powered by MCPO) to the internet using **Cloudflare Tunnel**, and how to connect it to **ChatGPT as a Custom GPT with tool access**. + +Once configured, ChatGPT becomes a powerful **coding agent** with direct access to your codebase, shell, and file system — so **read the security notes carefully**. + +--- +## Prerequisites + +Make sure you have [uv](https://docs.astral.sh/uv/getting-started/installation/) +and [cloudflared](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/) installed. + +## 1. Start the Serena MCP Server via MCPO + +Run the following command to launch Serena as http server (assuming port 8000): + +```bash +uvx mcpo --port 8000 --api-key -- \ + uvx --from git+https://github.com/oraios/serena \ + serena start-mcp-server --context chatgpt --project $(pwd) +``` + +- `--api-key` is required to secure the server. +- `--project` should point to the root of your codebase. + +You can also use other options, and you don't have to pass `--project` if you want to work on multiple projects +or want to activate it later. See + +```shell +uvx --from git+https://github.com/oraios/serena serena start-mcp-server --help +``` + +--- + +## 2. Expose the Server Using Cloudflare Tunnel + +Run: + +```bash +cloudflared tunnel --url http://localhost:8000 +``` + +This will give you a **public HTTPS URL** like: + +``` +https://serena-agent-tunnel.trycloudflare.com +``` + +Your server is now securely exposed to the internet. + +--- + +## 3. Connect It to ChatGPT (Custom GPT) + +### Steps: + +1. Go to [ChatGPT → Explore GPTs → Create](https://chat.openai.com/gpts/editor) +2. During setup, click **“Add APIs”** +3. Set up **API Key authentication** with the auth type as **Bearer** and enter the api key you used to start the MCPO server. +4. In the **Schema** section, click on **import from URL** and paste `/openapi.json` with the URL you got from the previous step. +5. Add the following line to the top of the imported JSON schema: + ``` + "servers": ["url": ""], + ``` + **Important**: don't include a trailing slash at the end of the URL! + +ChatGPT will read the schema and create functions automatically. + +--- + +## Security Warning — Read Carefully + +Depending on your configuration and enabled tools, Serena's MCP server may: +- Execute **arbitrary shell commands** +- Read, write, and modify **files in your codebase** + +This gives ChatGPT the same powers as a remote developer on your machine. + +### ⚠️ Key Rules: +- **NEVER expose your API key** +- **Only expose this server when needed**, and monitor its use. + +In your project’s `.serena/project.yml` or global config, you can disable tools like: + +```yaml +excluded_tools: + - execute_shell_command + - ... +read_only: true +``` + +This is strongly recommended if you want a read-only or safer agent. + + +--- + +## Final Thoughts + +With this setup, ChatGPT becomes a coding assistant **running on your local code** — able to index, search, edit, and even run shell commands depending on your configuration. + +Use responsibly, and keep security in mind. diff --git a/pyproject.toml b/pyproject.toml index fc20b70f..e9b318dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -260,7 +260,9 @@ markers = [ "elixir: language server running for Elixir", "terraform: language server running for Terraform", "swift: language server running for Swift", + "bash: language server running for Bash", "snapshot: snapshot tests for symbolic editing operations", + "ruby: language server running for Ruby", ] [tool.codespell] diff --git a/scripts/demo_run_tools.py b/scripts/demo_run_tools.py index 394fe553..d52cfb00 100644 --- a/scripts/demo_run_tools.py +++ b/scripts/demo_run_tools.py @@ -9,7 +9,7 @@ from pprint import pprint from serena.agent import SerenaAgent from serena.config.serena_config import SerenaConfig from serena.constants import REPO_ROOT -from serena.tools import FindFileTool, FindReferencingSymbolsTool, SearchForPatternTool +from serena.tools import FindFileTool, FindReferencingSymbolsTool, GetSymbolsOverviewTool, SearchForPatternTool if __name__ == "__main__": agent = SerenaAgent(project=REPO_ROOT, serena_config=SerenaConfig(gui_log_window_enabled=False, web_dashboard=False)) @@ -18,13 +18,9 @@ if __name__ == "__main__": find_refs_tool = agent.get_tool(FindReferencingSymbolsTool) find_file_tool = agent.get_tool(FindFileTool) search_pattern_tool = agent.get_tool(SearchForPatternTool) + overview_tool = agent.get_tool(GetSymbolsOverviewTool) result = agent.execute_task( - lambda: search_pattern_tool.apply( - r"def request_full_.*?\).*?\)", - restrict_search_to_code_files=False, - relative_path="src/solidlsp", - paths_include_glob="**/ls.py", - ) + lambda: overview_tool.apply("src/solidlsp/ls.py"), ) pprint(json.loads(result)) diff --git a/src/serena/agent.py b/src/serena/agent.py index 66394d62..53fead1d 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any, Optional, TypeVar from sensai.util import logging from sensai.util.logging import LogTime +from interprompt.jinja_template import JinjaTemplate from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode @@ -25,7 +26,8 @@ from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, T from serena.dashboard import SerenaDashboardAPI from serena.project import Project from serena.prompt_factory import SerenaPromptFactory -from serena.tools import ActivateProjectTool, Tool, ToolRegistry +from serena.tools import ActivateProjectTool, Tool, ToolMarker, ToolRegistry +from serena.util.inspection import iter_subclasses from serena.util.logging import MemoryLogHandler from solidlsp import SolidLanguageServer @@ -91,6 +93,23 @@ class MemoriesManager: return f"Memory {name} deleted." +class AvailableTools: + def __init__(self, tools: list[Tool]): + """ + :param tools: the list of available tools + """ + self.tools = tools + self.tool_names = [tool.get_name_from_cls() for tool in tools] + self.tool_marker_names = set() + for marker_class in iter_subclasses(ToolMarker): + for tool in tools: + if isinstance(tool, marker_class): + self.tool_marker_names.add(marker_class.__name__) + + def __len__(self) -> int: + return len(self.tools) + + class SerenaAgent: def __init__( self, @@ -192,7 +211,7 @@ class SerenaAgent: tool_inclusion_definitions.append(SerenaAgentMode.from_name_internal("jetbrains")) self._base_tool_set = ToolSet.default().apply(*tool_inclusion_definitions) - self._exposed_tools = {tc: t for tc, t in self._all_tools.items() if self._base_tool_set.includes_name(t.get_name())} + self._exposed_tools = AvailableTools([t for t in self._all_tools.values() if self._base_tool_set.includes_name(t.get_name())]) log.info(f"Number of exposed tools: {len(self._exposed_tools)}") # create executor for starting the language server and running tools in another thread @@ -227,6 +246,12 @@ class SerenaAgent: except Exception as e: log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e) + def get_context(self) -> SerenaAgentContext: + return self._context + + def get_tool_description_override(self, tool_name: str) -> str | None: + return self._context.tool_description_overrides.get(tool_name, None) + def _check_shell_settings(self) -> None: # On Windows, Claude Code sets COMSPEC to Git-Bash (often even with a path containing spaces), # which causes all sorts of trouble, preventing language servers from being launched correctly. @@ -301,7 +326,7 @@ class SerenaAgent: If a client should attempt to use a tool that is dynamically disabled (e.g. because a project is activated that disables it), it will receive an error. """ - return list(self._exposed_tools.values()) + return list(self._exposed_tools.tools) def get_active_project(self) -> Project | None: """ @@ -335,11 +360,21 @@ class SerenaAgent: """ return list(self._modes) + def _format_prompt(self, prompt_template: str) -> str: + template = JinjaTemplate(prompt_template) + return template.render(available_tools=self._exposed_tools.tool_names, available_markers=self._exposed_tools.tool_marker_names) + def create_system_prompt(self) -> str: - return self.prompt_factory.create_system_prompt( - context_system_prompt=self._context.prompt, - mode_system_prompts=[mode.prompt for mode in self._modes], + available_markers = self._exposed_tools.tool_marker_names + log.info("Generating system prompt with available_tools=(see exposed tools), available_markers=%s", available_markers) + system_prompt = self.prompt_factory.create_system_prompt( + context_system_prompt=self._format_prompt(self._context.prompt), + mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self._modes], + available_tools=self._exposed_tools.tool_names, + available_markers=available_markers, ) + log.info("System prompt:\n%s", system_prompt) + return system_prompt def _update_active_tools(self) -> None: """ @@ -583,3 +618,7 @@ class SerenaAgent: if self._gui_log_viewer: log.info("Stopping the GUI log window ...") self._gui_log_viewer.stop() + + def get_tool_by_name(self, tool_name: str) -> Tool: + tool_class = ToolRegistry().get_tool_class_by_name(tool_name) + return self.get_tool(tool_class) diff --git a/src/serena/cli.py b/src/serena/cli.py index 6b46a8e8..457b3acb 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -26,7 +26,7 @@ from serena.constants import ( USER_CONTEXT_YAMLS_DIR, USER_MODE_YAMLS_DIR, ) -from serena.mcp import SerenaMCPFactorySingleProcess +from serena.mcp import SerenaMCPFactory, SerenaMCPFactorySingleProcess from serena.project import Project from serena.tools import ToolRegistry from serena.util.logging import MemoryLogHandler @@ -478,6 +478,28 @@ class ToolCommands(AutoRegisteringGroup): else: ToolRegistry().print_tool_overview() + @staticmethod + @click.command( + "description", + help="Print the description of a tool, optionally with a specific context (the latter may modify the default description).", + ) + @click.argument("tool_name", type=str) + @click.option("--context", type=str, default=None, help="Context name or path to context file.") + def description(tool_name: str, context: str | None = None) -> None: + # Load the context + serena_context = None + if context: + serena_context = SerenaAgentContext.load(context) + + agent = SerenaAgent( + project=None, + serena_config=SerenaConfig(web_dashboard=False, log_level=logging.INFO), + context=serena_context, + ) + tool = agent.get_tool_by_name(tool_name) + mcp_tool = SerenaMCPFactory.make_mcp_tool(tool) + click.echo(mcp_tool.description) + class PromptCommands(AutoRegisteringGroup): def __init__(self) -> None: diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index de84f6a7..44cd42fb 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -3,8 +3,7 @@ Context and Mode configuration loader """ import os -from copy import copy -from dataclasses import asdict, dataclass +from dataclasses import dataclass, field from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Self @@ -39,22 +38,15 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): name: str prompt: str + """ + a Jinja2 template for the generation of the system prompt. + It is formatted by the agent (see SerenaAgent._format_prompt()). + """ description: str = "" def _tostring_includes(self) -> list[str]: return ["name"] - def to_json_dict(self) -> dict[str, str | list[str]]: - result = asdict(self) - result["excluded_tools"] = list(result["excluded_tools"]) - return result - - @classmethod - def from_json_dict(cls, data: dict) -> Self: - data = copy(data) - data["excluded_tools"] = set(data["excluded_tools"]) - return cls(**data) - def print_overview(self) -> None: """Print an overview of the mode.""" print(f"{self.name}:\n {self.description}") @@ -133,28 +125,26 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): name: str prompt: str + """ + a Jinja2 template for the generation of the system prompt. + It is formatted by the agent (see SerenaAgent._format_prompt()). + """ description: str = "" + tool_description_overrides: dict[str, str] = field(default_factory=dict) + """Maps tool names to custom descriptions, default descriptions are extracted from the tool docstrings.""" def _tostring_includes(self) -> list[str]: return ["name"] - def to_json_dict(self) -> dict[str, str | list[str]]: - result = asdict(self) - result["excluded_tools"] = list(result["excluded_tools"]) - return result - - @classmethod - def from_json_dict(cls, data: dict) -> Self: - data = copy(data) - data["excluded_tools"] = set(data["excluded_tools"]) - return cls(**data) - @classmethod def from_yaml(cls, yaml_path: str | Path) -> Self: """Load a context from a YAML file.""" with open(yaml_path, encoding="utf-8") as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) + # Ensure backwards compatibility for tool_description_overrides + if "tool_description_overrides" not in data: + data["tool_description_overrides"] = {} return cls(name=name, **data) @classmethod diff --git a/src/serena/generated/generated_prompt_factory.py b/src/serena/generated/generated_prompt_factory.py index 6bb7a76b..3a5e1442 100644 --- a/src/serena/generated/generated_prompt_factory.py +++ b/src/serena/generated/generated_prompt_factory.py @@ -32,5 +32,7 @@ class PromptFactory(PromptFactoryBase): def create_prepare_for_new_conversation(self) -> str: return self._render_prompt("prepare_for_new_conversation", locals()) - def create_system_prompt(self, *, context_system_prompt: Any, mode_system_prompts: Any) -> str: + def create_system_prompt( + self, *, available_markers: Any, available_tools: Any, context_system_prompt: Any, mode_system_prompts: Any + ) -> str: return self._render_prompt("system_prompt", locals()) diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 9926af8e..fb42cbe9 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -68,10 +68,17 @@ class SerenaMCPFactory: # Mount the tool description as a combination of the docstring description and # the return value description, if it exists. - if docstring.description: - func_doc = f"{docstring.description.strip().strip('.')}." + overridden_description = tool.agent.get_context().tool_description_overrides.get(func_name, None) + + if overridden_description is not None: + func_doc = overridden_description + elif docstring.description: + func_doc = docstring.description else: func_doc = "" + func_doc = func_doc.strip().strip(".") + if func_doc: + func_doc += "." if docstring.returns and (docstring_returns_descr := docstring.returns.description): # Only add a space before "Returns" if func_doc is not empty prefix = " " if func_doc else "" @@ -113,6 +120,7 @@ class SerenaMCPFactory: for tool in self._iter_tools(): mcp_tool = self.make_mcp_tool(tool) mcp._tool_manager._tools[tool.get_name()] = mcp_tool + log.info(f"Starting MCP server with {len(mcp._tool_manager._tools)} tools: {list(mcp._tool_manager._tools.keys())}") @abstractmethod def _instantiate_agent(self, serena_config: SerenaConfig, modes: list[SerenaAgentMode]) -> None: diff --git a/src/serena/resources/config/contexts/agent.yml b/src/serena/resources/config/contexts/agent.yml index d3da9da9..e2984283 100644 --- a/src/serena/resources/config/contexts/agent.yml +++ b/src/serena/resources/config/contexts/agent.yml @@ -3,4 +3,6 @@ prompt: | You are running in agent context where the system prompt is provided externally. You should use symbolic tools when possible for code understanding and modification. excluded_tools: - - initial_instructions \ No newline at end of file + - initial_instructions + +tool_description_overrides: {} \ No newline at end of file diff --git a/src/serena/resources/config/contexts/chatgpt.yml b/src/serena/resources/config/contexts/chatgpt.yml new file mode 100644 index 00000000..c758b2c3 --- /dev/null +++ b/src/serena/resources/config/contexts/chatgpt.yml @@ -0,0 +1,28 @@ +description: A configuration specific for chatgpt, which has a limit of 30 tools and requires short descriptions. +prompt: | + You are running in desktop app context where the tools give you access to the code base as well as some + access to the file system, if configured. You interact with the user through a chat interface that is separated + from the code base. As a consequence, if you are in interactive mode, your communication with the user should + involve high-level thinking and planning as well as some summarization of any code edits that you make. + For viewing the code edits the user will view them in a separate code editor window, and the back-and-forth + between the chat and the code editor should be minimized as well as facilitated by you. + If complex changes have been made, advise the user on how to review them in the code editor. + If complex relationships that the user asked for should be visualized or explained, consider creating + a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering + options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions. +excluded_tools: [] +included_optional_tools: + - switch_modes + +tool_description_overrides: + find_symbol: | + Retrieves symbols matching `name_path` in a file. + Use `depth > 0` to include children. `name_path` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar" + replace_regex: | + Replaces text using regular expressions. Preferred for smaller edits where symbol-level tools aren't appropriate. + Use wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content. + Essential for multi-line replacements. + search_for_pattern: | + Flexible pattern search across codebase. Prefer symbolic operations when possible. + Uses DOTALL matching. Use non-greedy quantifiers (.*?) to avoid over-matching. + Supports file filtering via globs and code-only restriction. \ No newline at end of file diff --git a/src/serena/resources/config/contexts/context.template.yml b/src/serena/resources/config/contexts/context.template.yml index 52e94535..c01a1177 100644 --- a/src/serena/resources/config/contexts/context.template.yml +++ b/src/serena/resources/config/contexts/context.template.yml @@ -1,4 +1,11 @@ # See Serena's documentation for more details on concept of contexts. description: Description of the context, not used in the code. prompt: Prompt that will form part of the system prompt/initial instructions for agents started in this context. -excluded_tools: [] \ No newline at end of file +excluded_tools: [] + +# several tools are excluded by default and have to be explicitly included by the user +included_optional_tools: [] + +# mapping of tool names to an override of their descriptions (the default description is the docstring of the Tool's apply method). +# Sometimes, tool descriptions are too long (e.g., for ChatGPT), or users may want to override them for another reason. +tool_description_overrides: {} \ No newline at end of file diff --git a/src/serena/resources/config/contexts/desktop-app.yml b/src/serena/resources/config/contexts/desktop-app.yml index ea1567fb..c3adcfb4 100644 --- a/src/serena/resources/config/contexts/desktop-app.yml +++ b/src/serena/resources/config/contexts/desktop-app.yml @@ -10,4 +10,8 @@ prompt: | If complex relationships that the user asked for should be visualized or explained, consider creating a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions. -excluded_tools: [] \ No newline at end of file +excluded_tools: [] +included_optional_tools: + - switch_modes + +tool_description_overrides: {} diff --git a/src/serena/resources/config/contexts/ide-assistant.yml b/src/serena/resources/config/contexts/ide-assistant.yml index 2a6c9e2c..27a13673 100644 --- a/src/serena/resources/config/contexts/ide-assistant.yml +++ b/src/serena/resources/config/contexts/ide-assistant.yml @@ -19,10 +19,8 @@ prompt: | excluded_tools: - create_text_file - read_file - - delete_lines - - replace_lines - - insert_at_line - execute_shell_command - prepare_for_new_conversation - - summarize_changes - - get_current_config + +tool_description_overrides: {} + diff --git a/src/serena/resources/config/modes/editing.yml b/src/serena/resources/config/modes/editing.yml index d53750fd..464f4537 100644 --- a/src/serena/resources/config/modes/editing.yml +++ b/src/serena/resources/config/modes/editing.yml @@ -14,7 +14,7 @@ prompt: | use the regex-based approach that is described below. Let us first discuss the symbol-based approach. - Symbols are identified by their name path and relative file path, see the description of the `find_symbols` tool for more details + Symbols are identified by their name path and relative file path, see the description of the `find_symbol` tool for more details on how the `name_path` matches symbols. You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible @@ -34,7 +34,8 @@ prompt: | You will generally be able to use the info from the snippets and the regex-based approach to adjust the references as well. You can assume that all symbol editing tools are reliable, so you don't need to verify the results if the tool returns without error. - Now let us discuss the regex-based approach. + {% if 'replace_regex' in available_tools %} + Let us discuss the regex-based approach. The regex-based approach is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation. This is the case if you need to adjust just a few lines of code within a method, or a chunk that is much smaller than a whole symbol. You use other tools to find the relevant content and @@ -104,7 +105,8 @@ prompt: | Generally, I remind you that you rely on the regex tool with providing you the correct feedback, no need for more verification! IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE LONG REGEXES WITHOUT USING WILDCARDS INSTEAD! -excluded_tools: + {% endif %} +excluded_tools: - replace_lines - insert_at_line - delete_lines diff --git a/src/serena/resources/config/modes/mode.template.yml b/src/serena/resources/config/modes/mode.template.yml index 431afdeb..6d836ca3 100644 --- a/src/serena/resources/config/modes/mode.template.yml +++ b/src/serena/resources/config/modes/mode.template.yml @@ -2,3 +2,6 @@ description: Description of the mode, not used in the code. prompt: Prompt that will form part of the message sent to the model when activating this mode. excluded_tools: [] + +# several tools are excluded by default and have to be explicitly included by the user +included_optional_tools: [] \ No newline at end of file diff --git a/src/serena/resources/config/prompt_templates/system_prompt.yml b/src/serena/resources/config/prompt_templates/system_prompt.yml index 8f3dd504..e912f7ca 100644 --- a/src/serena/resources/config/prompt_templates/system_prompt.yml +++ b/src/serena/resources/config/prompt_templates/system_prompt.yml @@ -5,32 +5,38 @@ prompts: system_prompt: | You are a professional coding agent concerned with one particular codebase. You have access to semantic coding tools on which you rely heavily for all your work, as well as collection of memory - files containing general information about the codebase. You operate in a frugal and intelligent manner, always + files containing general information about the codebase. You operate in a resource-efficient and intelligent manner, always keeping in mind to not read or generate content that is not needed for the task at hand. When reading code in order to answer a user question or task, you should try reading only the necessary code. Some tasks may require you to understand the architecture of large parts of the codebase, while for others, it may be enough to read a small set of symbols or a single file. Generally, you should avoid reading entire files unless it is absolutely necessary, instead relying on - intelligent step-by-step acquisition of information. However, if you already read a file, it does not make + intelligent step-by-step acquisition of information. {% if 'ToolMarkerSymbolicRead' in available_markers %}However, if you already read a file, it does not make sense to further analyse it with the symbolic tools (except for the `find_referencing_symbols` tool), - as you already have the information. + as you already have the information.{% endif %} - I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! CONSIDER INSTEAD USING THE OVERVIEW TOOL AND - SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST! + I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! + {% if 'ToolMarkerSymbolicRead' in available_markers %} + CONSIDER INSTEAD USING THE OVERVIEW TOOL AND SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST! I WILL BE EVEN MORE UPSET IF AFTER HAVING READ AN ENTIRE FILE YOU KEEP READING THE SAME CONTENT WITH THE SYMBOLIC TOOLS! THE PURPOSE OF THE SYMBOLIC TOOLS IS TO HAVE TO READ LESS CODE, NOT READ THE SAME CONTENT MULTIPLE TIMES! + {% endif %} You can achieve the intelligent reading of code by using the symbolic tools for getting an overview of symbols and the relations between them, and then only reading the bodies of symbols that are necessary to answer the question - or complete the task. You can also use the standard tools like list_dir, find_file and search_for_pattern if you need to. + or complete the task. + You can use the standard tools like list_dir, find_file and search_for_pattern if you need to. When tools allow it, you pass the `relative_path` parameter to restrict the search to a specific file or directory. For some tools, `relative_path` can only be a file path, so make sure to properly read the tool descriptions. - If you are unsure about a symbol's name or location (to the extent that substring_matching for the symbol name is not enough), you can use the `search_for_pattern` tool, which allows fast - and flexible search for patterns in the codebase. This way you can first find candidates for symbols or files, - and then proceed with the symbolic tools. + {% if 'search_for_pattern' in available_tools %} + If you are unsure about a symbol's name or location{% if 'find_symbol' in available_tools %} (to the extent that substring_matching for the symbol name is not enough){% endif %}, you can use the `search_for_pattern` tool, which allows fast + and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %}This way you can first find candidates for symbols or files, + and then proceed with the symbolic tools.{% endif %} + {% endif %} - Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbols` tool for more details + {% if 'ToolMarkerSymbolicRead' in available_markers %} + Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbol` tool for more details on how the `name_path` matches symbols. You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible @@ -40,10 +46,13 @@ prompts: you can use `find_symbol` with the name path `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding to read the desired methods with `include_body=True` You can understand relationships between symbols by using the `find_referencing_symbols` tool. + {% endif %} + {% if 'read_memory' in available_tools %} You generally have access to memories and it may be useful for you to read them, but also only if they help you to answer the question or complete the task. You can infer which memories are relevant to the current task by reading the memory names and descriptions. + {% endif %} The context and modes of operation are described below. From them you can infer how to interact with your user and which tasks and kinds of interactions are expected of you. diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index b57226aa..4edc0868 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -1,5 +1,6 @@ class LogMessage { constructor(message, toolNames) { + message = this.escapeHtml(message); const logLevel = this.determineLogLevel(message); const highlightedMessage = this.highlightToolNames(message, toolNames); this.$elem = $('
').addClass('log-' + logLevel).html(highlightedMessage + '\n'); @@ -27,6 +28,21 @@ class LogMessage { }); return highlightedMessage; } + + escapeHtml (convertString) { + if (typeof convertString !== 'string') return convertString; + + const patterns = { + '<' : '<', + '>' : '>', + '&' : '&', + '"' : '"', + '\'' : ''', + '`' : '`' + }; + + return convertString.replace(/[<>&"'`]/g, match => patterns[match]); + }; } class Dashboard { @@ -36,6 +52,7 @@ class Dashboard { this.toolNames = []; this.currentMaxIdx = -1; this.pollInterval = null; + this.failureCount = 0; this.$logContainer = $('#log-container'); this.$errorContainer = $('#error-container'); this.$loadButton = $('#load-logs'); @@ -145,6 +162,7 @@ class Dashboard { start_idx: self.currentMaxIdx + 1 }), success: function(response) { + self.failureCount = 0; // Only append new messages if we have any if (response.messages && response.messages.length > 0) { let wasAtBottom = false; @@ -174,6 +192,11 @@ class Dashboard { }, error: function(xhr, status, error) { console.error('Error polling for new logs:', error); + self.failureCount++; + if (self.failureCount >= 3) { + console.log('Server appears to be down, closing tab'); + window.close(); + } } }); } diff --git a/src/serena/tools/cmd_tools.py b/src/serena/tools/cmd_tools.py index 0db0da08..214d54e2 100644 --- a/src/serena/tools/cmd_tools.py +++ b/src/serena/tools/cmd_tools.py @@ -19,15 +19,8 @@ class ExecuteShellCommandTool(Tool, ToolMarkerCanEdit): max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH, ) -> str: """ - Execute a shell command and return its output. - - IMPORTANT: you should always consider the memory about suggested shell commands before using this tool. - If this memory was not loaded in the current conversation, you should load it using the `read_memory` tool - before using this tool. - - You should have at least once looked at the suggested shell commands from the corresponding memory - created during the onboarding process before using this tool. - Never execute unsafe shell commands like `rm -rf /` or similar! Generally be very careful with deletions. + Execute a shell command and return its output. If there is a memory about suggested commands, read that first. + Never execute unsafe shell commands like `rm -rf /` or similar! :param command: the shell command to execute :param cwd: the working directory to execute the command in. If None, the project root will be used. diff --git a/src/serena/tools/config_tools.py b/src/serena/tools/config_tools.py index 51b782ad..8a3ddeb7 100644 --- a/src/serena/tools/config_tools.py +++ b/src/serena/tools/config_tools.py @@ -1,7 +1,7 @@ import json from serena.config.context_mode import SerenaAgentMode -from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject +from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional class ActivateProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): @@ -35,7 +35,7 @@ class ActivateProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): return result_str -class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): +class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional): """ Removes a project from the Serena configuration. """ @@ -50,7 +50,7 @@ class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): return f"Successfully removed project '{project_name}' from configuration." -class SwitchModesTool(Tool): +class SwitchModesTool(Tool, ToolMarkerOptional): """ Activates modes by providing a list of their names """ @@ -71,7 +71,7 @@ class SwitchModesTool(Tool): return result_str -class GetCurrentConfigTool(Tool): +class GetCurrentConfigTool(Tool, ToolMarkerOptional): """ Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. """ diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index f79ffe06..b5770ab5 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -14,7 +14,7 @@ from fnmatch import fnmatch from pathlib import Path from serena.text_utils import search_files -from serena.tools import SUCCESS_RESULT, TOOL_DEFAULT_MAX_ANSWER_LENGTH, EditedFileContext, Tool, ToolMarkerCanEdit +from serena.tools import SUCCESS_RESULT, TOOL_DEFAULT_MAX_ANSWER_LENGTH, EditedFileContext, Tool, ToolMarkerCanEdit, ToolMarkerOptional from serena.util.file_system import scan_directory @@ -29,7 +29,6 @@ class ReadFileTool(Tool): """ Reads the given file or a chunk of it. Generally, symbolic operations like find_symbol or find_referencing_symbols should be preferred if you know which symbols you are looking for. - Reading the entire file is only recommended if there is no other way to get the content required for the task. :param relative_path: the relative path to the file to read :param start_line: the 0-based index of the first line to be retrieved. @@ -60,14 +59,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit): def apply(self, relative_path: str, content: str) -> str: """ - Write a new file (or overwrite an existing file). For existing files, it is strongly recommended - to use symbolic operations like replace_symbol_body or insert_after_symbol/insert_before_symbol, if possible. - You can also use insert_at_line to insert content at a specific line for existing files if the symbolic operations - are not the right choice for what you want to do. - - If ever used on an existing file, the content has to be the complete content of that file (so it - may never end with something like "The remaining content of the file is left unchanged."). - For operations that just replace a part of a file, use the replace_lines or the symbolic editing tools instead. + Write a new file or overwrite an existing file. :param relative_path: the relative path to the file to create :param content: the (utf-8-encoded) content to write to the file @@ -200,7 +192,7 @@ class ReplaceRegexTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class DeleteLinesTool(Tool, ToolMarkerCanEdit): +class DeleteLinesTool(Tool, ToolMarkerCanEdit, ToolMarkerOptional): """ Deletes a range of lines within a file. """ @@ -228,7 +220,7 @@ class DeleteLinesTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class ReplaceLinesTool(Tool, ToolMarkerCanEdit): +class ReplaceLinesTool(Tool, ToolMarkerCanEdit, ToolMarkerOptional): """ Replaces a range of lines within a file with new content. """ @@ -259,7 +251,7 @@ class ReplaceLinesTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class InsertAtLineTool(Tool, ToolMarkerCanEdit): +class InsertAtLineTool(Tool, ToolMarkerCanEdit, ToolMarkerOptional): """ Inserts content at a given line in a file. """ @@ -351,7 +343,7 @@ class SearchForPatternTool(Tool): For example, for finding classes or methods from a name pattern. Setting to False is a better choice if you also want to search in non-code files, like in html or yaml files, which is why it is the default. - :return: A JSON object mapping file paths to lists of matched consecutive lines (with context, if requested). + :return: A mapping of file paths to lists of matched consecutive lines. """ abs_path = os.path.join(self.get_project_root(), relative_path) if not os.path.exists(abs_path): diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 319aa590..8b9f2d1a 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -1,10 +1,10 @@ import json -from serena.tools import TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerOptional +from serena.tools import TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerOptional, ToolMarkerSymbolicRead from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient -class JetBrainsFindSymbolTool(Tool, ToolMarkerOptional): +class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). """ @@ -70,7 +70,7 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerOptional): return self._limit_length(result, max_answer_chars) -class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerOptional): +class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Finds symbols that reference the given symbol """ @@ -100,7 +100,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerOptional): return self._limit_length(result, max_answer_chars) -class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerOptional): +class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Retrieves an overview of the top-level symbols within a specified file """ diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index 63bba4fe..3cf611f0 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -10,15 +10,8 @@ class WriteMemoryTool(Tool): def apply(self, memory_name: str, content: str, max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH) -> str: """ - Write some information about this project that can be useful for future tasks to a memory. - Use markdown formatting for the content. - The information should be short and to the point. - The memory name should be meaningful, such that from the name you can infer what the information is about. - It is better to have multiple small memories than to have a single large one because - memories will be read one by one and we only ever want to read relevant memories. - - This tool is either called during the onboarding process or when you have identified - something worth remembering about the project from the past conversation. + Write some information about this project that can be useful for future tasks to a memory in md format. + The memory name should be meaningful. """ if len(content) > max_answer_chars: raise ValueError( diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 4d2a5ffc..0c9b7868 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -4,11 +4,18 @@ Language server-related tools import dataclasses import json +import os from collections.abc import Sequence from copy import copy from typing import Any -from serena.tools import SUCCESS_RESULT, TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerCanEdit +from serena.tools import ( + SUCCESS_RESULT, + TOOL_DEFAULT_MAX_ANSWER_LENGTH, + Tool, + ToolMarkerSymbolicEdit, + ToolMarkerSymbolicRead, +) from solidlsp.ls_types import SymbolKind @@ -33,44 +40,45 @@ class RestartLanguageServerTool(Tool): def apply(self) -> str: """Use this tool only on explicit user request or after confirmation. - It may be necessary to restart the language server if the user performs edits - not through Serena, so the language server state becomes outdated and further editing attempts lead to errors. - - If such editing errors happen, you should suggest using this tool. + It may be necessary to restart the language server if it hangs. """ self.agent.reset_language_server() return SUCCESS_RESULT -class GetSymbolsOverviewTool(Tool): +class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): """ Gets an overview of the top-level symbols defined in a given file or directory. """ def apply(self, relative_path: str, max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH) -> str: """ - Gets an overview of the given file or directory. - For each analyzed file, we list the top-level symbols in the file (name_path, kind). - Use this tool to get a high-level understanding of the code symbols. - Calling this is often a good idea before more targeted reading, searching or editing operations on the code symbols. - Before requesting a symbol overview, it is usually a good idea to narrow down the scope of the overview - by first understanding the basic directory structure of the repository that you can get from memories - or by using the `list_dir` and `find_file` tools (or similar). + Use this tool to get a high-level understanding of the code symbols in a file. + This should be the first tool to call when you want to understand a new file, unless you already know + what you are looking for. :param relative_path: the relative path to the file or directory to get the overview of :param max_answer_chars: if the overview is longer than this number of characters, no content will be returned. Don't adjust unless there is really no other way to get the content required for the task. If the overview is too long, you should use a smaller directory instead, (e.g. a subdirectory). - :return: a JSON object mapping relative paths of all contained files to info about top-level symbols in the file (name_path, kind). + :return: a JSON object containing info about top-level symbols in the file """ symbol_retriever = self.create_language_server_symbol_retriever() - result = symbol_retriever.get_symbol_overview(relative_path) - result_json_str = json.dumps({k: [dataclasses.asdict(i) for i in l] for k, l in result.items()}) + file_path = os.path.join(self.project.project_root, relative_path) + + # The symbol overview is capable of working with both files and directories, + # but we want to ensure that the user provides a file path. + if not os.path.exists(file_path): + raise FileNotFoundError(f"File or directory {relative_path} does not exist in the project.") + if os.path.isdir(file_path): + raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ") + result = symbol_retriever.get_symbol_overview(relative_path)[relative_path] + result_json_str = json.dumps([dataclasses.asdict(i) for i in result]) return self._limit_length(result_json_str, max_answer_chars) -class FindSymbolTool(Tool): +class FindSymbolTool(Tool, ToolMarkerSymbolicRead): """ Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). """ @@ -132,7 +140,7 @@ class FindSymbolTool(Tool): :param exclude_kinds: Optional. List of LSP symbol kind integers to exclude. Takes precedence over `include_kinds`. :param substring_matching: If True, use substring matching for the last segment of `name`. :param max_answer_chars: Max characters for the JSON result. If exceeded, no content is returned. - :return: JSON string: a list of symbols (with locations) matching the name. + :return: a list of symbols (with locations) matching the name. """ parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None @@ -150,7 +158,7 @@ class FindSymbolTool(Tool): return self._limit_length(result, max_answer_chars) -class FindReferencingSymbolsTool(Tool): +class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): """ Finds symbols that reference the symbol at the given location (optionally filtered by type). """ @@ -164,10 +172,8 @@ class FindReferencingSymbolsTool(Tool): max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH, ) -> str: """ - Finds symbols that reference the symbol at the given `name_path`. The result will contain metadata about the referencing symbols - as well as a short code snippet around the reference (unless `include_body` is True, then the short snippet will be omitted). - Note that among other kinds of references, this function can be used to find (direct) subclasses of a class, - as subclasses are referencing symbols that have the kind class. + Finds references to the symbol at the given `name_path`. The result will contain metadata about the referencing symbols + as well as a short code snippet around the reference. :param name_path: for finding the symbol to find references for, same logic as in the `find_symbol` tool. :param relative_path: the relative path to the file containing the symbol for which to find references. @@ -204,7 +210,7 @@ class FindReferencingSymbolsTool(Tool): return self._limit_length(result, max_answer_chars) -class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit): +class ReplaceSymbolBodyTool(Tool, ToolMarkerSymbolicEdit): """ Replaces the full definition of a symbol. """ @@ -232,7 +238,7 @@ class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit): +class InsertAfterSymbolTool(Tool, ToolMarkerSymbolicEdit): """ Inserts content after the end of the definition of a given symbol. """ @@ -257,7 +263,7 @@ class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit): +class InsertBeforeSymbolTool(Tool, ToolMarkerSymbolicEdit): """ Inserts content before the beginning of the definition of a given symbol. """ @@ -269,9 +275,9 @@ class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit): body: str, ) -> str: """ - Inserts the given body/content before the beginning of the definition of the given symbol (via the symbol's location). - A typical use case is to insert a new class, function, method, field or variable assignment. - It also can be used to insert a new import statement before the first symbol in the file. + Inserts the given content before the beginning of the definition of the given symbol (via the symbol's location). + A typical use case is to insert a new class, function, method, field or variable assignment; or + a new import statement before the first symbol in the file. :param name_path: name path of the symbol before which to insert content (definitions in the `find_symbol` tool apply) :param relative_path: the relative path to the file containing the symbol diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index cd025970..1f55e604 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -73,22 +73,40 @@ class Component(ABC): TOOL_DEFAULT_MAX_ANSWER_LENGTH = int(2e5) -class ToolMarkerCanEdit: +class ToolMarker: + """ + Base class for tool markers. + """ + + +class ToolMarkerCanEdit(ToolMarker): """ Marker class for all tools that can perform editing operations on files. """ -class ToolMarkerDoesNotRequireActiveProject: +class ToolMarkerDoesNotRequireActiveProject(ToolMarker): pass -class ToolMarkerOptional: +class ToolMarkerOptional(ToolMarker): """ Marker class for optional tools that are disabled by default. """ +class ToolMarkerSymbolicRead(ToolMarker): + """ + Marker class for tools that perform symbol read operations. + """ + + +class ToolMarkerSymbolicEdit(ToolMarkerCanEdit): + """ + Marker class for tools that perform symbolic edit operations. + """ + + class ApplyMethodProtocol(Protocol): """Callable protocol for the apply method of a tool.""" diff --git a/src/serena/tools/workflow_tools.py b/src/serena/tools/workflow_tools.py index 38cc19be..be78d03d 100644 --- a/src/serena/tools/workflow_tools.py +++ b/src/serena/tools/workflow_tools.py @@ -96,7 +96,7 @@ class ThinkAboutWhetherYouAreDoneTool(Tool): return self.prompt_factory.create_think_about_whether_you_are_done() -class SummarizeChangesTool(Tool): +class SummarizeChangesTool(Tool, ToolMarkerOptional): """ Provides instructions for summarizing the changes made to the codebase. """ diff --git a/src/solidlsp/language_servers/bash_language_server.py b/src/solidlsp/language_servers/bash_language_server.py new file mode 100644 index 00000000..2b8d8549 --- /dev/null +++ b/src/solidlsp/language_servers/bash_language_server.py @@ -0,0 +1,382 @@ +""" +Provides Bash specific instantiation of the LanguageServer class using bash-language-server. +Contains various configurations and settings specific to Bash scripting. +""" + +import logging +import os +import pathlib +import shutil +import threading + +from solidlsp import ls_types +from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection +from solidlsp.ls import SolidLanguageServer +from solidlsp.ls_config import LanguageServerConfig +from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams +from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo +from solidlsp.settings import SolidLSPSettings + + +class BashLanguageServer(SolidLanguageServer): + """ + Provides Bash specific instantiation of the LanguageServer class using bash-language-server. + Contains various configurations and settings specific to Bash scripting. + """ + + def __init__( + self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings + ): + """ + Creates a BashLanguageServer instance. This class is not meant to be instantiated directly. + Use LanguageServer.create() instead. + """ + bash_lsp_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings) + super().__init__( + config, + logger, + repository_root_path, + ProcessLaunchInfo(cmd=bash_lsp_executable_path, cwd=repository_root_path), + "bash", + solidlsp_settings, + ) + self.server_ready = threading.Event() + self.initialize_searcher_command_available = threading.Event() + + @classmethod + def _setup_runtime_dependencies( + cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings + ) -> str: + """ + Setup runtime dependencies for Bash Language Server and return the command to start the server. + """ + # Verify both node and npm are installed + is_node_installed = shutil.which("node") is not None + assert is_node_installed, "node is not installed or isn't in PATH. Please install NodeJS and try again." + is_npm_installed = shutil.which("npm") is not None + assert is_npm_installed, "npm is not installed or isn't in PATH. Please install npm and try again." + + deps = RuntimeDependencyCollection( + [ + RuntimeDependency( + id="bash-language-server", + description="bash-language-server package", + command="npm install --prefix ./ bash-language-server@5.6.0", + platform_id="any", + ), + ] + ) + + # Install bash-language-server if not already installed + bash_ls_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "bash-lsp") + bash_executable_path = os.path.join(bash_ls_dir, "node_modules", ".bin", "bash-language-server") + + # Handle Windows executable extension + if os.name == "nt": + bash_executable_path += ".cmd" + + if not os.path.exists(bash_executable_path): + logger.log(f"Bash Language Server executable not found at {bash_executable_path}. Installing...", logging.INFO) + deps.install(logger, bash_ls_dir) + logger.log("Bash language server dependencies installed successfully", logging.INFO) + + if not os.path.exists(bash_executable_path): + raise FileNotFoundError( + f"bash-language-server executable not found at {bash_executable_path}, something went wrong with the installation." + ) + return f"{bash_executable_path} start" + + @staticmethod + def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: + """ + Returns the initialize params for the Bash Language Server. + """ + root_uri = pathlib.Path(repository_absolute_path).as_uri() + initialize_params = { + "locale": "en", + "capabilities": { + "textDocument": { + "synchronization": {"didSave": True, "dynamicRegistration": True}, + "completion": {"dynamicRegistration": True, "completionItem": {"snippetSupport": True}}, + "definition": {"dynamicRegistration": True}, + "references": {"dynamicRegistration": True}, + "documentSymbol": { + "dynamicRegistration": True, + "hierarchicalDocumentSymbolSupport": True, + "symbolKind": {"valueSet": list(range(1, 27))}, + }, + "hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]}, + "signatureHelp": {"dynamicRegistration": True}, + "codeAction": {"dynamicRegistration": True}, + }, + "workspace": { + "workspaceFolders": True, + "didChangeConfiguration": {"dynamicRegistration": True}, + "symbol": {"dynamicRegistration": True}, + }, + }, + "processId": os.getpid(), + "rootPath": repository_absolute_path, + "rootUri": root_uri, + "workspaceFolders": [ + { + "uri": root_uri, + "name": os.path.basename(repository_absolute_path), + } + ], + } + return initialize_params + + def _start_server(self): + """ + Starts the Bash Language Server, waits for the server to be ready and yields the LanguageServer instance. + """ + + def register_capability_handler(params): + assert "registrations" in params + for registration in params["registrations"]: + if registration["method"] == "workspace/executeCommand": + self.initialize_searcher_command_available.set() + return + + def execute_client_command_handler(params): + return [] + + def do_nothing(params): + return + + def window_log_message(msg): + self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO) + # Check for bash-language-server ready signals + message_text = msg.get("message", "") + if "Analyzing" in message_text or "analysis complete" in message_text.lower(): + self.logger.log("Bash language server analysis signals detected", logging.INFO) + self.server_ready.set() + self.completions_available.set() + + self.server.on_request("client/registerCapability", register_capability_handler) + self.server.on_notification("window/logMessage", window_log_message) + self.server.on_request("workspace/executeClientCommand", execute_client_command_handler) + self.server.on_notification("$/progress", do_nothing) + self.server.on_notification("textDocument/publishDiagnostics", do_nothing) + + self.logger.log("Starting Bash server process", logging.INFO) + self.server.start() + initialize_params = self._get_initialize_params(self.repository_root_path) + + self.logger.log( + "Sending initialize request from LSP client to LSP server and awaiting response", + logging.INFO, + ) + init_response = self.server.send.initialize(initialize_params) + self.logger.log(f"Received initialize response from bash server: {init_response}", logging.DEBUG) + + # Enhanced capability checks for bash-language-server 5.6.0 + assert init_response["capabilities"]["textDocumentSync"] in [1, 2] # Full or Incremental + assert "completionProvider" in init_response["capabilities"] + + # Verify document symbol support is available + if "documentSymbolProvider" in init_response["capabilities"]: + self.logger.log("Bash server supports document symbols", logging.INFO) + else: + self.logger.log("Warning: Bash server does not report document symbol support", logging.WARNING) + + self.server.notify.initialized({}) + + # Wait for server readiness with timeout + self.logger.log("Waiting for Bash language server to be ready...", logging.INFO) + if not self.server_ready.wait(timeout=3.0): + # Fallback: assume server is ready after timeout + self.logger.log("Timeout waiting for bash server ready signal, proceeding anyway", logging.WARNING) + self.server_ready.set() + self.completions_available.set() + else: + self.logger.log("Bash server initialization complete", logging.INFO) + + def request_document_symbols( + self, relative_file_path: str, include_body: bool = False + ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: + """ + Enhanced document symbol request with hybrid LSP + regex-based function detection for bash files. + + This method combines both LSP-based detection and regex-based detection to provide comprehensive + function discovery. This dual approach is necessary because: + + 1. bash-language-server (v5.6.0) has inconsistent function detection capabilities + 2. Some bash function syntaxes are not reliably detected by the LSP server + 3. Files may contain mixed function notation styles within the same file + 4. Different formatting or indentation can affect LSP detection + + The hybrid approach ensures maximum compatibility and comprehensive function discovery + for reliable symbolic editing operations in Serena. + """ + # First get symbols from the standard LSP approach + lsp_all_symbols, lsp_root_symbols = super().request_document_symbols(relative_file_path, include_body) + + # Always run regex-based function detection for comprehensive coverage + # This addresses bash-language-server limitations and ensures we catch all function patterns + self.logger.log(f"Running hybrid function detection (LSP + regex) for {relative_file_path}", logging.DEBUG) + + regex_detected_functions = self._detect_bash_functions(relative_file_path, include_body) + + # Merge LSP and regex results, avoiding duplicates + merged_all_symbols, merged_root_symbols = self._merge_function_detections( + lsp_all_symbols, lsp_root_symbols, regex_detected_functions + ) + + # Log detection results for debugging + lsp_functions = [s for s in lsp_all_symbols if s.get("kind") == 12] + total_functions = [s for s in merged_all_symbols if s.get("kind") == 12] + + self.logger.log( + f"Function detection for {relative_file_path}: LSP={len(lsp_functions)}, " + f"Regex={len(regex_detected_functions)}, Total={len(total_functions)}", + logging.INFO, + ) + + return merged_all_symbols, merged_root_symbols + + def _merge_function_detections( + self, + lsp_all_symbols: list[ls_types.UnifiedSymbolInformation], + lsp_root_symbols: list[ls_types.UnifiedSymbolInformation], + regex_detected_functions: list[ls_types.UnifiedSymbolInformation], + ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: + """ + Merge LSP-detected symbols with regex-detected functions, avoiding duplicates. + + This method: + 1. Keeps all non-function symbols from LSP detection + 2. Keeps all LSP-detected functions (they have more accurate positioning) + 3. Adds regex-detected functions that weren't found by LSP + 4. Uses function names to detect duplicates + + Args: + lsp_all_symbols: All symbols detected by LSP + lsp_root_symbols: Root-level symbols detected by LSP + regex_detected_functions: Functions detected by regex (all are root-level) + + Returns: + Tuple of (merged_all_symbols, merged_root_symbols) + + """ + # Extract function names that LSP already detected + lsp_function_names = set() + for symbol in lsp_all_symbols: + if symbol.get("kind") == 12: # LSP Symbol Kind 12 = Function + lsp_function_names.add(symbol["name"]) + + # Start with all LSP symbols (both functions and non-functions) + merged_all_symbols = lsp_all_symbols.copy() + merged_root_symbols = lsp_root_symbols.copy() + + # Add regex-detected functions that weren't found by LSP + added_functions = 0 + for regex_function in regex_detected_functions: + function_name = regex_function["name"] + + # Only add if this function wasn't detected by LSP + if function_name not in lsp_function_names: + merged_all_symbols.append(regex_function) + merged_root_symbols.append(regex_function) # All regex functions are root-level + added_functions += 1 + self.logger.log(f"Added regex-detected function '{function_name}' not found by LSP", logging.DEBUG) + + if added_functions > 0: + self.logger.log(f"Merged {added_functions} additional functions from regex detection", logging.INFO) + + return merged_all_symbols, merged_root_symbols + + def _detect_bash_functions(self, relative_file_path: str, include_body: bool = False) -> list[ls_types.UnifiedSymbolInformation]: + """ + Regex-based detection of bash functions as fallback when LSP doesn't provide them. + """ + import re + + from solidlsp import ls_types + + try: + # Read the file content directly from filesystem + abs_path = os.path.join(self.repository_root_path, relative_file_path) + with open(abs_path, encoding="utf-8") as f: + file_content = f.read() + lines = file_content.split("\n") + + detected_functions = [] + + # Regex patterns for bash function definitions + # Pattern 1: function name() { ... } + # Pattern 2: name() { ... } + function_patterns = [ + re.compile(r"^\s*function\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{"), + re.compile(r"^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{"), + ] + + for line_num, line in enumerate(lines): + for pattern in function_patterns: + match = pattern.match(line) + if match: + func_name = match.group(1) + + # Find the end of the function by matching braces + start_line = line_num + end_line = self._find_function_end(lines, start_line) + + # Create location information + location = ls_types.Location( + uri=pathlib.Path(abs_path).as_uri(), + range={"start": {"line": start_line, "character": match.start(1)}, "end": {"line": end_line, "character": 0}}, + absolutePath=abs_path, + relativePath=relative_file_path, + ) + + # Create the function symbol + func_symbol = ls_types.UnifiedSymbolInformation( + name=func_name, + kind=12, # LSP Symbol Kind for Function + location=location, + range=location["range"], + selectionRange={ + "start": {"line": start_line, "character": match.start(1)}, + "end": {"line": start_line, "character": match.end(1)}, + }, + children=[], + parent=None, + ) + + # Add function body if requested + if include_body: + func_symbol["body"] = "\n".join(lines[start_line : end_line + 1]) + + detected_functions.append(func_symbol) + break + + return detected_functions + + except Exception as e: + self.logger.log(f"Error in regex-based function detection for {relative_file_path}: {e}", logging.WARNING) + return [] + + def _find_function_end(self, lines: list[str], start_line: int) -> int: + """ + Find the end line of a bash function by matching opening and closing braces. + """ + brace_count = 0 + in_function = False + + for i in range(start_line, len(lines)): + line = lines[i] + + # Count braces, handling basic cases + for char in line: + if char == "{": + brace_count += 1 + in_function = True + elif char == "}" and in_function: + brace_count -= 1 + if brace_count == 0: + return i + + # Fallback: if we can't match braces properly, assume single line or small function + return min(start_line + 10, len(lines) - 1) diff --git a/src/solidlsp/language_servers/clangd_language_server.py b/src/solidlsp/language_servers/clangd_language_server.py index a770d78e..3879fae7 100644 --- a/src/solidlsp/language_servers/clangd_language_server.py +++ b/src/solidlsp/language_servers/clangd_language_server.py @@ -5,7 +5,6 @@ Provides C/C++ specific instantiation of the LanguageServer class. Contains vari import logging import os import pathlib -import stat import threading from solidlsp.ls import SolidLanguageServer @@ -95,7 +94,7 @@ class ClangdLanguageServer(SolidLanguageServer): f"Clangd executable not found at {clangd_executable_path}.\n" "Make sure you have installed clangd. See https://clangd.llvm.org/installation" ) - os.chmod(clangd_executable_path, stat.S_IEXEC) + os.chmod(clangd_executable_path, 0o755) return clangd_executable_path diff --git a/src/solidlsp/language_servers/clojure_lsp.py b/src/solidlsp/language_servers/clojure_lsp.py index f973db15..d81fa236 100644 --- a/src/solidlsp/language_servers/clojure_lsp.py +++ b/src/solidlsp/language_servers/clojure_lsp.py @@ -6,7 +6,6 @@ import logging import os import pathlib import shutil -import stat import subprocess import threading @@ -118,7 +117,7 @@ class ClojureLSP(SolidLanguageServer): deps.install(logger, clojurelsp_ls_dir) if not os.path.exists(clojurelsp_executable_path): raise FileNotFoundError(f"Download failed? Could not find clojure-lsp executable at {clojurelsp_executable_path}") - os.chmod(clojurelsp_executable_path, stat.S_IEXEC) + os.chmod(clojurelsp_executable_path, 0o755) return clojurelsp_executable_path @staticmethod diff --git a/src/solidlsp/language_servers/csharp_language_server.py b/src/solidlsp/language_servers/csharp_language_server.py index 66f8c3cb..da65cb83 100644 --- a/src/solidlsp/language_servers/csharp_language_server.py +++ b/src/solidlsp/language_servers/csharp_language_server.py @@ -7,7 +7,6 @@ import logging import os import platform import shutil -import stat import subprocess import tarfile import threading @@ -26,6 +25,7 @@ from solidlsp.ls_utils import PathUtils from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo from solidlsp.settings import SolidLSPSettings +from solidlsp.util.zip import SafeZipExtractor from .common import RuntimeDependency @@ -314,7 +314,7 @@ class CSharpLanguageServer(SolidLanguageServer): # Make executable on Unix systems if platform.system().lower() != "windows": - server_dll.chmod(server_dll.stat().st_mode | stat.S_IEXEC) + server_dll.chmod(0o755) logger.log(f"Successfully installed Microsoft.CodeAnalysis.LanguageServer to {server_dll}", logging.INFO) return str(server_dll) @@ -387,8 +387,9 @@ class CSharpLanguageServer(SolidLanguageServer): package_extract_dir = temp_dir / f"{package_name}.{package_version}" package_extract_dir.mkdir(exist_ok=True) - with zipfile.ZipFile(nupkg_file, "r") as zip_ref: - zip_ref.extractall(package_extract_dir) + # Use SafeZipExtractor to handle long paths and skip errors + extractor = SafeZipExtractor(archive_path=nupkg_file, extract_dir=package_extract_dir, verbose=False) + extractor.extract_all() # Clean up the nupkg file nupkg_file.unlink() @@ -456,7 +457,7 @@ class CSharpLanguageServer(SolidLanguageServer): # Make dotnet executable on Unix if platform.system().lower() != "windows": - dotnet_exe.chmod(dotnet_exe.stat().st_mode | stat.S_IEXEC) + dotnet_exe.chmod(0o755) logger.log(f"Successfully installed .NET 9 runtime to {dotnet_exe}", logging.INFO) return str(dotnet_exe) diff --git a/src/solidlsp/language_servers/dart_language_server.py b/src/solidlsp/language_servers/dart_language_server.py index c9cfcad1..297294a4 100644 --- a/src/solidlsp/language_servers/dart_language_server.py +++ b/src/solidlsp/language_servers/dart_language_server.py @@ -1,7 +1,6 @@ import logging import os import pathlib -import stat from solidlsp.ls import SolidLanguageServer from solidlsp.ls_logger import LanguageServerLogger @@ -84,7 +83,7 @@ class DartLanguageServer(SolidLanguageServer): deps.install(logger, dart_ls_dir) assert os.path.exists(dart_executable_path) - os.chmod(dart_executable_path, stat.S_IEXEC) + os.chmod(dart_executable_path, 0o755) return f"{dart_executable_path} language-server --client-id multilspy.dart --client-version 1.2" diff --git a/src/solidlsp/language_servers/eclipse_jdtls.py b/src/solidlsp/language_servers/eclipse_jdtls.py index bf87aab7..d664fb7a 100644 --- a/src/solidlsp/language_servers/eclipse_jdtls.py +++ b/src/solidlsp/language_servers/eclipse_jdtls.py @@ -7,7 +7,6 @@ import logging import os import pathlib import shutil -import stat import threading import uuid from pathlib import PurePath @@ -279,7 +278,7 @@ class EclipseJDTLS(SolidLanguageServer): ): FileUtils.download_and_extract_archive(logger, dependency["url"], vscode_java_path, dependency["archiveType"]) - os.chmod(jre_path, stat.S_IEXEC) + os.chmod(jre_path, 0o755) assert os.path.exists(vscode_java_path) assert os.path.exists(jre_home_path) diff --git a/src/solidlsp/language_servers/omnisharp.py b/src/solidlsp/language_servers/omnisharp.py index c9755fa8..e43287c5 100644 --- a/src/solidlsp/language_servers/omnisharp.py +++ b/src/solidlsp/language_servers/omnisharp.py @@ -6,7 +6,6 @@ import json import logging import os import pathlib -import stat import threading from collections.abc import Iterable @@ -194,7 +193,7 @@ class OmniSharp(SolidLanguageServer): FileUtils.download_and_extract_archive(logger, runtime_dependencies["OmniSharp"]["url"], omnisharp_ls_dir, "zip") omnisharp_executable_path = os.path.join(omnisharp_ls_dir, runtime_dependencies["OmniSharp"]["binaryName"]) assert os.path.exists(omnisharp_executable_path) - os.chmod(omnisharp_executable_path, stat.S_IEXEC) + os.chmod(omnisharp_executable_path, 0o755) razor_omnisharp_ls_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "RazorOmnisharp") if not os.path.exists(razor_omnisharp_ls_dir): diff --git a/src/solidlsp/language_servers/rust_analyzer.py b/src/solidlsp/language_servers/rust_analyzer.py index 48bae28a..d9b09090 100644 --- a/src/solidlsp/language_servers/rust_analyzer.py +++ b/src/solidlsp/language_servers/rust_analyzer.py @@ -5,7 +5,6 @@ Provides Rust specific instantiation of the LanguageServer class. Contains vario import logging import os import pathlib -import stat import threading from overrides import override @@ -58,10 +57,18 @@ class RustAnalyzer(SolidLanguageServer): """ deps = RuntimeDependencyCollection( [ + RuntimeDependency( + id="RustAnalyzer", + description="RustAnalyzer for macOS (x64)", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-x86_64-apple-darwin.gz", + platform_id="osx-x64", + archive_type="gz", + binary_name="rust_analyzer", + ), RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for macOS (arm64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-aarch64-apple-darwin.gz", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-aarch64-apple-darwin.gz", platform_id="osx-arm64", archive_type="gz", binary_name="rust_analyzer", @@ -69,7 +76,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for Linux (x64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-x86_64-unknown-linux-gnu.gz", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-x86_64-unknown-linux-gnu.gz", platform_id="linux-x64", archive_type="gz", binary_name="rust_analyzer", @@ -77,7 +84,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for Linux (arm64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-aarch64-unknown-linux-gnu.gz", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-aarch64-unknown-linux-gnu.gz", platform_id="linux-arm64", archive_type="gz", binary_name="rust_analyzer", @@ -85,7 +92,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for Windows (x64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-x86_64-pc-windows-msvc.zip", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-x86_64-pc-windows-msvc.zip", platform_id="win-x64", archive_type="zip", binary_name="rust-analyzer.exe", @@ -104,7 +111,7 @@ class RustAnalyzer(SolidLanguageServer): os.makedirs(rustanalyzer_ls_dir) deps.install(logger, rustanalyzer_ls_dir) assert os.path.exists(rustanalyzer_executable_path) - os.chmod(rustanalyzer_executable_path, stat.S_IEXEC) + os.chmod(rustanalyzer_executable_path, 0o755) return rustanalyzer_executable_path diff --git a/src/solidlsp/language_servers/solargraph.py b/src/solidlsp/language_servers/solargraph.py index 7b79c324..b75b19e9 100644 --- a/src/solidlsp/language_servers/solargraph.py +++ b/src/solidlsp/language_servers/solargraph.py @@ -7,7 +7,7 @@ import json import logging import os import pathlib -import stat +import shutil import subprocess import threading @@ -48,6 +48,9 @@ class Solargraph(SolidLanguageServer): self.initialize_searcher_command_available = threading.Event() self.resolve_main_method_available = threading.Event() + # Set timeout for Solargraph requests - Bundler environments may need more time + self.set_request_timeout(120.0) # 120 seconds for initialization and requests + @override def is_ignored_dirname(self, dirname: str) -> bool: return super().is_ignored_dirname(dirname) or dirname in ["vendor"] @@ -57,17 +60,6 @@ class Solargraph(SolidLanguageServer): """ Setup runtime dependencies for Solargraph and return the command to start the server. """ - runtime_dependencies = [ - { - "url": "https://rubygems.org/downloads/solargraph-0.51.1.gem", - "installCommand": "gem install solargraph -v 0.51.1", - "binaryName": "solargraph", - "archiveType": "gem", - } - ] - - dependency = runtime_dependencies[0] - # Check if Ruby is installed try: result = subprocess.run(["ruby", "--version"], check=True, capture_output=True, cwd=repository_root_path) @@ -78,18 +70,84 @@ class Solargraph(SolidLanguageServer): except FileNotFoundError as e: raise RuntimeError("Ruby is not installed. Please install Ruby before continuing.") from e - # Check if solargraph is installed - try: - result = subprocess.run( - ["gem", "list", "^solargraph$", "-i"], check=False, capture_output=True, text=True, cwd=repository_root_path - ) - if result.stdout.strip() == "false": - logger.log("Installing Solargraph...", logging.INFO) - subprocess.run(dependency["installCommand"].split(), check=True, capture_output=True, cwd=repository_root_path) + # Check for Bundler project (Gemfile exists) + gemfile_path = os.path.join(repository_root_path, "Gemfile") + gemfile_lock_path = os.path.join(repository_root_path, "Gemfile.lock") + is_bundler_project = os.path.exists(gemfile_path) - return "gem exec solargraph" - except subprocess.CalledProcessError as e: - raise RuntimeError(f"Failed to check or install Solargraph. {e.stderr}") from e + if is_bundler_project: + logger.log("Detected Bundler project (Gemfile found)", logging.INFO) + + # Check if bundle command is available + bundle_path = shutil.which("bundle") + if not bundle_path: + # Try common bundle executables + for bundle_cmd in ["bin/bundle", "bundle"]: + bundle_full_path = ( + os.path.join(repository_root_path, bundle_cmd) if bundle_cmd.startswith("bin/") else shutil.which(bundle_cmd) + ) + if bundle_full_path and os.path.exists(bundle_full_path): + bundle_path = bundle_full_path if bundle_cmd.startswith("bin/") else bundle_cmd + break + + if not bundle_path: + raise RuntimeError("Bundler project detected but 'bundle' command not found. Please install Bundler.") + + # Check if solargraph is in Gemfile.lock + solargraph_in_bundle = False + if os.path.exists(gemfile_lock_path): + try: + with open(gemfile_lock_path) as f: + content = f.read() + solargraph_in_bundle = "solargraph" in content.lower() + except Exception as e: + logger.log(f"Warning: Could not read Gemfile.lock: {e}", logging.WARNING) + + if solargraph_in_bundle: + logger.log("Found solargraph in Gemfile.lock", logging.INFO) + return f"{bundle_path} exec solargraph" + else: + logger.log( + "solargraph not found in Gemfile.lock. Please add 'gem \"solargraph\"' to your Gemfile and run 'bundle install'", + logging.WARNING, + ) + # Fall through to global installation check + + # Check if solargraph is installed globally + # First, try to find solargraph in PATH (includes asdf shims) + solargraph_path = shutil.which("solargraph") + if solargraph_path: + logger.log(f"Found solargraph at: {solargraph_path}", logging.INFO) + return solargraph_path + + # Fallback to gem exec (for non-Bundler projects or when global solargraph not found) + if not is_bundler_project: + runtime_dependencies = [ + { + "url": "https://rubygems.org/downloads/solargraph-0.51.1.gem", + "installCommand": "gem install solargraph -v 0.51.1", + "binaryName": "solargraph", + "archiveType": "gem", + } + ] + + dependency = runtime_dependencies[0] + try: + result = subprocess.run( + ["gem", "list", "^solargraph$", "-i"], check=False, capture_output=True, text=True, cwd=repository_root_path + ) + if result.stdout.strip() == "false": + logger.log("Installing Solargraph...", logging.INFO) + subprocess.run(dependency["installCommand"].split(), check=True, capture_output=True, cwd=repository_root_path) + + return "gem exec solargraph" + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Failed to check or install Solargraph. {e.stderr}") from e + else: + raise RuntimeError( + "This appears to be a Bundler project, but solargraph is not available. " + "Please add 'gem \"solargraph\"' to your Gemfile and run 'bundle install'." + ) @staticmethod def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: @@ -170,4 +228,16 @@ class Solargraph(SolidLanguageServer): self.completions_available.set() self.server_ready.set() - self.server_ready.wait() + + # Wait for server to be ready with timeout, especially important for Bundler environments + server_ready_timeout = 60.0 + self.logger.log(f"Waiting up to {server_ready_timeout} seconds for Solargraph to become ready...", logging.INFO) + + if self.server_ready.wait(timeout=server_ready_timeout): + self.logger.log("Solargraph is ready and available for requests", logging.INFO) + else: + self.logger.log( + f"Timeout waiting for Solargraph to become ready within {server_ready_timeout} seconds, proceeding anyway. " + "This may indicate slow initialization in Bundler environment or large project indexing.", + logging.WARNING, + ) diff --git a/src/solidlsp/language_servers/terraform_ls.py b/src/solidlsp/language_servers/terraform_ls.py index e40d9af1..6a374df6 100644 --- a/src/solidlsp/language_servers/terraform_ls.py +++ b/src/solidlsp/language_servers/terraform_ls.py @@ -1,7 +1,6 @@ import logging import os import shutil -import stat import threading from overrides import override @@ -112,7 +111,7 @@ class TerraformLS(SolidLanguageServer): # Make the executable file executable on Unix-like systems if platform_id.value != "win-x64": - os.chmod(terraform_ls_executable_path, stat.S_IEXEC | stat.S_IREAD) + os.chmod(terraform_ls_executable_path, 0o755) return terraform_ls_executable_path diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 5e82b2ef..53ed740a 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -231,6 +231,11 @@ class SolidLanguageServer(ABC): ls = SourceKitLSP(config, logger, repository_root_path, solidlsp_settings=solidlsp_settings) + elif config.code_language == Language.BASH: + from solidlsp.language_servers.bash_language_server import BashLanguageServer + + ls = BashLanguageServer(config, logger, repository_root_path, solidlsp_settings=solidlsp_settings) + else: logger.log(f"Language {config.code_language} is not supported", logging.ERROR) raise SolidLSPException(f"Language {config.code_language} is not supported") diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index 363027c8..4bed832e 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -43,6 +43,7 @@ class Language(str, Enum): ELIXIR = "elixir" TERRAFORM = "terraform" SWIFT = "swift" + BASH = "bash" # Experimental or deprecated Language Servers TYPESCRIPT_VTS = "typescript_vts" """Use the typescript language server through the natively bundled vscode extension via https://github.com/yioneko/vtsls""" @@ -106,6 +107,8 @@ class Language(str, Enum): return FilenameMatcher("*.tf", "*.tfvars", "*.tfstate") case self.SWIFT: return FilenameMatcher("*.swift") + case self.BASH: + return FilenameMatcher("*.sh", "*.bash") case _: raise ValueError(f"Unhandled language: {self}") diff --git a/src/solidlsp/ls_handler.py b/src/solidlsp/ls_handler.py index 6a994ed3..c1f50fa1 100644 --- a/src/solidlsp/ls_handler.py +++ b/src/solidlsp/ls_handler.py @@ -370,7 +370,13 @@ class SolidLanguageServerHandler: line = self.process.stderr.readline() if not line: continue - log.error(line.decode(ENCODING, errors="replace")) + line = line.decode(ENCODING, errors="replace") + line_lower = line.lower() + if "error" in line_lower or "exception" in line_lower or line.startswith("E["): + level = logging.ERROR + else: + level = logging.INFO + log.log(level, line) except Exception as e: log.error("Error while reading stderr from language server process: %s", e, exc_info=e) if not self._is_shutting_down: diff --git a/src/solidlsp/util/zip.py b/src/solidlsp/util/zip.py new file mode 100644 index 00000000..d93d9421 --- /dev/null +++ b/src/solidlsp/util/zip.py @@ -0,0 +1,128 @@ +import fnmatch +import logging +import os +import sys +import zipfile +from pathlib import Path +from typing import Optional + +log = logging.getLogger(__name__) + + +class SafeZipExtractor: + """ + A utility class for extracting ZIP archives safely. + + Features: + - Handles long file paths on Windows + - Skips files that fail to extract, continuing with the rest + - Creates necessary directories automatically + - Optional include/exclude pattern filters + """ + + def __init__( + self, + archive_path: Path, + extract_dir: Path, + verbose: bool = True, + include_patterns: Optional[list[str]] = None, + exclude_patterns: Optional[list[str]] = None, + ) -> None: + """ + Initialize the SafeZipExtractor. + + :param archive_path: Path to the ZIP archive file + :param extract_dir: Directory where files will be extracted + :param verbose: Whether to log status messages + :param include_patterns: List of glob patterns for files to extract (None = all files) + :param exclude_patterns: List of glob patterns for files to skip + """ + self.archive_path = Path(archive_path) + self.extract_dir = Path(extract_dir) + self.verbose = verbose + self.include_patterns = include_patterns or [] + self.exclude_patterns = exclude_patterns or [] + + def extract_all(self) -> None: + """ + Extract all files from the archive, skipping any that fail. + """ + if not self.archive_path.exists(): + raise FileNotFoundError(f"Archive not found: {self.archive_path}") + + if self.verbose: + log.info(f"Extracting from: {self.archive_path} to {self.extract_dir}") + + with zipfile.ZipFile(self.archive_path, "r") as zip_ref: + for member in zip_ref.infolist(): + if self._should_extract(member.filename): + self._extract_member(zip_ref, member) + elif self.verbose: + log.info(f"Skipped: {member.filename}") + + def _should_extract(self, filename: str) -> bool: + """ + Determine whether a file should be extracted based on include/exclude patterns. + + :param filename: The file name from the archive + :return: True if the file should be extracted + """ + # If include_patterns is set, only extract if it matches at least one pattern + if self.include_patterns: + if not any(fnmatch.fnmatch(filename, pattern) for pattern in self.include_patterns): + return False + + # If exclude_patterns is set, skip if it matches any pattern + if self.exclude_patterns: + if any(fnmatch.fnmatch(filename, pattern) for pattern in self.exclude_patterns): + return False + + return True + + def _extract_member(self, zip_ref: zipfile.ZipFile, member: zipfile.ZipInfo) -> None: + """ + Extract a single member from the archive with error handling. + + :param zip_ref: Open ZipFile object + :param member: ZipInfo object representing the file + """ + try: + target_path = self.extract_dir / member.filename + + # Ensure directory structure exists + target_path.parent.mkdir(parents=True, exist_ok=True) + + # Handle long paths on Windows + final_path = self._normalize_path(target_path) + + # Extract file + with zip_ref.open(member) as source, open(final_path, "wb") as target: + target.write(source.read()) + + if self.verbose: + log.info(f"Extracted: {member.filename}") + + except Exception as e: + log.error(f"Failed to extract {member.filename}: {e}") + + @staticmethod + def _normalize_path(path: Path) -> Path: + """ + Adjust path to handle long paths on Windows. + + :param path: Original path + :return: Normalized path + """ + if sys.platform.startswith("win"): + return Path(rf"\\?\{os.path.abspath(path)}") + return path + + +# Example usage: +# extractor = SafeZipExtractor( +# archive_path=Path("file.nupkg"), +# extract_dir=Path("extract_dir"), +# include_patterns=["*.dll", "*.xml"], +# exclude_patterns=["*.pdb"] +# ) +# extractor.extract_all() diff --git a/test/resources/repos/bash/test_repo/config.sh b/test/resources/repos/bash/test_repo/config.sh new file mode 100644 index 00000000..440867e5 --- /dev/null +++ b/test/resources/repos/bash/test_repo/config.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# Configuration script for project setup + +# Environment variables +export PROJECT_NAME="bash-test-project" +export PROJECT_VERSION="1.0.0" +export LOG_LEVEL="INFO" +export CONFIG_DIR="./config" + +# Default settings +DEFAULT_TIMEOUT=30 +DEFAULT_RETRIES=3 +DEFAULT_PORT=8080 + +# Configuration arrays +declare -A ENVIRONMENTS=( + ["dev"]="development" + ["prod"]="production" + ["test"]="testing" +) + +declare -A DATABASE_CONFIGS=( + ["host"]="localhost" + ["port"]="5432" + ["name"]="myapp_db" + ["user"]="dbuser" +) + +# Function to load configuration +load_config() { + local env="${1:-dev}" + local config_file="${CONFIG_DIR}/${env}.conf" + + if [[ -f "$config_file" ]]; then + echo "Loading configuration from $config_file" + source "$config_file" + else + echo "Warning: Configuration file $config_file not found, using defaults" + fi +} + +# Function to validate configuration +validate_config() { + local errors=0 + + if [[ -z "$PROJECT_NAME" ]]; then + echo "Error: PROJECT_NAME is not set" >&2 + ((errors++)) + fi + + if [[ -z "$PROJECT_VERSION" ]]; then + echo "Error: PROJECT_VERSION is not set" >&2 + ((errors++)) + fi + + if [[ $DEFAULT_PORT -lt 1024 || $DEFAULT_PORT -gt 65535 ]]; then + echo "Error: Invalid port number $DEFAULT_PORT" >&2 + ((errors++)) + fi + + return $errors +} + +# Function to print configuration +print_config() { + echo "=== Current Configuration ===" + echo "Project Name: $PROJECT_NAME" + echo "Version: $PROJECT_VERSION" + echo "Log Level: $LOG_LEVEL" + echo "Default Port: $DEFAULT_PORT" + echo "Default Timeout: $DEFAULT_TIMEOUT" + echo "Default Retries: $DEFAULT_RETRIES" + + echo "\n=== Environments ===" + for env in "${!ENVIRONMENTS[@]}"; do + echo " $env: ${ENVIRONMENTS[$env]}" + done + + echo "\n=== Database Configuration ===" + for key in "${!DATABASE_CONFIGS[@]}"; do + echo " $key: ${DATABASE_CONFIGS[$key]}" + done +} + +# Initialize configuration if this script is run directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + load_config "$1" + validate_config + print_config +fi diff --git a/test/resources/repos/bash/test_repo/main.sh b/test/resources/repos/bash/test_repo/main.sh new file mode 100644 index 00000000..56d2e72e --- /dev/null +++ b/test/resources/repos/bash/test_repo/main.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Main script demonstrating various bash features + +# Global variables +readonly SCRIPT_NAME="Main Script" +COUNTER=0 +declare -a ITEMS=("item1" "item2" "item3") + +# Function definitions +function greet_user() { + local username="$1" + local greeting_type="${2:-default}" + + case "$greeting_type" in + "formal") + echo "Good day, ${username}!" + ;; + "casual") + echo "Hey ${username}!" + ;; + *) + echo "Hello, ${username}!" + ;; + esac +} + +function process_items() { + local -n items_ref=$1 + local operation="$2" + + for item in "${items_ref[@]}"; do + case "$operation" in + "count") + ((COUNTER++)) + echo "Processing item $COUNTER: $item" + ;; + "uppercase") + echo "${item^^}" + ;; + *) + echo "Unknown operation: $operation" + return 1 + ;; + esac + done +} + +# Main execution +main() { + echo "Starting $SCRIPT_NAME" + + if [[ $# -eq 0 ]]; then + echo "Usage: $0 [greeting_type]" + exit 1 + fi + + local user="$1" + local greeting="${2:-default}" + + greet_user "$user" "$greeting" + + echo "Processing items..." + process_items ITEMS "count" + + echo "Script completed successfully" +} + +# Run main function with all arguments +main "$@" diff --git a/test/resources/repos/bash/test_repo/utils.sh b/test/resources/repos/bash/test_repo/utils.sh new file mode 100644 index 00000000..77fc8fed --- /dev/null +++ b/test/resources/repos/bash/test_repo/utils.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Utility functions for bash scripting + +# String manipulation functions +function to_uppercase() { + echo "${1^^}" +} + +function to_lowercase() { + echo "${1,,}" +} + +function trim_whitespace() { + local var="$1" + var="${var#"${var%%[![:space:]]*}"}" + var="${var%"${var##*[![:space:]]}"}" + echo "$var" +} + +# File operations +function backup_file() { + local file="$1" + local backup_dir="${2:-./backups}" + + if [[ ! -f "$file" ]]; then + echo "Error: File '$file' does not exist" >&2 + return 1 + fi + + mkdir -p "$backup_dir" + cp "$file" "${backup_dir}/$(basename "$file").$(date +%Y%m%d_%H%M%S).bak" + echo "Backup created for $file" +} + +# Array operations +function contains_element() { + local element="$1" + shift + local array=("$@") + + for item in "${array[@]}"; do + if [[ "$item" == "$element" ]]; then + return 0 + fi + done + return 1 +} + +# Logging functions +function log_message() { + local level="$1" + local message="$2" + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + + case "$level" in + "ERROR") + echo "[$timestamp] ERROR: $message" >&2 + ;; + "WARN") + echo "[$timestamp] WARN: $message" >&2 + ;; + "INFO") + echo "[$timestamp] INFO: $message" + ;; + "DEBUG") + [[ "${DEBUG:-false}" == "true" ]] && echo "[$timestamp] DEBUG: $message" + ;; + *) + echo "[$timestamp] $message" + ;; + esac +} + +# Validation functions +function is_valid_email() { + local email="$1" + [[ "$email" =~ ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ ]] +} + +function is_number() { + [[ $1 =~ ^[0-9]+$ ]] +} diff --git a/test/resources/repos/ruby/test_repo/.solargraph.yml b/test/resources/repos/ruby/test_repo/.solargraph.yml new file mode 100644 index 00000000..b4678eec --- /dev/null +++ b/test/resources/repos/ruby/test_repo/.solargraph.yml @@ -0,0 +1,4 @@ +--- +include: + - "main.rb" + - "lib.rb" diff --git a/test/resources/repos/ruby/test_repo/lib.rb b/test/resources/repos/ruby/test_repo/lib.rb new file mode 100644 index 00000000..5dd8504b --- /dev/null +++ b/test/resources/repos/ruby/test_repo/lib.rb @@ -0,0 +1,9 @@ +class Calculator + def add(a, b) + a + b + end + + def subtract(a, b) + a - b + end +end diff --git a/test/resources/repos/ruby/test_repo/main.rb b/test/resources/repos/ruby/test_repo/main.rb new file mode 100644 index 00000000..dcf1b0f2 --- /dev/null +++ b/test/resources/repos/ruby/test_repo/main.rb @@ -0,0 +1,21 @@ +require './lib.rb' + +class DemoClass + attr_accessor :value + + def initialize(value) + @value = value + end + + def print_value + puts @value + end +end + +def helper_function(number = 42) + demo = DemoClass.new(number) + Calculator.new.add(demo.value, 10) + demo.print_value +end + +helper_function diff --git a/test/resources/repos/rust/test_repo_2024/Cargo.lock b/test/resources/repos/rust/test_repo_2024/Cargo.lock new file mode 100644 index 00000000..88beb720 --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "rsandbox_2024" +version = "0.1.0" diff --git a/test/resources/repos/rust/test_repo_2024/Cargo.toml b/test/resources/repos/rust/test_repo_2024/Cargo.toml new file mode 100644 index 00000000..2f20bf2e --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "rsandbox_2024" +version = "0.1.0" +edition = "2024" + +[dependencies] \ No newline at end of file diff --git a/test/resources/repos/rust/test_repo_2024/src/lib.rs b/test/resources/repos/rust/test_repo_2024/src/lib.rs new file mode 100644 index 00000000..28fac4c7 --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/src/lib.rs @@ -0,0 +1,21 @@ +pub fn multiply(a: i32, b: i32) -> i32 { + a * b +} + +pub struct Calculator { + pub result: i32, +} + +impl Calculator { + pub fn new() -> Self { + Calculator { result: 0 } + } + + pub fn add(&mut self, value: i32) { + self.result += value; + } + + pub fn get_result(&self) -> i32 { + self.result + } +} \ No newline at end of file diff --git a/test/resources/repos/rust/test_repo_2024/src/main.rs b/test/resources/repos/rust/test_repo_2024/src/main.rs new file mode 100644 index 00000000..ec20c47a --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/src/main.rs @@ -0,0 +1,19 @@ +fn main() { + println!("Hello, Rust 2024 edition!"); + let result = add(2, 3); + println!("2 + 3 = {}", result); +} + +pub fn add(a: i32, b: i32) -> i32 { + a + b +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_add() { + assert_eq!(add(2, 3), 5); + } +} \ No newline at end of file diff --git a/test/serena/test_mcp.py b/test/serena/test_mcp.py index 37b6eca8..731a1cc6 100644 --- a/test/serena/test_mcp.py +++ b/test/serena/test_mcp.py @@ -4,16 +4,28 @@ import pytest from mcp.server.fastmcp.tools.base import Tool as MCPTool from serena.agent import Tool, ToolRegistry +from serena.config.context_mode import SerenaAgentContext from serena.mcp import SerenaMCPFactory make_tool = SerenaMCPFactory.make_mcp_tool +# Create a mock agent for tool initialization +class MockAgent: + def __init__(self): + self.project_config = None + self.serena_config = None + + @staticmethod + def get_context() -> SerenaAgentContext: + return SerenaAgentContext.load_default() + + class BaseMockTool(Tool): """A mock Tool class for testing.""" - def __init__(self) -> None: - pass + def __init__(self): + super().__init__(MockAgent()) class BasicTool(BaseMockTool): @@ -274,13 +286,6 @@ def is_test_mock_class(tool_class: type) -> bool: @pytest.mark.parametrize("tool_class", ToolRegistry().get_all_tool_classes()) def test_make_tool_all_tools(tool_class) -> None: """Test that make_tool works for all tools in the codebase.""" - - # Create a mock agent for tool initialization - class MockAgent: - def __init__(self): - self.project_config = None - self.serena_config = None - # Create an instance of the tool tool_instance = tool_class(MockAgent()) diff --git a/test/solidlsp/bash/__init__.py b/test/solidlsp/bash/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/solidlsp/bash/test_bash_basic.py b/test/solidlsp/bash/test_bash_basic.py new file mode 100644 index 00000000..8db362e9 --- /dev/null +++ b/test/solidlsp/bash/test_bash_basic.py @@ -0,0 +1,151 @@ +""" +Basic integration tests for the bash language server functionality. + +These tests validate the functionality of the language server APIs +like request_document_symbols using the bash test repository. +""" + +import pytest + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language + + +@pytest.mark.bash +class TestBashLanguageServerBasics: + """Test basic functionality of the bash language server.""" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_language_server_initialization(self, language_server: SolidLanguageServer) -> None: + """Test that bash language server can be initialized successfully.""" + assert language_server is not None + assert language_server.language == Language.BASH + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_request_document_symbols(self, language_server: SolidLanguageServer) -> None: + """Test request_document_symbols for bash files.""" + # Test getting symbols from main.sh + all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=False) + + # Extract function symbols (LSP Symbol Kind 12) + function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] + function_names = [symbol["name"] for symbol in function_symbols] + + # Should detect all 3 functions from main.sh + assert "greet_user" in function_names, "Should find greet_user function" + assert "process_items" in function_names, "Should find process_items function" + assert "main" in function_names, "Should find main function" + assert len(function_symbols) >= 3, f"Should find at least 3 functions, found {len(function_symbols)}" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: + """Test request_document_symbols with body extraction.""" + # Test with include_body=True + all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=True) + + function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] + + # Find greet_user function and check it has body + greet_user_symbol = next((sym for sym in function_symbols if sym["name"] == "greet_user"), None) + assert greet_user_symbol is not None, "Should find greet_user function" + + if "body" in greet_user_symbol: + body = greet_user_symbol["body"] + assert "function greet_user()" in body, "Function body should contain function definition" + assert "case" in body.lower(), "Function body should contain case statement" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_utils_functions(self, language_server: SolidLanguageServer) -> None: + """Test function detection in utils.sh file.""" + # Test with utils.sh as well + utils_all_symbols, utils_root_symbols = language_server.request_document_symbols("utils.sh", include_body=False) + + utils_function_symbols = [symbol for symbol in utils_all_symbols if symbol.get("kind") == 12] + utils_function_names = [symbol["name"] for symbol in utils_function_symbols] + + # Should detect functions from utils.sh + expected_utils_functions = [ + "to_uppercase", + "to_lowercase", + "trim_whitespace", + "backup_file", + "contains_element", + "log_message", + "is_valid_email", + "is_number", + ] + + for func_name in expected_utils_functions: + assert func_name in utils_function_names, f"Should find {func_name} function in utils.sh" + + assert len(utils_function_symbols) >= 8, f"Should find at least 8 functions in utils.sh, found {len(utils_function_symbols)}" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_regex_fallback_detection(self, language_server: SolidLanguageServer) -> None: + """Test the regex-based fallback function detection directly.""" + # Check if the method exists (it should if we have our enhanced bash server) + if not hasattr(language_server, "_detect_bash_functions"): + pytest.fail("Enhanced bash function detection not available - this may be due to test environment setup") + + # Test the _detect_bash_functions method directly + detected_functions = language_server._detect_bash_functions("main.sh", include_body=True) + + # Should detect functions even if LSP doesn't provide them + assert len(detected_functions) == 3, f"Should detect exactly 3 functions, found {len(detected_functions)}" + + function_names = [func["name"] for func in detected_functions] + assert "greet_user" in function_names, "Regex should detect greet_user function" + assert "process_items" in function_names, "Regex should detect process_items function" + assert "main" in function_names, "Regex should detect main function" + + # Check that symbols have proper structure + for func in detected_functions: + assert func["kind"] == 12, f"Function {func['name']} should have LSP Symbol Kind 12" + assert "location" in func, f"Function {func['name']} should have location info" + assert "range" in func, f"Function {func['name']} should have range info" + assert "selectionRange" in func, f"Function {func['name']} should have selectionRange info" + + # Check location structure + location = func["location"] + assert "uri" in location, f"Function {func['name']} location should have URI" + assert "range" in location, f"Function {func['name']} location should have range" + assert "relativePath" in location, f"Function {func['name']} location should have relative path" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_function_syntax_patterns(self, language_server: SolidLanguageServer) -> None: + """Test detection of different bash function syntax patterns.""" + # Check if the method exists (it should if we have our enhanced bash server) + if not hasattr(language_server, "_detect_bash_functions"): + pytest.fail("Enhanced bash function detection not available - this may be due to test environment setup") + + # Test main.sh (has 'function' keyword functions and regular function) + main_functions = language_server._detect_bash_functions("main.sh", include_body=False) + + # Test utils.sh (all use 'function' keyword) + utils_functions = language_server._detect_bash_functions("utils.sh", include_body=False) + + # Verify we detect both syntax patterns + main_function_names = [func["name"] for func in main_functions] + utils_function_names = [func["name"] for func in utils_functions] + + # main() uses regular syntax: main() { + assert "main" in main_function_names, "Should detect regular function syntax" + + # Functions with 'function' keyword: function name() { + assert "greet_user" in main_function_names, "Should detect function keyword syntax" + assert "to_uppercase" in utils_function_names, "Should detect function keyword syntax in utils" + + # Verify all expected utils functions are detected + expected_utils = [ + "to_uppercase", + "to_lowercase", + "trim_whitespace", + "backup_file", + "contains_element", + "log_message", + "is_valid_email", + "is_number", + ] + + for expected_func in expected_utils: + assert expected_func in utils_function_names, f"Should detect {expected_func} function" diff --git a/test/solidlsp/ruby/test_ruby_basic.py b/test/solidlsp/ruby/test_ruby_basic.py new file mode 100644 index 00000000..2edf95f2 --- /dev/null +++ b/test/solidlsp/ruby/test_ruby_basic.py @@ -0,0 +1,44 @@ +import os +from pathlib import Path + +import pytest + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language +from solidlsp.ls_utils import SymbolUtils + + +@pytest.mark.ruby +class TestRubyLanguageServer: + @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) + def test_find_symbol(self, language_server: SolidLanguageServer) -> None: + symbols = language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "DemoClass"), "DemoClass not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "helper_function"), "helper_function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "print_value"), "print_value not found in symbol tree" + + @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) + def test_find_referencing_symbols(self, language_server: SolidLanguageServer) -> None: + file_path = os.path.join("main.rb") + symbols = language_server.request_document_symbols(file_path) + helper_symbol = None + for sym in symbols[0]: + if sym.get("name") == "helper_function": + helper_symbol = sym + break + print(helper_symbol) + assert helper_symbol is not None, "Could not find 'helper_function' symbol in main.rb" + + @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) + @pytest.mark.parametrize("repo_path", [Language.RUBY], indirect=True) + def test_find_definition_across_files(self, language_server: SolidLanguageServer, repo_path: Path) -> None: + # Test finding Calculator.add method definition from line 17: Calculator.new.add(demo.value, 10) + definition_location_list = language_server.request_definition( + str(repo_path / "main.rb"), 16, 17 + ) # add method at line 17 (0-indexed 16), position 17 + + assert len(definition_location_list) == 1 + definition_location = definition_location_list[0] + print(f"Found definition: {definition_location}") + assert definition_location["uri"].endswith("lib.rb") + assert definition_location["range"]["start"]["line"] == 1 # add method on line 2 (0-indexed 1) diff --git a/test/solidlsp/rust/test_rust_2024_edition.py b/test/solidlsp/rust/test_rust_2024_edition.py new file mode 100644 index 00000000..e73c8d6d --- /dev/null +++ b/test/solidlsp/rust/test_rust_2024_edition.py @@ -0,0 +1,112 @@ +import os +from pathlib import Path + +import pytest + +from solidlsp.ls_config import Language +from solidlsp.ls_utils import SymbolUtils +from test.conftest import create_ls + + +@pytest.mark.rust +class TestRust2024EditionLanguageServer: + @classmethod + def setup_class(cls): + """Set up the test class with the Rust 2024 edition test repository.""" + cls.test_repo_2024_path = Path(__file__).parent.parent.parent / "resources" / "repos" / "rust" / "test_repo_2024" + + if not cls.test_repo_2024_path.exists(): + pytest.skip("Rust 2024 edition test repository not found") + + # Create and start the language server for the 2024 edition repo + cls.language_server = create_ls(Language.RUST, str(cls.test_repo_2024_path)) + cls.language_server.start() + + @classmethod + def teardown_class(cls): + """Clean up the language server.""" + if hasattr(cls, "language_server"): + cls.language_server.stop() + + def test_find_references_raw(self) -> None: + # Test finding references to the 'add' function defined in main.rs + file_path = os.path.join("src", "main.rs") + symbols = self.language_server.request_document_symbols(file_path) + add_symbol = None + for sym in symbols[0]: + if sym.get("name") == "add": + add_symbol = sym + break + assert add_symbol is not None, "Could not find 'add' function symbol in main.rs" + sel_start = add_symbol["selectionRange"]["start"] + refs = self.language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + # The add function should be referenced within main.rs itself (in the main function) + assert any("main.rs" in ref.get("relativePath", "") for ref in refs), "main.rs should reference add function" + + def test_find_symbol(self) -> None: + symbols = self.language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "main"), "main function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "add"), "add function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "multiply"), "multiply function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Calculator"), "Calculator struct not found in symbol tree" + + def test_find_referencing_symbols_multiply(self) -> None: + # Find references to 'multiply' function defined in lib.rs + file_path = os.path.join("src", "lib.rs") + symbols = self.language_server.request_document_symbols(file_path) + multiply_symbol = None + for sym in symbols[0]: + if sym.get("name") == "multiply": + multiply_symbol = sym + break + assert multiply_symbol is not None, "Could not find 'multiply' function symbol in lib.rs" + sel_start = multiply_symbol["selectionRange"]["start"] + refs = self.language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + # The multiply function exists but may not be referenced anywhere, which is fine + # This test just verifies we can find the symbol and request references without error + assert isinstance(refs, list), "Should return a list of references (even if empty)" + + def test_find_calculator_struct_and_impl(self) -> None: + # Test finding the Calculator struct and its impl block + file_path = os.path.join("src", "lib.rs") + symbols = self.language_server.request_document_symbols(file_path) + + # Find the Calculator struct + calculator_struct = None + calculator_impl = None + for sym in symbols[0]: + if sym.get("name") == "Calculator" and sym.get("kind") == 23: # Struct kind + calculator_struct = sym + elif sym.get("name") == "Calculator" and sym.get("kind") == 11: # Interface/Impl kind + calculator_impl = sym + + assert calculator_struct is not None, "Could not find 'Calculator' struct symbol in lib.rs" + + # The struct should have the 'result' field + struct_children = calculator_struct.get("children", []) + field_names = [child.get("name") for child in struct_children] + assert "result" in field_names, "Calculator struct should have 'result' field" + + # Find the impl block and check its methods + if calculator_impl is not None: + impl_children = calculator_impl.get("children", []) + method_names = [child.get("name") for child in impl_children] + assert "new" in method_names, "Calculator impl should have 'new' method" + assert "add" in method_names, "Calculator impl should have 'add' method" + assert "get_result" in method_names, "Calculator impl should have 'get_result' method" + + def test_overview_methods(self) -> None: + symbols = self.language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "main"), "main missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "add"), "add missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "multiply"), "multiply missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Calculator"), "Calculator missing from overview" + + def test_rust_2024_edition_specific(self) -> None: + # Verify we're actually working with the 2024 edition repository + cargo_toml_path = self.test_repo_2024_path / "Cargo.toml" + assert cargo_toml_path.exists(), "Cargo.toml should exist in test repository" + + with open(cargo_toml_path) as f: + content = f.read() + assert 'edition = "2024"' in content, "Should be using Rust 2024 edition" diff --git a/test/solidlsp/util/test_zip.py b/test/solidlsp/util/test_zip.py new file mode 100644 index 00000000..5fa545ba --- /dev/null +++ b/test/solidlsp/util/test_zip.py @@ -0,0 +1,99 @@ +import sys +import zipfile +from pathlib import Path + +import pytest + +from solidlsp.util.zip import SafeZipExtractor + + +@pytest.fixture +def temp_zip_file(tmp_path: Path) -> Path: + """Create a temporary ZIP file for testing.""" + zip_path = tmp_path / "test.zip" + with zipfile.ZipFile(zip_path, "w") as zipf: + zipf.writestr("file1.txt", "Hello World 1") + zipf.writestr("file2.txt", "Hello World 2") + zipf.writestr("folder/file3.txt", "Hello World 3") + return zip_path + + +def test_extract_all_success(temp_zip_file: Path, tmp_path: Path) -> None: + """All files should extract without error.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False) + extractor.extract_all() + + assert (dest_dir / "file1.txt").read_text() == "Hello World 1" + assert (dest_dir / "file2.txt").read_text() == "Hello World 2" + assert (dest_dir / "folder" / "file3.txt").read_text() == "Hello World 3" + + +def test_include_patterns(temp_zip_file: Path, tmp_path: Path) -> None: + """Only files matching include_patterns should be extracted.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False, include_patterns=["*.txt"]) + extractor.extract_all() + + assert (dest_dir / "file1.txt").exists() + assert (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + + +def test_exclude_patterns(temp_zip_file: Path, tmp_path: Path) -> None: + """Files matching exclude_patterns should be skipped.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False, exclude_patterns=["file2.txt"]) + extractor.extract_all() + + assert (dest_dir / "file1.txt").exists() + assert not (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + + +def test_include_and_exclude_patterns(temp_zip_file: Path, tmp_path: Path) -> None: + """Exclude should override include if both match.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor( + temp_zip_file, + dest_dir, + verbose=False, + include_patterns=["*.txt"], + exclude_patterns=["file1.txt"], + ) + extractor.extract_all() + + assert not (dest_dir / "file1.txt").exists() + assert (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + + +def test_skip_on_error(monkeypatch, temp_zip_file: Path, tmp_path: Path) -> None: + """Should skip a file that raises an error and continue extracting others.""" + dest_dir = tmp_path / "extracted" + + original_open = zipfile.ZipFile.open + + def failing_open(self, member, *args, **kwargs): + if member.filename == "file2.txt": + raise OSError("Simulated failure") + return original_open(self, member, *args, **kwargs) + + # Patch the method on the class, not on an instance + monkeypatch.setattr(zipfile.ZipFile, "open", failing_open) + + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False) + extractor.extract_all() + + assert (dest_dir / "file1.txt").exists() + assert not (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + + +@pytest.mark.skipif(not sys.platform.startswith("win"), reason="Windows-only test") +def test_long_path_normalization(temp_zip_file: Path, tmp_path: Path) -> None: + r"""Ensure _normalize_path adds \\?\\ prefix on Windows.""" + dest_dir = tmp_path / ("a" * 250) # Simulate long path + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False) + norm_path = extractor._normalize_path(dest_dir / "file.txt") + assert str(norm_path).startswith("\\\\?\\")