From 9b9eab68bd169077eb4934ba0fabed36bc38fa1f Mon Sep 17 00:00:00 2001 From: architectureman <111078057+architectureman@users.noreply.github.com> Date: Sun, 27 Jul 2025 23:05:20 +0700 Subject: [PATCH 01/27] Update rust_analyzer.py for macOS x64 intel/amd --- src/solidlsp/language_servers/rust_analyzer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/solidlsp/language_servers/rust_analyzer.py b/src/solidlsp/language_servers/rust_analyzer.py index 0f2f2cf1..5a181fc3 100644 --- a/src/solidlsp/language_servers/rust_analyzer.py +++ b/src/solidlsp/language_servers/rust_analyzer.py @@ -52,6 +52,14 @@ class RustAnalyzer(SolidLanguageServer): """ deps = RuntimeDependencyCollection( [ + RuntimeDependency( + id="RustAnalyzer", + description="RustAnalyzer for macOS (x64)", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-x86_64-apple-darwin.gz", + platform_id="osx-x64", + archive_type="gz", + binary_name="rust_analyzer", + ), RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for macOS (arm64)", From c44a6be58f4a4447a9d057b3ca98331edb2297c5 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sat, 2 Aug 2025 17:38:20 +0200 Subject: [PATCH 02/27] Docs on connecting Serena to chatgpt --- docs/serena_on_chatgpt.md | 101 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 docs/serena_on_chatgpt.md diff --git a/docs/serena_on_chatgpt.md b/docs/serena_on_chatgpt.md new file mode 100644 index 00000000..64f7e248 --- /dev/null +++ b/docs/serena_on_chatgpt.md @@ -0,0 +1,101 @@ + +# Connecting Serena MCP Server to ChatGPT via MCPO & Cloudflare Tunnel + +This guide explains how to expose a **locally running Serena MCP server** (powered by MCPO) to the internet using **Cloudflare Tunnel**, and how to connect it to **ChatGPT as a Custom GPT with tool access**. + +Once configured, ChatGPT becomes a powerful **coding agent** with direct access to your codebase, shell, and file system — so **read the security notes carefully**. + +--- +## Prerequisites + +Make sure you have [uv](https://docs.astral.sh/uv/getting-started/installation/) +and [cloudflared](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/) installed. + +## 1. Start the Serena MCP Server via MCPO + +Run the following command to launch Serena as http server (assuming port 8000): + +```bash +uvx mcpo --port 8000 --api-key -- \ + uvx --from git+https://github.com/oraios/serena \ + serena start-mcp-server --project $(pwd) +``` + +- `--api-key` is required to secure the server. +- `--project` should point to the root of your codebase. + +You can also use other options, and you don't have to pass `--project` if you want to work on multiple projects +or want to activate it later. See + +```shell +uvx --from git+https://github.com/oraios/serena serena start-mcp-server --help +``` + +--- + +## 2. Expose the Server Using Cloudflare Tunnel + +Run: + +```bash +cloudflared tunnel --url http://localhost:8000 +``` + +This will give you a **public HTTPS URL** like: + +``` +https://serena-agent-tunnel.trycloudflare.com +``` + +Your server is now securely exposed to the internet. + +--- + +## 3. Connect It to ChatGPT (Custom GPT) + +### Steps: + +1. Go to [ChatGPT → Explore GPTs → Create](https://chat.openai.com/gpts/editor) +2. During setup, click **“Add APIs”** +3. Set up **API Key authentication** with the auth type as **Bearer** and enter the api key you used to start the MCPO server. +4. In the **Schema** section, click on **import from URL** and paste `/openapi.json` with the URL you got from the previous step. +5. Add the following line to the top of the imported JSON schema: + ``` + "servers": ["url": ""], + ``` + +ChatGPT will read the schema and create functions automatically. + +--- + +## Security Warning — Read Carefully + +Depending on your configuration and enabled tools, Serena's MCP server may: +- Execute **arbitrary shell commands** +- Read, write, and modify **files in your codebase** + +This gives ChatGPT the same powers as a remote developer on your machine. + +### ⚠️ Key Rules: +- **NEVER expose your API key** +- **Only expose this server when needed**, and monitor its use. + +In your project’s `.serena/project.yml` or global config, you can disable tools like: + +```yaml +excluded_tools: + - execute_shell_command + - ... +read_only: true +``` + +This is strongly recommended if you want a read-only or safer agent. + + +--- + +## Final Thoughts + +With this setup, ChatGPT becomes a coding assistant **running on your local code** — able to index, search, edit, and even run shell commands depending on your configuration. + +Use responsibly, and keep security in mind. From e41d09d9372d580e92835ca4980c00a432a9b03d Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sat, 2 Aug 2025 18:55:34 +0200 Subject: [PATCH 03/27] Formatting --- src/solidlsp/language_servers/solargraph.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/solidlsp/language_servers/solargraph.py b/src/solidlsp/language_servers/solargraph.py index 7b79c324..a4f7e694 100644 --- a/src/solidlsp/language_servers/solargraph.py +++ b/src/solidlsp/language_servers/solargraph.py @@ -7,7 +7,6 @@ import json import logging import os import pathlib -import stat import subprocess import threading From 46967fa3cbe5af325bcc9c07722ee9ee7d42aff9 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sat, 2 Aug 2025 18:56:09 +0200 Subject: [PATCH 04/27] Allow overriding tool descriptions in the context, cli command for tool description --- src/serena/agent.py | 10 ++++++++ src/serena/cli.py | 24 ++++++++++++++++++- src/serena/config/context_mode.py | 12 ++++++++-- src/serena/mcp.py | 11 +++++++-- .../resources/config/contexts/agent.yml | 4 +++- .../config/contexts/context.template.yml | 6 ++++- .../resources/config/contexts/desktop-app.yml | 3 ++- .../config/contexts/ide-assistant.yml | 3 +++ 8 files changed, 65 insertions(+), 8 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 66394d62..7e900f34 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -227,6 +227,12 @@ class SerenaAgent: except Exception as e: log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e) + def get_context(self) -> SerenaAgentContext: + return self._context + + def get_tool_description_override(self, tool_name: str) -> str | None: + return self._context.tool_description_overrides.get(tool_name, None) + def _check_shell_settings(self) -> None: # On Windows, Claude Code sets COMSPEC to Git-Bash (often even with a path containing spaces), # which causes all sorts of trouble, preventing language servers from being launched correctly. @@ -583,3 +589,7 @@ class SerenaAgent: if self._gui_log_viewer: log.info("Stopping the GUI log window ...") self._gui_log_viewer.stop() + + def get_tool_by_name(self, tool_name: str) -> Tool: + tool_class = ToolRegistry().get_tool_class_by_name(tool_name) + return self.get_tool(tool_class) diff --git a/src/serena/cli.py b/src/serena/cli.py index 6b46a8e8..457b3acb 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -26,7 +26,7 @@ from serena.constants import ( USER_CONTEXT_YAMLS_DIR, USER_MODE_YAMLS_DIR, ) -from serena.mcp import SerenaMCPFactorySingleProcess +from serena.mcp import SerenaMCPFactory, SerenaMCPFactorySingleProcess from serena.project import Project from serena.tools import ToolRegistry from serena.util.logging import MemoryLogHandler @@ -478,6 +478,28 @@ class ToolCommands(AutoRegisteringGroup): else: ToolRegistry().print_tool_overview() + @staticmethod + @click.command( + "description", + help="Print the description of a tool, optionally with a specific context (the latter may modify the default description).", + ) + @click.argument("tool_name", type=str) + @click.option("--context", type=str, default=None, help="Context name or path to context file.") + def description(tool_name: str, context: str | None = None) -> None: + # Load the context + serena_context = None + if context: + serena_context = SerenaAgentContext.load(context) + + agent = SerenaAgent( + project=None, + serena_config=SerenaConfig(web_dashboard=False, log_level=logging.INFO), + context=serena_context, + ) + tool = agent.get_tool_by_name(tool_name) + mcp_tool = SerenaMCPFactory.make_mcp_tool(tool) + click.echo(mcp_tool.description) + class PromptCommands(AutoRegisteringGroup): def __init__(self) -> None: diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index de84f6a7..30caba0b 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -4,7 +4,7 @@ Context and Mode configuration loader import os from copy import copy -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Self @@ -134,6 +134,8 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): name: str prompt: str description: str = "" + tool_description_overrides: dict[str, str] = field(default_factory=dict) + """Maps tool names to custom descriptions, default descriptions are extracted from the tool docstrings.""" def _tostring_includes(self) -> list[str]: return ["name"] @@ -147,6 +149,9 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): def from_json_dict(cls, data: dict) -> Self: data = copy(data) data["excluded_tools"] = set(data["excluded_tools"]) + # Ensure backwards compatibility for tool_description_overrides + if "tool_description_overrides" not in data: + data["tool_description_overrides"] = {} return cls(**data) @classmethod @@ -155,7 +160,10 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): with open(yaml_path, encoding="utf-8") as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) - return cls(name=name, **data) + prompt = data["prompt"] + description = data.get("description", "") + tool_description_overrides = data.get("tool_description_overrides", {}) + return cls(name=name, prompt=prompt, description=description, tool_description_overrides=tool_description_overrides) @classmethod def get_path(cls, name: str) -> str: diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 9926af8e..6c75556f 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -68,10 +68,16 @@ class SerenaMCPFactory: # Mount the tool description as a combination of the docstring description and # the return value description, if it exists. - if docstring.description: - func_doc = f"{docstring.description.strip().strip('.')}." + agent = cast(SerenaAgent, tool.agent) + overridden_description = agent.get_context().tool_description_overrides.get(func_name, None) + + if overridden_description is not None: + func_doc = overridden_description + elif docstring.description: + func_doc = docstring.description else: func_doc = "" + func_doc = func_doc.strip().strip(".") if docstring.returns and (docstring_returns_descr := docstring.returns.description): # Only add a space before "Returns" if func_doc is not empty prefix = " " if func_doc else "" @@ -113,6 +119,7 @@ class SerenaMCPFactory: for tool in self._iter_tools(): mcp_tool = self.make_mcp_tool(tool) mcp._tool_manager._tools[tool.get_name()] = mcp_tool + log.info(f"Starting MCP server with {len(mcp._tool_manager._tools)} tools: {list(mcp._tool_manager._tools.keys())}") @abstractmethod def _instantiate_agent(self, serena_config: SerenaConfig, modes: list[SerenaAgentMode]) -> None: diff --git a/src/serena/resources/config/contexts/agent.yml b/src/serena/resources/config/contexts/agent.yml index d3da9da9..e2984283 100644 --- a/src/serena/resources/config/contexts/agent.yml +++ b/src/serena/resources/config/contexts/agent.yml @@ -3,4 +3,6 @@ prompt: | You are running in agent context where the system prompt is provided externally. You should use symbolic tools when possible for code understanding and modification. excluded_tools: - - initial_instructions \ No newline at end of file + - initial_instructions + +tool_description_overrides: {} \ No newline at end of file diff --git a/src/serena/resources/config/contexts/context.template.yml b/src/serena/resources/config/contexts/context.template.yml index 52e94535..f9dee588 100644 --- a/src/serena/resources/config/contexts/context.template.yml +++ b/src/serena/resources/config/contexts/context.template.yml @@ -1,4 +1,8 @@ # See Serena's documentation for more details on concept of contexts. description: Description of the context, not used in the code. prompt: Prompt that will form part of the system prompt/initial instructions for agents started in this context. -excluded_tools: [] \ No newline at end of file +excluded_tools: [] + +# mapping of tool names to an override of their descriptions (the default description is the docstring of the Tool's apply method). +# Sometimes, tool descriptions are too long (e.g., for ChatGPT), or users may want to override them for another reason. +tool_description_overrides: {} \ No newline at end of file diff --git a/src/serena/resources/config/contexts/desktop-app.yml b/src/serena/resources/config/contexts/desktop-app.yml index ea1567fb..8105ca8d 100644 --- a/src/serena/resources/config/contexts/desktop-app.yml +++ b/src/serena/resources/config/contexts/desktop-app.yml @@ -10,4 +10,5 @@ prompt: | If complex relationships that the user asked for should be visualized or explained, consider creating a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions. -excluded_tools: [] \ No newline at end of file +excluded_tools: [] +tool_description_overrides: {} diff --git a/src/serena/resources/config/contexts/ide-assistant.yml b/src/serena/resources/config/contexts/ide-assistant.yml index 2a6c9e2c..14d65130 100644 --- a/src/serena/resources/config/contexts/ide-assistant.yml +++ b/src/serena/resources/config/contexts/ide-assistant.yml @@ -26,3 +26,6 @@ excluded_tools: - prepare_for_new_conversation - summarize_changes - get_current_config + +tool_description_overrides: {} + From e9e05c2f3fca4a72b10900359c2644f138b412bc Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sat, 2 Aug 2025 20:32:34 +0200 Subject: [PATCH 05/27] Allow system prompt to depend on available tools and tool categories (as indicated by ToolMarkers) * Introduce additional tool markers that represent relevant categories * Apply some conditions to current system prompt --- src/serena/agent.py | 18 ++++++++++-- .../generated/generated_prompt_factory.py | 4 ++- .../config/prompt_templates/system_prompt.yml | 29 ++++++++++++------- src/serena/tools/jetbrains_tools.py | 8 ++--- src/serena/tools/symbol_tools.py | 20 ++++++++----- src/serena/tools/tools_base.py | 24 +++++++++++++-- src/solidlsp/language_servers/solargraph.py | 1 - 7 files changed, 76 insertions(+), 28 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 66394d62..5f27a1d1 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -25,7 +25,8 @@ from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, T from serena.dashboard import SerenaDashboardAPI from serena.project import Project from serena.prompt_factory import SerenaPromptFactory -from serena.tools import ActivateProjectTool, Tool, ToolRegistry +from serena.tools import ActivateProjectTool, Tool, ToolMarker, ToolRegistry +from serena.util.inspection import iter_subclasses from serena.util.logging import MemoryLogHandler from solidlsp import SolidLanguageServer @@ -336,10 +337,23 @@ class SerenaAgent: return list(self._modes) def create_system_prompt(self) -> str: - return self.prompt_factory.create_system_prompt( + available_tools = [tool.get_name_from_cls() for tool in self.get_exposed_tool_instances()] + + # determine available tool markers (i.e. tool categories) + available_markers = set() + for marker_class in iter_subclasses(ToolMarker): + for tool in self.get_exposed_tool_instances(): + if isinstance(tool, marker_class): + available_markers.add(marker_class.__name__) + + log.info("Generating system prompt with available_tools=(see exposed tools), available_markers=%s", available_markers) + system_prompt = self.prompt_factory.create_system_prompt( context_system_prompt=self._context.prompt, mode_system_prompts=[mode.prompt for mode in self._modes], + available_tools=available_tools, + available_markers=available_markers, ) + log.info("System prompt:\n%s", system_prompt) def _update_active_tools(self) -> None: """ diff --git a/src/serena/generated/generated_prompt_factory.py b/src/serena/generated/generated_prompt_factory.py index 6bb7a76b..3a5e1442 100644 --- a/src/serena/generated/generated_prompt_factory.py +++ b/src/serena/generated/generated_prompt_factory.py @@ -32,5 +32,7 @@ class PromptFactory(PromptFactoryBase): def create_prepare_for_new_conversation(self) -> str: return self._render_prompt("prepare_for_new_conversation", locals()) - def create_system_prompt(self, *, context_system_prompt: Any, mode_system_prompts: Any) -> str: + def create_system_prompt( + self, *, available_markers: Any, available_tools: Any, context_system_prompt: Any, mode_system_prompts: Any + ) -> str: return self._render_prompt("system_prompt", locals()) diff --git a/src/serena/resources/config/prompt_templates/system_prompt.yml b/src/serena/resources/config/prompt_templates/system_prompt.yml index 8f3dd504..e912f7ca 100644 --- a/src/serena/resources/config/prompt_templates/system_prompt.yml +++ b/src/serena/resources/config/prompt_templates/system_prompt.yml @@ -5,32 +5,38 @@ prompts: system_prompt: | You are a professional coding agent concerned with one particular codebase. You have access to semantic coding tools on which you rely heavily for all your work, as well as collection of memory - files containing general information about the codebase. You operate in a frugal and intelligent manner, always + files containing general information about the codebase. You operate in a resource-efficient and intelligent manner, always keeping in mind to not read or generate content that is not needed for the task at hand. When reading code in order to answer a user question or task, you should try reading only the necessary code. Some tasks may require you to understand the architecture of large parts of the codebase, while for others, it may be enough to read a small set of symbols or a single file. Generally, you should avoid reading entire files unless it is absolutely necessary, instead relying on - intelligent step-by-step acquisition of information. However, if you already read a file, it does not make + intelligent step-by-step acquisition of information. {% if 'ToolMarkerSymbolicRead' in available_markers %}However, if you already read a file, it does not make sense to further analyse it with the symbolic tools (except for the `find_referencing_symbols` tool), - as you already have the information. + as you already have the information.{% endif %} - I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! CONSIDER INSTEAD USING THE OVERVIEW TOOL AND - SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST! + I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! + {% if 'ToolMarkerSymbolicRead' in available_markers %} + CONSIDER INSTEAD USING THE OVERVIEW TOOL AND SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST! I WILL BE EVEN MORE UPSET IF AFTER HAVING READ AN ENTIRE FILE YOU KEEP READING THE SAME CONTENT WITH THE SYMBOLIC TOOLS! THE PURPOSE OF THE SYMBOLIC TOOLS IS TO HAVE TO READ LESS CODE, NOT READ THE SAME CONTENT MULTIPLE TIMES! + {% endif %} You can achieve the intelligent reading of code by using the symbolic tools for getting an overview of symbols and the relations between them, and then only reading the bodies of symbols that are necessary to answer the question - or complete the task. You can also use the standard tools like list_dir, find_file and search_for_pattern if you need to. + or complete the task. + You can use the standard tools like list_dir, find_file and search_for_pattern if you need to. When tools allow it, you pass the `relative_path` parameter to restrict the search to a specific file or directory. For some tools, `relative_path` can only be a file path, so make sure to properly read the tool descriptions. - If you are unsure about a symbol's name or location (to the extent that substring_matching for the symbol name is not enough), you can use the `search_for_pattern` tool, which allows fast - and flexible search for patterns in the codebase. This way you can first find candidates for symbols or files, - and then proceed with the symbolic tools. + {% if 'search_for_pattern' in available_tools %} + If you are unsure about a symbol's name or location{% if 'find_symbol' in available_tools %} (to the extent that substring_matching for the symbol name is not enough){% endif %}, you can use the `search_for_pattern` tool, which allows fast + and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %}This way you can first find candidates for symbols or files, + and then proceed with the symbolic tools.{% endif %} + {% endif %} - Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbols` tool for more details + {% if 'ToolMarkerSymbolicRead' in available_markers %} + Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbol` tool for more details on how the `name_path` matches symbols. You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible @@ -40,10 +46,13 @@ prompts: you can use `find_symbol` with the name path `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding to read the desired methods with `include_body=True` You can understand relationships between symbols by using the `find_referencing_symbols` tool. + {% endif %} + {% if 'read_memory' in available_tools %} You generally have access to memories and it may be useful for you to read them, but also only if they help you to answer the question or complete the task. You can infer which memories are relevant to the current task by reading the memory names and descriptions. + {% endif %} The context and modes of operation are described below. From them you can infer how to interact with your user and which tasks and kinds of interactions are expected of you. diff --git a/src/serena/tools/jetbrains_tools.py b/src/serena/tools/jetbrains_tools.py index 319aa590..8b9f2d1a 100644 --- a/src/serena/tools/jetbrains_tools.py +++ b/src/serena/tools/jetbrains_tools.py @@ -1,10 +1,10 @@ import json -from serena.tools import TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerOptional +from serena.tools import TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerOptional, ToolMarkerSymbolicRead from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient -class JetBrainsFindSymbolTool(Tool, ToolMarkerOptional): +class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). """ @@ -70,7 +70,7 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerOptional): return self._limit_length(result, max_answer_chars) -class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerOptional): +class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Finds symbols that reference the given symbol """ @@ -100,7 +100,7 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerOptional): return self._limit_length(result, max_answer_chars) -class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerOptional): +class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional): """ Retrieves an overview of the top-level symbols within a specified file """ diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 4d2a5ffc..45e04f3e 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -8,7 +8,13 @@ from collections.abc import Sequence from copy import copy from typing import Any -from serena.tools import SUCCESS_RESULT, TOOL_DEFAULT_MAX_ANSWER_LENGTH, Tool, ToolMarkerCanEdit +from serena.tools import ( + SUCCESS_RESULT, + TOOL_DEFAULT_MAX_ANSWER_LENGTH, + Tool, + ToolMarkerSymbolicEdit, + ToolMarkerSymbolicRead, +) from solidlsp.ls_types import SymbolKind @@ -42,7 +48,7 @@ class RestartLanguageServerTool(Tool): return SUCCESS_RESULT -class GetSymbolsOverviewTool(Tool): +class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): """ Gets an overview of the top-level symbols defined in a given file or directory. """ @@ -70,7 +76,7 @@ class GetSymbolsOverviewTool(Tool): return self._limit_length(result_json_str, max_answer_chars) -class FindSymbolTool(Tool): +class FindSymbolTool(Tool, ToolMarkerSymbolicRead): """ Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). """ @@ -150,7 +156,7 @@ class FindSymbolTool(Tool): return self._limit_length(result, max_answer_chars) -class FindReferencingSymbolsTool(Tool): +class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead): """ Finds symbols that reference the symbol at the given location (optionally filtered by type). """ @@ -204,7 +210,7 @@ class FindReferencingSymbolsTool(Tool): return self._limit_length(result, max_answer_chars) -class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit): +class ReplaceSymbolBodyTool(Tool, ToolMarkerSymbolicEdit): """ Replaces the full definition of a symbol. """ @@ -232,7 +238,7 @@ class ReplaceSymbolBodyTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit): +class InsertAfterSymbolTool(Tool, ToolMarkerSymbolicEdit): """ Inserts content after the end of the definition of a given symbol. """ @@ -257,7 +263,7 @@ class InsertAfterSymbolTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit): +class InsertBeforeSymbolTool(Tool, ToolMarkerSymbolicEdit): """ Inserts content before the beginning of the definition of a given symbol. """ diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index cd025970..1f55e604 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -73,22 +73,40 @@ class Component(ABC): TOOL_DEFAULT_MAX_ANSWER_LENGTH = int(2e5) -class ToolMarkerCanEdit: +class ToolMarker: + """ + Base class for tool markers. + """ + + +class ToolMarkerCanEdit(ToolMarker): """ Marker class for all tools that can perform editing operations on files. """ -class ToolMarkerDoesNotRequireActiveProject: +class ToolMarkerDoesNotRequireActiveProject(ToolMarker): pass -class ToolMarkerOptional: +class ToolMarkerOptional(ToolMarker): """ Marker class for optional tools that are disabled by default. """ +class ToolMarkerSymbolicRead(ToolMarker): + """ + Marker class for tools that perform symbol read operations. + """ + + +class ToolMarkerSymbolicEdit(ToolMarkerCanEdit): + """ + Marker class for tools that perform symbolic edit operations. + """ + + class ApplyMethodProtocol(Protocol): """Callable protocol for the apply method of a tool.""" diff --git a/src/solidlsp/language_servers/solargraph.py b/src/solidlsp/language_servers/solargraph.py index 7b79c324..a4f7e694 100644 --- a/src/solidlsp/language_servers/solargraph.py +++ b/src/solidlsp/language_servers/solargraph.py @@ -7,7 +7,6 @@ import json import logging import os import pathlib -import stat import subprocess import threading From 1bff8fa77422c8566692a4d67050f411ffa7ef67 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sat, 2 Aug 2025 20:42:47 +0200 Subject: [PATCH 06/27] Support use of available_tools and available_markers in context/mode prompts --- src/serena/agent.py | 43 +++++++++++++------ src/serena/config/context_mode.py | 8 ++++ src/serena/resources/config/modes/editing.yml | 8 ++-- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 5f27a1d1..96d769e8 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any, Optional, TypeVar from sensai.util import logging from sensai.util.logging import LogTime +from interprompt.jinja_template import JinjaTemplate from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode @@ -92,6 +93,23 @@ class MemoriesManager: return f"Memory {name} deleted." +class AvailableTools: + def __init__(self, tools: list[Tool]): + """ + :param tools: the list of available tools + """ + self.tools = tools + self.tool_names = [tool.get_name_from_cls() for tool in tools] + self.tool_marker_names = set() + for marker_class in iter_subclasses(ToolMarker): + for tool in tools: + if isinstance(tool, marker_class): + self.tool_marker_names.add(marker_class.__name__) + + def __len__(self) -> int: + return len(self.tools) + + class SerenaAgent: def __init__( self, @@ -193,7 +211,7 @@ class SerenaAgent: tool_inclusion_definitions.append(SerenaAgentMode.from_name_internal("jetbrains")) self._base_tool_set = ToolSet.default().apply(*tool_inclusion_definitions) - self._exposed_tools = {tc: t for tc, t in self._all_tools.items() if self._base_tool_set.includes_name(t.get_name())} + self._exposed_tools = AvailableTools([t for t in self._all_tools.values() if self._base_tool_set.includes_name(t.get_name())]) log.info(f"Number of exposed tools: {len(self._exposed_tools)}") # create executor for starting the language server and running tools in another thread @@ -302,7 +320,7 @@ class SerenaAgent: If a client should attempt to use a tool that is dynamically disabled (e.g. because a project is activated that disables it), it will receive an error. """ - return list(self._exposed_tools.values()) + return list(self._exposed_tools.tools) def get_active_project(self) -> Project | None: """ @@ -336,24 +354,21 @@ class SerenaAgent: """ return list(self._modes) + def _format_prompt(self, prompt_template: str) -> str: + template = JinjaTemplate(prompt_template) + return template.render(available_tools=self._exposed_tools.tool_names, available_markers=self._exposed_tools.tool_marker_names) + def create_system_prompt(self) -> str: - available_tools = [tool.get_name_from_cls() for tool in self.get_exposed_tool_instances()] - - # determine available tool markers (i.e. tool categories) - available_markers = set() - for marker_class in iter_subclasses(ToolMarker): - for tool in self.get_exposed_tool_instances(): - if isinstance(tool, marker_class): - available_markers.add(marker_class.__name__) - + available_markers = self._exposed_tools.tool_marker_names log.info("Generating system prompt with available_tools=(see exposed tools), available_markers=%s", available_markers) system_prompt = self.prompt_factory.create_system_prompt( - context_system_prompt=self._context.prompt, - mode_system_prompts=[mode.prompt for mode in self._modes], - available_tools=available_tools, + context_system_prompt=self._format_prompt(self._context.prompt), + mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self._modes], + available_tools=self._exposed_tools.tool_names, available_markers=available_markers, ) log.info("System prompt:\n%s", system_prompt) + return system_prompt def _update_active_tools(self) -> None: """ diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index de84f6a7..394c609e 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -39,6 +39,10 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): name: str prompt: str + """ + a Jinja2 template for the generation of the system prompt. + It is formatted by the agent (see SerenaAgent._format_prompt()). + """ description: str = "" def _tostring_includes(self) -> list[str]: @@ -133,6 +137,10 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): name: str prompt: str + """ + a Jinja2 template for the generation of the system prompt. + It is formatted by the agent (see SerenaAgent._format_prompt()). + """ description: str = "" def _tostring_includes(self) -> list[str]: diff --git a/src/serena/resources/config/modes/editing.yml b/src/serena/resources/config/modes/editing.yml index d53750fd..464f4537 100644 --- a/src/serena/resources/config/modes/editing.yml +++ b/src/serena/resources/config/modes/editing.yml @@ -14,7 +14,7 @@ prompt: | use the regex-based approach that is described below. Let us first discuss the symbol-based approach. - Symbols are identified by their name path and relative file path, see the description of the `find_symbols` tool for more details + Symbols are identified by their name path and relative file path, see the description of the `find_symbol` tool for more details on how the `name_path` matches symbols. You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible @@ -34,7 +34,8 @@ prompt: | You will generally be able to use the info from the snippets and the regex-based approach to adjust the references as well. You can assume that all symbol editing tools are reliable, so you don't need to verify the results if the tool returns without error. - Now let us discuss the regex-based approach. + {% if 'replace_regex' in available_tools %} + Let us discuss the regex-based approach. The regex-based approach is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation. This is the case if you need to adjust just a few lines of code within a method, or a chunk that is much smaller than a whole symbol. You use other tools to find the relevant content and @@ -104,7 +105,8 @@ prompt: | Generally, I remind you that you rely on the regex tool with providing you the correct feedback, no need for more verification! IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE LONG REGEXES WITHOUT USING WILDCARDS INSTEAD! -excluded_tools: + {% endif %} +excluded_tools: - replace_lines - insert_at_line - delete_lines From eb0f44cb793e2fb34603d53fb471396ca40e74f5 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sat, 2 Aug 2025 20:03:19 +0200 Subject: [PATCH 07/27] Fixed appending of "." to docstring and mock in test... We should get rid of mocks eventually --- src/serena/mcp.py | 5 +++-- test/serena/test_mcp.py | 23 ++++++++++++++--------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 6c75556f..fb42cbe9 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -68,8 +68,7 @@ class SerenaMCPFactory: # Mount the tool description as a combination of the docstring description and # the return value description, if it exists. - agent = cast(SerenaAgent, tool.agent) - overridden_description = agent.get_context().tool_description_overrides.get(func_name, None) + overridden_description = tool.agent.get_context().tool_description_overrides.get(func_name, None) if overridden_description is not None: func_doc = overridden_description @@ -78,6 +77,8 @@ class SerenaMCPFactory: else: func_doc = "" func_doc = func_doc.strip().strip(".") + if func_doc: + func_doc += "." if docstring.returns and (docstring_returns_descr := docstring.returns.description): # Only add a space before "Returns" if func_doc is not empty prefix = " " if func_doc else "" diff --git a/test/serena/test_mcp.py b/test/serena/test_mcp.py index 37b6eca8..731a1cc6 100644 --- a/test/serena/test_mcp.py +++ b/test/serena/test_mcp.py @@ -4,16 +4,28 @@ import pytest from mcp.server.fastmcp.tools.base import Tool as MCPTool from serena.agent import Tool, ToolRegistry +from serena.config.context_mode import SerenaAgentContext from serena.mcp import SerenaMCPFactory make_tool = SerenaMCPFactory.make_mcp_tool +# Create a mock agent for tool initialization +class MockAgent: + def __init__(self): + self.project_config = None + self.serena_config = None + + @staticmethod + def get_context() -> SerenaAgentContext: + return SerenaAgentContext.load_default() + + class BaseMockTool(Tool): """A mock Tool class for testing.""" - def __init__(self) -> None: - pass + def __init__(self): + super().__init__(MockAgent()) class BasicTool(BaseMockTool): @@ -274,13 +286,6 @@ def is_test_mock_class(tool_class: type) -> bool: @pytest.mark.parametrize("tool_class", ToolRegistry().get_all_tool_classes()) def test_make_tool_all_tools(tool_class) -> None: """Test that make_tool works for all tools in the codebase.""" - - # Create a mock agent for tool initialization - class MockAgent: - def __init__(self): - self.project_config = None - self.serena_config = None - # Create an instance of the tool tool_instance = tool_class(MockAgent()) From 4410f37ab3b26f97a11903b1faa0e99f01c93742 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sat, 2 Aug 2025 22:49:42 +0200 Subject: [PATCH 08/27] Context: back to using **data in from_yaml, remove to/from json_dict methods Needed since some fields come from parent class, easy to make mistakes (like I did in the past commit) --- src/serena/config/context_mode.py | 33 ++++--------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index 30caba0b..e04f3383 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -44,17 +44,6 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): def _tostring_includes(self) -> list[str]: return ["name"] - def to_json_dict(self) -> dict[str, str | list[str]]: - result = asdict(self) - result["excluded_tools"] = list(result["excluded_tools"]) - return result - - @classmethod - def from_json_dict(cls, data: dict) -> Self: - data = copy(data) - data["excluded_tools"] = set(data["excluded_tools"]) - return cls(**data) - def print_overview(self) -> None: """Print an overview of the mode.""" print(f"{self.name}:\n {self.description}") @@ -140,30 +129,16 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): def _tostring_includes(self) -> list[str]: return ["name"] - def to_json_dict(self) -> dict[str, str | list[str]]: - result = asdict(self) - result["excluded_tools"] = list(result["excluded_tools"]) - return result - - @classmethod - def from_json_dict(cls, data: dict) -> Self: - data = copy(data) - data["excluded_tools"] = set(data["excluded_tools"]) - # Ensure backwards compatibility for tool_description_overrides - if "tool_description_overrides" not in data: - data["tool_description_overrides"] = {} - return cls(**data) - @classmethod def from_yaml(cls, yaml_path: str | Path) -> Self: """Load a context from a YAML file.""" with open(yaml_path, encoding="utf-8") as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) - prompt = data["prompt"] - description = data.get("description", "") - tool_description_overrides = data.get("tool_description_overrides", {}) - return cls(name=name, prompt=prompt, description=description, tool_description_overrides=tool_description_overrides) + # Ensure backwards compatibility for tool_description_overrides + if "tool_description_overrides" not in data: + data["tool_description_overrides"] = {} + return cls(name=name, **data) @classmethod def get_path(cls, name: str) -> str: From 99757f35ebd53173c397379d4884c70cacc3981e Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 00:14:11 +0200 Subject: [PATCH 09/27] Made more tools optional and shortened several docstrings --- docs/serena_on_chatgpt.md | 3 +- scripts/demo_run_tools.py | 10 ++--- src/serena/config/context_mode.py | 3 +- .../config/contexts/context.template.yml | 3 ++ .../resources/config/contexts/desktop-app.yml | 3 ++ .../config/contexts/ide-assistant.yml | 5 --- .../resources/config/modes/mode.template.yml | 3 ++ src/serena/tools/cmd_tools.py | 11 +---- src/serena/tools/config_tools.py | 8 ++-- src/serena/tools/file_tools.py | 20 +++------ src/serena/tools/memory_tools.py | 11 +---- src/serena/tools/symbol_tools.py | 44 +++++++++---------- src/serena/tools/workflow_tools.py | 2 +- 13 files changed, 52 insertions(+), 74 deletions(-) diff --git a/docs/serena_on_chatgpt.md b/docs/serena_on_chatgpt.md index 64f7e248..ec872696 100644 --- a/docs/serena_on_chatgpt.md +++ b/docs/serena_on_chatgpt.md @@ -18,7 +18,7 @@ Run the following command to launch Serena as http server (assuming port 8000): ```bash uvx mcpo --port 8000 --api-key -- \ uvx --from git+https://github.com/oraios/serena \ - serena start-mcp-server --project $(pwd) + serena start-mcp-server --context chatgpt --project $(pwd) ``` - `--api-key` is required to secure the server. @@ -63,6 +63,7 @@ Your server is now securely exposed to the internet. ``` "servers": ["url": ""], ``` + **Important**: don't include a trailing slash at the end of the URL! ChatGPT will read the schema and create functions automatically. diff --git a/scripts/demo_run_tools.py b/scripts/demo_run_tools.py index 394fe553..d52cfb00 100644 --- a/scripts/demo_run_tools.py +++ b/scripts/demo_run_tools.py @@ -9,7 +9,7 @@ from pprint import pprint from serena.agent import SerenaAgent from serena.config.serena_config import SerenaConfig from serena.constants import REPO_ROOT -from serena.tools import FindFileTool, FindReferencingSymbolsTool, SearchForPatternTool +from serena.tools import FindFileTool, FindReferencingSymbolsTool, GetSymbolsOverviewTool, SearchForPatternTool if __name__ == "__main__": agent = SerenaAgent(project=REPO_ROOT, serena_config=SerenaConfig(gui_log_window_enabled=False, web_dashboard=False)) @@ -18,13 +18,9 @@ if __name__ == "__main__": find_refs_tool = agent.get_tool(FindReferencingSymbolsTool) find_file_tool = agent.get_tool(FindFileTool) search_pattern_tool = agent.get_tool(SearchForPatternTool) + overview_tool = agent.get_tool(GetSymbolsOverviewTool) result = agent.execute_task( - lambda: search_pattern_tool.apply( - r"def request_full_.*?\).*?\)", - restrict_search_to_code_files=False, - relative_path="src/solidlsp", - paths_include_glob="**/ls.py", - ) + lambda: overview_tool.apply("src/solidlsp/ls.py"), ) pprint(json.loads(result)) diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index e04f3383..fbe34091 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -3,8 +3,7 @@ Context and Mode configuration loader """ import os -from copy import copy -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Self diff --git a/src/serena/resources/config/contexts/context.template.yml b/src/serena/resources/config/contexts/context.template.yml index f9dee588..c01a1177 100644 --- a/src/serena/resources/config/contexts/context.template.yml +++ b/src/serena/resources/config/contexts/context.template.yml @@ -3,6 +3,9 @@ description: Description of the context, not used in the code. prompt: Prompt that will form part of the system prompt/initial instructions for agents started in this context. excluded_tools: [] +# several tools are excluded by default and have to be explicitly included by the user +included_optional_tools: [] + # mapping of tool names to an override of their descriptions (the default description is the docstring of the Tool's apply method). # Sometimes, tool descriptions are too long (e.g., for ChatGPT), or users may want to override them for another reason. tool_description_overrides: {} \ No newline at end of file diff --git a/src/serena/resources/config/contexts/desktop-app.yml b/src/serena/resources/config/contexts/desktop-app.yml index 8105ca8d..c3adcfb4 100644 --- a/src/serena/resources/config/contexts/desktop-app.yml +++ b/src/serena/resources/config/contexts/desktop-app.yml @@ -11,4 +11,7 @@ prompt: | a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions. excluded_tools: [] +included_optional_tools: + - switch_modes + tool_description_overrides: {} diff --git a/src/serena/resources/config/contexts/ide-assistant.yml b/src/serena/resources/config/contexts/ide-assistant.yml index 14d65130..27a13673 100644 --- a/src/serena/resources/config/contexts/ide-assistant.yml +++ b/src/serena/resources/config/contexts/ide-assistant.yml @@ -19,13 +19,8 @@ prompt: | excluded_tools: - create_text_file - read_file - - delete_lines - - replace_lines - - insert_at_line - execute_shell_command - prepare_for_new_conversation - - summarize_changes - - get_current_config tool_description_overrides: {} diff --git a/src/serena/resources/config/modes/mode.template.yml b/src/serena/resources/config/modes/mode.template.yml index 431afdeb..6d836ca3 100644 --- a/src/serena/resources/config/modes/mode.template.yml +++ b/src/serena/resources/config/modes/mode.template.yml @@ -2,3 +2,6 @@ description: Description of the mode, not used in the code. prompt: Prompt that will form part of the message sent to the model when activating this mode. excluded_tools: [] + +# several tools are excluded by default and have to be explicitly included by the user +included_optional_tools: [] \ No newline at end of file diff --git a/src/serena/tools/cmd_tools.py b/src/serena/tools/cmd_tools.py index 0db0da08..214d54e2 100644 --- a/src/serena/tools/cmd_tools.py +++ b/src/serena/tools/cmd_tools.py @@ -19,15 +19,8 @@ class ExecuteShellCommandTool(Tool, ToolMarkerCanEdit): max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH, ) -> str: """ - Execute a shell command and return its output. - - IMPORTANT: you should always consider the memory about suggested shell commands before using this tool. - If this memory was not loaded in the current conversation, you should load it using the `read_memory` tool - before using this tool. - - You should have at least once looked at the suggested shell commands from the corresponding memory - created during the onboarding process before using this tool. - Never execute unsafe shell commands like `rm -rf /` or similar! Generally be very careful with deletions. + Execute a shell command and return its output. If there is a memory about suggested commands, read that first. + Never execute unsafe shell commands like `rm -rf /` or similar! :param command: the shell command to execute :param cwd: the working directory to execute the command in. If None, the project root will be used. diff --git a/src/serena/tools/config_tools.py b/src/serena/tools/config_tools.py index 51b782ad..8a3ddeb7 100644 --- a/src/serena/tools/config_tools.py +++ b/src/serena/tools/config_tools.py @@ -1,7 +1,7 @@ import json from serena.config.context_mode import SerenaAgentMode -from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject +from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional class ActivateProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): @@ -35,7 +35,7 @@ class ActivateProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): return result_str -class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): +class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional): """ Removes a project from the Serena configuration. """ @@ -50,7 +50,7 @@ class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject): return f"Successfully removed project '{project_name}' from configuration." -class SwitchModesTool(Tool): +class SwitchModesTool(Tool, ToolMarkerOptional): """ Activates modes by providing a list of their names """ @@ -71,7 +71,7 @@ class SwitchModesTool(Tool): return result_str -class GetCurrentConfigTool(Tool): +class GetCurrentConfigTool(Tool, ToolMarkerOptional): """ Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. """ diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index f79ffe06..b5770ab5 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -14,7 +14,7 @@ from fnmatch import fnmatch from pathlib import Path from serena.text_utils import search_files -from serena.tools import SUCCESS_RESULT, TOOL_DEFAULT_MAX_ANSWER_LENGTH, EditedFileContext, Tool, ToolMarkerCanEdit +from serena.tools import SUCCESS_RESULT, TOOL_DEFAULT_MAX_ANSWER_LENGTH, EditedFileContext, Tool, ToolMarkerCanEdit, ToolMarkerOptional from serena.util.file_system import scan_directory @@ -29,7 +29,6 @@ class ReadFileTool(Tool): """ Reads the given file or a chunk of it. Generally, symbolic operations like find_symbol or find_referencing_symbols should be preferred if you know which symbols you are looking for. - Reading the entire file is only recommended if there is no other way to get the content required for the task. :param relative_path: the relative path to the file to read :param start_line: the 0-based index of the first line to be retrieved. @@ -60,14 +59,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit): def apply(self, relative_path: str, content: str) -> str: """ - Write a new file (or overwrite an existing file). For existing files, it is strongly recommended - to use symbolic operations like replace_symbol_body or insert_after_symbol/insert_before_symbol, if possible. - You can also use insert_at_line to insert content at a specific line for existing files if the symbolic operations - are not the right choice for what you want to do. - - If ever used on an existing file, the content has to be the complete content of that file (so it - may never end with something like "The remaining content of the file is left unchanged."). - For operations that just replace a part of a file, use the replace_lines or the symbolic editing tools instead. + Write a new file or overwrite an existing file. :param relative_path: the relative path to the file to create :param content: the (utf-8-encoded) content to write to the file @@ -200,7 +192,7 @@ class ReplaceRegexTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class DeleteLinesTool(Tool, ToolMarkerCanEdit): +class DeleteLinesTool(Tool, ToolMarkerCanEdit, ToolMarkerOptional): """ Deletes a range of lines within a file. """ @@ -228,7 +220,7 @@ class DeleteLinesTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class ReplaceLinesTool(Tool, ToolMarkerCanEdit): +class ReplaceLinesTool(Tool, ToolMarkerCanEdit, ToolMarkerOptional): """ Replaces a range of lines within a file with new content. """ @@ -259,7 +251,7 @@ class ReplaceLinesTool(Tool, ToolMarkerCanEdit): return SUCCESS_RESULT -class InsertAtLineTool(Tool, ToolMarkerCanEdit): +class InsertAtLineTool(Tool, ToolMarkerCanEdit, ToolMarkerOptional): """ Inserts content at a given line in a file. """ @@ -351,7 +343,7 @@ class SearchForPatternTool(Tool): For example, for finding classes or methods from a name pattern. Setting to False is a better choice if you also want to search in non-code files, like in html or yaml files, which is why it is the default. - :return: A JSON object mapping file paths to lists of matched consecutive lines (with context, if requested). + :return: A mapping of file paths to lists of matched consecutive lines. """ abs_path = os.path.join(self.get_project_root(), relative_path) if not os.path.exists(abs_path): diff --git a/src/serena/tools/memory_tools.py b/src/serena/tools/memory_tools.py index 63bba4fe..3cf611f0 100644 --- a/src/serena/tools/memory_tools.py +++ b/src/serena/tools/memory_tools.py @@ -10,15 +10,8 @@ class WriteMemoryTool(Tool): def apply(self, memory_name: str, content: str, max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH) -> str: """ - Write some information about this project that can be useful for future tasks to a memory. - Use markdown formatting for the content. - The information should be short and to the point. - The memory name should be meaningful, such that from the name you can infer what the information is about. - It is better to have multiple small memories than to have a single large one because - memories will be read one by one and we only ever want to read relevant memories. - - This tool is either called during the onboarding process or when you have identified - something worth remembering about the project from the past conversation. + Write some information about this project that can be useful for future tasks to a memory in md format. + The memory name should be meaningful. """ if len(content) > max_answer_chars: raise ValueError( diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 4d2a5ffc..0dfb41a3 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -4,6 +4,7 @@ Language server-related tools import dataclasses import json +import os from collections.abc import Sequence from copy import copy from typing import Any @@ -33,10 +34,7 @@ class RestartLanguageServerTool(Tool): def apply(self) -> str: """Use this tool only on explicit user request or after confirmation. - It may be necessary to restart the language server if the user performs edits - not through Serena, so the language server state becomes outdated and further editing attempts lead to errors. - - If such editing errors happen, you should suggest using this tool. + It may be necessary to restart the language server if it hangs. """ self.agent.reset_language_server() return SUCCESS_RESULT @@ -49,24 +47,28 @@ class GetSymbolsOverviewTool(Tool): def apply(self, relative_path: str, max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH) -> str: """ - Gets an overview of the given file or directory. - For each analyzed file, we list the top-level symbols in the file (name_path, kind). - Use this tool to get a high-level understanding of the code symbols. - Calling this is often a good idea before more targeted reading, searching or editing operations on the code symbols. - Before requesting a symbol overview, it is usually a good idea to narrow down the scope of the overview - by first understanding the basic directory structure of the repository that you can get from memories - or by using the `list_dir` and `find_file` tools (or similar). + Use this tool to get a high-level understanding of the code symbols in a file. + This should be the first tool to call when you want to understand a new file, unless you already know + what you are looking for. :param relative_path: the relative path to the file or directory to get the overview of :param max_answer_chars: if the overview is longer than this number of characters, no content will be returned. Don't adjust unless there is really no other way to get the content required for the task. If the overview is too long, you should use a smaller directory instead, (e.g. a subdirectory). - :return: a JSON object mapping relative paths of all contained files to info about top-level symbols in the file (name_path, kind). + :return: a JSON object containing info about top-level symbols in the file """ symbol_retriever = self.create_language_server_symbol_retriever() - result = symbol_retriever.get_symbol_overview(relative_path) - result_json_str = json.dumps({k: [dataclasses.asdict(i) for i in l] for k, l in result.items()}) + file_path = os.path.join(self.project.project_root, relative_path) + + # The symbol overview is capable of working with both files and directories, + # but we want to ensure that the user provides a file path. + if not os.path.exists(file_path): + raise FileNotFoundError(f"File or directory {relative_path} does not exist in the project.") + if os.path.isdir(file_path): + raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ") + result = symbol_retriever.get_symbol_overview(relative_path)[relative_path] + result_json_str = json.dumps([dataclasses.asdict(i) for i in result]) return self._limit_length(result_json_str, max_answer_chars) @@ -132,7 +134,7 @@ class FindSymbolTool(Tool): :param exclude_kinds: Optional. List of LSP symbol kind integers to exclude. Takes precedence over `include_kinds`. :param substring_matching: If True, use substring matching for the last segment of `name`. :param max_answer_chars: Max characters for the JSON result. If exceeded, no content is returned. - :return: JSON string: a list of symbols (with locations) matching the name. + :return: a list of symbols (with locations) matching the name. """ parsed_include_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in include_kinds] if include_kinds else None parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None @@ -164,10 +166,8 @@ class FindReferencingSymbolsTool(Tool): max_answer_chars: int = TOOL_DEFAULT_MAX_ANSWER_LENGTH, ) -> str: """ - Finds symbols that reference the symbol at the given `name_path`. The result will contain metadata about the referencing symbols - as well as a short code snippet around the reference (unless `include_body` is True, then the short snippet will be omitted). - Note that among other kinds of references, this function can be used to find (direct) subclasses of a class, - as subclasses are referencing symbols that have the kind class. + Finds references to the symbol at the given `name_path`. The result will contain metadata about the referencing symbols + as well as a short code snippet around the reference. :param name_path: for finding the symbol to find references for, same logic as in the `find_symbol` tool. :param relative_path: the relative path to the file containing the symbol for which to find references. @@ -269,9 +269,9 @@ class InsertBeforeSymbolTool(Tool, ToolMarkerCanEdit): body: str, ) -> str: """ - Inserts the given body/content before the beginning of the definition of the given symbol (via the symbol's location). - A typical use case is to insert a new class, function, method, field or variable assignment. - It also can be used to insert a new import statement before the first symbol in the file. + Inserts the given content before the beginning of the definition of the given symbol (via the symbol's location). + A typical use case is to insert a new class, function, method, field or variable assignment; or + a new import statement before the first symbol in the file. :param name_path: name path of the symbol before which to insert content (definitions in the `find_symbol` tool apply) :param relative_path: the relative path to the file containing the symbol diff --git a/src/serena/tools/workflow_tools.py b/src/serena/tools/workflow_tools.py index 38cc19be..be78d03d 100644 --- a/src/serena/tools/workflow_tools.py +++ b/src/serena/tools/workflow_tools.py @@ -96,7 +96,7 @@ class ThinkAboutWhetherYouAreDoneTool(Tool): return self.prompt_factory.create_think_about_whether_you_are_done() -class SummarizeChangesTool(Tool): +class SummarizeChangesTool(Tool, ToolMarkerOptional): """ Provides instructions for summarizing the changes made to the codebase. """ From 3d8f5f7446508bd5cd68db826b66109ffa685524 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 00:14:23 +0200 Subject: [PATCH 10/27] Added chatgpt context --- .../resources/config/contexts/chatgpt.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/serena/resources/config/contexts/chatgpt.yml diff --git a/src/serena/resources/config/contexts/chatgpt.yml b/src/serena/resources/config/contexts/chatgpt.yml new file mode 100644 index 00000000..c758b2c3 --- /dev/null +++ b/src/serena/resources/config/contexts/chatgpt.yml @@ -0,0 +1,28 @@ +description: A configuration specific for chatgpt, which has a limit of 30 tools and requires short descriptions. +prompt: | + You are running in desktop app context where the tools give you access to the code base as well as some + access to the file system, if configured. You interact with the user through a chat interface that is separated + from the code base. As a consequence, if you are in interactive mode, your communication with the user should + involve high-level thinking and planning as well as some summarization of any code edits that you make. + For viewing the code edits the user will view them in a separate code editor window, and the back-and-forth + between the chat and the code editor should be minimized as well as facilitated by you. + If complex changes have been made, advise the user on how to review them in the code editor. + If complex relationships that the user asked for should be visualized or explained, consider creating + a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering + options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions. +excluded_tools: [] +included_optional_tools: + - switch_modes + +tool_description_overrides: + find_symbol: | + Retrieves symbols matching `name_path` in a file. + Use `depth > 0` to include children. `name_path` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar" + replace_regex: | + Replaces text using regular expressions. Preferred for smaller edits where symbol-level tools aren't appropriate. + Use wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content. + Essential for multi-line replacements. + search_for_pattern: | + Flexible pattern search across codebase. Prefer symbolic operations when possible. + Uses DOTALL matching. Use non-greedy quantifiers (.*?) to avoid over-matching. + Supports file filtering via globs and code-only restriction. \ No newline at end of file From 069db44091afc90385606c72e07a5f6552b58b06 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 00:26:03 +0200 Subject: [PATCH 11/27] GH: fixing paths, hopefully cache starts working on unix --- .github/workflows/pytest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 91dcd610..31f4b372 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -36,8 +36,8 @@ jobs: uses: actions/cache@v3 with: path: | - ~/go/bin - ~/.cache/go-build + ${{ env.HOME }}/go/bin + ${{ env.HOME }}/.cache/go-build key: go-binaries-${{ runner.os }}-gopls-latest - name: Install gopls if: steps.cache-go-binaries.outputs.cache-hit != 'true' @@ -77,7 +77,7 @@ jobs: id: cache-language-servers uses: actions/cache@v3 with: - path: ~/.serena/language_servers/static + path: ${{ env.HOME }}/.serena/language_servers/static key: language-servers-${{ runner.os }}-v1 restore-keys: | language-servers-${{ runner.os }}- From 7990c3f30d416321e3c69d57fe165c1be86d6d01 Mon Sep 17 00:00:00 2001 From: Kazuki Hanai Date: Sun, 3 Aug 2025 19:24:42 +0900 Subject: [PATCH 12/27] Fix: Add HTML escaping to dashboard log messages Escape special HTML characters in log messages to prevent XSS vulnerabilities and ensure proper display of code snippets containing HTML tags. --- src/serena/resources/dashboard/dashboard.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index b57226aa..318d0c16 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -1,5 +1,6 @@ class LogMessage { constructor(message, toolNames) { + message = this.escapeHtml(message); const logLevel = this.determineLogLevel(message); const highlightedMessage = this.highlightToolNames(message, toolNames); this.$elem = $('
').addClass('log-' + logLevel).html(highlightedMessage + '\n'); @@ -27,6 +28,21 @@ class LogMessage { }); return highlightedMessage; } + + escapeHtml (convertString) { + if (typeof convertString !== 'string') return convertString; + + const patterns = { + '<' : '<', + '>' : '>', + '&' : '&', + '"' : '"', + '\'' : ''', + '`' : '`' + }; + + return convertString.replace(/[<>&"'`]/g, match => patterns[match]); + }; } class Dashboard { From 574eb7b327a7c765c3cfca5d74b59f811140546b Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 13:35:33 +0200 Subject: [PATCH 13/27] GH: make sure cached dirs exist --- .github/workflows/pytest.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 31f4b372..3eeac6be 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -28,6 +28,12 @@ jobs: - uses: actions/setup-go@v5 with: go-version: '>=1.17.0' + - name: Ensure cached directory exist before calling cache-related actions + shell: bash + run: | + mkdir -p $HOME/.serena/language_servers/static + mkdir -p $HOME/.cache/go-build + mkdir -p $HOME/go/bin # Add Go bin directory to PATH for this workflow # GITHUB_PATH is a special file that GitHub Actions uses to modify PATH # Writing to this file adds the directory to the PATH for subsequent steps @@ -36,8 +42,8 @@ jobs: uses: actions/cache@v3 with: path: | - ${{ env.HOME }}/go/bin - ${{ env.HOME }}/.cache/go-build + ~/go/bin + ~/.cache/go-build key: go-binaries-${{ runner.os }}-gopls-latest - name: Install gopls if: steps.cache-go-binaries.outputs.cache-hit != 'true' @@ -49,7 +55,6 @@ jobs: with: elixir-version: '1.18.4' otp-version: '26.1' - - name: Prepare java uses: actions/setup-java@v3 with: @@ -77,7 +82,7 @@ jobs: id: cache-language-servers uses: actions/cache@v3 with: - path: ${{ env.HOME }}/.serena/language_servers/static + path: ~/.serena/language_servers/static key: language-servers-${{ runner.os }}-v1 restore-keys: | language-servers-${{ runner.os }}- From a5db2f46162896b37d87a6a9b619fbc96a119153 Mon Sep 17 00:00:00 2001 From: MischaPanch Date: Sun, 3 Aug 2025 15:53:43 +0200 Subject: [PATCH 14/27] Permissions: set sane mode (755) instead of just executable for downloaded binaries --- src/solidlsp/language_servers/clangd_language_server.py | 3 +-- src/solidlsp/language_servers/clojure_lsp.py | 3 +-- src/solidlsp/language_servers/csharp_language_server.py | 5 ++--- src/solidlsp/language_servers/dart_language_server.py | 3 +-- src/solidlsp/language_servers/eclipse_jdtls.py | 3 +-- src/solidlsp/language_servers/omnisharp.py | 3 +-- src/solidlsp/language_servers/rust_analyzer.py | 3 +-- src/solidlsp/language_servers/terraform_ls.py | 3 +-- 8 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/solidlsp/language_servers/clangd_language_server.py b/src/solidlsp/language_servers/clangd_language_server.py index a770d78e..3879fae7 100644 --- a/src/solidlsp/language_servers/clangd_language_server.py +++ b/src/solidlsp/language_servers/clangd_language_server.py @@ -5,7 +5,6 @@ Provides C/C++ specific instantiation of the LanguageServer class. Contains vari import logging import os import pathlib -import stat import threading from solidlsp.ls import SolidLanguageServer @@ -95,7 +94,7 @@ class ClangdLanguageServer(SolidLanguageServer): f"Clangd executable not found at {clangd_executable_path}.\n" "Make sure you have installed clangd. See https://clangd.llvm.org/installation" ) - os.chmod(clangd_executable_path, stat.S_IEXEC) + os.chmod(clangd_executable_path, 0o755) return clangd_executable_path diff --git a/src/solidlsp/language_servers/clojure_lsp.py b/src/solidlsp/language_servers/clojure_lsp.py index f973db15..d81fa236 100644 --- a/src/solidlsp/language_servers/clojure_lsp.py +++ b/src/solidlsp/language_servers/clojure_lsp.py @@ -6,7 +6,6 @@ import logging import os import pathlib import shutil -import stat import subprocess import threading @@ -118,7 +117,7 @@ class ClojureLSP(SolidLanguageServer): deps.install(logger, clojurelsp_ls_dir) if not os.path.exists(clojurelsp_executable_path): raise FileNotFoundError(f"Download failed? Could not find clojure-lsp executable at {clojurelsp_executable_path}") - os.chmod(clojurelsp_executable_path, stat.S_IEXEC) + os.chmod(clojurelsp_executable_path, 0o755) return clojurelsp_executable_path @staticmethod diff --git a/src/solidlsp/language_servers/csharp_language_server.py b/src/solidlsp/language_servers/csharp_language_server.py index 66f8c3cb..f01d5b04 100644 --- a/src/solidlsp/language_servers/csharp_language_server.py +++ b/src/solidlsp/language_servers/csharp_language_server.py @@ -7,7 +7,6 @@ import logging import os import platform import shutil -import stat import subprocess import tarfile import threading @@ -314,7 +313,7 @@ class CSharpLanguageServer(SolidLanguageServer): # Make executable on Unix systems if platform.system().lower() != "windows": - server_dll.chmod(server_dll.stat().st_mode | stat.S_IEXEC) + server_dll.chmod(0o755) logger.log(f"Successfully installed Microsoft.CodeAnalysis.LanguageServer to {server_dll}", logging.INFO) return str(server_dll) @@ -456,7 +455,7 @@ class CSharpLanguageServer(SolidLanguageServer): # Make dotnet executable on Unix if platform.system().lower() != "windows": - dotnet_exe.chmod(dotnet_exe.stat().st_mode | stat.S_IEXEC) + dotnet_exe.chmod(0o755) logger.log(f"Successfully installed .NET 9 runtime to {dotnet_exe}", logging.INFO) return str(dotnet_exe) diff --git a/src/solidlsp/language_servers/dart_language_server.py b/src/solidlsp/language_servers/dart_language_server.py index c9cfcad1..297294a4 100644 --- a/src/solidlsp/language_servers/dart_language_server.py +++ b/src/solidlsp/language_servers/dart_language_server.py @@ -1,7 +1,6 @@ import logging import os import pathlib -import stat from solidlsp.ls import SolidLanguageServer from solidlsp.ls_logger import LanguageServerLogger @@ -84,7 +83,7 @@ class DartLanguageServer(SolidLanguageServer): deps.install(logger, dart_ls_dir) assert os.path.exists(dart_executable_path) - os.chmod(dart_executable_path, stat.S_IEXEC) + os.chmod(dart_executable_path, 0o755) return f"{dart_executable_path} language-server --client-id multilspy.dart --client-version 1.2" diff --git a/src/solidlsp/language_servers/eclipse_jdtls.py b/src/solidlsp/language_servers/eclipse_jdtls.py index bf87aab7..d664fb7a 100644 --- a/src/solidlsp/language_servers/eclipse_jdtls.py +++ b/src/solidlsp/language_servers/eclipse_jdtls.py @@ -7,7 +7,6 @@ import logging import os import pathlib import shutil -import stat import threading import uuid from pathlib import PurePath @@ -279,7 +278,7 @@ class EclipseJDTLS(SolidLanguageServer): ): FileUtils.download_and_extract_archive(logger, dependency["url"], vscode_java_path, dependency["archiveType"]) - os.chmod(jre_path, stat.S_IEXEC) + os.chmod(jre_path, 0o755) assert os.path.exists(vscode_java_path) assert os.path.exists(jre_home_path) diff --git a/src/solidlsp/language_servers/omnisharp.py b/src/solidlsp/language_servers/omnisharp.py index c9755fa8..e43287c5 100644 --- a/src/solidlsp/language_servers/omnisharp.py +++ b/src/solidlsp/language_servers/omnisharp.py @@ -6,7 +6,6 @@ import json import logging import os import pathlib -import stat import threading from collections.abc import Iterable @@ -194,7 +193,7 @@ class OmniSharp(SolidLanguageServer): FileUtils.download_and_extract_archive(logger, runtime_dependencies["OmniSharp"]["url"], omnisharp_ls_dir, "zip") omnisharp_executable_path = os.path.join(omnisharp_ls_dir, runtime_dependencies["OmniSharp"]["binaryName"]) assert os.path.exists(omnisharp_executable_path) - os.chmod(omnisharp_executable_path, stat.S_IEXEC) + os.chmod(omnisharp_executable_path, 0o755) razor_omnisharp_ls_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "RazorOmnisharp") if not os.path.exists(razor_omnisharp_ls_dir): diff --git a/src/solidlsp/language_servers/rust_analyzer.py b/src/solidlsp/language_servers/rust_analyzer.py index 48bae28a..308b653f 100644 --- a/src/solidlsp/language_servers/rust_analyzer.py +++ b/src/solidlsp/language_servers/rust_analyzer.py @@ -5,7 +5,6 @@ Provides Rust specific instantiation of the LanguageServer class. Contains vario import logging import os import pathlib -import stat import threading from overrides import override @@ -104,7 +103,7 @@ class RustAnalyzer(SolidLanguageServer): os.makedirs(rustanalyzer_ls_dir) deps.install(logger, rustanalyzer_ls_dir) assert os.path.exists(rustanalyzer_executable_path) - os.chmod(rustanalyzer_executable_path, stat.S_IEXEC) + os.chmod(rustanalyzer_executable_path, 0o755) return rustanalyzer_executable_path diff --git a/src/solidlsp/language_servers/terraform_ls.py b/src/solidlsp/language_servers/terraform_ls.py index e40d9af1..6a374df6 100644 --- a/src/solidlsp/language_servers/terraform_ls.py +++ b/src/solidlsp/language_servers/terraform_ls.py @@ -1,7 +1,6 @@ import logging import os import shutil -import stat import threading from overrides import override @@ -112,7 +111,7 @@ class TerraformLS(SolidLanguageServer): # Make the executable file executable on Unix-like systems if platform_id.value != "win-x64": - os.chmod(terraform_ls_executable_path, stat.S_IEXEC | stat.S_IREAD) + os.chmod(terraform_ls_executable_path, 0o755) return terraform_ls_executable_path From b50d8215864afd8f782a4bc8785d059ceb45649a Mon Sep 17 00:00:00 2001 From: KeisukeKaji <113495285+Keisuke05410@users.noreply.github.com> Date: Mon, 4 Aug 2025 23:07:35 +0900 Subject: [PATCH 15/27] Add PATH lookup for solargraph binary before falling back to 'gem exec' --- src/solidlsp/language_servers/solargraph.py | 29 +++++++++++++-------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/solidlsp/language_servers/solargraph.py b/src/solidlsp/language_servers/solargraph.py index a4f7e694..ca148f14 100644 --- a/src/solidlsp/language_servers/solargraph.py +++ b/src/solidlsp/language_servers/solargraph.py @@ -7,6 +7,7 @@ import json import logging import os import pathlib +import shutil import subprocess import threading @@ -56,17 +57,6 @@ class Solargraph(SolidLanguageServer): """ Setup runtime dependencies for Solargraph and return the command to start the server. """ - runtime_dependencies = [ - { - "url": "https://rubygems.org/downloads/solargraph-0.51.1.gem", - "installCommand": "gem install solargraph -v 0.51.1", - "binaryName": "solargraph", - "archiveType": "gem", - } - ] - - dependency = runtime_dependencies[0] - # Check if Ruby is installed try: result = subprocess.run(["ruby", "--version"], check=True, capture_output=True, cwd=repository_root_path) @@ -78,6 +68,23 @@ class Solargraph(SolidLanguageServer): raise RuntimeError("Ruby is not installed. Please install Ruby before continuing.") from e # Check if solargraph is installed + # First, try to find solargraph in PATH (includes asdf shims) + solargraph_path = shutil.which("solargraph") + if solargraph_path: + logger.log(f"Found solargraph at: {solargraph_path}", logging.INFO) + return solargraph_path + + # Fallback to gem exec + runtime_dependencies = [ + { + "url": "https://rubygems.org/downloads/solargraph-0.51.1.gem", + "installCommand": "gem install solargraph -v 0.51.1", + "binaryName": "solargraph", + "archiveType": "gem", + } + ] + + dependency = runtime_dependencies[0] try: result = subprocess.run( ["gem", "list", "^solargraph$", "-i"], check=False, capture_output=True, text=True, cwd=repository_root_path From 1d22a377978812bb71fb347731fd60a2ed962525 Mon Sep 17 00:00:00 2001 From: kght6123 Date: Tue, 5 Aug 2025 00:20:36 +0900 Subject: [PATCH 16/27] Docker: improve documentation for quick setup (#394) --- DOCKER.md | 20 ++++++++++++++++++++ compose.yaml | 6 ++++++ 2 files changed, 26 insertions(+) diff --git a/DOCKER.md b/DOCKER.md index 5ee8829c..e2647503 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -68,6 +68,8 @@ SERENA_DASHBOARD_PORT=8080 docker-compose up serena docker-compose up serena-dev ``` +Note: Edit the `compose.yaml` file to customize volume mounts for your projects. + ### Using Docker directly ```bash @@ -83,6 +85,24 @@ docker run -it --rm \ serena ``` +### Using Docker Compose with Merge Compose files + +To use Docker Compose with merge files, you can create a `compose.override.yml` file to customize the configuration: + +```yaml +services: + serena: + # To work with projects, you must mount them as volumes: + volumes: + - ./my-project:/workspace/my-project + - /path/to/another/project:/workspace/another-project + # Add the context for the IDE assistant option: + command: + - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide-assistant" +``` + +See the [Docker Merge Compose files documentation](https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/) for more details on using merge files. + ## Accessing the Dashboard Once running, access the web dashboard at: diff --git a/compose.yaml b/compose.yaml index bb8f683e..48923e27 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,10 @@ services: serena: image: serena:latest + # To work with projects, you must mount them as volumes: + # volumes: + # - ./my-project:/workspace/my-project + # - /path/to/another/project:/workspace/another-project build: context: ./ dockerfile: Dockerfile @@ -12,6 +16,8 @@ services: - SERENA_DOCKER=1 command: - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0" + # Add the context for the IDE assistant + # - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide-assistant" serena-dev: image: serena:dev From d56665d35919488f1ddbacef08981071b13fe90b Mon Sep 17 00:00:00 2001 From: liebertar Date: Tue, 5 Aug 2025 16:20:18 +0900 Subject: [PATCH 17/27] Auto-close dashboard tabs when server stops responding This addresses the dashboard tab accumulation issue that's been brought up. Having multiple tabs open gets disruptive during development. Implemented the approach @opcode81 suggested - detecting when the server stops responding and letting the tab close itself. The dashboard now monitors its connection and auto-closes after 3 failed polling attempts. Should help with the workflow disruption without being overly complex. Hope this helps\! Fixes #388 --- src/serena/resources/dashboard/dashboard.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index 318d0c16..4edc0868 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -52,6 +52,7 @@ class Dashboard { this.toolNames = []; this.currentMaxIdx = -1; this.pollInterval = null; + this.failureCount = 0; this.$logContainer = $('#log-container'); this.$errorContainer = $('#error-container'); this.$loadButton = $('#load-logs'); @@ -161,6 +162,7 @@ class Dashboard { start_idx: self.currentMaxIdx + 1 }), success: function(response) { + self.failureCount = 0; // Only append new messages if we have any if (response.messages && response.messages.length > 0) { let wasAtBottom = false; @@ -190,6 +192,11 @@ class Dashboard { }, error: function(xhr, status, error) { console.error('Error polling for new logs:', error); + self.failureCount++; + if (self.failureCount >= 3) { + console.log('Server appears to be down, closing tab'); + window.close(); + } } }); } From b931e972434a7c50ddf4edf92fee77e5db8d8053 Mon Sep 17 00:00:00 2001 From: Kai Yao Date: Tue, 5 Aug 2025 11:15:35 -0500 Subject: [PATCH 18/27] added support for rust 2024 edition (#412) --- .../language_servers/rust_analyzer.py | 10 +- .../repos/rust/test_repo_2024/Cargo.lock | 7 ++ .../repos/rust/test_repo_2024/Cargo.toml | 6 + .../repos/rust/test_repo_2024/src/lib.rs | 21 ++++ .../repos/rust/test_repo_2024/src/main.rs | 19 +++ test/solidlsp/rust/test_rust_2024_edition.py | 112 ++++++++++++++++++ 6 files changed, 170 insertions(+), 5 deletions(-) create mode 100644 test/resources/repos/rust/test_repo_2024/Cargo.lock create mode 100644 test/resources/repos/rust/test_repo_2024/Cargo.toml create mode 100644 test/resources/repos/rust/test_repo_2024/src/lib.rs create mode 100644 test/resources/repos/rust/test_repo_2024/src/main.rs create mode 100644 test/solidlsp/rust/test_rust_2024_edition.py diff --git a/src/solidlsp/language_servers/rust_analyzer.py b/src/solidlsp/language_servers/rust_analyzer.py index de73f045..d9b09090 100644 --- a/src/solidlsp/language_servers/rust_analyzer.py +++ b/src/solidlsp/language_servers/rust_analyzer.py @@ -60,7 +60,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for macOS (x64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-x86_64-apple-darwin.gz", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-x86_64-apple-darwin.gz", platform_id="osx-x64", archive_type="gz", binary_name="rust_analyzer", @@ -68,7 +68,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for macOS (arm64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-aarch64-apple-darwin.gz", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-aarch64-apple-darwin.gz", platform_id="osx-arm64", archive_type="gz", binary_name="rust_analyzer", @@ -76,7 +76,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for Linux (x64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-x86_64-unknown-linux-gnu.gz", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-x86_64-unknown-linux-gnu.gz", platform_id="linux-x64", archive_type="gz", binary_name="rust_analyzer", @@ -84,7 +84,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for Linux (arm64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-aarch64-unknown-linux-gnu.gz", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-aarch64-unknown-linux-gnu.gz", platform_id="linux-arm64", archive_type="gz", binary_name="rust_analyzer", @@ -92,7 +92,7 @@ class RustAnalyzer(SolidLanguageServer): RuntimeDependency( id="RustAnalyzer", description="RustAnalyzer for Windows (x64)", - url="https://github.com/rust-lang/rust-analyzer/releases/download/2023-10-09/rust-analyzer-x86_64-pc-windows-msvc.zip", + url="https://github.com/rust-lang/rust-analyzer/releases/download/2025-08-04/rust-analyzer-x86_64-pc-windows-msvc.zip", platform_id="win-x64", archive_type="zip", binary_name="rust-analyzer.exe", diff --git a/test/resources/repos/rust/test_repo_2024/Cargo.lock b/test/resources/repos/rust/test_repo_2024/Cargo.lock new file mode 100644 index 00000000..88beb720 --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "rsandbox_2024" +version = "0.1.0" diff --git a/test/resources/repos/rust/test_repo_2024/Cargo.toml b/test/resources/repos/rust/test_repo_2024/Cargo.toml new file mode 100644 index 00000000..2f20bf2e --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "rsandbox_2024" +version = "0.1.0" +edition = "2024" + +[dependencies] \ No newline at end of file diff --git a/test/resources/repos/rust/test_repo_2024/src/lib.rs b/test/resources/repos/rust/test_repo_2024/src/lib.rs new file mode 100644 index 00000000..28fac4c7 --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/src/lib.rs @@ -0,0 +1,21 @@ +pub fn multiply(a: i32, b: i32) -> i32 { + a * b +} + +pub struct Calculator { + pub result: i32, +} + +impl Calculator { + pub fn new() -> Self { + Calculator { result: 0 } + } + + pub fn add(&mut self, value: i32) { + self.result += value; + } + + pub fn get_result(&self) -> i32 { + self.result + } +} \ No newline at end of file diff --git a/test/resources/repos/rust/test_repo_2024/src/main.rs b/test/resources/repos/rust/test_repo_2024/src/main.rs new file mode 100644 index 00000000..ec20c47a --- /dev/null +++ b/test/resources/repos/rust/test_repo_2024/src/main.rs @@ -0,0 +1,19 @@ +fn main() { + println!("Hello, Rust 2024 edition!"); + let result = add(2, 3); + println!("2 + 3 = {}", result); +} + +pub fn add(a: i32, b: i32) -> i32 { + a + b +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_add() { + assert_eq!(add(2, 3), 5); + } +} \ No newline at end of file diff --git a/test/solidlsp/rust/test_rust_2024_edition.py b/test/solidlsp/rust/test_rust_2024_edition.py new file mode 100644 index 00000000..e73c8d6d --- /dev/null +++ b/test/solidlsp/rust/test_rust_2024_edition.py @@ -0,0 +1,112 @@ +import os +from pathlib import Path + +import pytest + +from solidlsp.ls_config import Language +from solidlsp.ls_utils import SymbolUtils +from test.conftest import create_ls + + +@pytest.mark.rust +class TestRust2024EditionLanguageServer: + @classmethod + def setup_class(cls): + """Set up the test class with the Rust 2024 edition test repository.""" + cls.test_repo_2024_path = Path(__file__).parent.parent.parent / "resources" / "repos" / "rust" / "test_repo_2024" + + if not cls.test_repo_2024_path.exists(): + pytest.skip("Rust 2024 edition test repository not found") + + # Create and start the language server for the 2024 edition repo + cls.language_server = create_ls(Language.RUST, str(cls.test_repo_2024_path)) + cls.language_server.start() + + @classmethod + def teardown_class(cls): + """Clean up the language server.""" + if hasattr(cls, "language_server"): + cls.language_server.stop() + + def test_find_references_raw(self) -> None: + # Test finding references to the 'add' function defined in main.rs + file_path = os.path.join("src", "main.rs") + symbols = self.language_server.request_document_symbols(file_path) + add_symbol = None + for sym in symbols[0]: + if sym.get("name") == "add": + add_symbol = sym + break + assert add_symbol is not None, "Could not find 'add' function symbol in main.rs" + sel_start = add_symbol["selectionRange"]["start"] + refs = self.language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + # The add function should be referenced within main.rs itself (in the main function) + assert any("main.rs" in ref.get("relativePath", "") for ref in refs), "main.rs should reference add function" + + def test_find_symbol(self) -> None: + symbols = self.language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "main"), "main function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "add"), "add function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "multiply"), "multiply function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Calculator"), "Calculator struct not found in symbol tree" + + def test_find_referencing_symbols_multiply(self) -> None: + # Find references to 'multiply' function defined in lib.rs + file_path = os.path.join("src", "lib.rs") + symbols = self.language_server.request_document_symbols(file_path) + multiply_symbol = None + for sym in symbols[0]: + if sym.get("name") == "multiply": + multiply_symbol = sym + break + assert multiply_symbol is not None, "Could not find 'multiply' function symbol in lib.rs" + sel_start = multiply_symbol["selectionRange"]["start"] + refs = self.language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + # The multiply function exists but may not be referenced anywhere, which is fine + # This test just verifies we can find the symbol and request references without error + assert isinstance(refs, list), "Should return a list of references (even if empty)" + + def test_find_calculator_struct_and_impl(self) -> None: + # Test finding the Calculator struct and its impl block + file_path = os.path.join("src", "lib.rs") + symbols = self.language_server.request_document_symbols(file_path) + + # Find the Calculator struct + calculator_struct = None + calculator_impl = None + for sym in symbols[0]: + if sym.get("name") == "Calculator" and sym.get("kind") == 23: # Struct kind + calculator_struct = sym + elif sym.get("name") == "Calculator" and sym.get("kind") == 11: # Interface/Impl kind + calculator_impl = sym + + assert calculator_struct is not None, "Could not find 'Calculator' struct symbol in lib.rs" + + # The struct should have the 'result' field + struct_children = calculator_struct.get("children", []) + field_names = [child.get("name") for child in struct_children] + assert "result" in field_names, "Calculator struct should have 'result' field" + + # Find the impl block and check its methods + if calculator_impl is not None: + impl_children = calculator_impl.get("children", []) + method_names = [child.get("name") for child in impl_children] + assert "new" in method_names, "Calculator impl should have 'new' method" + assert "add" in method_names, "Calculator impl should have 'add' method" + assert "get_result" in method_names, "Calculator impl should have 'get_result' method" + + def test_overview_methods(self) -> None: + symbols = self.language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "main"), "main missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "add"), "add missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "multiply"), "multiply missing from overview" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Calculator"), "Calculator missing from overview" + + def test_rust_2024_edition_specific(self) -> None: + # Verify we're actually working with the 2024 edition repository + cargo_toml_path = self.test_repo_2024_path / "Cargo.toml" + assert cargo_toml_path.exists(), "Cargo.toml should exist in test repository" + + with open(cargo_toml_path) as f: + content = f.read() + assert 'edition = "2024"' in content, "Should be using Rust 2024 edition" From 396ca3f356a2fea7a429ba6ed675bb4e650fb1d1 Mon Sep 17 00:00:00 2001 From: VH <80348187+vhbui02@users.noreply.github.com> Date: Wed, 6 Aug 2025 11:34:01 +0000 Subject: [PATCH 19/27] Add NodeJS inside Docker image (#421) --- Dockerfile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d88d4e26..205f4ee6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,8 @@ FROM python:3.11-slim AS base SHELL ["/bin/bash", "-c"] # Set environment variables to make Python print directly to the terminal and avoid .pyc files. -ENV PYTHONUNBUFFERED 1 -ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 # Install system dependencies required for package manager and build tools. RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -18,6 +18,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN python3 -m pip install --no-cache-dir pipx \ && pipx ensurepath +# Install nodejs +ENV NVM_VERSION=0.40.3 +ENV NODE_VERSION=22.18.0 +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh | bash +# standard location +ENV NVM_DIR=/root/.nvm +RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} +ENV PATH="${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH}" + # Add local bin to the path ENV PATH="${PATH}:/root/.local/bin" From 7158aa4870f0c1e08117322c0eff72b80d4fd9bb Mon Sep 17 00:00:00 2001 From: hyuraku <32809703+hyuraku@users.noreply.github.com> Date: Wed, 6 Aug 2025 22:21:43 +0900 Subject: [PATCH 20/27] Adds tests for Ruby language server integration (#416) --- .github/workflows/pytest.yml | 7 ++++ pyproject.toml | 1 + .../repos/ruby/test_repo/.solargraph.yml | 4 ++ test/resources/repos/ruby/test_repo/lib.rb | 9 ++++ test/resources/repos/ruby/test_repo/main.rb | 21 ++++++++++ test/solidlsp/ruby/test_ruby_basic.py | 42 +++++++++++++++++++ 6 files changed, 84 insertions(+) create mode 100644 test/resources/repos/ruby/test_repo/.solargraph.yml create mode 100644 test/resources/repos/ruby/test_repo/lib.rb create mode 100644 test/resources/repos/ruby/test_repo/main.rb create mode 100644 test/solidlsp/ruby/test_ruby_basic.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3eeac6be..ddfe7a32 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -69,6 +69,13 @@ jobs: with: terraform_version: "1.5.0" terraform_wrapper: false + - name: Install Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.4' + - name: Install Ruby language server + shell: bash + run: gem install solargraph - name: Install uv shell: bash run: curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/pyproject.toml b/pyproject.toml index 8872384c..d46c678f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -260,6 +260,7 @@ markers = [ "elixir: language server running for Elixir", "terraform: language server running for Terraform", "snapshot: snapshot tests for symbolic editing operations", + "ruby: language server running for Ruby", ] [tool.codespell] diff --git a/test/resources/repos/ruby/test_repo/.solargraph.yml b/test/resources/repos/ruby/test_repo/.solargraph.yml new file mode 100644 index 00000000..b4678eec --- /dev/null +++ b/test/resources/repos/ruby/test_repo/.solargraph.yml @@ -0,0 +1,4 @@ +--- +include: + - "main.rb" + - "lib.rb" diff --git a/test/resources/repos/ruby/test_repo/lib.rb b/test/resources/repos/ruby/test_repo/lib.rb new file mode 100644 index 00000000..5dd8504b --- /dev/null +++ b/test/resources/repos/ruby/test_repo/lib.rb @@ -0,0 +1,9 @@ +class Calculator + def add(a, b) + a + b + end + + def subtract(a, b) + a - b + end +end diff --git a/test/resources/repos/ruby/test_repo/main.rb b/test/resources/repos/ruby/test_repo/main.rb new file mode 100644 index 00000000..dcf1b0f2 --- /dev/null +++ b/test/resources/repos/ruby/test_repo/main.rb @@ -0,0 +1,21 @@ +require './lib.rb' + +class DemoClass + attr_accessor :value + + def initialize(value) + @value = value + end + + def print_value + puts @value + end +end + +def helper_function(number = 42) + demo = DemoClass.new(number) + Calculator.new.add(demo.value, 10) + demo.print_value +end + +helper_function diff --git a/test/solidlsp/ruby/test_ruby_basic.py b/test/solidlsp/ruby/test_ruby_basic.py new file mode 100644 index 00000000..435c0538 --- /dev/null +++ b/test/solidlsp/ruby/test_ruby_basic.py @@ -0,0 +1,42 @@ +import os +from pathlib import Path + +import pytest + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language +from solidlsp.ls_utils import SymbolUtils + + +@pytest.mark.ruby +class TestRubyLanguageServer: + @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) + def test_find_symbol(self, language_server: SolidLanguageServer) -> None: + symbols = language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "DemoClass"), "DemoClass not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "helper_function"), "helper_function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "print_value"), "print_value not found in symbol tree" + + @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) + def test_find_referencing_symbols(self, language_server: SolidLanguageServer) -> None: + file_path = os.path.join("main.rb") + symbols = language_server.request_document_symbols(file_path) + helper_symbol = None + for sym in symbols[0]: + if sym.get("name") == "helper_function": + helper_symbol = sym + break + print(helper_symbol) + assert helper_symbol is not None, "Could not find 'helper_function' symbol in main.rb" + + @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) + @pytest.mark.parametrize("repo_path", [Language.RUBY], indirect=True) + def test_find_definition_across_files(self, language_server: SolidLanguageServer, repo_path: Path) -> None: + # Test finding Calculator.add method definition from line 17: Calculator.new.add(demo.value, 10) + definition_location_list = language_server.request_definition(str(repo_path / "main.rb"), 16,17) # add method at line 17 (0-indexed 16), position 17 + + assert len(definition_location_list) == 1 + definition_location = definition_location_list[0] + print(f"Found definition: {definition_location}") + assert definition_location["uri"].endswith("lib.rb") + assert definition_location["range"]["start"]["line"] == 1 # add method on line 2 (0-indexed 1) From ec1277167b7ec26c833bc4decd5df7c7ec007da0 Mon Sep 17 00:00:00 2001 From: Yen <5915590+antigenius0910@users.noreply.github.com> Date: Wed, 6 Aug 2025 09:41:04 -0500 Subject: [PATCH 21/27] Add comprehensive bash language support (#375) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add comprehensive bash language support Implement BashLanguageServer using bash-language-server (v5.4.0) - Add automatic Node.js/npm dependency management - Support .sh and .bash file extensions - Include cross-platform compatibility (Windows .cmd handling) - Add comprehensive test infrastructure with realistic bash scripts - Integrate bash into Language enum and language server factory - Add bash pytest marker for selective testing - Include bash_language_server fixture for tests Features: * LSP capabilities: text sync, completion, hover, document symbols * Runtime dependency installation via npm * Symbol recognition for functions, variables, conditionals * Proper LSP initialization and error handling * Test repository with main.sh, utils.sh, and config.sh examples Files: + src/solidlsp/language_servers/bash_language_server.py (new implementation) + test/solidlsp/bash/test_bash_basic.py (comprehensive test suite) + test/resources/repos/bash/test_repo/ (test scripts) * src/solidlsp/ls_config.py (add BASH enum and file patterns) * src/solidlsp/ls.py (integrate bash language server) * pyproject.toml (add bash pytest marker) * test/conftest.py (add bash_language_server fixture) Tests: 2 passed, 7 appropriately skipped All existing tests continue to pass (288 passed total) * trigger flow * feat: enhance bash function detection with hybrid LSP + regex approach - Upgrade bash-language-server from 5.4.0 to 5.6.0 for better capabilities - Add hybrid function detection: primary LSP + regex fallback when needed - Implement _detect_bash_functions() with regex patterns for both syntax types: * function name() { } (function keyword syntax) * name() { } (regular function syntax) - Add _find_function_end() with brace matching for accurate function boundaries - Enhance request_document_symbols() with intelligent fallback logic - Add comprehensive test suite with 4 new test functions: * test_bash_function_detection_hybrid_approach * test_bash_function_detection_with_body * test_bash_regex_fallback_detection * test_bash_function_syntax_patterns - All tests passing: 6 passed, 7 skipped, 0 failed - Maintains 100% backward compatibility - Enables full symbolic operations for bash functions: find_symbol, replace_symbol_body, etc. * remove trash * clean: remove debug changes from Python services.py - Remove 'MODIFIED VIA SYMBOLIC OPERATIONS!' debug comments - Remove debug print statements added during testing - Restore original clean docstring and code - No functional changes to bash language support implementation * fix: Remove non-bash specific directory ignoring from BashLanguageServer - Removes is_ignored_dirname override that included generic directories like node_modules, .cache, tmp, temp - These directories aren't bash-specific and gitignore already handles file exclusion - Addresses PR feedback: 'that doesn't seem bash related. Note that gitignored files are ignored anyway' * fix: Restructure bash tests to follow standard patterns and remove pytest.skip - Replace custom bash_language_server fixture with standard parametrized language_server fixture - Remove all pytest.skip statements and replace with proper assertions or failures - Remove tests for non-existent methods (get_symbols_for_document, get_workspace_symbols, etc.) - Keep only tests that use actual SolidLanguageServer methods like request_document_symbols - All bash tests now pass successfully - Addresses PR feedback about test structure and pytest.skip removal * docs: Add bash language support to README - Add Bash to the list of directly supported languages - Note that it requires Node.js and npm for bash-language-server installation - Addresses PR feedback about missing README extension for bash support * docs: Add bash language support to CHANGELOG - Document comprehensive bash language support with bash-language-server integration - Highlight key features: .sh/.bash extensions, hybrid function detection, automatic dependency management - Note cross-platform compatibility including Windows support - Addresses PR feedback about missing CHANGELOG extension for bash support * refactor: Change regex fallback to always-on hybrid detection for bash functions - Remove conditional logic that only ran regex when LSP found no functions - Implement always-on hybrid approach that runs both LSP and regex detection - Add comprehensive documentation explaining bash-language-server limitations - Add new _merge_function_detections method to combine results without duplicates - Addresses reviewer feedback about files with mixed function notation styles 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --------- Co-authored-by: Claude --- CHANGELOG.md | 5 + README.md | 1 + pyproject.toml | 1 + .../language_servers/bash_language_server.py | 388 ++++++++++++++++++ src/solidlsp/ls.py | 5 + src/solidlsp/ls_config.py | 3 + test/conftest.py | 3 + test/resources/repos/bash/test_repo/config.sh | 91 ++++ test/resources/repos/bash/test_repo/main.sh | 70 ++++ test/resources/repos/bash/test_repo/utils.sh | 83 ++++ test/solidlsp/bash/__init__.py | 0 test/solidlsp/bash/test_bash_basic.py | 137 +++++++ 12 files changed, 787 insertions(+) create mode 100644 src/solidlsp/language_servers/bash_language_server.py create mode 100644 test/resources/repos/bash/test_repo/config.sh create mode 100644 test/resources/repos/bash/test_repo/main.sh create mode 100644 test/resources/repos/bash/test_repo/utils.sh create mode 100644 test/solidlsp/bash/__init__.py create mode 100644 test/solidlsp/bash/test_bash_basic.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dfbf52c4..a32f46cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Status of the `main` branch. Changes prior to the next official version change w * Reliably detect language server termination and propagate the respective error all the way back to the tool application, where an unexpected termination is handled by restarting the language server and subsequently retrying the tool application. + * Add comprehensive bash language support with bash-language-server integration + * Support for .sh and .bash file extensions + * Hybrid function detection (LSP + regex fallback) for reliable symbolic editing + * Automatic Node.js/npm dependency management + * Cross-platform compatibility including Windows Fixes: * Ignore `.git` in check for ignored paths and improve performance of `find_all_non_ignored_files` diff --git a/README.md b/README.md index 1b6777cb..1d8d1d3d 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ With Serena, we provide * Java (_Note_: startup is slow, initial startup especially so. There may be issues with java on macos and linux, we are working on it.) * Elixir (Requires NextLS and Elixir install; **Windows not supported** - Next LS does not provide Windows binaries) * Clojure + * Bash (Requires Node.js and npm for bash-language-server installation) * C/C++ (You may experience issues with finding references, we are working on it) * indirect support (may require some code changes/manual installation) for: * Ruby (untested) diff --git a/pyproject.toml b/pyproject.toml index d46c678f..8e4d563c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -259,6 +259,7 @@ markers = [ "csharp: language server running for C#", "elixir: language server running for Elixir", "terraform: language server running for Terraform", + "bash: language server running for Bash", "snapshot: snapshot tests for symbolic editing operations", "ruby: language server running for Ruby", ] diff --git a/src/solidlsp/language_servers/bash_language_server.py b/src/solidlsp/language_servers/bash_language_server.py new file mode 100644 index 00000000..04ffe355 --- /dev/null +++ b/src/solidlsp/language_servers/bash_language_server.py @@ -0,0 +1,388 @@ +""" +Provides Bash specific instantiation of the LanguageServer class using bash-language-server. +Contains various configurations and settings specific to Bash scripting. +""" + +import logging +import os +import pathlib +import shutil +import threading + +from overrides import override + +from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection +from solidlsp.ls import SolidLanguageServer +from solidlsp.ls_config import LanguageServerConfig +from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams +from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo +from solidlsp.settings import SolidLSPSettings + + +from solidlsp import ls_types + +class BashLanguageServer(SolidLanguageServer): + """ + Provides Bash specific instantiation of the LanguageServer class using bash-language-server. + Contains various configurations and settings specific to Bash scripting. + """ + + def __init__( + self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings + ): + """ + Creates a BashLanguageServer instance. This class is not meant to be instantiated directly. + Use LanguageServer.create() instead. + """ + bash_lsp_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings) + super().__init__( + config, + logger, + repository_root_path, + ProcessLaunchInfo(cmd=bash_lsp_executable_path, cwd=repository_root_path), + "bash", + solidlsp_settings, + ) + self.server_ready = threading.Event() + self.initialize_searcher_command_available = threading.Event() + + + + @classmethod + def _setup_runtime_dependencies( + cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings + ) -> str: + """ + Setup runtime dependencies for Bash Language Server and return the command to start the server. + """ + # Verify both node and npm are installed + is_node_installed = shutil.which("node") is not None + assert is_node_installed, "node is not installed or isn't in PATH. Please install NodeJS and try again." + is_npm_installed = shutil.which("npm") is not None + assert is_npm_installed, "npm is not installed or isn't in PATH. Please install npm and try again." + + deps = RuntimeDependencyCollection( + [ + RuntimeDependency( + id="bash-language-server", + description="bash-language-server package", + command="npm install --prefix ./ bash-language-server@5.6.0", + platform_id="any", + ), + ] + ) + + # Install bash-language-server if not already installed + bash_ls_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "bash-lsp") + bash_executable_path = os.path.join(bash_ls_dir, "node_modules", ".bin", "bash-language-server") + + # Handle Windows executable extension + if os.name == "nt": + bash_executable_path += ".cmd" + + if not os.path.exists(bash_executable_path): + logger.log(f"Bash Language Server executable not found at {bash_executable_path}. Installing...", logging.INFO) + deps.install(logger, bash_ls_dir) + logger.log("Bash language server dependencies installed successfully", logging.INFO) + + if not os.path.exists(bash_executable_path): + raise FileNotFoundError( + f"bash-language-server executable not found at {bash_executable_path}, something went wrong with the installation." + ) + return f"{bash_executable_path} start" + + @staticmethod + def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: + """ + Returns the initialize params for the Bash Language Server. + """ + root_uri = pathlib.Path(repository_absolute_path).as_uri() + initialize_params = { + "locale": "en", + "capabilities": { + "textDocument": { + "synchronization": {"didSave": True, "dynamicRegistration": True}, + "completion": {"dynamicRegistration": True, "completionItem": {"snippetSupport": True}}, + "definition": {"dynamicRegistration": True}, + "references": {"dynamicRegistration": True}, + "documentSymbol": { + "dynamicRegistration": True, + "hierarchicalDocumentSymbolSupport": True, + "symbolKind": {"valueSet": list(range(1, 27))}, + }, + "hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]}, + "signatureHelp": {"dynamicRegistration": True}, + "codeAction": {"dynamicRegistration": True}, + }, + "workspace": { + "workspaceFolders": True, + "didChangeConfiguration": {"dynamicRegistration": True}, + "symbol": {"dynamicRegistration": True}, + }, + }, + "processId": os.getpid(), + "rootPath": repository_absolute_path, + "rootUri": root_uri, + "workspaceFolders": [ + { + "uri": root_uri, + "name": os.path.basename(repository_absolute_path), + } + ], + } + return initialize_params + + def _start_server(self): + """ + Starts the Bash Language Server, waits for the server to be ready and yields the LanguageServer instance. + """ + + def register_capability_handler(params): + assert "registrations" in params + for registration in params["registrations"]: + if registration["method"] == "workspace/executeCommand": + self.initialize_searcher_command_available.set() + return + + def execute_client_command_handler(params): + return [] + + def do_nothing(params): + return + + def window_log_message(msg): + self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO) + # Check for bash-language-server ready signals + message_text = msg.get("message", "") + if "Analyzing" in message_text or "analysis complete" in message_text.lower(): + self.logger.log("Bash language server analysis signals detected", logging.INFO) + self.server_ready.set() + self.completions_available.set() + + self.server.on_request("client/registerCapability", register_capability_handler) + self.server.on_notification("window/logMessage", window_log_message) + self.server.on_request("workspace/executeClientCommand", execute_client_command_handler) + self.server.on_notification("$/progress", do_nothing) + self.server.on_notification("textDocument/publishDiagnostics", do_nothing) + + self.logger.log("Starting Bash server process", logging.INFO) + self.server.start() + initialize_params = self._get_initialize_params(self.repository_root_path) + + self.logger.log( + "Sending initialize request from LSP client to LSP server and awaiting response", + logging.INFO, + ) + init_response = self.server.send.initialize(initialize_params) + self.logger.log(f"Received initialize response from bash server: {init_response}", logging.DEBUG) + + # Enhanced capability checks for bash-language-server 5.6.0 + assert init_response["capabilities"]["textDocumentSync"] in [1, 2] # Full or Incremental + assert "completionProvider" in init_response["capabilities"] + + # Verify document symbol support is available + if "documentSymbolProvider" in init_response["capabilities"]: + self.logger.log("Bash server supports document symbols", logging.INFO) + else: + self.logger.log("Warning: Bash server does not report document symbol support", logging.WARNING) + + self.server.notify.initialized({}) + + # Wait for server readiness with timeout + self.logger.log("Waiting for Bash language server to be ready...", logging.INFO) + if not self.server_ready.wait(timeout=3.0): + # Fallback: assume server is ready after timeout + self.logger.log("Timeout waiting for bash server ready signal, proceeding anyway", logging.WARNING) + self.server_ready.set() + self.completions_available.set() + else: + self.logger.log("Bash server initialization complete", logging.INFO) + + def request_document_symbols( + self, relative_file_path: str, include_body: bool = False + ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: + """ + Enhanced document symbol request with hybrid LSP + regex-based function detection for bash files. + + This method combines both LSP-based detection and regex-based detection to provide comprehensive + function discovery. This dual approach is necessary because: + + 1. bash-language-server (v5.6.0) has inconsistent function detection capabilities + 2. Some bash function syntaxes are not reliably detected by the LSP server + 3. Files may contain mixed function notation styles within the same file + 4. Different formatting or indentation can affect LSP detection + + The hybrid approach ensures maximum compatibility and comprehensive function discovery + for reliable symbolic editing operations in Serena. + """ + # First get symbols from the standard LSP approach + lsp_all_symbols, lsp_root_symbols = super().request_document_symbols(relative_file_path, include_body) + + # Always run regex-based function detection for comprehensive coverage + # This addresses bash-language-server limitations and ensures we catch all function patterns + self.logger.log(f"Running hybrid function detection (LSP + regex) for {relative_file_path}", logging.DEBUG) + + regex_detected_functions = self._detect_bash_functions(relative_file_path, include_body) + + # Merge LSP and regex results, avoiding duplicates + merged_all_symbols, merged_root_symbols = self._merge_function_detections( + lsp_all_symbols, lsp_root_symbols, regex_detected_functions + ) + + # Log detection results for debugging + lsp_functions = [s for s in lsp_all_symbols if s.get("kind") == 12] + total_functions = [s for s in merged_all_symbols if s.get("kind") == 12] + + self.logger.log( + f"Function detection for {relative_file_path}: LSP={len(lsp_functions)}, " + f"Regex={len(regex_detected_functions)}, Total={len(total_functions)}", + logging.INFO + ) + + return merged_all_symbols, merged_root_symbols + + def _merge_function_detections( + self, + lsp_all_symbols: list[ls_types.UnifiedSymbolInformation], + lsp_root_symbols: list[ls_types.UnifiedSymbolInformation], + regex_detected_functions: list[ls_types.UnifiedSymbolInformation] + ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: + """ + Merge LSP-detected symbols with regex-detected functions, avoiding duplicates. + + This method: + 1. Keeps all non-function symbols from LSP detection + 2. Keeps all LSP-detected functions (they have more accurate positioning) + 3. Adds regex-detected functions that weren't found by LSP + 4. Uses function names to detect duplicates + + Args: + lsp_all_symbols: All symbols detected by LSP + lsp_root_symbols: Root-level symbols detected by LSP + regex_detected_functions: Functions detected by regex (all are root-level) + + Returns: + Tuple of (merged_all_symbols, merged_root_symbols) + """ + # Extract function names that LSP already detected + lsp_function_names = set() + for symbol in lsp_all_symbols: + if symbol.get("kind") == 12: # LSP Symbol Kind 12 = Function + lsp_function_names.add(symbol["name"]) + + # Start with all LSP symbols (both functions and non-functions) + merged_all_symbols = lsp_all_symbols.copy() + merged_root_symbols = lsp_root_symbols.copy() + + # Add regex-detected functions that weren't found by LSP + added_functions = 0 + for regex_function in regex_detected_functions: + function_name = regex_function["name"] + + # Only add if this function wasn't detected by LSP + if function_name not in lsp_function_names: + merged_all_symbols.append(regex_function) + merged_root_symbols.append(regex_function) # All regex functions are root-level + added_functions += 1 + self.logger.log(f"Added regex-detected function '{function_name}' not found by LSP", logging.DEBUG) + + if added_functions > 0: + self.logger.log(f"Merged {added_functions} additional functions from regex detection", logging.INFO) + + return merged_all_symbols, merged_root_symbols + + def _detect_bash_functions(self, relative_file_path: str, include_body: bool = False) -> list[ls_types.UnifiedSymbolInformation]: + """ + Regex-based detection of bash functions as fallback when LSP doesn't provide them. + """ + import re + from solidlsp import ls_types + + try: + # Read the file content directly from filesystem + abs_path = os.path.join(self.repository_root_path, relative_file_path) + with open(abs_path, 'r', encoding='utf-8') as f: + file_content = f.read() + lines = file_content.split("\n") + + detected_functions = [] + + # Regex patterns for bash function definitions + # Pattern 1: function name() { ... } + # Pattern 2: name() { ... } + function_patterns = [ + re.compile(r'^\s*function\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{'), + re.compile(r'^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{') + ] + + for line_num, line in enumerate(lines): + for pattern in function_patterns: + match = pattern.match(line) + if match: + func_name = match.group(1) + + # Find the end of the function by matching braces + start_line = line_num + end_line = self._find_function_end(lines, start_line) + + # Create location information + location = ls_types.Location( + uri=pathlib.Path(abs_path).as_uri(), + range={ + "start": {"line": start_line, "character": match.start(1)}, + "end": {"line": end_line, "character": 0} + }, + absolutePath=abs_path, + relativePath=relative_file_path + ) + + # Create the function symbol + func_symbol = ls_types.UnifiedSymbolInformation( + name=func_name, + kind=12, # LSP Symbol Kind for Function + location=location, + range=location["range"], + selectionRange={ + "start": {"line": start_line, "character": match.start(1)}, + "end": {"line": start_line, "character": match.end(1)} + }, + children=[], + parent=None + ) + + # Add function body if requested + if include_body: + func_symbol["body"] = "\n".join(lines[start_line:end_line + 1]) + + detected_functions.append(func_symbol) + break + + return detected_functions + + except Exception as e: + self.logger.log(f"Error in regex-based function detection for {relative_file_path}: {e}", logging.WARNING) + return [] + + def _find_function_end(self, lines: list[str], start_line: int) -> int: + """ + Find the end line of a bash function by matching opening and closing braces. + """ + brace_count = 0 + in_function = False + + for i in range(start_line, len(lines)): + line = lines[i] + + # Count braces, handling basic cases + for char in line: + if char == '{': + brace_count += 1 + in_function = True + elif char == '}' and in_function: + brace_count -= 1 + if brace_count == 0: + return i + + # Fallback: if we can't match braces properly, assume single line or small function + return min(start_line + 10, len(lines) - 1) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index ded2645a..edda2d6e 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -226,6 +226,11 @@ class SolidLanguageServer(ABC): ls = TerraformLS(config, logger, repository_root_path, solidlsp_settings=solidlsp_settings) + elif config.code_language == Language.BASH: + from solidlsp.language_servers.bash_language_server import BashLanguageServer + + ls = BashLanguageServer(config, logger, repository_root_path, solidlsp_settings=solidlsp_settings) + else: logger.log(f"Language {config.code_language} is not supported", logging.ERROR) raise SolidLSPException(f"Language {config.code_language} is not supported") diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index efb55a87..3965d066 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -42,6 +42,7 @@ class Language(str, Enum): CLOJURE = "clojure" ELIXIR = "elixir" TERRAFORM = "terraform" + BASH = "bash" # Experimental or deprecated Language Servers TYPESCRIPT_VTS = "typescript_vts" """Use the typescript language server through the natively bundled vscode extension via https://github.com/yioneko/vtsls""" @@ -103,6 +104,8 @@ class Language(str, Enum): return FilenameMatcher("*.ex", "*.exs") case self.TERRAFORM: return FilenameMatcher("*.tf", "*.tfvars", "*.tfstate") + case self.BASH: + return FilenameMatcher("*.sh", "*.bash") case _: raise ValueError(f"Unhandled language: {self}") diff --git a/test/conftest.py b/test/conftest.py index 849af543..da91ebdd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -144,3 +144,6 @@ def project(request: LanguageParamRequest): language = request.param yield create_default_project(language) + + + diff --git a/test/resources/repos/bash/test_repo/config.sh b/test/resources/repos/bash/test_repo/config.sh new file mode 100644 index 00000000..440867e5 --- /dev/null +++ b/test/resources/repos/bash/test_repo/config.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# Configuration script for project setup + +# Environment variables +export PROJECT_NAME="bash-test-project" +export PROJECT_VERSION="1.0.0" +export LOG_LEVEL="INFO" +export CONFIG_DIR="./config" + +# Default settings +DEFAULT_TIMEOUT=30 +DEFAULT_RETRIES=3 +DEFAULT_PORT=8080 + +# Configuration arrays +declare -A ENVIRONMENTS=( + ["dev"]="development" + ["prod"]="production" + ["test"]="testing" +) + +declare -A DATABASE_CONFIGS=( + ["host"]="localhost" + ["port"]="5432" + ["name"]="myapp_db" + ["user"]="dbuser" +) + +# Function to load configuration +load_config() { + local env="${1:-dev}" + local config_file="${CONFIG_DIR}/${env}.conf" + + if [[ -f "$config_file" ]]; then + echo "Loading configuration from $config_file" + source "$config_file" + else + echo "Warning: Configuration file $config_file not found, using defaults" + fi +} + +# Function to validate configuration +validate_config() { + local errors=0 + + if [[ -z "$PROJECT_NAME" ]]; then + echo "Error: PROJECT_NAME is not set" >&2 + ((errors++)) + fi + + if [[ -z "$PROJECT_VERSION" ]]; then + echo "Error: PROJECT_VERSION is not set" >&2 + ((errors++)) + fi + + if [[ $DEFAULT_PORT -lt 1024 || $DEFAULT_PORT -gt 65535 ]]; then + echo "Error: Invalid port number $DEFAULT_PORT" >&2 + ((errors++)) + fi + + return $errors +} + +# Function to print configuration +print_config() { + echo "=== Current Configuration ===" + echo "Project Name: $PROJECT_NAME" + echo "Version: $PROJECT_VERSION" + echo "Log Level: $LOG_LEVEL" + echo "Default Port: $DEFAULT_PORT" + echo "Default Timeout: $DEFAULT_TIMEOUT" + echo "Default Retries: $DEFAULT_RETRIES" + + echo "\n=== Environments ===" + for env in "${!ENVIRONMENTS[@]}"; do + echo " $env: ${ENVIRONMENTS[$env]}" + done + + echo "\n=== Database Configuration ===" + for key in "${!DATABASE_CONFIGS[@]}"; do + echo " $key: ${DATABASE_CONFIGS[$key]}" + done +} + +# Initialize configuration if this script is run directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + load_config "$1" + validate_config + print_config +fi diff --git a/test/resources/repos/bash/test_repo/main.sh b/test/resources/repos/bash/test_repo/main.sh new file mode 100644 index 00000000..56d2e72e --- /dev/null +++ b/test/resources/repos/bash/test_repo/main.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Main script demonstrating various bash features + +# Global variables +readonly SCRIPT_NAME="Main Script" +COUNTER=0 +declare -a ITEMS=("item1" "item2" "item3") + +# Function definitions +function greet_user() { + local username="$1" + local greeting_type="${2:-default}" + + case "$greeting_type" in + "formal") + echo "Good day, ${username}!" + ;; + "casual") + echo "Hey ${username}!" + ;; + *) + echo "Hello, ${username}!" + ;; + esac +} + +function process_items() { + local -n items_ref=$1 + local operation="$2" + + for item in "${items_ref[@]}"; do + case "$operation" in + "count") + ((COUNTER++)) + echo "Processing item $COUNTER: $item" + ;; + "uppercase") + echo "${item^^}" + ;; + *) + echo "Unknown operation: $operation" + return 1 + ;; + esac + done +} + +# Main execution +main() { + echo "Starting $SCRIPT_NAME" + + if [[ $# -eq 0 ]]; then + echo "Usage: $0 [greeting_type]" + exit 1 + fi + + local user="$1" + local greeting="${2:-default}" + + greet_user "$user" "$greeting" + + echo "Processing items..." + process_items ITEMS "count" + + echo "Script completed successfully" +} + +# Run main function with all arguments +main "$@" diff --git a/test/resources/repos/bash/test_repo/utils.sh b/test/resources/repos/bash/test_repo/utils.sh new file mode 100644 index 00000000..77fc8fed --- /dev/null +++ b/test/resources/repos/bash/test_repo/utils.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Utility functions for bash scripting + +# String manipulation functions +function to_uppercase() { + echo "${1^^}" +} + +function to_lowercase() { + echo "${1,,}" +} + +function trim_whitespace() { + local var="$1" + var="${var#"${var%%[![:space:]]*}"}" + var="${var%"${var##*[![:space:]]}"}" + echo "$var" +} + +# File operations +function backup_file() { + local file="$1" + local backup_dir="${2:-./backups}" + + if [[ ! -f "$file" ]]; then + echo "Error: File '$file' does not exist" >&2 + return 1 + fi + + mkdir -p "$backup_dir" + cp "$file" "${backup_dir}/$(basename "$file").$(date +%Y%m%d_%H%M%S).bak" + echo "Backup created for $file" +} + +# Array operations +function contains_element() { + local element="$1" + shift + local array=("$@") + + for item in "${array[@]}"; do + if [[ "$item" == "$element" ]]; then + return 0 + fi + done + return 1 +} + +# Logging functions +function log_message() { + local level="$1" + local message="$2" + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + + case "$level" in + "ERROR") + echo "[$timestamp] ERROR: $message" >&2 + ;; + "WARN") + echo "[$timestamp] WARN: $message" >&2 + ;; + "INFO") + echo "[$timestamp] INFO: $message" + ;; + "DEBUG") + [[ "${DEBUG:-false}" == "true" ]] && echo "[$timestamp] DEBUG: $message" + ;; + *) + echo "[$timestamp] $message" + ;; + esac +} + +# Validation functions +function is_valid_email() { + local email="$1" + [[ "$email" =~ ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ ]] +} + +function is_number() { + [[ $1 =~ ^[0-9]+$ ]] +} diff --git a/test/solidlsp/bash/__init__.py b/test/solidlsp/bash/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/solidlsp/bash/test_bash_basic.py b/test/solidlsp/bash/test_bash_basic.py new file mode 100644 index 00000000..3fd1f637 --- /dev/null +++ b/test/solidlsp/bash/test_bash_basic.py @@ -0,0 +1,137 @@ +""" +Basic integration tests for the bash language server functionality. + +These tests validate the functionality of the language server APIs +like request_document_symbols using the bash test repository. +""" + +import pytest + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language + + +@pytest.mark.bash +class TestBashLanguageServerBasics: + """Test basic functionality of the bash language server.""" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_language_server_initialization(self, language_server: SolidLanguageServer) -> None: + """Test that bash language server can be initialized successfully.""" + assert language_server is not None + assert language_server.language == Language.BASH + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_request_document_symbols(self, language_server: SolidLanguageServer) -> None: + """Test request_document_symbols for bash files.""" + # Test getting symbols from main.sh + all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=False) + + # Extract function symbols (LSP Symbol Kind 12) + function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] + function_names = [symbol["name"] for symbol in function_symbols] + + # Should detect all 3 functions from main.sh + assert "greet_user" in function_names, "Should find greet_user function" + assert "process_items" in function_names, "Should find process_items function" + assert "main" in function_names, "Should find main function" + assert len(function_symbols) >= 3, f"Should find at least 3 functions, found {len(function_symbols)}" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: + """Test request_document_symbols with body extraction.""" + # Test with include_body=True + all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=True) + + function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] + + # Find greet_user function and check it has body + greet_user_symbol = next((sym for sym in function_symbols if sym["name"] == "greet_user"), None) + assert greet_user_symbol is not None, "Should find greet_user function" + + if "body" in greet_user_symbol: + body = greet_user_symbol["body"] + assert "function greet_user()" in body, "Function body should contain function definition" + assert "case" in body.lower(), "Function body should contain case statement" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_utils_functions(self, language_server: SolidLanguageServer) -> None: + """Test function detection in utils.sh file.""" + # Test with utils.sh as well + utils_all_symbols, utils_root_symbols = language_server.request_document_symbols( + "utils.sh", include_body=False + ) + + utils_function_symbols = [symbol for symbol in utils_all_symbols if symbol.get("kind") == 12] + utils_function_names = [symbol["name"] for symbol in utils_function_symbols] + + # Should detect functions from utils.sh + expected_utils_functions = ["to_uppercase", "to_lowercase", "trim_whitespace", "backup_file", + "contains_element", "log_message", "is_valid_email", "is_number"] + + for func_name in expected_utils_functions: + assert func_name in utils_function_names, f"Should find {func_name} function in utils.sh" + + assert len(utils_function_symbols) >= 8, f"Should find at least 8 functions in utils.sh, found {len(utils_function_symbols)}" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_regex_fallback_detection(self, language_server: SolidLanguageServer) -> None: + """Test the regex-based fallback function detection directly.""" + # Check if the method exists (it should if we have our enhanced bash server) + if not hasattr(language_server, '_detect_bash_functions'): + pytest.fail("Enhanced bash function detection not available - this may be due to test environment setup") + + # Test the _detect_bash_functions method directly + detected_functions = language_server._detect_bash_functions("main.sh", include_body=True) + + # Should detect functions even if LSP doesn't provide them + assert len(detected_functions) == 3, f"Should detect exactly 3 functions, found {len(detected_functions)}" + + function_names = [func["name"] for func in detected_functions] + assert "greet_user" in function_names, "Regex should detect greet_user function" + assert "process_items" in function_names, "Regex should detect process_items function" + assert "main" in function_names, "Regex should detect main function" + + # Check that symbols have proper structure + for func in detected_functions: + assert func["kind"] == 12, f"Function {func['name']} should have LSP Symbol Kind 12" + assert "location" in func, f"Function {func['name']} should have location info" + assert "range" in func, f"Function {func['name']} should have range info" + assert "selectionRange" in func, f"Function {func['name']} should have selectionRange info" + + # Check location structure + location = func["location"] + assert "uri" in location, f"Function {func['name']} location should have URI" + assert "range" in location, f"Function {func['name']} location should have range" + assert "relativePath" in location, f"Function {func['name']} location should have relative path" + + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + def test_bash_function_syntax_patterns(self, language_server: SolidLanguageServer) -> None: + """Test detection of different bash function syntax patterns.""" + # Check if the method exists (it should if we have our enhanced bash server) + if not hasattr(language_server, '_detect_bash_functions'): + pytest.fail("Enhanced bash function detection not available - this may be due to test environment setup") + + # Test main.sh (has 'function' keyword functions and regular function) + main_functions = language_server._detect_bash_functions("main.sh", include_body=False) + + # Test utils.sh (all use 'function' keyword) + utils_functions = language_server._detect_bash_functions("utils.sh", include_body=False) + + # Verify we detect both syntax patterns + main_function_names = [func["name"] for func in main_functions] + utils_function_names = [func["name"] for func in utils_functions] + + # main() uses regular syntax: main() { + assert "main" in main_function_names, "Should detect regular function syntax" + + # Functions with 'function' keyword: function name() { + assert "greet_user" in main_function_names, "Should detect function keyword syntax" + assert "to_uppercase" in utils_function_names, "Should detect function keyword syntax in utils" + + # Verify all expected utils functions are detected + expected_utils = ["to_uppercase", "to_lowercase", "trim_whitespace", "backup_file", + "contains_element", "log_message", "is_valid_email", "is_number"] + + for expected_func in expected_utils: + assert expected_func in utils_function_names, f"Should detect {expected_func} function" \ No newline at end of file From f2d81812c787fab44ac4b8c0f8539401efc839cb Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 6 Aug 2025 20:58:14 +0200 Subject: [PATCH 22/27] Apply formatter --- .../language_servers/bash_language_server.py | 110 +++++++++--------- test/conftest.py | 3 - test/solidlsp/bash/test_bash_basic.py | 82 +++++++------ test/solidlsp/ruby/test_ruby_basic.py | 12 +- 4 files changed, 107 insertions(+), 100 deletions(-) diff --git a/src/solidlsp/language_servers/bash_language_server.py b/src/solidlsp/language_servers/bash_language_server.py index 04ffe355..2b8d8549 100644 --- a/src/solidlsp/language_servers/bash_language_server.py +++ b/src/solidlsp/language_servers/bash_language_server.py @@ -9,8 +9,7 @@ import pathlib import shutil import threading -from overrides import override - +from solidlsp import ls_types from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection from solidlsp.ls import SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig @@ -20,8 +19,6 @@ from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo from solidlsp.settings import SolidLSPSettings -from solidlsp import ls_types - class BashLanguageServer(SolidLanguageServer): """ Provides Bash specific instantiation of the LanguageServer class using bash-language-server. @@ -47,8 +44,6 @@ class BashLanguageServer(SolidLanguageServer): self.server_ready = threading.Event() self.initialize_searcher_command_available = threading.Event() - - @classmethod def _setup_runtime_dependencies( cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings @@ -180,7 +175,7 @@ class BashLanguageServer(SolidLanguageServer): # Enhanced capability checks for bash-language-server 5.6.0 assert init_response["capabilities"]["textDocumentSync"] in [1, 2] # Full or Incremental assert "completionProvider" in init_response["capabilities"] - + # Verify document symbol support is available if "documentSymbolProvider" in init_response["capabilities"]: self.logger.log("Bash server supports document symbols", logging.INFO) @@ -204,139 +199,138 @@ class BashLanguageServer(SolidLanguageServer): ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: """ Enhanced document symbol request with hybrid LSP + regex-based function detection for bash files. - + This method combines both LSP-based detection and regex-based detection to provide comprehensive function discovery. This dual approach is necessary because: - + 1. bash-language-server (v5.6.0) has inconsistent function detection capabilities 2. Some bash function syntaxes are not reliably detected by the LSP server 3. Files may contain mixed function notation styles within the same file 4. Different formatting or indentation can affect LSP detection - + The hybrid approach ensures maximum compatibility and comprehensive function discovery for reliable symbolic editing operations in Serena. """ # First get symbols from the standard LSP approach lsp_all_symbols, lsp_root_symbols = super().request_document_symbols(relative_file_path, include_body) - + # Always run regex-based function detection for comprehensive coverage # This addresses bash-language-server limitations and ensures we catch all function patterns self.logger.log(f"Running hybrid function detection (LSP + regex) for {relative_file_path}", logging.DEBUG) - + regex_detected_functions = self._detect_bash_functions(relative_file_path, include_body) - + # Merge LSP and regex results, avoiding duplicates merged_all_symbols, merged_root_symbols = self._merge_function_detections( lsp_all_symbols, lsp_root_symbols, regex_detected_functions ) - + # Log detection results for debugging lsp_functions = [s for s in lsp_all_symbols if s.get("kind") == 12] total_functions = [s for s in merged_all_symbols if s.get("kind") == 12] - + self.logger.log( f"Function detection for {relative_file_path}: LSP={len(lsp_functions)}, " - f"Regex={len(regex_detected_functions)}, Total={len(total_functions)}", - logging.INFO + f"Regex={len(regex_detected_functions)}, Total={len(total_functions)}", + logging.INFO, ) - + return merged_all_symbols, merged_root_symbols def _merge_function_detections( - self, - lsp_all_symbols: list[ls_types.UnifiedSymbolInformation], + self, + lsp_all_symbols: list[ls_types.UnifiedSymbolInformation], lsp_root_symbols: list[ls_types.UnifiedSymbolInformation], - regex_detected_functions: list[ls_types.UnifiedSymbolInformation] + regex_detected_functions: list[ls_types.UnifiedSymbolInformation], ) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]: """ Merge LSP-detected symbols with regex-detected functions, avoiding duplicates. - + This method: 1. Keeps all non-function symbols from LSP detection 2. Keeps all LSP-detected functions (they have more accurate positioning) 3. Adds regex-detected functions that weren't found by LSP 4. Uses function names to detect duplicates - + Args: lsp_all_symbols: All symbols detected by LSP lsp_root_symbols: Root-level symbols detected by LSP regex_detected_functions: Functions detected by regex (all are root-level) - + Returns: Tuple of (merged_all_symbols, merged_root_symbols) + """ # Extract function names that LSP already detected lsp_function_names = set() for symbol in lsp_all_symbols: if symbol.get("kind") == 12: # LSP Symbol Kind 12 = Function lsp_function_names.add(symbol["name"]) - + # Start with all LSP symbols (both functions and non-functions) merged_all_symbols = lsp_all_symbols.copy() merged_root_symbols = lsp_root_symbols.copy() - + # Add regex-detected functions that weren't found by LSP added_functions = 0 for regex_function in regex_detected_functions: function_name = regex_function["name"] - + # Only add if this function wasn't detected by LSP if function_name not in lsp_function_names: merged_all_symbols.append(regex_function) merged_root_symbols.append(regex_function) # All regex functions are root-level added_functions += 1 self.logger.log(f"Added regex-detected function '{function_name}' not found by LSP", logging.DEBUG) - + if added_functions > 0: self.logger.log(f"Merged {added_functions} additional functions from regex detection", logging.INFO) - + return merged_all_symbols, merged_root_symbols - + def _detect_bash_functions(self, relative_file_path: str, include_body: bool = False) -> list[ls_types.UnifiedSymbolInformation]: """ Regex-based detection of bash functions as fallback when LSP doesn't provide them. """ import re + from solidlsp import ls_types - + try: # Read the file content directly from filesystem abs_path = os.path.join(self.repository_root_path, relative_file_path) - with open(abs_path, 'r', encoding='utf-8') as f: + with open(abs_path, encoding="utf-8") as f: file_content = f.read() lines = file_content.split("\n") - + detected_functions = [] - + # Regex patterns for bash function definitions # Pattern 1: function name() { ... } # Pattern 2: name() { ... } function_patterns = [ - re.compile(r'^\s*function\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{'), - re.compile(r'^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{') + re.compile(r"^\s*function\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{"), + re.compile(r"^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\(\s*\)\s*\{"), ] - + for line_num, line in enumerate(lines): for pattern in function_patterns: match = pattern.match(line) if match: func_name = match.group(1) - + # Find the end of the function by matching braces start_line = line_num end_line = self._find_function_end(lines, start_line) - + # Create location information location = ls_types.Location( uri=pathlib.Path(abs_path).as_uri(), - range={ - "start": {"line": start_line, "character": match.start(1)}, - "end": {"line": end_line, "character": 0} - }, + range={"start": {"line": start_line, "character": match.start(1)}, "end": {"line": end_line, "character": 0}}, absolutePath=abs_path, - relativePath=relative_file_path + relativePath=relative_file_path, ) - + # Create the function symbol func_symbol = ls_types.UnifiedSymbolInformation( name=func_name, @@ -345,44 +339,44 @@ class BashLanguageServer(SolidLanguageServer): range=location["range"], selectionRange={ "start": {"line": start_line, "character": match.start(1)}, - "end": {"line": start_line, "character": match.end(1)} + "end": {"line": start_line, "character": match.end(1)}, }, children=[], - parent=None + parent=None, ) - + # Add function body if requested if include_body: - func_symbol["body"] = "\n".join(lines[start_line:end_line + 1]) - + func_symbol["body"] = "\n".join(lines[start_line : end_line + 1]) + detected_functions.append(func_symbol) break - + return detected_functions - + except Exception as e: self.logger.log(f"Error in regex-based function detection for {relative_file_path}: {e}", logging.WARNING) return [] - + def _find_function_end(self, lines: list[str], start_line: int) -> int: """ Find the end line of a bash function by matching opening and closing braces. """ brace_count = 0 in_function = False - + for i in range(start_line, len(lines)): line = lines[i] - + # Count braces, handling basic cases for char in line: - if char == '{': + if char == "{": brace_count += 1 in_function = True - elif char == '}' and in_function: + elif char == "}" and in_function: brace_count -= 1 if brace_count == 0: return i - + # Fallback: if we can't match braces properly, assume single line or small function return min(start_line + 10, len(lines) - 1) diff --git a/test/conftest.py b/test/conftest.py index da91ebdd..849af543 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -144,6 +144,3 @@ def project(request: LanguageParamRequest): language = request.param yield create_default_project(language) - - - diff --git a/test/solidlsp/bash/test_bash_basic.py b/test/solidlsp/bash/test_bash_basic.py index 3fd1f637..8db362e9 100644 --- a/test/solidlsp/bash/test_bash_basic.py +++ b/test/solidlsp/bash/test_bash_basic.py @@ -26,14 +26,14 @@ class TestBashLanguageServerBasics: """Test request_document_symbols for bash files.""" # Test getting symbols from main.sh all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=False) - + # Extract function symbols (LSP Symbol Kind 12) function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] function_names = [symbol["name"] for symbol in function_symbols] - + # Should detect all 3 functions from main.sh assert "greet_user" in function_names, "Should find greet_user function" - assert "process_items" in function_names, "Should find process_items function" + assert "process_items" in function_names, "Should find process_items function" assert "main" in function_names, "Should find main function" assert len(function_symbols) >= 3, f"Should find at least 3 functions, found {len(function_symbols)}" @@ -42,13 +42,13 @@ class TestBashLanguageServerBasics: """Test request_document_symbols with body extraction.""" # Test with include_body=True all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=True) - + function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] - + # Find greet_user function and check it has body greet_user_symbol = next((sym for sym in function_symbols if sym["name"] == "greet_user"), None) assert greet_user_symbol is not None, "Should find greet_user function" - + if "body" in greet_user_symbol: body = greet_user_symbol["body"] assert "function greet_user()" in body, "Function body should contain function definition" @@ -58,47 +58,53 @@ class TestBashLanguageServerBasics: def test_bash_utils_functions(self, language_server: SolidLanguageServer) -> None: """Test function detection in utils.sh file.""" # Test with utils.sh as well - utils_all_symbols, utils_root_symbols = language_server.request_document_symbols( - "utils.sh", include_body=False - ) - + utils_all_symbols, utils_root_symbols = language_server.request_document_symbols("utils.sh", include_body=False) + utils_function_symbols = [symbol for symbol in utils_all_symbols if symbol.get("kind") == 12] utils_function_names = [symbol["name"] for symbol in utils_function_symbols] - + # Should detect functions from utils.sh - expected_utils_functions = ["to_uppercase", "to_lowercase", "trim_whitespace", "backup_file", - "contains_element", "log_message", "is_valid_email", "is_number"] - + expected_utils_functions = [ + "to_uppercase", + "to_lowercase", + "trim_whitespace", + "backup_file", + "contains_element", + "log_message", + "is_valid_email", + "is_number", + ] + for func_name in expected_utils_functions: assert func_name in utils_function_names, f"Should find {func_name} function in utils.sh" - + assert len(utils_function_symbols) >= 8, f"Should find at least 8 functions in utils.sh, found {len(utils_function_symbols)}" - @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) + @pytest.mark.parametrize("language_server", [Language.BASH], indirect=True) def test_bash_regex_fallback_detection(self, language_server: SolidLanguageServer) -> None: """Test the regex-based fallback function detection directly.""" # Check if the method exists (it should if we have our enhanced bash server) - if not hasattr(language_server, '_detect_bash_functions'): + if not hasattr(language_server, "_detect_bash_functions"): pytest.fail("Enhanced bash function detection not available - this may be due to test environment setup") - - # Test the _detect_bash_functions method directly + + # Test the _detect_bash_functions method directly detected_functions = language_server._detect_bash_functions("main.sh", include_body=True) - + # Should detect functions even if LSP doesn't provide them assert len(detected_functions) == 3, f"Should detect exactly 3 functions, found {len(detected_functions)}" - + function_names = [func["name"] for func in detected_functions] assert "greet_user" in function_names, "Regex should detect greet_user function" assert "process_items" in function_names, "Regex should detect process_items function" assert "main" in function_names, "Regex should detect main function" - + # Check that symbols have proper structure for func in detected_functions: assert func["kind"] == 12, f"Function {func['name']} should have LSP Symbol Kind 12" assert "location" in func, f"Function {func['name']} should have location info" assert "range" in func, f"Function {func['name']} should have range info" assert "selectionRange" in func, f"Function {func['name']} should have selectionRange info" - + # Check location structure location = func["location"] assert "uri" in location, f"Function {func['name']} location should have URI" @@ -109,29 +115,37 @@ class TestBashLanguageServerBasics: def test_bash_function_syntax_patterns(self, language_server: SolidLanguageServer) -> None: """Test detection of different bash function syntax patterns.""" # Check if the method exists (it should if we have our enhanced bash server) - if not hasattr(language_server, '_detect_bash_functions'): + if not hasattr(language_server, "_detect_bash_functions"): pytest.fail("Enhanced bash function detection not available - this may be due to test environment setup") - + # Test main.sh (has 'function' keyword functions and regular function) main_functions = language_server._detect_bash_functions("main.sh", include_body=False) - + # Test utils.sh (all use 'function' keyword) utils_functions = language_server._detect_bash_functions("utils.sh", include_body=False) - + # Verify we detect both syntax patterns main_function_names = [func["name"] for func in main_functions] utils_function_names = [func["name"] for func in utils_functions] - + # main() uses regular syntax: main() { assert "main" in main_function_names, "Should detect regular function syntax" - + # Functions with 'function' keyword: function name() { assert "greet_user" in main_function_names, "Should detect function keyword syntax" assert "to_uppercase" in utils_function_names, "Should detect function keyword syntax in utils" - + # Verify all expected utils functions are detected - expected_utils = ["to_uppercase", "to_lowercase", "trim_whitespace", "backup_file", - "contains_element", "log_message", "is_valid_email", "is_number"] - + expected_utils = [ + "to_uppercase", + "to_lowercase", + "trim_whitespace", + "backup_file", + "contains_element", + "log_message", + "is_valid_email", + "is_number", + ] + for expected_func in expected_utils: - assert expected_func in utils_function_names, f"Should detect {expected_func} function" \ No newline at end of file + assert expected_func in utils_function_names, f"Should detect {expected_func} function" diff --git a/test/solidlsp/ruby/test_ruby_basic.py b/test/solidlsp/ruby/test_ruby_basic.py index 435c0538..2edf95f2 100644 --- a/test/solidlsp/ruby/test_ruby_basic.py +++ b/test/solidlsp/ruby/test_ruby_basic.py @@ -27,14 +27,16 @@ class TestRubyLanguageServer: helper_symbol = sym break print(helper_symbol) - assert helper_symbol is not None, "Could not find 'helper_function' symbol in main.rb" + assert helper_symbol is not None, "Could not find 'helper_function' symbol in main.rb" @pytest.mark.parametrize("language_server", [Language.RUBY], indirect=True) - @pytest.mark.parametrize("repo_path", [Language.RUBY], indirect=True) - def test_find_definition_across_files(self, language_server: SolidLanguageServer, repo_path: Path) -> None: + @pytest.mark.parametrize("repo_path", [Language.RUBY], indirect=True) + def test_find_definition_across_files(self, language_server: SolidLanguageServer, repo_path: Path) -> None: # Test finding Calculator.add method definition from line 17: Calculator.new.add(demo.value, 10) - definition_location_list = language_server.request_definition(str(repo_path / "main.rb"), 16,17) # add method at line 17 (0-indexed 16), position 17 - + definition_location_list = language_server.request_definition( + str(repo_path / "main.rb"), 16, 17 + ) # add method at line 17 (0-indexed 16), position 17 + assert len(definition_location_list) == 1 definition_location = definition_location_list[0] print(f"Found definition: {definition_location}") From 2b4d68bea940f9d58c7f938ed7248e4aa921ca22 Mon Sep 17 00:00:00 2001 From: Akira TSURUDA Date: Thu, 7 Aug 2025 07:05:35 +0900 Subject: [PATCH 23/27] Add Bundler support and timeout configuration for Ruby projects (#419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Bundler support for Ruby projects This change enhances the Solargraph language server implementation to properly support Ruby projects using Bundler with vendor/bundle configurations. Key improvements: - Detects Bundler projects by checking for Gemfile existence - Supports both 'bin/bundle' and 'bundle' commands - Checks Gemfile.lock for solargraph availability - Uses 'bundle exec solargraph' when appropriate - Provides clear error messages for missing solargraph in Bundler projects - Maintains backward compatibility with non-Bundler projects Fixes permission issues encountered with rbenv/system-managed Ruby installations when trying to install gems globally. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude * Add timeout configuration for Solargraph language server This change adds proper timeout handling to the Solargraph Ruby language server to address initialization timeout issues, especially in Bundler environments. Key improvements: - Set 120-second request timeout for LSP requests and initialization - Add 60-second server readiness timeout with graceful fallback - Enhanced logging for timeout scenarios - Better handling of slow initialization in large projects or Bundler environments This resolves connection timeout issues where Solargraph takes longer than the default 30-second MCP timeout to fully initialize and become ready. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --------- Co-authored-by: Claude --- src/solidlsp/language_servers/solargraph.py | 108 ++++++++++++++++---- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/src/solidlsp/language_servers/solargraph.py b/src/solidlsp/language_servers/solargraph.py index ca148f14..b75b19e9 100644 --- a/src/solidlsp/language_servers/solargraph.py +++ b/src/solidlsp/language_servers/solargraph.py @@ -48,6 +48,9 @@ class Solargraph(SolidLanguageServer): self.initialize_searcher_command_available = threading.Event() self.resolve_main_method_available = threading.Event() + # Set timeout for Solargraph requests - Bundler environments may need more time + self.set_request_timeout(120.0) # 120 seconds for initialization and requests + @override def is_ignored_dirname(self, dirname: str) -> bool: return super().is_ignored_dirname(dirname) or dirname in ["vendor"] @@ -67,35 +70,84 @@ class Solargraph(SolidLanguageServer): except FileNotFoundError as e: raise RuntimeError("Ruby is not installed. Please install Ruby before continuing.") from e - # Check if solargraph is installed + # Check for Bundler project (Gemfile exists) + gemfile_path = os.path.join(repository_root_path, "Gemfile") + gemfile_lock_path = os.path.join(repository_root_path, "Gemfile.lock") + is_bundler_project = os.path.exists(gemfile_path) + + if is_bundler_project: + logger.log("Detected Bundler project (Gemfile found)", logging.INFO) + + # Check if bundle command is available + bundle_path = shutil.which("bundle") + if not bundle_path: + # Try common bundle executables + for bundle_cmd in ["bin/bundle", "bundle"]: + bundle_full_path = ( + os.path.join(repository_root_path, bundle_cmd) if bundle_cmd.startswith("bin/") else shutil.which(bundle_cmd) + ) + if bundle_full_path and os.path.exists(bundle_full_path): + bundle_path = bundle_full_path if bundle_cmd.startswith("bin/") else bundle_cmd + break + + if not bundle_path: + raise RuntimeError("Bundler project detected but 'bundle' command not found. Please install Bundler.") + + # Check if solargraph is in Gemfile.lock + solargraph_in_bundle = False + if os.path.exists(gemfile_lock_path): + try: + with open(gemfile_lock_path) as f: + content = f.read() + solargraph_in_bundle = "solargraph" in content.lower() + except Exception as e: + logger.log(f"Warning: Could not read Gemfile.lock: {e}", logging.WARNING) + + if solargraph_in_bundle: + logger.log("Found solargraph in Gemfile.lock", logging.INFO) + return f"{bundle_path} exec solargraph" + else: + logger.log( + "solargraph not found in Gemfile.lock. Please add 'gem \"solargraph\"' to your Gemfile and run 'bundle install'", + logging.WARNING, + ) + # Fall through to global installation check + + # Check if solargraph is installed globally # First, try to find solargraph in PATH (includes asdf shims) solargraph_path = shutil.which("solargraph") if solargraph_path: logger.log(f"Found solargraph at: {solargraph_path}", logging.INFO) return solargraph_path - # Fallback to gem exec - runtime_dependencies = [ - { - "url": "https://rubygems.org/downloads/solargraph-0.51.1.gem", - "installCommand": "gem install solargraph -v 0.51.1", - "binaryName": "solargraph", - "archiveType": "gem", - } - ] + # Fallback to gem exec (for non-Bundler projects or when global solargraph not found) + if not is_bundler_project: + runtime_dependencies = [ + { + "url": "https://rubygems.org/downloads/solargraph-0.51.1.gem", + "installCommand": "gem install solargraph -v 0.51.1", + "binaryName": "solargraph", + "archiveType": "gem", + } + ] - dependency = runtime_dependencies[0] - try: - result = subprocess.run( - ["gem", "list", "^solargraph$", "-i"], check=False, capture_output=True, text=True, cwd=repository_root_path + dependency = runtime_dependencies[0] + try: + result = subprocess.run( + ["gem", "list", "^solargraph$", "-i"], check=False, capture_output=True, text=True, cwd=repository_root_path + ) + if result.stdout.strip() == "false": + logger.log("Installing Solargraph...", logging.INFO) + subprocess.run(dependency["installCommand"].split(), check=True, capture_output=True, cwd=repository_root_path) + + return "gem exec solargraph" + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Failed to check or install Solargraph. {e.stderr}") from e + else: + raise RuntimeError( + "This appears to be a Bundler project, but solargraph is not available. " + "Please add 'gem \"solargraph\"' to your Gemfile and run 'bundle install'." ) - if result.stdout.strip() == "false": - logger.log("Installing Solargraph...", logging.INFO) - subprocess.run(dependency["installCommand"].split(), check=True, capture_output=True, cwd=repository_root_path) - - return "gem exec solargraph" - except subprocess.CalledProcessError as e: - raise RuntimeError(f"Failed to check or install Solargraph. {e.stderr}") from e @staticmethod def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: @@ -176,4 +228,16 @@ class Solargraph(SolidLanguageServer): self.completions_available.set() self.server_ready.set() - self.server_ready.wait() + + # Wait for server to be ready with timeout, especially important for Bundler environments + server_ready_timeout = 60.0 + self.logger.log(f"Waiting up to {server_ready_timeout} seconds for Solargraph to become ready...", logging.INFO) + + if self.server_ready.wait(timeout=server_ready_timeout): + self.logger.log("Solargraph is ready and available for requests", logging.INFO) + else: + self.logger.log( + f"Timeout waiting for Solargraph to become ready within {server_ready_timeout} seconds, proceeding anyway. " + "This may indicate slow initialization in Bundler environment or large project indexing.", + logging.WARNING, + ) From b09cbc089da79731f17a99d7c800ae0b7a783a4e Mon Sep 17 00:00:00 2001 From: johngambleubind <127568872+johngambleubind@users.noreply.github.com> Date: Thu, 7 Aug 2025 18:52:01 +1000 Subject: [PATCH 24/27] Fix #384: Use safe zip extraction for C# dependencies (#418) --- .../csharp_language_server.py | 10 +- src/solidlsp/util/zip.py | 127 ++++++++++++++++++ test/solidlsp/util/test_zip.py | 103 ++++++++++++++ 3 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 src/solidlsp/util/zip.py create mode 100644 test/solidlsp/util/test_zip.py diff --git a/src/solidlsp/language_servers/csharp_language_server.py b/src/solidlsp/language_servers/csharp_language_server.py index f01d5b04..b8f0cc38 100644 --- a/src/solidlsp/language_servers/csharp_language_server.py +++ b/src/solidlsp/language_servers/csharp_language_server.py @@ -25,6 +25,7 @@ from solidlsp.ls_utils import PathUtils from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo from solidlsp.settings import SolidLSPSettings +from solidlsp.util.zip import SafeZipExtractor from .common import RuntimeDependency @@ -386,8 +387,13 @@ class CSharpLanguageServer(SolidLanguageServer): package_extract_dir = temp_dir / f"{package_name}.{package_version}" package_extract_dir.mkdir(exist_ok=True) - with zipfile.ZipFile(nupkg_file, "r") as zip_ref: - zip_ref.extractall(package_extract_dir) + # Use SafeZipExtractor to handle long paths and skip errors + extractor = SafeZipExtractor( + archive_path=nupkg_file, + extract_dir=package_extract_dir, + verbose=False + ) + extractor.extract_all() # Clean up the nupkg file nupkg_file.unlink() diff --git a/src/solidlsp/util/zip.py b/src/solidlsp/util/zip.py new file mode 100644 index 00000000..ae0c3183 --- /dev/null +++ b/src/solidlsp/util/zip.py @@ -0,0 +1,127 @@ +import logging +import os +import sys +import zipfile +from pathlib import Path +from typing import Optional, List +import fnmatch + +log = logging.getLogger(__name__) + +class SafeZipExtractor: + """ + A utility class for extracting ZIP archives safely. + + Features: + - Handles long file paths on Windows + - Skips files that fail to extract, continuing with the rest + - Creates necessary directories automatically + - Optional include/exclude pattern filters + """ + + def __init__( + self, + archive_path: Path, + extract_dir: Path, + verbose: bool = True, + include_patterns: Optional[List[str]] = None, + exclude_patterns: Optional[List[str]] = None + ) -> None: + """ + Initialize the SafeZipExtractor. + + :param archive_path: Path to the ZIP archive file + :param extract_dir: Directory where files will be extracted + :param verbose: Whether to log status messages + :param include_patterns: List of glob patterns for files to extract (None = all files) + :param exclude_patterns: List of glob patterns for files to skip + """ + self.archive_path = Path(archive_path) + self.extract_dir = Path(extract_dir) + self.verbose = verbose + self.include_patterns = include_patterns or [] + self.exclude_patterns = exclude_patterns or [] + + def extract_all(self) -> None: + """ + Extract all files from the archive, skipping any that fail. + """ + if not self.archive_path.exists(): + raise FileNotFoundError(f"Archive not found: {self.archive_path}") + + if self.verbose: + log.info(f"Extracting from: {self.archive_path} to {self.extract_dir}") + + with zipfile.ZipFile(self.archive_path, "r") as zip_ref: + for member in zip_ref.infolist(): + if self._should_extract(member.filename): + self._extract_member(zip_ref, member) + elif self.verbose: + log.info(f"Skipped: {member.filename}") + + def _should_extract(self, filename: str) -> bool: + """ + Determine whether a file should be extracted based on include/exclude patterns. + + :param filename: The file name from the archive + :return: True if the file should be extracted + """ + # If include_patterns is set, only extract if it matches at least one pattern + if self.include_patterns: + if not any(fnmatch.fnmatch(filename, pattern) for pattern in self.include_patterns): + return False + + # If exclude_patterns is set, skip if it matches any pattern + if self.exclude_patterns: + if any(fnmatch.fnmatch(filename, pattern) for pattern in self.exclude_patterns): + return False + + return True + + def _extract_member(self, zip_ref: zipfile.ZipFile, member: zipfile.ZipInfo) -> None: + """ + Extract a single member from the archive with error handling. + + :param zip_ref: Open ZipFile object + :param member: ZipInfo object representing the file + """ + try: + target_path = self.extract_dir / member.filename + + # Ensure directory structure exists + target_path.parent.mkdir(parents=True, exist_ok=True) + + # Handle long paths on Windows + final_path = self._normalize_path(target_path) + + # Extract file + with zip_ref.open(member) as source, open(final_path, "wb") as target: + target.write(source.read()) + + if self.verbose: + log.info(f"Extracted: {member.filename}") + + except Exception as e: + log.error(f"Failed to extract {member.filename}: {e}") + + @staticmethod + def _normalize_path(path: Path) -> Path: + """ + Adjust path to handle long paths on Windows. + + :param path: Original path + :return: Normalized path + """ + if sys.platform.startswith("win"): + return Path(r"\\?\{}".format(os.path.abspath(path))) + return path + + +# Example usage: +# extractor = SafeZipExtractor( +# archive_path=Path("file.nupkg"), +# extract_dir=Path("extract_dir"), +# include_patterns=["*.dll", "*.xml"], +# exclude_patterns=["*.pdb"] +# ) +# extractor.extract_all() diff --git a/test/solidlsp/util/test_zip.py b/test/solidlsp/util/test_zip.py new file mode 100644 index 00000000..4498781a --- /dev/null +++ b/test/solidlsp/util/test_zip.py @@ -0,0 +1,103 @@ +import io +import os +import sys +import zipfile +from pathlib import Path +import pytest + +from solidlsp.util.zip import SafeZipExtractor + + +@pytest.fixture +def temp_zip_file(tmp_path: Path) -> Path: + """Create a temporary ZIP file for testing.""" + zip_path = tmp_path / "test.zip" + with zipfile.ZipFile(zip_path, "w") as zipf: + zipf.writestr("file1.txt", "Hello World 1") + zipf.writestr("file2.txt", "Hello World 2") + zipf.writestr("folder/file3.txt", "Hello World 3") + return zip_path + + +def test_extract_all_success(temp_zip_file: Path, tmp_path: Path) -> None: + """All files should extract without error.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False) + extractor.extract_all() + + assert (dest_dir / "file1.txt").read_text() == "Hello World 1" + assert (dest_dir / "file2.txt").read_text() == "Hello World 2" + assert (dest_dir / "folder" / "file3.txt").read_text() == "Hello World 3" + + +def test_include_patterns(temp_zip_file: Path, tmp_path: Path) -> None: + """Only files matching include_patterns should be extracted.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor( + temp_zip_file, dest_dir, verbose=False, include_patterns=["*.txt"] + ) + extractor.extract_all() + + assert (dest_dir / "file1.txt").exists() + assert (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + + +def test_exclude_patterns(temp_zip_file: Path, tmp_path: Path) -> None: + """Files matching exclude_patterns should be skipped.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor( + temp_zip_file, dest_dir, verbose=False, exclude_patterns=["file2.txt"] + ) + extractor.extract_all() + + assert (dest_dir / "file1.txt").exists() + assert not (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + + +def test_include_and_exclude_patterns(temp_zip_file: Path, tmp_path: Path) -> None: + """Exclude should override include if both match.""" + dest_dir = tmp_path / "extracted" + extractor = SafeZipExtractor( + temp_zip_file, + dest_dir, + verbose=False, + include_patterns=["*.txt"], + exclude_patterns=["file1.txt"], + ) + extractor.extract_all() + + assert not (dest_dir / "file1.txt").exists() + assert (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + + +def test_skip_on_error(monkeypatch, temp_zip_file: Path, tmp_path: Path) -> None: + """Should skip a file that raises an error and continue extracting others.""" + dest_dir = tmp_path / "extracted" + + original_open = zipfile.ZipFile.open + + def failing_open(self, member, *args, **kwargs): + if member.filename == "file2.txt": + raise IOError("Simulated failure") + return original_open(self, member, *args, **kwargs) + + # Patch the method on the class, not on an instance + monkeypatch.setattr(zipfile.ZipFile, "open", failing_open) + + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False) + extractor.extract_all() + + assert (dest_dir / "file1.txt").exists() + assert not (dest_dir / "file2.txt").exists() + assert (dest_dir / "folder" / "file3.txt").exists() + +@pytest.mark.skipif(not sys.platform.startswith("win"), reason="Windows-only test") +def test_long_path_normalization(temp_zip_file: Path, tmp_path: Path) -> None: + """Ensure _normalize_path adds \\?\ prefix on Windows.""" + dest_dir = tmp_path / ("a" * 250) # Simulate long path + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False) + norm_path = extractor._normalize_path(dest_dir / "file.txt") + assert str(norm_path).startswith("\\\\?\\") From e5641de0ff09c2b7ab7aac9514040550374b0bf0 Mon Sep 17 00:00:00 2001 From: "jetbrains-junie[bot]" <201638009+jetbrains-junie[bot]@users.noreply.github.com> Date: Thu, 7 Aug 2025 12:48:04 +0200 Subject: [PATCH 25/27] feat(junie): added .junie workflow (#430) Co-authored-by: jetbrains-junie[bot] <201638009+jetbrains-junie[bot]@users.noreply.github.com> --- .github/workflows/junie.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/junie.yml diff --git a/.github/workflows/junie.yml b/.github/workflows/junie.yml new file mode 100644 index 00000000..9860e79e --- /dev/null +++ b/.github/workflows/junie.yml @@ -0,0 +1,21 @@ +name: Junie +run-name: Junie run ${{ inputs.run_id }} + +permissions: + contents: write + +on: + workflow_dispatch: + inputs: + run_id: + description: "id of workflow process" + required: true + workflow_params: + description: "stringified params" + required: true + +jobs: + call-workflow-passing-data: + uses: jetbrains-junie/junie-workflows/.github/workflows/ej-issue.yml@main + with: + workflow_params: ${{ inputs.workflow_params }} From 0f4942900284a9560e1354af6f2c4a53f2442356 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 7 Aug 2025 12:55:11 +0200 Subject: [PATCH 26/27] Formatting --- .../language_servers/csharp_language_server.py | 6 +----- src/solidlsp/util/zip.py | 11 ++++++----- test/solidlsp/util/test_zip.py | 16 ++++++---------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/solidlsp/language_servers/csharp_language_server.py b/src/solidlsp/language_servers/csharp_language_server.py index b8f0cc38..da65cb83 100644 --- a/src/solidlsp/language_servers/csharp_language_server.py +++ b/src/solidlsp/language_servers/csharp_language_server.py @@ -388,11 +388,7 @@ class CSharpLanguageServer(SolidLanguageServer): package_extract_dir.mkdir(exist_ok=True) # Use SafeZipExtractor to handle long paths and skip errors - extractor = SafeZipExtractor( - archive_path=nupkg_file, - extract_dir=package_extract_dir, - verbose=False - ) + extractor = SafeZipExtractor(archive_path=nupkg_file, extract_dir=package_extract_dir, verbose=False) extractor.extract_all() # Clean up the nupkg file diff --git a/src/solidlsp/util/zip.py b/src/solidlsp/util/zip.py index ae0c3183..d93d9421 100644 --- a/src/solidlsp/util/zip.py +++ b/src/solidlsp/util/zip.py @@ -1,13 +1,14 @@ +import fnmatch import logging import os import sys import zipfile from pathlib import Path -from typing import Optional, List -import fnmatch +from typing import Optional log = logging.getLogger(__name__) + class SafeZipExtractor: """ A utility class for extracting ZIP archives safely. @@ -24,8 +25,8 @@ class SafeZipExtractor: archive_path: Path, extract_dir: Path, verbose: bool = True, - include_patterns: Optional[List[str]] = None, - exclude_patterns: Optional[List[str]] = None + include_patterns: Optional[list[str]] = None, + exclude_patterns: Optional[list[str]] = None, ) -> None: """ Initialize the SafeZipExtractor. @@ -113,7 +114,7 @@ class SafeZipExtractor: :return: Normalized path """ if sys.platform.startswith("win"): - return Path(r"\\?\{}".format(os.path.abspath(path))) + return Path(rf"\\?\{os.path.abspath(path)}") return path diff --git a/test/solidlsp/util/test_zip.py b/test/solidlsp/util/test_zip.py index 4498781a..5fa545ba 100644 --- a/test/solidlsp/util/test_zip.py +++ b/test/solidlsp/util/test_zip.py @@ -1,8 +1,7 @@ -import io -import os import sys import zipfile from pathlib import Path + import pytest from solidlsp.util.zip import SafeZipExtractor @@ -33,9 +32,7 @@ def test_extract_all_success(temp_zip_file: Path, tmp_path: Path) -> None: def test_include_patterns(temp_zip_file: Path, tmp_path: Path) -> None: """Only files matching include_patterns should be extracted.""" dest_dir = tmp_path / "extracted" - extractor = SafeZipExtractor( - temp_zip_file, dest_dir, verbose=False, include_patterns=["*.txt"] - ) + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False, include_patterns=["*.txt"]) extractor.extract_all() assert (dest_dir / "file1.txt").exists() @@ -46,9 +43,7 @@ def test_include_patterns(temp_zip_file: Path, tmp_path: Path) -> None: def test_exclude_patterns(temp_zip_file: Path, tmp_path: Path) -> None: """Files matching exclude_patterns should be skipped.""" dest_dir = tmp_path / "extracted" - extractor = SafeZipExtractor( - temp_zip_file, dest_dir, verbose=False, exclude_patterns=["file2.txt"] - ) + extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False, exclude_patterns=["file2.txt"]) extractor.extract_all() assert (dest_dir / "file1.txt").exists() @@ -81,7 +76,7 @@ def test_skip_on_error(monkeypatch, temp_zip_file: Path, tmp_path: Path) -> None def failing_open(self, member, *args, **kwargs): if member.filename == "file2.txt": - raise IOError("Simulated failure") + raise OSError("Simulated failure") return original_open(self, member, *args, **kwargs) # Patch the method on the class, not on an instance @@ -94,9 +89,10 @@ def test_skip_on_error(monkeypatch, temp_zip_file: Path, tmp_path: Path) -> None assert not (dest_dir / "file2.txt").exists() assert (dest_dir / "folder" / "file3.txt").exists() + @pytest.mark.skipif(not sys.platform.startswith("win"), reason="Windows-only test") def test_long_path_normalization(temp_zip_file: Path, tmp_path: Path) -> None: - """Ensure _normalize_path adds \\?\ prefix on Windows.""" + r"""Ensure _normalize_path adds \\?\\ prefix on Windows.""" dest_dir = tmp_path / ("a" * 250) # Simulate long path extractor = SafeZipExtractor(temp_zip_file, dest_dir, verbose=False) norm_path = extractor._normalize_path(dest_dir / "file.txt") From c1d77012233d4f8b3e48f696c448008e9652b0e3 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 7 Aug 2025 12:57:46 +0200 Subject: [PATCH 27/27] Add basic log level detection to stderr stream reader --- src/solidlsp/ls_handler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/solidlsp/ls_handler.py b/src/solidlsp/ls_handler.py index 6a994ed3..c1f50fa1 100644 --- a/src/solidlsp/ls_handler.py +++ b/src/solidlsp/ls_handler.py @@ -370,7 +370,13 @@ class SolidLanguageServerHandler: line = self.process.stderr.readline() if not line: continue - log.error(line.decode(ENCODING, errors="replace")) + line = line.decode(ENCODING, errors="replace") + line_lower = line.lower() + if "error" in line_lower or "exception" in line_lower or line.startswith("E["): + level = logging.ERROR + else: + level = logging.INFO + log.log(level, line) except Exception as e: log.error("Error while reading stderr from language server process: %s", e, exc_info=e) if not self._is_shutting_down: