From e523c4225986ac1d741af975cf49d5f74f48c741 Mon Sep 17 00:00:00 2001 From: Akira TSURUDA Date: Sat, 11 Oct 2025 17:36:35 +0900 Subject: [PATCH 01/10] feat: add rbenv integration for Ruby LSP server (#628) * feat: add rbenv integration for Ruby LSP server Add support for rbenv Ruby version management in the Ruby LSP language server. This allows Serena to respect project-specific .ruby-version files and use the appropriate Ruby and bundler versions as specified by rbenv. --- src/solidlsp/language_servers/ruby_lsp.py | 51 +++++++++++++++++++---- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/src/solidlsp/language_servers/ruby_lsp.py b/src/solidlsp/language_servers/ruby_lsp.py index 0046c7f5..0bf2c207 100644 --- a/src/solidlsp/language_servers/ruby_lsp.py +++ b/src/solidlsp/language_servers/ruby_lsp.py @@ -103,9 +103,44 @@ class RubyLsp(SolidLanguageServer): Setup runtime dependencies for ruby-lsp and return the command list to start the server. Installation strategy: Bundler project > global ruby-lsp > gem install ruby-lsp """ + # Detect rbenv-managed Ruby environment + # When .ruby-version exists, it indicates the project uses rbenv for version management. + # rbenv automatically reads .ruby-version to determine which Ruby version to use. + # Using "rbenv exec" ensures commands run with the correct Ruby version and its gems. + # + # Why rbenv is preferred over system Ruby: + # - Respects project-specific Ruby versions + # - Avoids bundler version mismatches between system and project + # - Ensures consistent environment across developers + # + # Fallback behavior: + # If .ruby-version doesn't exist or rbenv isn't installed, we fall back to system Ruby. + # This may cause issues if: + # - System Ruby version differs from what the project expects + # - System bundler version is incompatible with Gemfile.lock + # - Project gems aren't installed in system Ruby + ruby_version_file = os.path.join(repository_root_path, ".ruby-version") + use_rbenv = os.path.exists(ruby_version_file) and shutil.which("rbenv") is not None + + if use_rbenv: + ruby_cmd = ["rbenv", "exec", "ruby"] + bundle_cmd = ["rbenv", "exec", "bundle"] + logger.log(f"Using rbenv-managed Ruby (found {ruby_version_file})", logging.INFO) + else: + ruby_cmd = ["ruby"] + bundle_cmd = ["bundle"] + if os.path.exists(ruby_version_file): + logger.log( + f"Found {ruby_version_file} but rbenv is not installed. " + "Using system Ruby. Consider installing rbenv for better version management: https://github.com/rbenv/rbenv", + logging.WARNING, + ) + else: + logger.log("No .ruby-version file found, using system Ruby", logging.INFO) + # Check if Ruby is installed try: - result = subprocess.run(["ruby", "--version"], check=True, capture_output=True, cwd=repository_root_path, text=True) + result = subprocess.run(ruby_cmd + ["--version"], check=True, capture_output=True, cwd=repository_root_path, text=True) ruby_version = result.stdout.strip() logger.log(f"Ruby version: {ruby_version}", logging.INFO) @@ -141,16 +176,16 @@ class RubyLsp(SolidLanguageServer): logger.log("Detected Bundler project (Gemfile found)", logging.INFO) # Check if bundle command is available using Windows-compatible search - bundle_path = RubyLsp._find_executable_with_extensions("bundle") + bundle_path = RubyLsp._find_executable_with_extensions(bundle_cmd[0] if len(bundle_cmd) == 1 else "bundle") if not bundle_path: # Try common bundle executables - for bundle_cmd in ["bin/bundle", "bundle"]: - if bundle_cmd.startswith("bin/"): - bundle_full_path = os.path.join(repository_root_path, bundle_cmd) + for bundle_executable in ["bin/bundle", "bundle"]: + if bundle_executable.startswith("bin/"): + bundle_full_path = os.path.join(repository_root_path, bundle_executable) else: - bundle_full_path = RubyLsp._find_executable_with_extensions(bundle_cmd) + bundle_full_path = RubyLsp._find_executable_with_extensions(bundle_executable) if bundle_full_path and os.path.exists(bundle_full_path): - bundle_path = bundle_full_path if bundle_cmd.startswith("bin/") else bundle_cmd + bundle_path = bundle_full_path if bundle_executable.startswith("bin/") else bundle_executable break if not bundle_path: @@ -171,7 +206,7 @@ class RubyLsp(SolidLanguageServer): if ruby_lsp_in_bundle: logger.log("Found ruby-lsp in Gemfile.lock", logging.INFO) - return [bundle_path, "exec", "ruby-lsp"] + return bundle_cmd + ["exec", "ruby-lsp"] else: logger.log( "ruby-lsp not found in Gemfile.lock. Consider adding 'gem \"ruby-lsp\"' to your Gemfile for better compatibility.", From dd239b9f988c7bb7ef545a8ac98c0b58df118ade Mon Sep 17 00:00:00 2001 From: intoffset <4951782+intoffset@users.noreply.github.com> Date: Sun, 12 Oct 2025 05:36:37 +0900 Subject: [PATCH 02/10] Add Markdown language server support with marksman (#655) --- README.md | 1 + pyproject.toml | 1 + src/solidlsp/language_servers/common.py | 2 +- src/solidlsp/language_servers/marksman.py | 193 ++++++++++++++++++ src/solidlsp/ls_config.py | 13 +- src/solidlsp/ls_utils.py | 3 + .../repos/markdown/test_repo/CONTRIBUTING.md | 74 +++++++ .../repos/markdown/test_repo/README.md | 53 +++++ .../resources/repos/markdown/test_repo/api.md | 78 +++++++ .../repos/markdown/test_repo/guide.md | 59 ++++++ test/solidlsp/markdown/__init__.py | 1 + test/solidlsp/markdown/test_markdown_basic.py | 65 ++++++ 12 files changed, 541 insertions(+), 2 deletions(-) create mode 100644 src/solidlsp/language_servers/marksman.py create mode 100644 test/resources/repos/markdown/test_repo/CONTRIBUTING.md create mode 100644 test/resources/repos/markdown/test_repo/README.md create mode 100644 test/resources/repos/markdown/test_repo/api.md create mode 100644 test/resources/repos/markdown/test_repo/guide.md create mode 100644 test/solidlsp/markdown/__init__.py create mode 100644 test/solidlsp/markdown/test_markdown_basic.py diff --git a/README.md b/README.md index 114c5f64..eee4c4df 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ With Serena, we provide direct, out-of-the-box support for: * Elixir (requires installation of NextLS and Elixir; **Windows not supported**) * Erlang (requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls), experimental, might be slow or hang) * AL + * Markdown (automatically downloads marksman if not installed, experimental, must be explicitly specified via `--language markdown` when generating project config - primarily useful for documentation-heavy projects) Support for further languages can easily be added by providing a shallow adapter for a new language server implementation, see Serena's [memory on that](.serena/memories/adding_new_language_support_guide.md). diff --git a/pyproject.toml b/pyproject.toml index 95f13566..f82a1f9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -273,6 +273,7 @@ markers = [ "dart: language server running for Dart", "erlang: language server running for Erlang", "al: language server running for AL (Microsoft Dynamics 365 Business Central)", + "markdown: language server running for Markdown", ] [tool.codespell] diff --git a/src/solidlsp/language_servers/common.py b/src/solidlsp/language_servers/common.py index a6bcbb3e..8505141e 100644 --- a/src/solidlsp/language_servers/common.py +++ b/src/solidlsp/language_servers/common.py @@ -133,7 +133,7 @@ class RuntimeDependencyCollection: @staticmethod def _install_from_url(dep: RuntimeDependency, logger: LanguageServerLogger, target_dir: str) -> None: assert dep.url is not None - if dep.archive_type == "gz" and dep.binary_name: + if dep.archive_type in ("gz", "binary") and dep.binary_name: dest = os.path.join(target_dir, dep.binary_name) FileUtils.download_and_extract_archive(logger, dep.url, dest, dep.archive_type) else: diff --git a/src/solidlsp/language_servers/marksman.py b/src/solidlsp/language_servers/marksman.py new file mode 100644 index 00000000..c35d2ca6 --- /dev/null +++ b/src/solidlsp/language_servers/marksman.py @@ -0,0 +1,193 @@ +""" +Provides Markdown specific instantiation of the LanguageServer class using marksman. +Contains various configurations and settings specific to Markdown. +""" + +import logging +import os +import pathlib +import threading + +from overrides import override + +from solidlsp.ls import SolidLanguageServer +from solidlsp.ls_config import LanguageServerConfig +from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams +from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo +from solidlsp.settings import SolidLSPSettings + +from .common import RuntimeDependency, RuntimeDependencyCollection + + +class Marksman(SolidLanguageServer): + """ + Provides Markdown specific instantiation of the LanguageServer class using marksman. + """ + + marksman_releases = "https://github.com/artempyanykh/marksman/releases/download/2024-12-18" + runtime_dependencies = RuntimeDependencyCollection( + [ + RuntimeDependency( + id="marksman", + url=f"{marksman_releases}/marksman-linux-x64", + platform_id="linux-x64", + archive_type="binary", + binary_name="marksman", + ), + RuntimeDependency( + id="marksman", + url=f"{marksman_releases}/marksman-linux-arm64", + platform_id="linux-arm64", + archive_type="binary", + binary_name="marksman", + ), + RuntimeDependency( + id="marksman", + url=f"{marksman_releases}/marksman-macos", + platform_id="osx-x64", + archive_type="binary", + binary_name="marksman", + ), + RuntimeDependency( + id="marksman", + url=f"{marksman_releases}/marksman-macos", + platform_id="osx-arm64", + archive_type="binary", + binary_name="marksman", + ), + RuntimeDependency( + id="marksman", + url=f"{marksman_releases}/marksman.exe", + platform_id="win-x64", + archive_type="binary", + binary_name="marksman.exe", + ), + ] + ) + + @classmethod + def _setup_runtime_dependencies( + cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings + ) -> str: + """Setup runtime dependencies for marksman and return the command to start the server.""" + deps = cls.runtime_dependencies + dependency = deps.get_single_dep_for_current_platform() + + marksman_ls_dir = cls.ls_resources_dir(solidlsp_settings) + marksman_executable_path = deps.binary_path(marksman_ls_dir) + if not os.path.exists(marksman_executable_path): + logger.log( + f"Downloading marksman from {dependency.url} to {marksman_ls_dir}", + logging.INFO, + ) + deps.install(logger, marksman_ls_dir) + if not os.path.exists(marksman_executable_path): + raise FileNotFoundError(f"Download failed? Could not find marksman executable at {marksman_executable_path}") + os.chmod(marksman_executable_path, 0o755) + return marksman_executable_path + + def __init__( + self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings + ): + """ + Creates a Marksman instance. This class is not meant to be instantiated directly. + Use LanguageServer.create() instead. + """ + marksman_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings) + + super().__init__( + config, + logger, + repository_root_path, + ProcessLaunchInfo(cmd=f"{marksman_executable_path} server", cwd=repository_root_path), + "markdown", + solidlsp_settings, + ) + self.server_ready = threading.Event() + + @override + def is_ignored_dirname(self, dirname: str) -> bool: + return super().is_ignored_dirname(dirname) or dirname in ["node_modules", ".obsidian", ".vitepress", ".vuepress"] + + @staticmethod + def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: + """ + Returns the initialize params for the Marksman Language Server. + """ + root_uri = pathlib.Path(repository_absolute_path).as_uri() + initialize_params: InitializeParams = { # type: ignore + "processId": os.getpid(), + "locale": "en", + "rootPath": repository_absolute_path, + "rootUri": root_uri, + "capabilities": { + "textDocument": { + "synchronization": {"didSave": True, "dynamicRegistration": True}, + "completion": {"dynamicRegistration": True, "completionItem": {"snippetSupport": True}}, + "definition": {"dynamicRegistration": True}, + "references": {"dynamicRegistration": True}, + "documentSymbol": { + "dynamicRegistration": True, + "hierarchicalDocumentSymbolSupport": True, + "symbolKind": {"valueSet": list(range(1, 27))}, + }, + "hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]}, + "codeAction": {"dynamicRegistration": True}, + }, + "workspace": { + "workspaceFolders": True, + "didChangeConfiguration": {"dynamicRegistration": True}, + "symbol": {"dynamicRegistration": True}, + }, + }, + "workspaceFolders": [ + { + "uri": root_uri, + "name": os.path.basename(repository_absolute_path), + } + ], + } + return initialize_params + + def _start_server(self): + """ + Starts the Marksman Language Server and waits for it to be ready. + """ + + def register_capability_handler(_params): + return + + def window_log_message(msg): + self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO) + + def do_nothing(_params): + return + + self.server.on_request("client/registerCapability", register_capability_handler) + self.server.on_notification("window/logMessage", window_log_message) + self.server.on_notification("$/progress", do_nothing) + self.server.on_notification("textDocument/publishDiagnostics", do_nothing) + + self.logger.log("Starting marksman server process", logging.INFO) + self.server.start() + initialize_params = self._get_initialize_params(self.repository_root_path) + + self.logger.log( + "Sending initialize request from LSP client to marksman server and awaiting response", + logging.INFO, + ) + init_response = self.server.send.initialize(initialize_params) + self.logger.log(f"Received initialize response from marksman server: {init_response}", logging.DEBUG) + + # Verify server capabilities + assert "textDocumentSync" in init_response["capabilities"] + assert "completionProvider" in init_response["capabilities"] + assert "definitionProvider" in init_response["capabilities"] + + self.server.notify.initialized({}) + + # marksman is typically ready immediately after initialization + self.logger.log("Marksman server initialization complete", logging.INFO) + self.server_ready.set() + self.completions_available.set() diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index 4e51c1c3..11c4e4ff 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -66,6 +66,11 @@ class Language(str, Enum): """Solargraph language server for Ruby (legacy, experimental). Use Language.RUBY (ruby-lsp) for better performance and modern LSP features. """ + MARKDOWN = "markdown" + """Marksman language server for Markdown (experimental). + Must be explicitly specified as the main language, not auto-detected. + This is an edge case primarily useful when working on documentation-heavy projects. + """ @classmethod def iter_all(cls, include_experimental: bool = False) -> Iterable[Self]: @@ -77,7 +82,7 @@ class Language(str, Enum): """ Check if the language server is experimental or deprecated. """ - return self in {self.TYPESCRIPT_VTS, self.PYTHON_JEDI, self.CSHARP_OMNISHARP, self.RUBY_SOLARGRAPH} + return self in {self.TYPESCRIPT_VTS, self.PYTHON_JEDI, self.CSHARP_OMNISHARP, self.RUBY_SOLARGRAPH, self.MARKDOWN} def __str__(self) -> str: return self.value @@ -136,6 +141,8 @@ class Language(str, Enum): return FilenameMatcher("*.erl", "*.hrl", "*.escript", "*.config", "*.app", "*.app.src") case self.AL: return FilenameMatcher("*.al", "*.dal") + case self.MARKDOWN: + return FilenameMatcher("*.md", "*.markdown") case _: raise ValueError(f"Unhandled language: {self}") @@ -241,6 +248,10 @@ class Language(str, Enum): from solidlsp.language_servers.al_language_server import ALLanguageServer return ALLanguageServer + case self.MARKDOWN: + from solidlsp.language_servers.marksman import Marksman + + return Marksman case self.R: from solidlsp.language_servers.r_language_server import RLanguageServer diff --git a/src/solidlsp/ls_utils.py b/src/solidlsp/ls_utils.py index 0202dbc0..11fe88f3 100644 --- a/src/solidlsp/ls_utils.py +++ b/src/solidlsp/ls_utils.py @@ -233,6 +233,9 @@ class FileUtils: elif archive_type == "gz": with gzip.open(tmp_file_name, "rb") as f_in, open(target_path, "wb") as f_out: shutil.copyfileobj(f_in, f_out) + elif archive_type == "binary": + # For single binary files, just move to target without extraction + shutil.move(tmp_file_name, target_path) else: logger.log(f"Unknown archive type '{archive_type}' for extraction", logging.ERROR) raise SolidLSPException(f"Unknown archive type '{archive_type}'") diff --git a/test/resources/repos/markdown/test_repo/CONTRIBUTING.md b/test/resources/repos/markdown/test_repo/CONTRIBUTING.md new file mode 100644 index 00000000..2b533556 --- /dev/null +++ b/test/resources/repos/markdown/test_repo/CONTRIBUTING.md @@ -0,0 +1,74 @@ +# Contributing Guidelines + +Thank you for considering contributing to this project! + +## Table of Contents + +- [Code of Conduct](#code-of-conduct) +- [Getting Started](#getting-started) +- [Development Setup](#development-setup) +- [Submitting Changes](#submitting-changes) + +## Code of Conduct + +Please be respectful and considerate in all interactions. + +## Getting Started + +To contribute: + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Submit a pull request + +## Development Setup + +### Prerequisites + +- Git +- Node.js (v16+) +- npm or yarn + +### Installation Steps + +```bash +git clone https://github.com/example/repo.git +cd repo +npm install +``` + +## Submitting Changes + +### Pull Request Process + +1. Update documentation +2. Add tests for new features +3. Ensure all tests pass +4. Update the [README](README.md) + +### Commit Messages + +Use clear and descriptive commit messages: + +- feat: Add new feature +- fix: Bug fix +- docs: Documentation changes +- test: Add or update tests + +## Testing + +Run the test suite before submitting: + +```bash +npm test +``` + +For more information, see: + +- [User Guide](guide.md) +- [API Reference](api.md) + +## Questions? + +Contact the maintainers or open an issue. diff --git a/test/resources/repos/markdown/test_repo/README.md b/test/resources/repos/markdown/test_repo/README.md new file mode 100644 index 00000000..fbf9a5dc --- /dev/null +++ b/test/resources/repos/markdown/test_repo/README.md @@ -0,0 +1,53 @@ +# Test Repository + +This is a test repository for markdown language server testing. + +## Overview + +This repository contains sample markdown files for testing LSP features. + +## Features + +- Document symbol detection +- Link navigation +- Reference finding +- Code completion + +### Installation + +To use this test repository: + +1. Clone the repository +2. Install dependencies +3. Run tests + +### Usage + +See [guide.md](guide.md) for detailed usage instructions. + +## Code Examples + +Here's a simple example: + +```python +def hello_world(): + print("Hello, World!") +``` + +### JavaScript Example + +```javascript +function greet(name) { + console.log(`Hello, ${name}!`); +} +``` + +## References + +- [Official Documentation](https://example.com/docs) +- [API Reference](api.md) +- [Contributing Guide](CONTRIBUTING.md) + +## License + +MIT License diff --git a/test/resources/repos/markdown/test_repo/api.md b/test/resources/repos/markdown/test_repo/api.md new file mode 100644 index 00000000..81d0e8af --- /dev/null +++ b/test/resources/repos/markdown/test_repo/api.md @@ -0,0 +1,78 @@ +# API Reference + +Complete API documentation for the test repository. + +## Classes + +### Client + +The main client class for interacting with the API. + +#### Methods + +##### `connect()` + +Establishes a connection to the server. + +**Parameters:** +- `host`: Server hostname +- `port`: Server port number + +**Returns:** Connection object + +##### `disconnect()` + +Closes the connection to the server. + +**Returns:** None + +### Server + +Server-side implementation. + +#### Configuration + +```json +{ + "host": "localhost", + "port": 8080, + "timeout": 30 +} +``` + +## Functions + +### `initialize(config)` + +Initializes the system with the provided configuration. + +**Parameters:** +- `config`: Configuration dictionary + +**Example:** + +```python +config = { + "host": "localhost", + "port": 8080 +} +initialize(config) +``` + +### `shutdown()` + +Gracefully shuts down the system. + +## Error Handling + +Common errors and their solutions: + +- `ConnectionError`: Check network connectivity +- `TimeoutError`: Increase timeout value +- `ConfigError`: Validate configuration file + +## See Also + +- [User Guide](guide.md) +- [README](README.md) +- [Contributing](CONTRIBUTING.md) diff --git a/test/resources/repos/markdown/test_repo/guide.md b/test/resources/repos/markdown/test_repo/guide.md new file mode 100644 index 00000000..025f4302 --- /dev/null +++ b/test/resources/repos/markdown/test_repo/guide.md @@ -0,0 +1,59 @@ +# User Guide + +This guide provides detailed instructions for using the test repository. + +## Getting Started + +Welcome to the user guide. This document covers: + +- Basic concepts +- Advanced features +- Troubleshooting + +### Basic Concepts + +The fundamental concepts you need to understand: + +#### Headers and Structure + +Markdown documents use headers to create structure. Headers are created using `#` symbols. + +#### Links and References + +Internal links can reference other documents: + +- [Back to README](README.md) +- [See API documentation](api.md) + +### Advanced Features + +For advanced users, we provide: + +1. Custom extensions +2. Plugin support +3. Theme customization + +## Configuration + +Configuration options are stored in `config.yaml`: + +```yaml +server: + port: 8080 + host: localhost +``` + +## Troubleshooting + +If you encounter issues: + +1. Check the [README](README.md) first +2. Review [common issues](CONTRIBUTING.md) +3. Contact support + +## Next Steps + +After reading this guide, check out: + +- [API Reference](api.md) +- [Contributing Guidelines](CONTRIBUTING.md) diff --git a/test/solidlsp/markdown/__init__.py b/test/solidlsp/markdown/__init__.py new file mode 100644 index 00000000..9f2adc19 --- /dev/null +++ b/test/solidlsp/markdown/__init__.py @@ -0,0 +1 @@ +"""Tests for markdown language server functionality.""" diff --git a/test/solidlsp/markdown/test_markdown_basic.py b/test/solidlsp/markdown/test_markdown_basic.py new file mode 100644 index 00000000..a3e8c42e --- /dev/null +++ b/test/solidlsp/markdown/test_markdown_basic.py @@ -0,0 +1,65 @@ +""" +Basic integration tests for the markdown language server functionality. + +These tests validate the functionality of the language server APIs +like request_document_symbols using the markdown test repository. +""" + +import pytest + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language + + +@pytest.mark.markdown +class TestMarkdownLanguageServerBasics: + """Test basic functionality of the markdown language server.""" + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_language_server_initialization(self, language_server: SolidLanguageServer) -> None: + """Test that markdown language server can be initialized successfully.""" + assert language_server is not None + assert language_server.language == Language.MARKDOWN + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_request_document_symbols(self, language_server: SolidLanguageServer) -> None: + """Test request_document_symbols for markdown files.""" + # Test getting symbols from README.md + all_symbols, root_symbols = language_server.request_document_symbols("README.md", include_body=False) + + # Extract heading symbols (LSP Symbol Kind 15 is String, but marksman uses kind 15 for headings) + # Note: Different markdown LSPs may use different symbol kinds for headings + # Marksman typically uses kind 15 (String) for markdown headings + heading_names = [symbol["name"] for symbol in all_symbols] + + # Should detect headings from README.md + assert "Test Repository" in heading_names or len(all_symbols) > 0, "Should find at least one heading" + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_request_symbols_from_guide(self, language_server: SolidLanguageServer) -> None: + """Test symbol detection in guide.md file.""" + all_symbols, root_symbols = language_server.request_document_symbols("guide.md", include_body=False) + + # At least some headings should be found + assert len(all_symbols) > 0, f"Should find headings in guide.md, found {len(all_symbols)}" + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_request_symbols_from_api(self, language_server: SolidLanguageServer) -> None: + """Test symbol detection in api.md file.""" + all_symbols, root_symbols = language_server.request_document_symbols("api.md", include_body=False) + + # Should detect headings from api.md + assert len(all_symbols) > 0, f"Should find headings in api.md, found {len(all_symbols)}" + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: + """Test request_document_symbols with body extraction.""" + # Test with include_body=True + all_symbols, root_symbols = language_server.request_document_symbols("README.md", include_body=True) + + # Should have found some symbols + assert len(all_symbols) > 0, "Should find symbols in README.md" + + # Note: Not all markdown LSPs provide body information for symbols + # This test is more lenient and just verifies the API works + assert all_symbols is not None, "Should return symbols even if body extraction is limited" From 9b622bcfd59827bee8f459fd317a14fb1fa60a42 Mon Sep 17 00:00:00 2001 From: Ryuhei Nakano <57317148+asunaro276@users.noreply.github.com> Date: Sun, 12 Oct 2025 23:07:34 +0900 Subject: [PATCH 03/10] added support for Perl via Perl::LanguageServer (#656) --- .github/workflows/pytest.yml | 17 ++ CHANGELOG.md | 1 + CLAUDE.md | 2 +- README.md | 1 + pyproject.toml | 1 + .../language_servers/perl_language_server.py | 225 ++++++++++++++++++ src/solidlsp/ls_config.py | 7 + src/solidlsp/ls_handler.py | 9 +- test/resources/repos/perl/test_repo/helper.pl | 9 + test/resources/repos/perl/test_repo/main.pl | 22 ++ test/solidlsp/perl/test_perl_basic.py | 76 ++++++ 11 files changed, 368 insertions(+), 2 deletions(-) create mode 100644 src/solidlsp/language_servers/perl_language_server.py create mode 100644 test/resources/repos/perl/test_repo/helper.pl create mode 100644 test/resources/repos/perl/test_repo/main.pl create mode 100644 test/solidlsp/perl/test_perl_basic.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0ab82056..e73b76e9 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -213,6 +213,23 @@ jobs: echo "$LUA_LS_DIR/bin" >> $GITHUB_PATH rm lua-ls.zip fi + - name: Install Perl::LanguageServer + if: runner.os != 'Windows' + shell: bash + run: | + if [[ "${{ runner.os }}" == "Linux" ]]; then + sudo apt-get update + sudo apt-get install -y cpanminus build-essential libanyevent-perl libio-aio-perl + elif [[ "${{ runner.os }}" == "macOS" ]]; then + brew install cpanminus + fi + PERL_MM_USE_DEFAULT=1 cpanm --notest --force Perl::LanguageServer + # Set up Perl local::lib environment for subsequent steps + echo "PERL5LIB=$HOME/perl5/lib/perl5${PERL5LIB:+:${PERL5LIB}}" >> $GITHUB_ENV + echo "PERL_LOCAL_LIB_ROOT=$HOME/perl5${PERL_LOCAL_LIB_ROOT:+:${PERL_LOCAL_LIB_ROOT}}" >> $GITHUB_ENV + echo "PERL_MB_OPT=--install_base \"$HOME/perl5\"" >> $GITHUB_ENV + echo "PERL_MM_OPT=INSTALL_BASE=$HOME/perl5" >> $GITHUB_ENV + echo "$HOME/perl5/bin" >> $GITHUB_PATH - name: Install Nix if: runner.os != 'Windows' # Nix doesn't support Windows natively uses: cachix/install-nix-action@v30 diff --git a/CHANGELOG.md b/CHANGELOG.md index cac7b245..11696a9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ Status of the `main` branch. Changes prior to the next official version change will appear here. * Language support: + * **Add support for Perl** via Perl::LanguageServer with LSP integration for .pl, .pm, and .t files * **Add support for AL (Application Language)** for Microsoft Dynamics 365 Business Central development. Requires VS Code AL extension (ms-dynamics-smb.al). * **Add support for R** via the R languageserver package with LSP integration, performance optimizations, and fallback symbol extraction * **Add support for Zig** via ZLS (cross-file references may not fully work on Windows) diff --git a/CLAUDE.md b/CLAUDE.md index 876c16f7..edf91cd1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co **Test Markers:** Available pytest markers for selective testing: -- `python`, `go`, `java`, `rust`, `typescript`, `php`, `csharp`, `elixir`, `terraform`, `clojure`, `swift`, `bash`, `ruby`, `ruby_solargraph` +- `python`, `go`, `java`, `rust`, `typescript`, `php`, `perl`, `csharp`, `elixir`, `terraform`, `clojure`, `swift`, `bash`, `ruby`, `ruby_solargraph` - `snapshot` - for symbolic editing operation tests **Project Management:** diff --git a/README.md b/README.md index eee4c4df..d1e6c73c 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ With Serena, we provide direct, out-of-the-box support for: * Nix (requires nixd installation) * Elixir (requires installation of NextLS and Elixir; **Windows not supported**) * Erlang (requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls), experimental, might be slow or hang) + * Perl (requires installation of Perl::LanguageServer) * AL * Markdown (automatically downloads marksman if not installed, experimental, must be explicitly specified via `--language markdown` when generating project config - primarily useful for documentation-heavy projects) diff --git a/pyproject.toml b/pyproject.toml index f82a1f9a..c9147dea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -259,6 +259,7 @@ markers = [ "rust: language server running for Rust", "typescript: language server running for TypeScript", "php: language server running for PHP", + "perl: language server running for Perl", "csharp: language server running for C#", "elixir: language server running for Elixir", "terraform: language server running for Terraform", diff --git a/src/solidlsp/language_servers/perl_language_server.py b/src/solidlsp/language_servers/perl_language_server.py new file mode 100644 index 00000000..0552c834 --- /dev/null +++ b/src/solidlsp/language_servers/perl_language_server.py @@ -0,0 +1,225 @@ +""" +Provides Perl specific instantiation of the LanguageServer class using Perl::LanguageServer. + +Note: Windows is not supported as Nix itself doesn't support Windows natively. +""" + +import logging +import os +import pathlib +import subprocess +import time + +from overrides import override + +from solidlsp.ls import SolidLanguageServer +from solidlsp.ls_config import LanguageServerConfig +from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.ls_utils import PlatformId, PlatformUtils +from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams +from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo +from solidlsp.settings import SolidLSPSettings + + +class PerlLanguageServer(SolidLanguageServer): + """ + Provides Perl specific instantiation of the LanguageServer class using Perl::LanguageServer. + """ + + @staticmethod + def _get_perl_version(): + """Get the installed Perl version or None if not found.""" + try: + result = subprocess.run(["perl", "-v"], capture_output=True, text=True, check=False) + if result.returncode == 0: + return result.stdout.strip() + except FileNotFoundError: + return None + return None + + @staticmethod + def _get_perl_language_server_version(): + """Get the installed Perl::LanguageServer version or None if not found.""" + try: + result = subprocess.run( + ["perl", "-MPerl::LanguageServer", "-e", "print $Perl::LanguageServer::VERSION"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + return result.stdout.strip() + except FileNotFoundError: + return None + return None + + @override + def is_ignored_dirname(self, dirname: str) -> bool: + # For Perl projects, we should ignore: + # - blib: build library directory + # - local: local Perl module installation + # - .carton: Carton dependency manager cache + # - vendor: vendored dependencies + # - _build: Module::Build output + return super().is_ignored_dirname(dirname) or dirname in ["blib", "local", ".carton", "vendor", "_build", "cover_db"] + + @classmethod + def _setup_runtime_dependencies(cls) -> str: + """ + Check if required Perl runtime dependencies are available. + Raises RuntimeError with helpful message if dependencies are missing. + """ + platform_id = PlatformUtils.get_platform_id() + + valid_platforms = [ + PlatformId.LINUX_x64, + PlatformId.LINUX_arm64, + PlatformId.OSX, + PlatformId.OSX_x64, + PlatformId.OSX_arm64, + ] + if platform_id not in valid_platforms: + raise RuntimeError(f"Platform {platform_id} is not supported for Perl at the moment") + + perl_version = cls._get_perl_version() + if not perl_version: + raise RuntimeError( + "Perl is not installed. Please install Perl from https://www.perl.org/get.html and make sure it is added to your PATH." + ) + + perl_ls_version = cls._get_perl_language_server_version() + if not perl_ls_version: + raise RuntimeError( + "Found a Perl version but Perl::LanguageServer is not installed.\n" + "Please install Perl::LanguageServer: cpanm Perl::LanguageServer\n" + "See: https://metacpan.org/pod/Perl::LanguageServer" + ) + + return "perl -MPerl::LanguageServer -e 'Perl::LanguageServer::run'" + + def __init__( + self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings + ): + # Setup runtime dependencies before initializing + perl_ls_cmd = self._setup_runtime_dependencies() + + super().__init__( + config, + logger, + repository_root_path, + ProcessLaunchInfo(cmd=perl_ls_cmd, cwd=repository_root_path), + "perl", + solidlsp_settings, + ) + self.request_id = 0 + + @staticmethod + def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: + """ + Returns the initialize params for Perl::LanguageServer. + Based on the expected structure from Perl::LanguageServer::Methods::_rpcreq_initialize. + """ + root_uri = pathlib.Path(repository_absolute_path).as_uri() + initialize_params = { + "processId": os.getpid(), + "rootPath": repository_absolute_path, + "rootUri": root_uri, + "capabilities": { + "textDocument": { + "synchronization": {"didSave": True, "dynamicRegistration": True}, + "definition": {"dynamicRegistration": True}, + "references": {"dynamicRegistration": True}, + "documentSymbol": {"dynamicRegistration": True}, + "hover": {"dynamicRegistration": True}, + }, + "workspace": { + "workspaceFolders": True, + "didChangeConfiguration": {"dynamicRegistration": True}, + "symbol": {"dynamicRegistration": True}, + }, + }, + "initializationOptions": {}, + "workspaceFolders": [ + { + "uri": root_uri, + "name": os.path.basename(repository_absolute_path), + } + ], + } + + return initialize_params + + def _start_server(self): + """Start Perl::LanguageServer process""" + + def register_capability_handler(params): + return + + def window_log_message(msg): + self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO) + + def do_nothing(params): + return + + def workspace_configuration_handler(params): + """Handle workspace/configuration request from Perl::LanguageServer.""" + self.logger.log(f"Received workspace/configuration request: {params}", logging.INFO) + + perl_config = { + "perlInc": [self.repository_root_path, "."], + "fileFilter": [".pm", ".pl"], + "ignoreDirs": [".git", ".svn", "blib", "local", ".carton", "vendor", "_build", "cover_db"], + } + + return [perl_config] + + self.server.on_request("client/registerCapability", register_capability_handler) + self.server.on_request("workspace/configuration", workspace_configuration_handler) + self.server.on_notification("window/logMessage", window_log_message) + self.server.on_notification("$/progress", do_nothing) + self.server.on_notification("textDocument/publishDiagnostics", do_nothing) + + self.logger.log("Starting Perl::LanguageServer process", logging.INFO) + self.server.start() + initialize_params = self._get_initialize_params(self.repository_root_path) + + self.logger.log( + "Sending initialize request from LSP client to LSP server and awaiting response", + logging.INFO, + ) + init_response = self.server.send.initialize(initialize_params) + self.logger.log( + "After sent initialize params", + logging.INFO, + ) + + # Verify server capabilities + assert "textDocumentSync" in init_response["capabilities"] + assert "definitionProvider" in init_response["capabilities"] + assert "referencesProvider" in init_response["capabilities"] + + self.server.notify.initialized({}) + + # Send workspace configuration to Perl::LanguageServer + # Perl::LanguageServer requires didChangeConfiguration to set perlInc, fileFilter, and ignoreDirs + # See: Perl::LanguageServer::Methods::workspace::_rpcnot_didChangeConfiguration + perl_config = { + "settings": { + "perl": { + "perlInc": [self.repository_root_path, "."], + "fileFilter": [".pm", ".pl"], + "ignoreDirs": [".git", ".svn", "blib", "local", ".carton", "vendor", "_build", "cover_db"], + } + } + } + self.logger.log(f"Sending workspace/didChangeConfiguration notification with config: {perl_config}", logging.INFO) + self.server.notify.workspace_did_change_configuration(perl_config) + + self.completions_available.set() + + # Perl::LanguageServer needs time to index files and resolve cross-file references + # Without this delay, requests for definitions/references may return empty results + settling_time = 0.5 + self.logger.log(f"Allowing {settling_time} seconds for Perl::LanguageServer to index files...", logging.INFO) + time.sleep(settling_time) + self.logger.log("Perl::LanguageServer settling period complete", logging.INFO) diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index 11c4e4ff..18f634a7 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -43,6 +43,7 @@ class Language(str, Enum): CPP = "cpp" PHP = "php" R = "r" + PERL = "perl" CLOJURE = "clojure" ELIXIR = "elixir" TERRAFORM = "terraform" @@ -121,6 +122,8 @@ class Language(str, Enum): return FilenameMatcher("*.php") case self.R: return FilenameMatcher("*.R", "*.r", "*.Rmd", "*.Rnw") + case self.PERL: + return FilenameMatcher("*.pl", "*.pm", "*.t") case self.CLOJURE: return FilenameMatcher("*.clj", "*.cljs", "*.cljc", "*.edn") # codespell:ignore edn case self.ELIXIR: @@ -208,6 +211,10 @@ class Language(str, Enum): from solidlsp.language_servers.intelephense import Intelephense return Intelephense + case self.PERL: + from solidlsp.language_servers.perl_language_server import PerlLanguageServer + + return PerlLanguageServer case self.CLOJURE: from solidlsp.language_servers.clojure_lsp import ClojureLSP diff --git a/src/solidlsp/ls_handler.py b/src/solidlsp/ls_handler.py index baa8b860..02363aaf 100644 --- a/src/solidlsp/ls_handler.py +++ b/src/solidlsp/ls_handler.py @@ -517,8 +517,15 @@ class SolidLanguageServerHandler: """ Handle the response received from the server for a request, using the id to determine the request """ + response_id = response["id"] with self._response_handlers_lock: - request = self._pending_requests.pop(response["id"]) + request = self._pending_requests.pop(response_id, None) + if request is None and isinstance(response_id, str) and response_id.isdigit(): + request = self._pending_requests.pop(int(response_id), None) + + if request is None: # need to convert response_id to the right type + log.debug("Request interrupted by user or not found for ID %s", response_id) + return if "result" in response and "error" not in response: request.on_result(response["result"]) diff --git a/test/resources/repos/perl/test_repo/helper.pl b/test/resources/repos/perl/test_repo/helper.pl new file mode 100644 index 00000000..2ee934e6 --- /dev/null +++ b/test/resources/repos/perl/test_repo/helper.pl @@ -0,0 +1,9 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +sub helper_function { + print "Helper function was called.\n"; +} + +1; diff --git a/test/resources/repos/perl/test_repo/main.pl b/test/resources/repos/perl/test_repo/main.pl new file mode 100644 index 00000000..e0e2be7f --- /dev/null +++ b/test/resources/repos/perl/test_repo/main.pl @@ -0,0 +1,22 @@ +#!/usr/bin/env perl +use strict; +use warnings; +use lib '.'; + +require helper; + +sub greet { + my ($name) = @_; + return "Hello, $name!"; +} + +my $user_name = "Perl User"; +my $greeting = greet($user_name); + +print "$greeting\n"; + +helper_function(); + +sub use_helper_function { + helper_function(); +} diff --git a/test/solidlsp/perl/test_perl_basic.py b/test/solidlsp/perl/test_perl_basic.py new file mode 100644 index 00000000..b7b936fb --- /dev/null +++ b/test/solidlsp/perl/test_perl_basic.py @@ -0,0 +1,76 @@ +import platform +from pathlib import Path + +import pytest + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language + + +@pytest.mark.perl +@pytest.mark.skipif(platform.system() == "Windows", reason="Perl::LanguageServer does not support native Windows operation") +class TestPerlLanguageServer: + """ + Tests for Perl::LanguageServer integration. + + Perl::LanguageServer provides comprehensive LSP support for Perl including: + - Document symbols (functions, variables) + - Go to definition (including cross-file) + - Find references (including cross-file) - this was not available in PLS + """ + + @pytest.mark.parametrize("language_server", [Language.PERL], indirect=True) + @pytest.mark.parametrize("repo_path", [Language.PERL], indirect=True) + def test_ls_is_running(self, language_server: SolidLanguageServer, repo_path: Path) -> None: + """Test that the language server starts and stops successfully.""" + # The fixture already handles start and stop + assert language_server.is_running() + assert Path(language_server.language_server.repository_root_path).resolve() == repo_path.resolve() + + @pytest.mark.parametrize("language_server", [Language.PERL], indirect=True) + def test_document_symbols(self, language_server: SolidLanguageServer) -> None: + """Test that document symbols are correctly identified.""" + # Request document symbols + all_symbols, _ = language_server.request_document_symbols("main.pl", include_body=False) + + assert all_symbols, "Expected to find symbols in main.pl" + assert len(all_symbols) > 0, "Expected at least one symbol" + + # DEBUG: Print all symbols + print("\n=== All symbols in main.pl ===") + for s in all_symbols: + line = s.get("range", {}).get("start", {}).get("line", "?") + print(f"Line {line}: {s.get('name')} (kind={s.get('kind')})") + + # Check that we can find function symbols + function_symbols = [s for s in all_symbols if s.get("kind") == 12] # 12 = Function/Method + assert len(function_symbols) >= 2, f"Expected at least 2 functions (greet, use_helper_function), found {len(function_symbols)}" + + function_names = [s.get("name") for s in function_symbols] + assert "greet" in function_names, f"Expected 'greet' function in symbols, found: {function_names}" + assert "use_helper_function" in function_names, f"Expected 'use_helper_function' in symbols, found: {function_names}" + + # @pytest.mark.skip(reason="Perl::LanguageServer cross-file definition tracking needs configuration") + @pytest.mark.parametrize("language_server", [Language.PERL], indirect=True) + def test_find_definition_across_files(self, language_server: SolidLanguageServer) -> None: + definition_location_list = language_server.request_definition("main.pl", 17, 0) + + assert len(definition_location_list) == 1 + definition_location = definition_location_list[0] + print(f"Found definition: {definition_location}") + assert definition_location["uri"].endswith("helper.pl") + assert definition_location["range"]["start"]["line"] == 4 # add method on line 2 (0-indexed 1) + + @pytest.mark.parametrize("language_server", [Language.PERL], indirect=True) + def test_find_references_across_files(self, language_server: SolidLanguageServer) -> None: + """Test finding references to a function across multiple files.""" + reference_locations = language_server.request_references("helper.pl", 4, 5) + + assert len(reference_locations) >= 2, f"Expected at least 2 references to helper_function, found {len(reference_locations)}" + + main_pl_refs = [ref for ref in reference_locations if ref["uri"].endswith("main.pl")] + assert len(main_pl_refs) >= 2, f"Expected at least 2 references in main.pl, found {len(main_pl_refs)}" + + main_pl_lines = sorted([ref["range"]["start"]["line"] for ref in main_pl_refs]) + assert 17 in main_pl_lines, f"Expected reference at line 18 (0-indexed 17), found: {main_pl_lines}" + assert 20 in main_pl_lines, f"Expected reference at line 21 (0-indexed 20), found: {main_pl_lines}" From 21af8bf4517dbd858681d0c6309fec3d3ed76762 Mon Sep 17 00:00:00 2001 From: Michael Panchenko <35432522+MischaPanch@users.noreply.github.com> Date: Mon, 13 Oct 2025 11:37:17 +0200 Subject: [PATCH 04/10] Removed LinesRead and mark_file_modified (#666) --- src/serena/agent.py | 23 ------------------- src/serena/code_editor.py | 3 --- src/serena/tools/file_tools.py | 4 ---- src/serena/tools/tools_base.py | 7 +----- .../custom_test/advanced_features.py | 2 +- test/solidlsp/al/test_al_basic.py | 6 ++--- test/solidlsp/bash/test_bash_basic.py | 10 ++++---- test/solidlsp/markdown/test_markdown_basic.py | 8 +++---- test/solidlsp/r/test_r_basic.py | 2 +- .../ruby/test_ruby_symbol_retrieval.py | 8 +++---- 10 files changed, 19 insertions(+), 54 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 90be1420..d8a491dd 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -8,7 +8,6 @@ import platform import sys import threading import webbrowser -from collections import defaultdict from collections.abc import Callable from concurrent.futures import Future, ThreadPoolExecutor from logging import Logger @@ -44,22 +43,6 @@ class ProjectNotFoundError(Exception): pass -class LinesRead: - def __init__(self) -> None: - self.files: dict[str, set[tuple[int, int]]] = defaultdict(lambda: set()) - - def add_lines_read(self, relative_path: str, lines: tuple[int, int]) -> None: - self.files[relative_path].add(lines) - - def were_lines_read(self, relative_path: str, lines: tuple[int, int]) -> bool: - lines_read_in_file = self.files[relative_path] - return lines in lines_read_in_file - - def invalidate_lines_read(self, relative_path: str) -> None: - if relative_path in self.files: - del self.files[relative_path] - - class MemoriesManager: def __init__(self, project_root: str): self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" @@ -139,7 +122,6 @@ class SerenaAgent: self._active_project: Project | None = None self.language_server: SolidLanguageServer | None = None self.memories_manager: MemoriesManager | None = None - self.lines_read: LinesRead | None = None # adjust log level serena_log_level = self.serena_config.log_level @@ -444,7 +426,6 @@ class SerenaAgent: # initialize project-specific instances which do not depend on the language server self.memories_manager = MemoriesManager(project.project_root) - self.lines_read = LinesRead() def init_language_server() -> None: # start the language server @@ -604,10 +585,6 @@ class SerenaAgent: def print_tool_overview(self) -> None: ToolRegistry().print_tool_overview(self._active_tools.values()) - def mark_file_modified(self, relative_path: str) -> None: - assert self.lines_read is not None - self.lines_read.invalidate_lines_read(relative_path) - def __del__(self) -> None: """ Destructor to clean up the language server instance and GUI logger diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index 04e3c127..6fe41da1 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -60,9 +60,6 @@ class CodeEditor(Generic[TSymbol], ABC): abs_path = os.path.join(self.project_root, relative_path) with open(abs_path, "w", encoding="utf-8") as f: f.write(edited_file.get_contents()) - # notify agent (if provided) - if self.agent is not None: - self.agent.mark_file_modified(relative_path) @abstractmethod def _find_unique_symbol(self, name_path: str, relative_file_path: str) -> TSymbol: diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index d521512f..56d2cf94 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -43,7 +43,6 @@ class ReadFileTool(Tool): if end_line is None: result_lines = result_lines[start_line:] else: - self.lines_read.add_lines_read(relative_path, (start_line, end_line)) result_lines = result_lines[start_line : end_line + 1] result = "\n".join(result_lines) @@ -228,9 +227,6 @@ class DeleteLinesTool(Tool, ToolMarkerCanEdit, ToolMarkerOptional): :param start_line: the 0-based index of the first line to be deleted :param end_line: the 0-based index of the last line to be deleted """ - if not self.lines_read.were_lines_read(relative_path, (start_line, end_line)): - read_lines_tool = self.agent.get_tool(ReadFileTool) - return f"Error: Must call `{read_lines_tool.get_name_from_cls()}` first to read exactly the affected lines." code_editor = self.create_code_editor() code_editor.delete_lines(relative_path, start_line, end_line) return SUCCESS_RESULT diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index a914cf1d..3f2d2c25 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -18,7 +18,7 @@ from serena.util.inspection import iter_subclasses from solidlsp.ls_exceptions import SolidLSPException if TYPE_CHECKING: - from serena.agent import LinesRead, MemoriesManager, SerenaAgent + from serena.agent import MemoriesManager, SerenaAgent from serena.code_editor import CodeEditor log = logging.getLogger(__name__) @@ -64,11 +64,6 @@ class Component(ABC): else: return JetBrainsCodeEditor(project=self.project, agent=self.agent) - @property - def lines_read(self) -> "LinesRead": - assert self.agent.lines_read is not None - return self.agent.lines_read - class ToolMarker: """ diff --git a/test/resources/repos/python/test_repo/custom_test/advanced_features.py b/test/resources/repos/python/test_repo/custom_test/advanced_features.py index 07d4ae4e..b2fa7c9d 100644 --- a/test/resources/repos/python/test_repo/custom_test/advanced_features.py +++ b/test/resources/repos/python/test_repo/custom_test/advanced_features.py @@ -458,7 +458,7 @@ def main() -> None: processor.process(task.name) # Use advanced search - results, total = advanced_search(query="test", filters={"status": Status.PENDING}, sort_by="priority", page=1, include_metadata=True) + _results, _total = advanced_search(query="test", filters={"status": Status.PENDING}, sort_by="priority", page=1, include_metadata=True) # Create a tree root = TreeNode("root") diff --git a/test/solidlsp/al/test_al_basic.py b/test/solidlsp/al/test_al_basic.py index 44687bbc..1fb95e42 100644 --- a/test/solidlsp/al/test_al_basic.py +++ b/test/solidlsp/al/test_al_basic.py @@ -47,7 +47,7 @@ class TestALLanguageServer: # AL tables should have their fields as child symbols customer_table = None - all_symbols, root_symbols = symbols + _all_symbols, root_symbols = symbols for sym in root_symbols: if "TEST Customer" in sym.get("name", ""): customer_table = sym @@ -80,7 +80,7 @@ class TestALLanguageServer: # Find the codeunit symbol - AL returns 'Codeunit 50000 CustomerMgt' codeunit_symbol = None - all_symbols, root_symbols = symbols + _all_symbols, root_symbols = symbols for sym in root_symbols: if "CustomerMgt" in sym.get("name", ""): codeunit_symbol = sym @@ -104,7 +104,7 @@ class TestALLanguageServer: # Find the Customer table symbol customer_symbol = None - all_symbols, root_symbols = symbols + _all_symbols, root_symbols = symbols for sym in root_symbols: if "TEST Customer" in sym.get("name", ""): customer_symbol = sym diff --git a/test/solidlsp/bash/test_bash_basic.py b/test/solidlsp/bash/test_bash_basic.py index ebba71e0..4060f212 100644 --- a/test/solidlsp/bash/test_bash_basic.py +++ b/test/solidlsp/bash/test_bash_basic.py @@ -25,7 +25,7 @@ class TestBashLanguageServerBasics: def test_bash_request_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols for bash files.""" # Test getting symbols from main.sh - all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=False) + all_symbols, _root_symbols = language_server.request_document_symbols("main.sh", include_body=False) # Extract function symbols (LSP Symbol Kind 12) function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] @@ -41,7 +41,7 @@ class TestBashLanguageServerBasics: def test_bash_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols with body extraction.""" # Test with include_body=True - all_symbols, root_symbols = language_server.request_document_symbols("main.sh", include_body=True) + all_symbols, _root_symbols = language_server.request_document_symbols("main.sh", include_body=True) function_symbols = [symbol for symbol in all_symbols if symbol.get("kind") == 12] @@ -58,7 +58,7 @@ class TestBashLanguageServerBasics: def test_bash_utils_functions(self, language_server: SolidLanguageServer) -> None: """Test function detection in utils.sh file.""" # Test with utils.sh as well - utils_all_symbols, utils_root_symbols = language_server.request_document_symbols("utils.sh", include_body=False) + utils_all_symbols, _utils_root_symbols = language_server.request_document_symbols("utils.sh", include_body=False) utils_function_symbols = [symbol for symbol in utils_all_symbols if symbol.get("kind") == 12] utils_function_names = [symbol["name"] for symbol in utils_function_symbols] @@ -84,12 +84,12 @@ class TestBashLanguageServerBasics: def test_bash_function_syntax_patterns(self, language_server: SolidLanguageServer) -> None: """Test that LSP detects different bash function syntax patterns correctly.""" # Test main.sh (has both 'function' keyword and traditional syntax) - main_all_symbols, main_root_symbols = language_server.request_document_symbols("main.sh", include_body=False) + main_all_symbols, _main_root_symbols = language_server.request_document_symbols("main.sh", include_body=False) main_functions = [symbol for symbol in main_all_symbols if symbol.get("kind") == 12] main_function_names = [func["name"] for func in main_functions] # Test utils.sh (all use 'function' keyword) - utils_all_symbols, utils_root_symbols = language_server.request_document_symbols("utils.sh", include_body=False) + utils_all_symbols, _utils_root_symbols = language_server.request_document_symbols("utils.sh", include_body=False) utils_functions = [symbol for symbol in utils_all_symbols if symbol.get("kind") == 12] utils_function_names = [func["name"] for func in utils_functions] diff --git a/test/solidlsp/markdown/test_markdown_basic.py b/test/solidlsp/markdown/test_markdown_basic.py index a3e8c42e..2f472638 100644 --- a/test/solidlsp/markdown/test_markdown_basic.py +++ b/test/solidlsp/markdown/test_markdown_basic.py @@ -25,7 +25,7 @@ class TestMarkdownLanguageServerBasics: def test_markdown_request_document_symbols(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols for markdown files.""" # Test getting symbols from README.md - all_symbols, root_symbols = language_server.request_document_symbols("README.md", include_body=False) + all_symbols, _root_symbols = language_server.request_document_symbols("README.md", include_body=False) # Extract heading symbols (LSP Symbol Kind 15 is String, but marksman uses kind 15 for headings) # Note: Different markdown LSPs may use different symbol kinds for headings @@ -38,7 +38,7 @@ class TestMarkdownLanguageServerBasics: @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) def test_markdown_request_symbols_from_guide(self, language_server: SolidLanguageServer) -> None: """Test symbol detection in guide.md file.""" - all_symbols, root_symbols = language_server.request_document_symbols("guide.md", include_body=False) + all_symbols, _root_symbols = language_server.request_document_symbols("guide.md", include_body=False) # At least some headings should be found assert len(all_symbols) > 0, f"Should find headings in guide.md, found {len(all_symbols)}" @@ -46,7 +46,7 @@ class TestMarkdownLanguageServerBasics: @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) def test_markdown_request_symbols_from_api(self, language_server: SolidLanguageServer) -> None: """Test symbol detection in api.md file.""" - all_symbols, root_symbols = language_server.request_document_symbols("api.md", include_body=False) + all_symbols, _root_symbols = language_server.request_document_symbols("api.md", include_body=False) # Should detect headings from api.md assert len(all_symbols) > 0, f"Should find headings in api.md, found {len(all_symbols)}" @@ -55,7 +55,7 @@ class TestMarkdownLanguageServerBasics: def test_markdown_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: """Test request_document_symbols with body extraction.""" # Test with include_body=True - all_symbols, root_symbols = language_server.request_document_symbols("README.md", include_body=True) + all_symbols, _root_symbols = language_server.request_document_symbols("README.md", include_body=True) # Should have found some symbols assert len(all_symbols) > 0, "Should find symbols in README.md" diff --git a/test/solidlsp/r/test_r_basic.py b/test/solidlsp/r/test_r_basic.py index 56ff345a..fc01d58c 100644 --- a/test/solidlsp/r/test_r_basic.py +++ b/test/solidlsp/r/test_r_basic.py @@ -27,7 +27,7 @@ class TestRLanguageServer: @pytest.mark.parametrize("language_server", [Language.R], indirect=True) def test_symbol_retrieval(self, language_server: SolidLanguageServer): """Test R document symbol extraction.""" - all_symbols, root_symbols = language_server.request_document_symbols(os.path.join("R", "utils.R")) + all_symbols, _root_symbols = language_server.request_document_symbols(os.path.join("R", "utils.R")) # Should find the three exported functions function_symbols = [s for s in all_symbols if s.get("kind") == 12] # Function kind diff --git a/test/solidlsp/ruby/test_ruby_symbol_retrieval.py b/test/solidlsp/ruby/test_ruby_symbol_retrieval.py index d75af32f..9ac4133e 100644 --- a/test/solidlsp/ruby/test_ruby_symbol_retrieval.py +++ b/test/solidlsp/ruby/test_ruby_symbol_retrieval.py @@ -89,7 +89,7 @@ class TestRubyLanguageServerSymbols: """Test request_containing_symbol for a module context.""" # Test that we can find the Services module in document symbols file_path = os.path.join("services.rb") - symbols, roots = language_server.request_document_symbols(file_path) + symbols, _roots = language_server.request_document_symbols(file_path) # Verify Services module appears in document symbols services_module = None @@ -141,7 +141,7 @@ class TestRubyLanguageServerSymbols: # Test referencing symbols for create_user method file_path = os.path.join("services.rb") # Line containing the create_user method definition - symbols, roots = language_server.request_document_symbols(file_path) + symbols, _roots = language_server.request_document_symbols(file_path) create_user_symbol = None # Find create_user method in the document symbols (Ruby returns flat list) @@ -169,7 +169,7 @@ class TestRubyLanguageServerSymbols: # Test referencing symbols for User class file_path = os.path.join("models.rb") # Find User class in document symbols - symbols, roots = language_server.request_document_symbols(file_path) + symbols, _roots = language_server.request_document_symbols(file_path) user_symbol = None for symbol in symbols: @@ -363,7 +363,7 @@ class TestRubyLanguageServerSymbols: # Test referencing symbols for a method parameter in get_user method file_path = os.path.join("services.rb") # Find get_user method and test parameter references - symbols, roots = language_server.request_document_symbols(file_path) + symbols, _roots = language_server.request_document_symbols(file_path) get_user_symbol = None for symbol in symbols: From 675a42128779f11b70778eaa06aa8d941c2d05ca Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Tue, 14 Oct 2025 10:15:53 +0200 Subject: [PATCH 05/10] feat: Elm language support (#667) Also added Node.js setup to GitHub Actions workflow --------- Co-authored-by: Jordan Munch O'Hare --- .github/workflows/pytest.yml | 7 + .gitignore | 4 + CHANGELOG.md | 1 + README.md | 1 + pyproject.toml | 1 + .../language_servers/elm_language_server.py | 189 ++++++++++++++++++ src/solidlsp/ls_config.py | 7 + test/resources/repos/elm/test_repo/Main.elm | 72 +++++++ test/resources/repos/elm/test_repo/Utils.elm | 28 +++ test/resources/repos/elm/test_repo/elm.json | 24 +++ test/solidlsp/elm/test_elm_basic.py | 54 +++++ 11 files changed, 388 insertions(+) create mode 100644 src/solidlsp/language_servers/elm_language_server.py create mode 100644 test/resources/repos/elm/test_repo/Main.elm create mode 100644 test/resources/repos/elm/test_repo/Utils.elm create mode 100644 test/resources/repos/elm/test_repo/elm.json create mode 100644 test/solidlsp/elm/test_elm_basic.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index e73b76e9..e1db6ce2 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -28,6 +28,10 @@ jobs: - uses: actions/setup-go@v5 with: go-version: ">=1.17.0" + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20.x' - name: Ensure cached directory exist before calling cache-related actions shell: bash run: | @@ -230,6 +234,9 @@ jobs: echo "PERL_MB_OPT=--install_base \"$HOME/perl5\"" >> $GITHUB_ENV echo "PERL_MM_OPT=INSTALL_BASE=$HOME/perl5" >> $GITHUB_ENV echo "$HOME/perl5/bin" >> $GITHUB_PATH + - name: Install Elm + shell: bash + run: npm install -g elm@0.19.1-6 - name: Install Nix if: runner.os != 'Windows' # Nix doesn't support Windows natively uses: cachix/install-nix-action@v30 diff --git a/.gitignore b/.gitignore index 1ac36b26..196476e6 100644 --- a/.gitignore +++ b/.gitignore @@ -234,3 +234,7 @@ tmp/ # Swift /test/resources/repos/swift/test_repo/.build /test/resources/repos/swift/test_repo/.swiftpm + +# Elm +/test/resources/repos/elm/test_repo/.elm/ +/test/resources/repos/elm/test_repo/elm-stuff/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 11696a9b..d4a464f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ Status of the `main` branch. Changes prior to the next official version change will appear here. * Language support: + * **Add support for Elm** via @elm-tooling/elm-language-server (automatically downloads if not installed; requires Elm compiler) * **Add support for Perl** via Perl::LanguageServer with LSP integration for .pl, .pm, and .t files * **Add support for AL (Application Language)** for Microsoft Dynamics 365 Business Central development. Requires VS Code AL extension (ms-dynamics-smb.al). * **Add support for R** via the R languageserver package with LSP integration, performance optimizations, and fallback symbol extraction diff --git a/README.md b/README.md index d1e6c73c..07c92081 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ With Serena, we provide direct, out-of-the-box support for: * Lua (automatically downloads lua-language-server if not installed) * Nix (requires nixd installation) * Elixir (requires installation of NextLS and Elixir; **Windows not supported**) + * Elm (automatically downloads elm-language-server if not installed; requires Elm compiler) * Erlang (requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls), experimental, might be slow or hang) * Perl (requires installation of Perl::LanguageServer) * AL diff --git a/pyproject.toml b/pyproject.toml index c9147dea..e6bbdf1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -262,6 +262,7 @@ markers = [ "perl: language server running for Perl", "csharp: language server running for C#", "elixir: language server running for Elixir", + "elm: language server running for Elm", "terraform: language server running for Terraform", "swift: language server running for Swift", "bash: language server running for Bash", diff --git a/src/solidlsp/language_servers/elm_language_server.py b/src/solidlsp/language_servers/elm_language_server.py new file mode 100644 index 00000000..6f5a6baa --- /dev/null +++ b/src/solidlsp/language_servers/elm_language_server.py @@ -0,0 +1,189 @@ +""" +Provides Elm specific instantiation of the LanguageServer class. Contains various configurations and settings specific to Elm. +""" + +import logging +import os +import pathlib +import shutil +import threading + +from overrides import override +from sensai.util.logging import LogTime + +from solidlsp.ls import SolidLanguageServer +from solidlsp.ls_config import LanguageServerConfig +from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams +from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo +from solidlsp.settings import SolidLSPSettings + +from .common import RuntimeDependency, RuntimeDependencyCollection + + +class ElmLanguageServer(SolidLanguageServer): + """ + Provides Elm specific instantiation of the LanguageServer class. Contains various configurations and settings specific to Elm. + """ + + def __init__( + self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings + ): + """ + Creates an ElmLanguageServer instance. This class is not meant to be instantiated directly. Use LanguageServer.create() instead. + """ + elm_lsp_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings) + super().__init__( + config, + logger, + repository_root_path, + ProcessLaunchInfo(cmd=elm_lsp_executable_path, cwd=repository_root_path), + "elm", + solidlsp_settings, + ) + self.server_ready = threading.Event() + + @override + def is_ignored_dirname(self, dirname: str) -> bool: + return super().is_ignored_dirname(dirname) or dirname in [ + "elm-stuff", + "node_modules", + "dist", + "build", + ] + + @classmethod + def _setup_runtime_dependencies( + cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings + ) -> list[str]: + """ + Setup runtime dependencies for Elm Language Server and return the command to start the server. + """ + # Check if elm-language-server is already installed globally + system_elm_ls = shutil.which("elm-language-server") + if system_elm_ls: + logger.log(f"Found system-installed elm-language-server at {system_elm_ls}", logging.INFO) + return [system_elm_ls, "--stdio"] + + # Verify node and npm are installed + is_node_installed = shutil.which("node") is not None + assert is_node_installed, "node is not installed or isn't in PATH. Please install NodeJS and try again." + is_npm_installed = shutil.which("npm") is not None + assert is_npm_installed, "npm is not installed or isn't in PATH. Please install npm and try again." + + deps = RuntimeDependencyCollection( + [ + RuntimeDependency( + id="elm-language-server", + description="@elm-tooling/elm-language-server package", + command=["npm", "install", "--prefix", "./", "@elm-tooling/elm-language-server@2.8.0"], + platform_id="any", + ), + ] + ) + + # Install elm-language-server if not already installed + elm_ls_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "elm-lsp") + elm_ls_executable_path = os.path.join(elm_ls_dir, "node_modules", ".bin", "elm-language-server") + if not os.path.exists(elm_ls_executable_path): + logger.log(f"Elm Language Server executable not found at {elm_ls_executable_path}. Installing...", logging.INFO) + with LogTime("Installation of Elm language server dependencies", logger=logger.logger): + deps.install(logger, elm_ls_dir) + + if not os.path.exists(elm_ls_executable_path): + raise FileNotFoundError( + f"elm-language-server executable not found at {elm_ls_executable_path}, something went wrong with the installation." + ) + return [elm_ls_executable_path, "--stdio"] + + @staticmethod + def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: + """ + Returns the initialize params for the Elm Language Server. + """ + root_uri = pathlib.Path(repository_absolute_path).as_uri() + + initialize_params = { + "locale": "en", + "capabilities": { + "textDocument": { + "synchronization": {"didSave": True, "dynamicRegistration": True}, + "completion": {"dynamicRegistration": True, "completionItem": {"snippetSupport": True}}, + "definition": {"dynamicRegistration": True}, + "references": {"dynamicRegistration": True}, + "documentSymbol": { + "dynamicRegistration": True, + "hierarchicalDocumentSymbolSupport": True, + "symbolKind": {"valueSet": list(range(1, 27))}, + }, + "hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]}, + "codeAction": {"dynamicRegistration": True}, + "rename": {"dynamicRegistration": True}, + }, + "workspace": { + "workspaceFolders": True, + "didChangeConfiguration": {"dynamicRegistration": True}, + "symbol": {"dynamicRegistration": True}, + }, + }, + "initializationOptions": { + "elmPath": "elm", + "elmFormatPath": "elm-format", + "elmTestPath": "elm-test", + "skipInstallPackageConfirmation": True, + "onlyUpdateDiagnosticsOnSave": False, + }, + "processId": os.getpid(), + "rootPath": repository_absolute_path, + "rootUri": root_uri, + "workspaceFolders": [ + { + "uri": root_uri, + "name": os.path.basename(repository_absolute_path), + } + ], + } + return initialize_params + + def _start_server(self): + """ + Starts the Elm Language Server, waits for the server to be ready and yields the LanguageServer instance. + """ + + def do_nothing(params): + return + + def window_log_message(msg): + self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO) + + self.server.on_notification("window/logMessage", window_log_message) + self.server.on_notification("$/progress", do_nothing) + self.server.on_notification("textDocument/publishDiagnostics", do_nothing) + + self.logger.log("Starting Elm server process", logging.INFO) + self.server.start() + initialize_params = self._get_initialize_params(self.repository_root_path) + + self.logger.log( + "Sending initialize request from LSP client to LSP server and awaiting response", + logging.INFO, + ) + init_response = self.server.send.initialize(initialize_params) + + # Elm-specific capability checks + assert "textDocumentSync" in init_response["capabilities"] + assert "completionProvider" in init_response["capabilities"] + assert "definitionProvider" in init_response["capabilities"] + assert "referencesProvider" in init_response["capabilities"] + assert "documentSymbolProvider" in init_response["capabilities"] + + self.server.notify.initialized({}) + self.logger.log("Elm server initialized successfully, waiting for workspace scan...", logging.INFO) + + self.server_ready.set() + self.completions_available.set() + self.logger.log("Elm server ready", logging.INFO) + + @override + def _get_wait_time_for_cross_file_referencing(self) -> float: + return 1.0 diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index 18f634a7..e2dfd970 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -46,6 +46,7 @@ class Language(str, Enum): PERL = "perl" CLOJURE = "clojure" ELIXIR = "elixir" + ELM = "elm" TERRAFORM = "terraform" SWIFT = "swift" BASH = "bash" @@ -128,6 +129,8 @@ class Language(str, Enum): return FilenameMatcher("*.clj", "*.cljs", "*.cljc", "*.edn") # codespell:ignore edn case self.ELIXIR: return FilenameMatcher("*.ex", "*.exs") + case self.ELM: + return FilenameMatcher("*.elm") case self.TERRAFORM: return FilenameMatcher("*.tf", "*.tfvars", "*.tfstate") case self.SWIFT: @@ -223,6 +226,10 @@ class Language(str, Enum): from solidlsp.language_servers.elixir_tools.elixir_tools import ElixirTools return ElixirTools + case self.ELM: + from solidlsp.language_servers.elm_language_server import ElmLanguageServer + + return ElmLanguageServer case self.TERRAFORM: from solidlsp.language_servers.terraform_ls import TerraformLS diff --git a/test/resources/repos/elm/test_repo/Main.elm b/test/resources/repos/elm/test_repo/Main.elm new file mode 100644 index 00000000..76a34781 --- /dev/null +++ b/test/resources/repos/elm/test_repo/Main.elm @@ -0,0 +1,72 @@ +module Main exposing (main, greet, calculateSum) + +{-| Main module for testing Elm language server functionality. + +This module contains basic functions to test: + + - Symbol discovery + - Reference finding + - Cross-file imports + +-} + +import Browser +import Html exposing (Html, div, h1, p, text) +import Utils exposing (formatMessage, addNumbers) + + +{-| The main entry point for the application +-} +main : Program () Model Msg +main = + Browser.sandbox + { init = init + , view = view + , update = update + } + + +type alias Model = + { message : String + , count : Int + } + + +init : Model +init = + { message = greet "World" + , count = calculateSum 5 10 + } + + +type Msg + = NoOp + + +update : Msg -> Model -> Model +update msg model = + case msg of + NoOp -> + model + + +view : Model -> Html Msg +view model = + div [] + [ h1 [] [ text (formatMessage model.message) ] + , p [] [ text ("Count: " ++ String.fromInt model.count) ] + ] + + +{-| Greet someone by name +-} +greet : String -> String +greet name = + "Hello, " ++ name ++ "!" + + +{-| Calculate the sum of two numbers +-} +calculateSum : Int -> Int -> Int +calculateSum a b = + addNumbers a b diff --git a/test/resources/repos/elm/test_repo/Utils.elm b/test/resources/repos/elm/test_repo/Utils.elm new file mode 100644 index 00000000..c54b8a90 --- /dev/null +++ b/test/resources/repos/elm/test_repo/Utils.elm @@ -0,0 +1,28 @@ +module Utils exposing (formatMessage, addNumbers, multiplyNumbers) + +{-| Utility functions for the Elm test application. + +This module provides helper functions used by other modules. + +-} + + +{-| Format a message by adding brackets around it +-} +formatMessage : String -> String +formatMessage msg = + "[ " ++ msg ++ " ]" + + +{-| Add two numbers together +-} +addNumbers : Int -> Int -> Int +addNumbers x y = + x + y + + +{-| Multiply two numbers +-} +multiplyNumbers : Int -> Int -> Int +multiplyNumbers x y = + x * y diff --git a/test/resources/repos/elm/test_repo/elm.json b/test/resources/repos/elm/test_repo/elm.json new file mode 100644 index 00000000..0a668c08 --- /dev/null +++ b/test/resources/repos/elm/test_repo/elm.json @@ -0,0 +1,24 @@ +{ + "type": "application", + "source-directories": [ + "." + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/browser": "1.0.2", + "elm/core": "1.0.5", + "elm/html": "1.0.0" + }, + "indirect": { + "elm/json": "1.1.3", + "elm/time": "1.0.0", + "elm/url": "1.0.0", + "elm/virtual-dom": "1.0.3" + } + }, + "test-dependencies": { + "direct": {}, + "indirect": {} + } +} diff --git a/test/solidlsp/elm/test_elm_basic.py b/test/solidlsp/elm/test_elm_basic.py new file mode 100644 index 00000000..87669d8f --- /dev/null +++ b/test/solidlsp/elm/test_elm_basic.py @@ -0,0 +1,54 @@ +import os + +import pytest + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language +from solidlsp.ls_utils import SymbolUtils + + +@pytest.mark.elm +class TestElmLanguageServer: + @pytest.mark.parametrize("language_server", [Language.ELM], indirect=True) + def test_find_symbol(self, language_server: SolidLanguageServer) -> None: + symbols = language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "greet"), "greet function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "calculateSum"), "calculateSum function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "formatMessage"), "formatMessage function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "addNumbers"), "addNumbers function not found in symbol tree" + + @pytest.mark.parametrize("language_server", [Language.ELM], indirect=True) + def test_find_references_within_file(self, language_server: SolidLanguageServer) -> None: + file_path = os.path.join("Main.elm") + symbols = language_server.request_document_symbols(file_path) + greet_symbol = None + for sym in symbols[0]: + if sym.get("name") == "greet": + greet_symbol = sym + break + assert greet_symbol is not None, "Could not find 'greet' symbol in Main.elm" + sel_start = greet_symbol["selectionRange"]["start"] + refs = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + assert any("Main.elm" in ref.get("relativePath", "") for ref in refs), "Main.elm should reference greet function" + + @pytest.mark.parametrize("language_server", [Language.ELM], indirect=True) + def test_find_references_across_files(self, language_server: SolidLanguageServer) -> None: + # Test formatMessage function which is defined in Utils.elm and used in Main.elm + utils_path = os.path.join("Utils.elm") + symbols = language_server.request_document_symbols(utils_path) + formatMessage_symbol = None + for sym in symbols[0]: + if sym.get("name") == "formatMessage": + formatMessage_symbol = sym + break + assert formatMessage_symbol is not None, "Could not find 'formatMessage' symbol in Utils.elm" + + # Get references from the definition in Utils.elm + sel_start = formatMessage_symbol["selectionRange"]["start"] + refs = language_server.request_references(utils_path, sel_start["line"], sel_start["character"]) + + # Verify that we found references + assert refs, "Expected to find references for formatMessage" + + # Verify that at least one reference is in Main.elm (where formatMessage is used) + assert any("Main.elm" in ref.get("relativePath", "") for ref in refs), "Expected to find usage of formatMessage in Main.elm" From 8abfe58c67731c58911892c59ff3e7aaa1407110 Mon Sep 17 00:00:00 2001 From: Drew Volz Date: Tue, 14 Oct 2025 04:15:39 -0700 Subject: [PATCH 06/10] feat(text_utils): add brace expansion for glob patterns (#652) * adds expand_braces function to handle glob patterns * updates search_files to use it for include/exclude patterns * allows for flexible file filtering `**/*.{js,ts}` to match ts/js --- src/serena/text_utils.py | 49 +++++++++++++++--- src/serena/tools/file_tools.py | 2 + test/serena/test_text_utils.py | 93 ++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 6 deletions(-) diff --git a/src/serena/text_utils.py b/src/serena/text_utils.py index 4be99d78..19987cee 100644 --- a/src/serena/text_utils.py +++ b/src/serena/text_utils.py @@ -241,6 +241,29 @@ def default_file_reader(file_path: str) -> str: return f.read() +def expand_braces(pattern: str) -> list[str]: + """ + Expands brace patterns in a glob string. + For example, "**/*.{js,jsx,ts,tsx}" becomes ["**/*.js", "**/*.jsx", "**/*.ts", "**/*.tsx"]. + Handles multiple brace sets as well. + """ + patterns = [pattern] + while any("{" in p for p in patterns): + new_patterns = [] + for p in patterns: + match = re.search(r"\{([^{}]+)\}", p) + if match: + prefix = p[: match.start()] + suffix = p[match.end() :] + options = match.group(1).split(",") + for option in options: + new_patterns.append(f"{prefix}{option}{suffix}") + else: + new_patterns.append(p) + patterns = new_patterns + return patterns + + def glob_match(pattern: str, path: str) -> bool: """ Match a file path against a glob pattern. @@ -251,6 +274,13 @@ def glob_match(pattern: str, path: str) -> bool: - ? matches a single character except / - [seq] matches any character in seq + Supports brace expansion: + - {a,b,c} expands to multiple patterns (including nesting) + + Unsupported patterns: + - Bash extended glob features are unavailable in Python's fnmatch + - Extended globs like !(), ?(), +(), *(), @() are not supported + :param pattern: Glob pattern (e.g., 'src/**/*.py', '**agent.py') :param path: File path to match against :return: True if path matches pattern @@ -313,14 +343,21 @@ def search_files( """ # Pre-filter paths (done sequentially to avoid overhead) # Use proper glob matching instead of gitignore patterns + include_patterns = expand_braces(paths_include_glob) if paths_include_glob else None + exclude_patterns = expand_braces(paths_exclude_glob) if paths_exclude_glob else None + filtered_paths = [] for path in relative_file_paths: - if paths_include_glob and not glob_match(paths_include_glob, path): - log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}") - continue - if paths_exclude_glob and glob_match(paths_exclude_glob, path): - log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}") - continue + if include_patterns: + if not any(glob_match(p, path) for p in include_patterns): + log.debug(f"Skipping {path}: does not match include pattern {paths_include_glob}") + continue + + if exclude_patterns: + if any(glob_match(p, path) for p in exclude_patterns): + log.debug(f"Skipping {path}: matches exclude pattern {paths_exclude_glob}") + continue + filtered_paths.append(path) log.info(f"Processing {len(filtered_paths)} files.") diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index 56d2cf94..f3db634e 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -339,9 +339,11 @@ class SearchForPatternTool(Tool): :param context_lines_after: Number of lines of context to include after each match :param paths_include_glob: optional glob pattern specifying files to include in the search. Matches against relative file paths from the project root (e.g., "*.py", "src/**/*.ts"). + Supports standard glob patterns (*, ?, [seq], **, etc.) and brace expansion {a,b,c}. Only matches files, not directories. If left empty, all non-ignored files will be included. :param paths_exclude_glob: optional glob pattern specifying files to exclude from the search. Matches against relative file paths from the project root (e.g., "*test*", "**/*_generated.py"). + Supports standard glob patterns (*, ?, [seq], **, etc.) and brace expansion {a,b,c}. Takes precedence over paths_include_glob. Only matches files, not directories. If left empty, no files are excluded. :param relative_path: only subpaths of this path (relative to the repo root) will be analyzed. If a path to a single file is passed, only that will be searched. The path must exist, otherwise a `FileNotFoundError` is raised. diff --git a/test/serena/test_text_utils.py b/test/serena/test_text_utils.py index d122f6a5..d3701ad9 100644 --- a/test/serena/test_text_utils.py +++ b/test/serena/test_text_utils.py @@ -358,6 +358,72 @@ class TestSearchFiles: assert result.matched_lines[0].line_content == "This line contains a match." assert result.matched_lines[0].match_type == LineType.MATCH + @pytest.mark.parametrize( + "file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description", + [ + # Brace expansion in include glob + ( + ["a.py", "b.js", "c.txt"], + "match", + "*.{py,js}", + None, + ["a.py", "b.js"], + "Brace expansion in include glob", + ), + # Brace expansion in exclude glob + ( + ["a.py", "b.log", "c.txt"], + "match", + "*.{py,log,txt}", + "*.{log,txt}", + ["a.py"], + "Brace expansion in exclude glob", + ), + # Brace expansion in both include and exclude + ( + ["src/a.ts", "src/b.js", "test/a.ts", "test/b.js"], + "match", + "**/*.{ts,js}", + "test/**/*.{ts,js}", + ["src/a.ts", "src/b.js"], + "Brace expansion in both include and exclude", + ), + # No matching files with brace expansion + ( + ["a.py", "b.js"], + "match", + "*.{c,h}", + None, + [], + "Brace expansion with no matching files", + ), + # Multiple brace expansions + ( + ["src/a/a.py", "src/b/b.py", "lib/a/a.py", "lib/b/b.py"], + "match", + "{src,lib}/{a,b}/*.py", + "lib/b/*.py", + ["src/a/a.py", "src/b/b.py", "lib/a/a.py"], + "Multiple brace expansions in include/exclude", + ), + ], + ids=lambda x: x if isinstance(x, str) else "", + ) + def test_search_files_with_brace_expansion( + self, file_paths, pattern, paths_include_glob, paths_exclude_glob, expected_matched_files, description + ): + """Test search_files with glob patterns containing brace expansions.""" + results = search_files( + relative_file_paths=file_paths, + pattern=pattern, + file_reader=mock_reader_always_match, + paths_include_glob=paths_include_glob, + paths_exclude_glob=paths_exclude_glob, + ) + + actual_matched_files = sorted([result.source_file_path for result in results if result.source_file_path]) + assert actual_matched_files == sorted(expected_matched_files), f"Test failed: {description}" + def test_search_files_no_pattern_match_in_content(self): """Test that no results are returned if the pattern doesn't match the file content, even if files pass filters.""" file_paths = ["a.py", "b.txt"] @@ -482,3 +548,30 @@ class TestGlobMatch: from src.serena.text_utils import glob_match assert glob_match(pattern, path) == expected + + +class TestExpandBraces: + """Test the expand_braces function.""" + + @pytest.mark.parametrize( + "pattern, expected", + [ + # Basic case + ("src/*.{js,ts}", ["src/*.js", "src/*.ts"]), + # No braces + ("src/*.py", ["src/*.py"]), + # Multiple brace sets + ("src/{a,b}/{c,d}.py", ["src/a/c.py", "src/a/d.py", "src/b/c.py", "src/b/d.py"]), + # Empty string + ("", [""]), + # Braces with empty elements + ("src/{a,,b}.py", ["src/a.py", "src/.py", "src/b.py"]), + # No commas + ("src/{a}.py", ["src/a.py"]), + ], + ) + def test_expand_braces(self, pattern, expected): + """Test brace expansion for glob patterns.""" + from serena.text_utils import expand_braces + + assert sorted(expand_braces(pattern)) == sorted(expected) From a82d9a9fb9d9ea7bdf17ff64ec4bdb97cf58b088 Mon Sep 17 00:00:00 2001 From: Anivie Michaelis <88354708+Anivie@users.noreply.github.com> Date: Wed, 15 Oct 2025 11:21:02 +0800 Subject: [PATCH 07/10] Update rust_analyzer.py Fix 'Failed to deserialize config field at /cargo/cfgs' error by rust analyzer --- src/solidlsp/language_servers/rust_analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solidlsp/language_servers/rust_analyzer.py b/src/solidlsp/language_servers/rust_analyzer.py index f07483d7..e332c2e6 100644 --- a/src/solidlsp/language_servers/rust_analyzer.py +++ b/src/solidlsp/language_servers/rust_analyzer.py @@ -406,7 +406,7 @@ class RustAnalyzer(SolidLanguageServer): "overrideCommand": None, "useRustcWrapper": True, }, - "cfgs": {}, + "cfgs": [], "extraArgs": [], "extraEnv": {}, "features": [], From dc31e4e27e01188a24485567b8a593be80cdd590 Mon Sep 17 00:00:00 2001 From: Anatolii Prylutskyi Date: Wed, 15 Oct 2025 12:14:32 +0300 Subject: [PATCH 08/10] fix(nix): resolve build issues and add CI verification (#668) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: add Nix build verification to pytest workflow Add a verification step to the CI pipeline that validates the Nix package builds successfully on each run. The step uses 'nix build --no-link' to verify the flake builds without creating artifacts. This provides early detection of Nix packaging issues with minimal overhead, reusing the existing Nix installation on Linux and macOS runners. 🤖 Generated with [Claude Code](https://claude.com/claude-code) via [Happy](https://happy.engineering) Co-Authored-By: Claude Co-Authored-By: Happy * fix(nix): add setuptools to ruamel-yaml-clib build dependencies Add pyproject override to ensure ruamel-yaml-clib has setuptools available during build. This fixes Nix build failures where the package requires setuptools but doesn't declare it as a native build input. 🤖 Generated with [Claude Code](https://claude.com/claude-code) via [Happy](https://happy.engineering) Co-Authored-By: Claude Co-Authored-By: Happy --------- Co-authored-by: Claude Co-authored-by: Happy --- .github/workflows/pytest.yml | 6 ++++++ flake.nix | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index e1db6ce2..e87eccfb 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -256,6 +256,12 @@ jobs: fi echo "$HOME/.nix-profile/bin" >> $GITHUB_PATH + - name: Verify Nix package build + if: runner.os != 'Windows' # Nix only supported on Linux/macOS + shell: bash + run: | + # Verify the flake builds successfully + nix build --no-link - name: Install uv shell: bash run: curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/flake.nix b/flake.nix index e92f1999..7529365b 100644 --- a/flake.nix +++ b/flake.nix @@ -50,7 +50,14 @@ sourcePreference = "wheel"; # or sourcePreference = "sdist"; }; - pyprojectOverrides = _final: _prev: {}; + pyprojectOverrides = final: prev: { + # Add setuptools for packages that need it during build + ruamel-yaml-clib = prev.ruamel-yaml-clib.overrideAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ + final.setuptools + ]; + }); + }; python = pkgs.python311; From 39a11657a98ab25e6ac055c4d38e3d97ec53e7ef Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 15 Oct 2025 18:16:49 +0200 Subject: [PATCH 09/10] Project.validate_relative_path: Make handling of ignored files explicit #672 --- src/serena/project.py | 12 ++++++++---- src/serena/tools/file_tools.py | 12 ++++++------ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/serena/project.py b/src/serena/project.py index 0673f05e..966fc8c7 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -170,18 +170,22 @@ class Project: abs_path = Path(self.project_root) / relative_path return abs_path.exists() - def validate_relative_path(self, relative_path: str) -> None: + def validate_relative_path(self, relative_path: str, require_not_ignored: bool = False) -> None: """ Validates that the given relative path to an existing file/dir is safe to read or edit, - meaning it's inside the project directory and is not ignored by git. + meaning it's inside the project directory. Passing a path to a non-existing file will lead to a `FileNotFoundError`. + + :param relative_path: the path to validate, relative to the project root + :param require_not_ignored: if True, the path must not be ignored according to the project's ignore settings """ if not self.is_path_in_project(relative_path): raise ValueError(f"{relative_path=} points to path outside of the repository root; cannot access for safety reasons") - if self.is_ignored_path(relative_path): - raise ValueError(f"Path {relative_path} is ignored; cannot access for safety reasons") + if require_not_ignored: + if self.is_ignored_path(relative_path): + raise ValueError(f"Path {relative_path} is ignored; cannot access for safety reasons") def gather_source_files(self, relative_path: str = "") -> list[str]: """Retrieves relative paths of all source files, optionally limited to the given path diff --git a/src/serena/tools/file_tools.py b/src/serena/tools/file_tools.py index f3db634e..d7ec5a41 100644 --- a/src/serena/tools/file_tools.py +++ b/src/serena/tools/file_tools.py @@ -36,7 +36,7 @@ class ReadFileTool(Tool): required for the task. :return: the full text of the file at the given relative path """ - self.project.validate_relative_path(relative_path) + self.project.validate_relative_path(relative_path, require_not_ignored=True) result = self.project.read_file(relative_path) result_lines = result.splitlines() @@ -67,7 +67,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit): will_overwrite_existing = abs_path.exists() if will_overwrite_existing: - self.project.validate_relative_path(relative_path) + self.project.validate_relative_path(relative_path, require_not_ignored=True) else: assert abs_path.is_relative_to( self.get_project_root() @@ -88,7 +88,7 @@ class ListDirTool(Tool): def apply(self, relative_path: str, recursive: bool, skip_ignored_files: bool = False, max_answer_chars: int = -1) -> str: """ - Lists all non-gitignored files and directories in the given directory (optionally with recursion). + Lists files and directories in the given directory (optionally with recursion). :param relative_path: the relative path to the directory to list; pass "." to scan the project root :param recursive: whether to scan subdirectories recursively @@ -107,7 +107,7 @@ class ListDirTool(Tool): } return json.dumps(error_info) - self.project.validate_relative_path(relative_path) + self.project.validate_relative_path(relative_path, require_not_ignored=skip_ignored_files) dirs, files = scan_directory( os.path.join(self.get_project_root(), relative_path), @@ -134,7 +134,7 @@ class FindFileTool(Tool): :param relative_path: the relative path to the directory to search in; pass "." to scan the project root :return: a JSON object with the list of matching files """ - self.project.validate_relative_path(relative_path) + self.project.validate_relative_path(relative_path, require_not_ignored=True) dir_to_scan = os.path.join(self.get_project_root(), relative_path) @@ -192,7 +192,7 @@ class ReplaceRegexTool(Tool, ToolMarkerCanEdit): If this is set to False and the regex matches multiple occurrences, an error will be returned (and you may retry with a revised, more specific regex). """ - self.project.validate_relative_path(relative_path) + self.project.validate_relative_path(relative_path, require_not_ignored=True) with EditedFileContext(relative_path, self.agent) as context: original_content = context.get_original_content() updated_content, n = re.subn(regex, repl, original_content, flags=re.DOTALL | re.MULTILINE) From f2919f11c5a5531962be549f67e053d8afd92399 Mon Sep 17 00:00:00 2001 From: Yen <5915590+antigenius0910@users.noreply.github.com> Date: Wed, 15 Oct 2025 15:46:25 -0500 Subject: [PATCH 10/10] Add rego language support (#670) Also added an ignore rule for ruff (about unused unpacked variables), we might reintroduce this rule later --- .github/workflows/pytest.yml | 31 +++- pyproject.toml | 2 + src/solidlsp/language_servers/regal_server.py | 140 ++++++++++++++++++ src/solidlsp/ls_config.py | 7 + .../repos/rego/test_repo/policies/authz.rego | 33 +++++ .../rego/test_repo/policies/validation.rego | 23 +++ .../repos/rego/test_repo/utils/helpers.rego | 29 ++++ test/solidlsp/rego/test_rego_basic.py | 138 +++++++++++++++++ 8 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 src/solidlsp/language_servers/regal_server.py create mode 100644 test/resources/repos/rego/test_repo/policies/authz.rego create mode 100644 test/resources/repos/rego/test_repo/policies/validation.rego create mode 100644 test/resources/repos/rego/test_repo/utils/helpers.rego create mode 100644 test/solidlsp/rego/test_rego_basic.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index e87eccfb..7936dbc9 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -248,13 +248,13 @@ jobs: run: | # Install nixd using nix nix profile install github:nix-community/nixd - + # Verify nixd is installed and working if ! command -v nixd &> /dev/null; then echo "nixd installation failed or not in PATH" exit 1 fi - + echo "$HOME/.nix-profile/bin" >> $GITHUB_PATH - name: Verify Nix package build if: runner.os != 'Windows' # Nix only supported on Linux/macOS @@ -262,6 +262,33 @@ jobs: run: | # Verify the flake builds successfully nix build --no-link + - name: Install Regal (Rego Language Server) + shell: bash + run: | + REGAL_VERSION="0.36.1" + + if [[ "${{ runner.os }}" == "Linux" ]]; then + if [[ "$(uname -m)" == "x86_64" ]]; then + curl -L -o regal https://github.com/StyraInc/regal/releases/download/v${REGAL_VERSION}/regal_Linux_x86_64 + else + curl -L -o regal https://github.com/StyraInc/regal/releases/download/v${REGAL_VERSION}/regal_Linux_arm64 + fi + chmod +x regal + sudo mv regal /usr/local/bin/ + elif [[ "${{ runner.os }}" == "macOS" ]]; then + if [[ "$(uname -m)" == "x86_64" ]]; then + curl -L -o regal https://github.com/StyraInc/regal/releases/download/v${REGAL_VERSION}/regal_Darwin_x86_64 + else + curl -L -o regal https://github.com/StyraInc/regal/releases/download/v${REGAL_VERSION}/regal_Darwin_arm64 + fi + chmod +x regal + sudo mv regal /usr/local/bin/ + elif [[ "${{ runner.os }}" == "Windows" ]]; then + curl -L -o regal.exe https://github.com/StyraInc/regal/releases/download/v${REGAL_VERSION}/regal_Windows_x86_64.exe + mkdir -p "$HOME/bin" + mv regal.exe "$HOME/bin/" + echo "$HOME/bin" >> $GITHUB_PATH + fi - name: Install uv shell: bash run: curl -LsSf https://astral.sh/uv/install.sh | sh diff --git a/pyproject.toml b/pyproject.toml index e6bbdf1e..fb7c230d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -167,6 +167,7 @@ ignore = [ "PLC0415", "RUF002", "RUF005", + "RUF059", "SIM118", "SIM108", "E501", @@ -275,6 +276,7 @@ markers = [ "dart: language server running for Dart", "erlang: language server running for Erlang", "al: language server running for AL (Microsoft Dynamics 365 Business Central)", + "rego: language server running for Rego", "markdown: language server running for Markdown", ] diff --git a/src/solidlsp/language_servers/regal_server.py b/src/solidlsp/language_servers/regal_server.py new file mode 100644 index 00000000..eb6abc1d --- /dev/null +++ b/src/solidlsp/language_servers/regal_server.py @@ -0,0 +1,140 @@ +"""Regal Language Server implementation for Rego policy files.""" + +import logging +import os +import shutil +import threading + +from overrides import override + +from solidlsp.ls import SolidLanguageServer +from solidlsp.ls_config import LanguageServerConfig +from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.ls_utils import PathUtils +from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams +from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo +from solidlsp.settings import SolidLSPSettings + + +class RegalLanguageServer(SolidLanguageServer): + """ + Provides Rego specific instantiation of the LanguageServer class using Regal. + + Regal is the official linter and language server for Rego (Open Policy Agent's policy language). + See: https://github.com/StyraInc/regal + """ + + @override + def is_ignored_dirname(self, dirname: str) -> bool: + return super().is_ignored_dirname(dirname) or dirname in [".regal", ".opa"] + + def __init__( + self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings + ): + """ + Creates a RegalLanguageServer instance. + + This class is not meant to be instantiated directly. Use LanguageServer.create() instead. + + :param config: Language server configuration + :param logger: Logger instance + :param repository_root_path: Path to the repository root + :param solidlsp_settings: Settings for solidlsp + """ + # Regal should be installed system-wide (via CI or user installation) + regal_executable_path = shutil.which("regal") + if not regal_executable_path: + raise RuntimeError( + "Regal language server not found. Please install it from https://github.com/StyraInc/regal or via your package manager." + ) + + super().__init__( + config, + logger, + repository_root_path, + ProcessLaunchInfo(cmd=f"{regal_executable_path} language-server", cwd=repository_root_path), + "rego", + solidlsp_settings, + ) + self.server_ready = threading.Event() + + @staticmethod + def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: + """ + Returns the initialize params for the Regal Language Server. + + :param repository_absolute_path: Absolute path to the repository + :return: LSP initialization parameters + """ + root_uri = PathUtils.path_to_uri(repository_absolute_path) + return { + "processId": os.getpid(), + "locale": "en", + "rootPath": repository_absolute_path, + "rootUri": root_uri, + "capabilities": { + "textDocument": { + "synchronization": {"didSave": True, "dynamicRegistration": True}, + "completion": {"dynamicRegistration": True, "completionItem": {"snippetSupport": True}}, + "definition": {"dynamicRegistration": True}, + "references": {"dynamicRegistration": True}, + "documentSymbol": { + "dynamicRegistration": True, + "hierarchicalDocumentSymbolSupport": True, + "symbolKind": {"valueSet": list(range(1, 27))}, # type: ignore[arg-type] + }, + "hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]}, # type: ignore[list-item] + "codeAction": {"dynamicRegistration": True}, + "formatting": {"dynamicRegistration": True}, + }, + "workspace": { + "workspaceFolders": True, + "didChangeConfiguration": {"dynamicRegistration": True}, + "symbol": {"dynamicRegistration": True}, + }, + }, + "workspaceFolders": [ + { + "name": os.path.basename(repository_absolute_path), + "uri": root_uri, + } + ], + } + + def _start_server(self) -> None: + """Start Regal language server process and wait for initialization.""" + + def register_capability_handler(params) -> None: # type: ignore[no-untyped-def] + return + + def window_log_message(msg) -> None: # type: ignore[no-untyped-def] + self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO) + + def do_nothing(params) -> None: # type: ignore[no-untyped-def] + return + + self.server.on_request("client/registerCapability", register_capability_handler) + self.server.on_notification("window/logMessage", window_log_message) + self.server.on_notification("$/progress", do_nothing) + self.server.on_notification("textDocument/publishDiagnostics", do_nothing) + + self.logger.log("Starting Regal language server process", logging.INFO) + self.server.start() + initialize_params = self._get_initialize_params(self.repository_root_path) + + self.logger.log( + "Sending initialize request from LSP client to LSP server and awaiting response", + logging.INFO, + ) + init_response = self.server.send.initialize(initialize_params) + + # Verify server capabilities + assert "capabilities" in init_response + assert "textDocumentSync" in init_response["capabilities"] + + self.server.notify.initialized({}) + self.completions_available.set() + + # Regal server is ready immediately after initialization + self.server_ready.set() + self.server_ready.wait() diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index e2dfd970..d70c1dce 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -55,6 +55,7 @@ class Language(str, Enum): NIX = "nix" ERLANG = "erlang" AL = "al" + REGO = "rego" # Experimental or deprecated Language Servers TYPESCRIPT_VTS = "typescript_vts" """Use the typescript language server through the natively bundled vscode extension via https://github.com/yioneko/vtsls""" @@ -147,6 +148,8 @@ class Language(str, Enum): return FilenameMatcher("*.erl", "*.hrl", "*.escript", "*.config", "*.app", "*.app.src") case self.AL: return FilenameMatcher("*.al", "*.dal") + case self.REGO: + return FilenameMatcher("*.rego") case self.MARKDOWN: return FilenameMatcher("*.md", "*.markdown") case _: @@ -262,6 +265,10 @@ class Language(str, Enum): from solidlsp.language_servers.al_language_server import ALLanguageServer return ALLanguageServer + case self.REGO: + from solidlsp.language_servers.regal_server import RegalLanguageServer + + return RegalLanguageServer case self.MARKDOWN: from solidlsp.language_servers.marksman import Marksman diff --git a/test/resources/repos/rego/test_repo/policies/authz.rego b/test/resources/repos/rego/test_repo/policies/authz.rego new file mode 100644 index 00000000..fbac4bab --- /dev/null +++ b/test/resources/repos/rego/test_repo/policies/authz.rego @@ -0,0 +1,33 @@ +package policies + +import data.utils + +# Default deny +default allow := false + +# Admin access rule +allow if { + input.user.role == "admin" + utils.is_valid_user(input.user) +} + +# Read access for authenticated users +allow_read if { + input.action == "read" + input.user.authenticated +} + +# User roles list +admin_roles := ["admin", "superuser"] + +# Helper function to check if user is admin +is_admin(user) if { + admin_roles[_] == user.role +} + +# Check if action is allowed for user +check_permission(user, action) if { + user.role == "admin" + allowed_actions := ["read", "write", "delete"] + allowed_actions[_] == action +} diff --git a/test/resources/repos/rego/test_repo/policies/validation.rego b/test/resources/repos/rego/test_repo/policies/validation.rego new file mode 100644 index 00000000..49e64221 --- /dev/null +++ b/test/resources/repos/rego/test_repo/policies/validation.rego @@ -0,0 +1,23 @@ +package policies + +import data.policies +import data.utils + +# Validate user input +validate_user_input if { + utils.is_valid_user(input.user) + utils.is_valid_email(input.user.email) +} + +# Check if user has valid credentials +has_valid_credentials(user) if { + user.username != "" + user.password != "" + utils.is_valid_email(user.email) +} + +# Validate request +validate_request if { + input.user.authenticated + policies.allow +} diff --git a/test/resources/repos/rego/test_repo/utils/helpers.rego b/test/resources/repos/rego/test_repo/utils/helpers.rego new file mode 100644 index 00000000..2fb60bd4 --- /dev/null +++ b/test/resources/repos/rego/test_repo/utils/helpers.rego @@ -0,0 +1,29 @@ +package utils + +# User validation +is_valid_user(user) if { + user.id != "" + user.email != "" +} + +# Email validation +is_valid_email(email) if { + contains(email, "@") + contains(email, ".") +} + +# Username validation +is_valid_username(username) if { + count(username) >= 3 + count(username) <= 32 +} + +# Check if string is empty +is_empty(str) if { + str == "" +} + +# Check if array contains element +array_contains(arr, elem) if { + arr[_] == elem +} diff --git a/test/solidlsp/rego/test_rego_basic.py b/test/solidlsp/rego/test_rego_basic.py new file mode 100644 index 00000000..814562dc --- /dev/null +++ b/test/solidlsp/rego/test_rego_basic.py @@ -0,0 +1,138 @@ +"""Tests for Rego language server (Regal) functionality.""" + +import os +import sys + +import pytest + +from solidlsp.ls import SolidLanguageServer +from solidlsp.ls_config import Language +from solidlsp.ls_utils import SymbolUtils + + +@pytest.mark.rego +@pytest.mark.skipif( + sys.platform == "win32", reason="Regal LSP has Windows path handling bug - see https://github.com/StyraInc/regal/issues/1683" +) +class TestRegoLanguageServer: + """Test Regal language server functionality for Rego.""" + + @pytest.mark.parametrize("language_server", [Language.REGO], indirect=True) + def test_request_document_symbols_authz(self, language_server: SolidLanguageServer) -> None: + """Test that document symbols can be retrieved from authz.rego.""" + file_path = os.path.join("policies", "authz.rego") + symbols = language_server.request_document_symbols(file_path) + + assert symbols is not None + assert len(symbols) > 0 + + # Extract symbol names + symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols + symbol_names = {sym.get("name") for sym in symbol_list if isinstance(sym, dict)} + + # Verify specific Rego rules/functions are found + assert "allow" in symbol_names, "allow rule not found" + assert "allow_read" in symbol_names, "allow_read rule not found" + assert "is_admin" in symbol_names, "is_admin function not found" + assert "admin_roles" in symbol_names, "admin_roles constant not found" + + @pytest.mark.parametrize("language_server", [Language.REGO], indirect=True) + def test_request_document_symbols_helpers(self, language_server: SolidLanguageServer) -> None: + """Test that document symbols can be retrieved from helpers.rego.""" + file_path = os.path.join("utils", "helpers.rego") + symbols = language_server.request_document_symbols(file_path) + + assert symbols is not None + assert len(symbols) > 0 + + # Extract symbol names + symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols + symbol_names = {sym.get("name") for sym in symbol_list if isinstance(sym, dict)} + + # Verify specific helper functions are found + assert "is_valid_user" in symbol_names, "is_valid_user function not found" + assert "is_valid_email" in symbol_names, "is_valid_email function not found" + assert "is_valid_username" in symbol_names, "is_valid_username function not found" + + @pytest.mark.parametrize("language_server", [Language.REGO], indirect=True) + def test_find_symbol_full_tree(self, language_server: SolidLanguageServer) -> None: + """Test finding symbols across entire workspace using symbol tree.""" + symbols = language_server.request_full_symbol_tree() + + # Use SymbolUtils to check for expected symbols + assert SymbolUtils.symbol_tree_contains_name(symbols, "allow"), "allow rule not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "is_valid_user"), "is_valid_user function not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "is_admin"), "is_admin function not found in symbol tree" + + @pytest.mark.parametrize("language_server", [Language.REGO], indirect=True) + def test_request_definition_within_file(self, language_server: SolidLanguageServer) -> None: + """Test go-to-definition for symbols within the same file.""" + # In authz.rego, check_permission references admin_roles + file_path = os.path.join("policies", "authz.rego") + + # Get document symbols + symbols = language_server.request_document_symbols(file_path) + symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols + + # Find the is_admin symbol which references admin_roles + is_admin_symbol = next((s for s in symbol_list if s.get("name") == "is_admin"), None) + assert is_admin_symbol is not None, "is_admin symbol should always be found in authz.rego" + assert "range" in is_admin_symbol, "is_admin symbol should have a range" + + # Request definition from within is_admin (line 25, which references admin_roles at line 21) + # Line 25 is: admin_roles[_] == user.role + line = is_admin_symbol["range"]["start"]["line"] + 1 + char = 4 # Position at "admin_roles" + + definitions = language_server.request_definition(file_path, line, char) + assert definitions is not None and len(definitions) > 0, "Should find definition for admin_roles" + + # Verify the definition points to admin_roles in the same file + assert any("authz.rego" in defn.get("relativePath", "") for defn in definitions), "Definition should be in authz.rego" + + @pytest.mark.parametrize("language_server", [Language.REGO], indirect=True) + def test_request_definition_across_files(self, language_server: SolidLanguageServer) -> None: + """Test go-to-definition for symbols across files (cross-file references).""" + # In authz.rego line 11, the allow rule calls utils.is_valid_user + # This function is defined in utils/helpers.rego + file_path = os.path.join("policies", "authz.rego") + + # Get document symbols + symbols = language_server.request_document_symbols(file_path) + symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols + + # Find the allow symbol + allow_symbol = next((s for s in symbol_list if s.get("name") == "allow"), None) + assert allow_symbol is not None, "allow symbol should always be found in authz.rego" + assert "range" in allow_symbol, "allow symbol should have a range" + + # Request definition from line 11 where utils.is_valid_user is called + # Line 11: utils.is_valid_user(input.user) + line = 10 # 0-indexed, so line 11 in file is line 10 in LSP + char = 7 # Position at "is_valid_user" in "utils.is_valid_user" + + definitions = language_server.request_definition(file_path, line, char) + assert definitions is not None and len(definitions) > 0, "Should find cross-file definition for is_valid_user" + + # Verify the definition points to helpers.rego (cross-file) + assert any( + "helpers.rego" in defn.get("relativePath", "") for defn in definitions + ), "Definition should be in utils/helpers.rego (cross-file reference)" + + @pytest.mark.parametrize("language_server", [Language.REGO], indirect=True) + def test_find_symbols_validation(self, language_server: SolidLanguageServer) -> None: + """Test finding symbols in validation.rego which has imports.""" + file_path = os.path.join("policies", "validation.rego") + symbols = language_server.request_document_symbols(file_path) + + assert symbols is not None + assert len(symbols) > 0 + + # Extract symbol names + symbol_list = symbols[0] if isinstance(symbols, tuple) else symbols + symbol_names = {sym.get("name") for sym in symbol_list if isinstance(sym, dict)} + + # Verify expected symbols + assert "validate_user_input" in symbol_names, "validate_user_input rule not found" + assert "has_valid_credentials" in symbol_names, "has_valid_credentials function not found" + assert "validate_request" in symbol_names, "validate_request rule not found"