From dd239b9f988c7bb7ef545a8ac98c0b58df118ade Mon Sep 17 00:00:00 2001 From: intoffset <4951782+intoffset@users.noreply.github.com> Date: Sun, 12 Oct 2025 05:36:37 +0900 Subject: [PATCH] Add Markdown language server support with marksman (#655) --- README.md | 1 + pyproject.toml | 1 + src/solidlsp/language_servers/common.py | 2 +- src/solidlsp/language_servers/marksman.py | 193 ++++++++++++++++++ src/solidlsp/ls_config.py | 13 +- src/solidlsp/ls_utils.py | 3 + .../repos/markdown/test_repo/CONTRIBUTING.md | 74 +++++++ .../repos/markdown/test_repo/README.md | 53 +++++ .../resources/repos/markdown/test_repo/api.md | 78 +++++++ .../repos/markdown/test_repo/guide.md | 59 ++++++ test/solidlsp/markdown/__init__.py | 1 + test/solidlsp/markdown/test_markdown_basic.py | 65 ++++++ 12 files changed, 541 insertions(+), 2 deletions(-) create mode 100644 src/solidlsp/language_servers/marksman.py create mode 100644 test/resources/repos/markdown/test_repo/CONTRIBUTING.md create mode 100644 test/resources/repos/markdown/test_repo/README.md create mode 100644 test/resources/repos/markdown/test_repo/api.md create mode 100644 test/resources/repos/markdown/test_repo/guide.md create mode 100644 test/solidlsp/markdown/__init__.py create mode 100644 test/solidlsp/markdown/test_markdown_basic.py diff --git a/README.md b/README.md index 114c5f64..eee4c4df 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ With Serena, we provide direct, out-of-the-box support for: * Elixir (requires installation of NextLS and Elixir; **Windows not supported**) * Erlang (requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls), experimental, might be slow or hang) * AL + * Markdown (automatically downloads marksman if not installed, experimental, must be explicitly specified via `--language markdown` when generating project config - primarily useful for documentation-heavy projects) Support for further languages can easily be added by providing a shallow adapter for a new language server implementation, see Serena's [memory on that](.serena/memories/adding_new_language_support_guide.md). diff --git a/pyproject.toml b/pyproject.toml index 95f13566..f82a1f9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -273,6 +273,7 @@ markers = [ "dart: language server running for Dart", "erlang: language server running for Erlang", "al: language server running for AL (Microsoft Dynamics 365 Business Central)", + "markdown: language server running for Markdown", ] [tool.codespell] diff --git a/src/solidlsp/language_servers/common.py b/src/solidlsp/language_servers/common.py index a6bcbb3e..8505141e 100644 --- a/src/solidlsp/language_servers/common.py +++ b/src/solidlsp/language_servers/common.py @@ -133,7 +133,7 @@ class RuntimeDependencyCollection: @staticmethod def _install_from_url(dep: RuntimeDependency, logger: LanguageServerLogger, target_dir: str) -> None: assert dep.url is not None - if dep.archive_type == "gz" and dep.binary_name: + if dep.archive_type in ("gz", "binary") and dep.binary_name: dest = os.path.join(target_dir, dep.binary_name) FileUtils.download_and_extract_archive(logger, dep.url, dest, dep.archive_type) else: diff --git a/src/solidlsp/language_servers/marksman.py b/src/solidlsp/language_servers/marksman.py new file mode 100644 index 00000000..c35d2ca6 --- /dev/null +++ b/src/solidlsp/language_servers/marksman.py @@ -0,0 +1,193 @@ +""" +Provides Markdown specific instantiation of the LanguageServer class using marksman. +Contains various configurations and settings specific to Markdown. +""" + +import logging +import os +import pathlib +import threading + +from overrides import override + +from solidlsp.ls import SolidLanguageServer +from solidlsp.ls_config import LanguageServerConfig +from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams +from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo +from solidlsp.settings import SolidLSPSettings + +from .common import RuntimeDependency, RuntimeDependencyCollection + + +class Marksman(SolidLanguageServer): + """ + Provides Markdown specific instantiation of the LanguageServer class using marksman. + """ + + marksman_releases = "https://github.com/artempyanykh/marksman/releases/download/2024-12-18" + runtime_dependencies = RuntimeDependencyCollection( + [ + RuntimeDependency( + id="marksman", + url=f"{marksman_releases}/marksman-linux-x64", + platform_id="linux-x64", + archive_type="binary", + binary_name="marksman", + ), + RuntimeDependency( + id="marksman", + url=f"{marksman_releases}/marksman-linux-arm64", + platform_id="linux-arm64", + archive_type="binary", + binary_name="marksman", + ), + RuntimeDependency( + id="marksman", + url=f"{marksman_releases}/marksman-macos", + platform_id="osx-x64", + archive_type="binary", + binary_name="marksman", + ), + RuntimeDependency( + id="marksman", + url=f"{marksman_releases}/marksman-macos", + platform_id="osx-arm64", + archive_type="binary", + binary_name="marksman", + ), + RuntimeDependency( + id="marksman", + url=f"{marksman_releases}/marksman.exe", + platform_id="win-x64", + archive_type="binary", + binary_name="marksman.exe", + ), + ] + ) + + @classmethod + def _setup_runtime_dependencies( + cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings + ) -> str: + """Setup runtime dependencies for marksman and return the command to start the server.""" + deps = cls.runtime_dependencies + dependency = deps.get_single_dep_for_current_platform() + + marksman_ls_dir = cls.ls_resources_dir(solidlsp_settings) + marksman_executable_path = deps.binary_path(marksman_ls_dir) + if not os.path.exists(marksman_executable_path): + logger.log( + f"Downloading marksman from {dependency.url} to {marksman_ls_dir}", + logging.INFO, + ) + deps.install(logger, marksman_ls_dir) + if not os.path.exists(marksman_executable_path): + raise FileNotFoundError(f"Download failed? Could not find marksman executable at {marksman_executable_path}") + os.chmod(marksman_executable_path, 0o755) + return marksman_executable_path + + def __init__( + self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings + ): + """ + Creates a Marksman instance. This class is not meant to be instantiated directly. + Use LanguageServer.create() instead. + """ + marksman_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings) + + super().__init__( + config, + logger, + repository_root_path, + ProcessLaunchInfo(cmd=f"{marksman_executable_path} server", cwd=repository_root_path), + "markdown", + solidlsp_settings, + ) + self.server_ready = threading.Event() + + @override + def is_ignored_dirname(self, dirname: str) -> bool: + return super().is_ignored_dirname(dirname) or dirname in ["node_modules", ".obsidian", ".vitepress", ".vuepress"] + + @staticmethod + def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: + """ + Returns the initialize params for the Marksman Language Server. + """ + root_uri = pathlib.Path(repository_absolute_path).as_uri() + initialize_params: InitializeParams = { # type: ignore + "processId": os.getpid(), + "locale": "en", + "rootPath": repository_absolute_path, + "rootUri": root_uri, + "capabilities": { + "textDocument": { + "synchronization": {"didSave": True, "dynamicRegistration": True}, + "completion": {"dynamicRegistration": True, "completionItem": {"snippetSupport": True}}, + "definition": {"dynamicRegistration": True}, + "references": {"dynamicRegistration": True}, + "documentSymbol": { + "dynamicRegistration": True, + "hierarchicalDocumentSymbolSupport": True, + "symbolKind": {"valueSet": list(range(1, 27))}, + }, + "hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]}, + "codeAction": {"dynamicRegistration": True}, + }, + "workspace": { + "workspaceFolders": True, + "didChangeConfiguration": {"dynamicRegistration": True}, + "symbol": {"dynamicRegistration": True}, + }, + }, + "workspaceFolders": [ + { + "uri": root_uri, + "name": os.path.basename(repository_absolute_path), + } + ], + } + return initialize_params + + def _start_server(self): + """ + Starts the Marksman Language Server and waits for it to be ready. + """ + + def register_capability_handler(_params): + return + + def window_log_message(msg): + self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO) + + def do_nothing(_params): + return + + self.server.on_request("client/registerCapability", register_capability_handler) + self.server.on_notification("window/logMessage", window_log_message) + self.server.on_notification("$/progress", do_nothing) + self.server.on_notification("textDocument/publishDiagnostics", do_nothing) + + self.logger.log("Starting marksman server process", logging.INFO) + self.server.start() + initialize_params = self._get_initialize_params(self.repository_root_path) + + self.logger.log( + "Sending initialize request from LSP client to marksman server and awaiting response", + logging.INFO, + ) + init_response = self.server.send.initialize(initialize_params) + self.logger.log(f"Received initialize response from marksman server: {init_response}", logging.DEBUG) + + # Verify server capabilities + assert "textDocumentSync" in init_response["capabilities"] + assert "completionProvider" in init_response["capabilities"] + assert "definitionProvider" in init_response["capabilities"] + + self.server.notify.initialized({}) + + # marksman is typically ready immediately after initialization + self.logger.log("Marksman server initialization complete", logging.INFO) + self.server_ready.set() + self.completions_available.set() diff --git a/src/solidlsp/ls_config.py b/src/solidlsp/ls_config.py index 4e51c1c3..11c4e4ff 100644 --- a/src/solidlsp/ls_config.py +++ b/src/solidlsp/ls_config.py @@ -66,6 +66,11 @@ class Language(str, Enum): """Solargraph language server for Ruby (legacy, experimental). Use Language.RUBY (ruby-lsp) for better performance and modern LSP features. """ + MARKDOWN = "markdown" + """Marksman language server for Markdown (experimental). + Must be explicitly specified as the main language, not auto-detected. + This is an edge case primarily useful when working on documentation-heavy projects. + """ @classmethod def iter_all(cls, include_experimental: bool = False) -> Iterable[Self]: @@ -77,7 +82,7 @@ class Language(str, Enum): """ Check if the language server is experimental or deprecated. """ - return self in {self.TYPESCRIPT_VTS, self.PYTHON_JEDI, self.CSHARP_OMNISHARP, self.RUBY_SOLARGRAPH} + return self in {self.TYPESCRIPT_VTS, self.PYTHON_JEDI, self.CSHARP_OMNISHARP, self.RUBY_SOLARGRAPH, self.MARKDOWN} def __str__(self) -> str: return self.value @@ -136,6 +141,8 @@ class Language(str, Enum): return FilenameMatcher("*.erl", "*.hrl", "*.escript", "*.config", "*.app", "*.app.src") case self.AL: return FilenameMatcher("*.al", "*.dal") + case self.MARKDOWN: + return FilenameMatcher("*.md", "*.markdown") case _: raise ValueError(f"Unhandled language: {self}") @@ -241,6 +248,10 @@ class Language(str, Enum): from solidlsp.language_servers.al_language_server import ALLanguageServer return ALLanguageServer + case self.MARKDOWN: + from solidlsp.language_servers.marksman import Marksman + + return Marksman case self.R: from solidlsp.language_servers.r_language_server import RLanguageServer diff --git a/src/solidlsp/ls_utils.py b/src/solidlsp/ls_utils.py index 0202dbc0..11fe88f3 100644 --- a/src/solidlsp/ls_utils.py +++ b/src/solidlsp/ls_utils.py @@ -233,6 +233,9 @@ class FileUtils: elif archive_type == "gz": with gzip.open(tmp_file_name, "rb") as f_in, open(target_path, "wb") as f_out: shutil.copyfileobj(f_in, f_out) + elif archive_type == "binary": + # For single binary files, just move to target without extraction + shutil.move(tmp_file_name, target_path) else: logger.log(f"Unknown archive type '{archive_type}' for extraction", logging.ERROR) raise SolidLSPException(f"Unknown archive type '{archive_type}'") diff --git a/test/resources/repos/markdown/test_repo/CONTRIBUTING.md b/test/resources/repos/markdown/test_repo/CONTRIBUTING.md new file mode 100644 index 00000000..2b533556 --- /dev/null +++ b/test/resources/repos/markdown/test_repo/CONTRIBUTING.md @@ -0,0 +1,74 @@ +# Contributing Guidelines + +Thank you for considering contributing to this project! + +## Table of Contents + +- [Code of Conduct](#code-of-conduct) +- [Getting Started](#getting-started) +- [Development Setup](#development-setup) +- [Submitting Changes](#submitting-changes) + +## Code of Conduct + +Please be respectful and considerate in all interactions. + +## Getting Started + +To contribute: + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Submit a pull request + +## Development Setup + +### Prerequisites + +- Git +- Node.js (v16+) +- npm or yarn + +### Installation Steps + +```bash +git clone https://github.com/example/repo.git +cd repo +npm install +``` + +## Submitting Changes + +### Pull Request Process + +1. Update documentation +2. Add tests for new features +3. Ensure all tests pass +4. Update the [README](README.md) + +### Commit Messages + +Use clear and descriptive commit messages: + +- feat: Add new feature +- fix: Bug fix +- docs: Documentation changes +- test: Add or update tests + +## Testing + +Run the test suite before submitting: + +```bash +npm test +``` + +For more information, see: + +- [User Guide](guide.md) +- [API Reference](api.md) + +## Questions? + +Contact the maintainers or open an issue. diff --git a/test/resources/repos/markdown/test_repo/README.md b/test/resources/repos/markdown/test_repo/README.md new file mode 100644 index 00000000..fbf9a5dc --- /dev/null +++ b/test/resources/repos/markdown/test_repo/README.md @@ -0,0 +1,53 @@ +# Test Repository + +This is a test repository for markdown language server testing. + +## Overview + +This repository contains sample markdown files for testing LSP features. + +## Features + +- Document symbol detection +- Link navigation +- Reference finding +- Code completion + +### Installation + +To use this test repository: + +1. Clone the repository +2. Install dependencies +3. Run tests + +### Usage + +See [guide.md](guide.md) for detailed usage instructions. + +## Code Examples + +Here's a simple example: + +```python +def hello_world(): + print("Hello, World!") +``` + +### JavaScript Example + +```javascript +function greet(name) { + console.log(`Hello, ${name}!`); +} +``` + +## References + +- [Official Documentation](https://example.com/docs) +- [API Reference](api.md) +- [Contributing Guide](CONTRIBUTING.md) + +## License + +MIT License diff --git a/test/resources/repos/markdown/test_repo/api.md b/test/resources/repos/markdown/test_repo/api.md new file mode 100644 index 00000000..81d0e8af --- /dev/null +++ b/test/resources/repos/markdown/test_repo/api.md @@ -0,0 +1,78 @@ +# API Reference + +Complete API documentation for the test repository. + +## Classes + +### Client + +The main client class for interacting with the API. + +#### Methods + +##### `connect()` + +Establishes a connection to the server. + +**Parameters:** +- `host`: Server hostname +- `port`: Server port number + +**Returns:** Connection object + +##### `disconnect()` + +Closes the connection to the server. + +**Returns:** None + +### Server + +Server-side implementation. + +#### Configuration + +```json +{ + "host": "localhost", + "port": 8080, + "timeout": 30 +} +``` + +## Functions + +### `initialize(config)` + +Initializes the system with the provided configuration. + +**Parameters:** +- `config`: Configuration dictionary + +**Example:** + +```python +config = { + "host": "localhost", + "port": 8080 +} +initialize(config) +``` + +### `shutdown()` + +Gracefully shuts down the system. + +## Error Handling + +Common errors and their solutions: + +- `ConnectionError`: Check network connectivity +- `TimeoutError`: Increase timeout value +- `ConfigError`: Validate configuration file + +## See Also + +- [User Guide](guide.md) +- [README](README.md) +- [Contributing](CONTRIBUTING.md) diff --git a/test/resources/repos/markdown/test_repo/guide.md b/test/resources/repos/markdown/test_repo/guide.md new file mode 100644 index 00000000..025f4302 --- /dev/null +++ b/test/resources/repos/markdown/test_repo/guide.md @@ -0,0 +1,59 @@ +# User Guide + +This guide provides detailed instructions for using the test repository. + +## Getting Started + +Welcome to the user guide. This document covers: + +- Basic concepts +- Advanced features +- Troubleshooting + +### Basic Concepts + +The fundamental concepts you need to understand: + +#### Headers and Structure + +Markdown documents use headers to create structure. Headers are created using `#` symbols. + +#### Links and References + +Internal links can reference other documents: + +- [Back to README](README.md) +- [See API documentation](api.md) + +### Advanced Features + +For advanced users, we provide: + +1. Custom extensions +2. Plugin support +3. Theme customization + +## Configuration + +Configuration options are stored in `config.yaml`: + +```yaml +server: + port: 8080 + host: localhost +``` + +## Troubleshooting + +If you encounter issues: + +1. Check the [README](README.md) first +2. Review [common issues](CONTRIBUTING.md) +3. Contact support + +## Next Steps + +After reading this guide, check out: + +- [API Reference](api.md) +- [Contributing Guidelines](CONTRIBUTING.md) diff --git a/test/solidlsp/markdown/__init__.py b/test/solidlsp/markdown/__init__.py new file mode 100644 index 00000000..9f2adc19 --- /dev/null +++ b/test/solidlsp/markdown/__init__.py @@ -0,0 +1 @@ +"""Tests for markdown language server functionality.""" diff --git a/test/solidlsp/markdown/test_markdown_basic.py b/test/solidlsp/markdown/test_markdown_basic.py new file mode 100644 index 00000000..a3e8c42e --- /dev/null +++ b/test/solidlsp/markdown/test_markdown_basic.py @@ -0,0 +1,65 @@ +""" +Basic integration tests for the markdown language server functionality. + +These tests validate the functionality of the language server APIs +like request_document_symbols using the markdown test repository. +""" + +import pytest + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language + + +@pytest.mark.markdown +class TestMarkdownLanguageServerBasics: + """Test basic functionality of the markdown language server.""" + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_language_server_initialization(self, language_server: SolidLanguageServer) -> None: + """Test that markdown language server can be initialized successfully.""" + assert language_server is not None + assert language_server.language == Language.MARKDOWN + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_request_document_symbols(self, language_server: SolidLanguageServer) -> None: + """Test request_document_symbols for markdown files.""" + # Test getting symbols from README.md + all_symbols, root_symbols = language_server.request_document_symbols("README.md", include_body=False) + + # Extract heading symbols (LSP Symbol Kind 15 is String, but marksman uses kind 15 for headings) + # Note: Different markdown LSPs may use different symbol kinds for headings + # Marksman typically uses kind 15 (String) for markdown headings + heading_names = [symbol["name"] for symbol in all_symbols] + + # Should detect headings from README.md + assert "Test Repository" in heading_names or len(all_symbols) > 0, "Should find at least one heading" + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_request_symbols_from_guide(self, language_server: SolidLanguageServer) -> None: + """Test symbol detection in guide.md file.""" + all_symbols, root_symbols = language_server.request_document_symbols("guide.md", include_body=False) + + # At least some headings should be found + assert len(all_symbols) > 0, f"Should find headings in guide.md, found {len(all_symbols)}" + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_request_symbols_from_api(self, language_server: SolidLanguageServer) -> None: + """Test symbol detection in api.md file.""" + all_symbols, root_symbols = language_server.request_document_symbols("api.md", include_body=False) + + # Should detect headings from api.md + assert len(all_symbols) > 0, f"Should find headings in api.md, found {len(all_symbols)}" + + @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True) + def test_markdown_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None: + """Test request_document_symbols with body extraction.""" + # Test with include_body=True + all_symbols, root_symbols = language_server.request_document_symbols("README.md", include_body=True) + + # Should have found some symbols + assert len(all_symbols) > 0, "Should find symbols in README.md" + + # Note: Not all markdown LSPs provide body information for symbols + # This test is more lenient and just verifies the API works + assert all_symbols is not None, "Should return symbols even if body extraction is limited"