Add Markdown language server support with marksman (#655)

README.md

@@ -97,6 +97,7 @@ With Serena, we provide direct, out-of-the-box support for:
   * Elixir (requires installation of NextLS and Elixir; **Windows not supported**)
   * Erlang (requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls), experimental, might be slow or hang)
   * AL
+  * Markdown (automatically downloads marksman if not installed, experimental, must be explicitly specified via `--language markdown` when generating project config - primarily useful for documentation-heavy projects)
 
 Support for further languages can easily be added by providing a shallow adapter for a new language server implementation,
 see Serena's [memory on that](.serena/memories/adding_new_language_support_guide.md).

pyproject.toml

@@ -273,6 +273,7 @@ markers = [
   "dart: language server running for Dart",
   "erlang: language server running for Erlang",
   "al: language server running for AL (Microsoft Dynamics 365 Business Central)",
+  "markdown: language server running for Markdown",
 ]
 
 [tool.codespell]

src/solidlsp/language_servers/common.py

@@ -133,7 +133,7 @@ class RuntimeDependencyCollection:
     @staticmethod
     def _install_from_url(dep: RuntimeDependency, logger: LanguageServerLogger, target_dir: str) -> None:
         assert dep.url is not None
-        if dep.archive_type == "gz" and dep.binary_name:
+        if dep.archive_type in ("gz", "binary") and dep.binary_name:
             dest = os.path.join(target_dir, dep.binary_name)
             FileUtils.download_and_extract_archive(logger, dep.url, dest, dep.archive_type)
         else:

src/solidlsp/language_servers/marksman.py

@@ -0,0 +1,193 @@
+"""
+Provides Markdown specific instantiation of the LanguageServer class using marksman.
+Contains various configurations and settings specific to Markdown.
+"""
+
+import logging
+import os
+import pathlib
+import threading
+
+from overrides import override
+
+from solidlsp.ls import SolidLanguageServer
+from solidlsp.ls_config import LanguageServerConfig
+from solidlsp.ls_logger import LanguageServerLogger
+from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
+from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
+from solidlsp.settings import SolidLSPSettings
+
+from .common import RuntimeDependency, RuntimeDependencyCollection
+
+
+class Marksman(SolidLanguageServer):
+    """
+    Provides Markdown specific instantiation of the LanguageServer class using marksman.
+    """
+
+    marksman_releases = "https://github.com/artempyanykh/marksman/releases/download/2024-12-18"
+    runtime_dependencies = RuntimeDependencyCollection(
+        [
+            RuntimeDependency(
+                id="marksman",
+                url=f"{marksman_releases}/marksman-linux-x64",
+                platform_id="linux-x64",
+                archive_type="binary",
+                binary_name="marksman",
+            ),
+            RuntimeDependency(
+                id="marksman",
+                url=f"{marksman_releases}/marksman-linux-arm64",
+                platform_id="linux-arm64",
+                archive_type="binary",
+                binary_name="marksman",
+            ),
+            RuntimeDependency(
+                id="marksman",
+                url=f"{marksman_releases}/marksman-macos",
+                platform_id="osx-x64",
+                archive_type="binary",
+                binary_name="marksman",
+            ),
+            RuntimeDependency(
+                id="marksman",
+                url=f"{marksman_releases}/marksman-macos",
+                platform_id="osx-arm64",
+                archive_type="binary",
+                binary_name="marksman",
+            ),
+            RuntimeDependency(
+                id="marksman",
+                url=f"{marksman_releases}/marksman.exe",
+                platform_id="win-x64",
+                archive_type="binary",
+                binary_name="marksman.exe",
+            ),
+        ]
+    )
+
+    @classmethod
+    def _setup_runtime_dependencies(
+        cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings
+    ) -> str:
+        """Setup runtime dependencies for marksman and return the command to start the server."""
+        deps = cls.runtime_dependencies
+        dependency = deps.get_single_dep_for_current_platform()
+
+        marksman_ls_dir = cls.ls_resources_dir(solidlsp_settings)
+        marksman_executable_path = deps.binary_path(marksman_ls_dir)
+        if not os.path.exists(marksman_executable_path):
+            logger.log(
+                f"Downloading marksman from {dependency.url} to {marksman_ls_dir}",
+                logging.INFO,
+            )
+            deps.install(logger, marksman_ls_dir)
+        if not os.path.exists(marksman_executable_path):
+            raise FileNotFoundError(f"Download failed? Could not find marksman executable at {marksman_executable_path}")
+        os.chmod(marksman_executable_path, 0o755)
+        return marksman_executable_path
+
+    def __init__(
+        self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings
+    ):
+        """
+        Creates a Marksman instance. This class is not meant to be instantiated directly.
+        Use LanguageServer.create() instead.
+        """
+        marksman_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings)
+
+        super().__init__(
+            config,
+            logger,
+            repository_root_path,
+            ProcessLaunchInfo(cmd=f"{marksman_executable_path} server", cwd=repository_root_path),
+            "markdown",
+            solidlsp_settings,
+        )
+        self.server_ready = threading.Event()
+
+    @override
+    def is_ignored_dirname(self, dirname: str) -> bool:
+        return super().is_ignored_dirname(dirname) or dirname in ["node_modules", ".obsidian", ".vitepress", ".vuepress"]
+
+    @staticmethod
+    def _get_initialize_params(repository_absolute_path: str) -> InitializeParams:
+        """
+        Returns the initialize params for the Marksman Language Server.
+        """
+        root_uri = pathlib.Path(repository_absolute_path).as_uri()
+        initialize_params: InitializeParams = {  # type: ignore
+            "processId": os.getpid(),
+            "locale": "en",
+            "rootPath": repository_absolute_path,
+            "rootUri": root_uri,
+            "capabilities": {
+                "textDocument": {
+                    "synchronization": {"didSave": True, "dynamicRegistration": True},
+                    "completion": {"dynamicRegistration": True, "completionItem": {"snippetSupport": True}},
+                    "definition": {"dynamicRegistration": True},
+                    "references": {"dynamicRegistration": True},
+                    "documentSymbol": {
+                        "dynamicRegistration": True,
+                        "hierarchicalDocumentSymbolSupport": True,
+                        "symbolKind": {"valueSet": list(range(1, 27))},
+                    },
+                    "hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]},
+                    "codeAction": {"dynamicRegistration": True},
+                },
+                "workspace": {
+                    "workspaceFolders": True,
+                    "didChangeConfiguration": {"dynamicRegistration": True},
+                    "symbol": {"dynamicRegistration": True},
+                },
+            },
+            "workspaceFolders": [
+                {
+                    "uri": root_uri,
+                    "name": os.path.basename(repository_absolute_path),
+                }
+            ],
+        }
+        return initialize_params
+
+    def _start_server(self):
+        """
+        Starts the Marksman Language Server and waits for it to be ready.
+        """
+
+        def register_capability_handler(_params):
+            return
+
+        def window_log_message(msg):
+            self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO)
+
+        def do_nothing(_params):
+            return
+
+        self.server.on_request("client/registerCapability", register_capability_handler)
+        self.server.on_notification("window/logMessage", window_log_message)
+        self.server.on_notification("$/progress", do_nothing)
+        self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
+
+        self.logger.log("Starting marksman server process", logging.INFO)
+        self.server.start()
+        initialize_params = self._get_initialize_params(self.repository_root_path)
+
+        self.logger.log(
+            "Sending initialize request from LSP client to marksman server and awaiting response",
+            logging.INFO,
+        )
+        init_response = self.server.send.initialize(initialize_params)
+        self.logger.log(f"Received initialize response from marksman server: {init_response}", logging.DEBUG)
+
+        # Verify server capabilities
+        assert "textDocumentSync" in init_response["capabilities"]
+        assert "completionProvider" in init_response["capabilities"]
+        assert "definitionProvider" in init_response["capabilities"]
+
+        self.server.notify.initialized({})
+
+        # marksman is typically ready immediately after initialization
+        self.logger.log("Marksman server initialization complete", logging.INFO)
+        self.server_ready.set()
+        self.completions_available.set()

src/solidlsp/ls_config.py

@@ -66,6 +66,11 @@ class Language(str, Enum):
     """Solargraph language server for Ruby (legacy, experimental).
     Use Language.RUBY (ruby-lsp) for better performance and modern LSP features.
     """
+    MARKDOWN = "markdown"
+    """Marksman language server for Markdown (experimental).
+    Must be explicitly specified as the main language, not auto-detected.
+    This is an edge case primarily useful when working on documentation-heavy projects.
+    """
 
     @classmethod
     def iter_all(cls, include_experimental: bool = False) -> Iterable[Self]:
@@ -77,7 +82,7 @@ class Language(str, Enum):
         """
         Check if the language server is experimental or deprecated.
         """
-        return self in {self.TYPESCRIPT_VTS, self.PYTHON_JEDI, self.CSHARP_OMNISHARP, self.RUBY_SOLARGRAPH}
+        return self in {self.TYPESCRIPT_VTS, self.PYTHON_JEDI, self.CSHARP_OMNISHARP, self.RUBY_SOLARGRAPH, self.MARKDOWN}
 
     def __str__(self) -> str:
         return self.value
@@ -136,6 +141,8 @@ class Language(str, Enum):
                 return FilenameMatcher("*.erl", "*.hrl", "*.escript", "*.config", "*.app", "*.app.src")
             case self.AL:
                 return FilenameMatcher("*.al", "*.dal")
+            case self.MARKDOWN:
+                return FilenameMatcher("*.md", "*.markdown")
             case _:
                 raise ValueError(f"Unhandled language: {self}")
 
@@ -241,6 +248,10 @@ class Language(str, Enum):
                 from solidlsp.language_servers.al_language_server import ALLanguageServer
 
                 return ALLanguageServer
+            case self.MARKDOWN:
+                from solidlsp.language_servers.marksman import Marksman
+
+                return Marksman
             case self.R:
                 from solidlsp.language_servers.r_language_server import RLanguageServer
 

src/solidlsp/ls_utils.py

@@ -233,6 +233,9 @@ class FileUtils:
             elif archive_type == "gz":
                 with gzip.open(tmp_file_name, "rb") as f_in, open(target_path, "wb") as f_out:
                     shutil.copyfileobj(f_in, f_out)
+            elif archive_type == "binary":
+                # For single binary files, just move to target without extraction
+                shutil.move(tmp_file_name, target_path)
             else:
                 logger.log(f"Unknown archive type '{archive_type}' for extraction", logging.ERROR)
                 raise SolidLSPException(f"Unknown archive type '{archive_type}'")

test/resources/repos/markdown/test_repo/CONTRIBUTING.md

@@ -0,0 +1,74 @@
+# Contributing Guidelines
+
+Thank you for considering contributing to this project!
+
+## Table of Contents
+
+- [Code of Conduct](#code-of-conduct)
+- [Getting Started](#getting-started)
+- [Development Setup](#development-setup)
+- [Submitting Changes](#submitting-changes)
+
+## Code of Conduct
+
+Please be respectful and considerate in all interactions.
+
+## Getting Started
+
+To contribute:
+
+1. Fork the repository
+2. Create a feature branch
+3. Make your changes
+4. Submit a pull request
+
+## Development Setup
+
+### Prerequisites
+
+- Git
+- Node.js (v16+)
+- npm or yarn
+
+### Installation Steps
+
+```bash
+git clone https://github.com/example/repo.git
+cd repo
+npm install
+```
+
+## Submitting Changes
+
+### Pull Request Process
+
+1. Update documentation
+2. Add tests for new features
+3. Ensure all tests pass
+4. Update the [README](README.md)
+
+### Commit Messages
+
+Use clear and descriptive commit messages:
+
+- feat: Add new feature
+- fix: Bug fix
+- docs: Documentation changes
+- test: Add or update tests
+
+## Testing
+
+Run the test suite before submitting:
+
+```bash
+npm test
+```
+
+For more information, see:
+
+- [User Guide](guide.md)
+- [API Reference](api.md)
+
+## Questions?
+
+Contact the maintainers or open an issue.

test/resources/repos/markdown/test_repo/README.md

@@ -0,0 +1,53 @@
+# Test Repository
+
+This is a test repository for markdown language server testing.
+
+## Overview
+
+This repository contains sample markdown files for testing LSP features.
+
+## Features
+
+- Document symbol detection
+- Link navigation
+- Reference finding
+- Code completion
+
+### Installation
+
+To use this test repository:
+
+1. Clone the repository
+2. Install dependencies
+3. Run tests
+
+### Usage
+
+See [guide.md](guide.md) for detailed usage instructions.
+
+## Code Examples
+
+Here's a simple example:
+
+```python
+def hello_world():
+    print("Hello, World!")
+```
+
+### JavaScript Example
+
+```javascript
+function greet(name) {
+    console.log(`Hello, ${name}!`);
+}
+```
+
+## References
+
+- [Official Documentation](https://example.com/docs)
+- [API Reference](api.md)
+- [Contributing Guide](CONTRIBUTING.md)
+
+## License
+
+MIT License

test/resources/repos/markdown/test_repo/api.md

@@ -0,0 +1,78 @@
+# API Reference
+
+Complete API documentation for the test repository.
+
+## Classes
+
+### Client
+
+The main client class for interacting with the API.
+
+#### Methods
+
+##### `connect()`
+
+Establishes a connection to the server.
+
+**Parameters:**
+- `host`: Server hostname
+- `port`: Server port number
+
+**Returns:** Connection object
+
+##### `disconnect()`
+
+Closes the connection to the server.
+
+**Returns:** None
+
+### Server
+
+Server-side implementation.
+
+#### Configuration
+
+```json
+{
+  "host": "localhost",
+  "port": 8080,
+  "timeout": 30
+}
+```
+
+## Functions
+
+### `initialize(config)`
+
+Initializes the system with the provided configuration.
+
+**Parameters:**
+- `config`: Configuration dictionary
+
+**Example:**
+
+```python
+config = {
+    "host": "localhost",
+    "port": 8080
+}
+initialize(config)
+```
+
+### `shutdown()`
+
+Gracefully shuts down the system.
+
+## Error Handling
+
+Common errors and their solutions:
+
+- `ConnectionError`: Check network connectivity
+- `TimeoutError`: Increase timeout value
+- `ConfigError`: Validate configuration file
+
+## See Also
+
+- [User Guide](guide.md)
+- [README](README.md)
+- [Contributing](CONTRIBUTING.md)

test/resources/repos/markdown/test_repo/guide.md

@@ -0,0 +1,59 @@
+# User Guide
+
+This guide provides detailed instructions for using the test repository.
+
+## Getting Started
+
+Welcome to the user guide. This document covers:
+
+- Basic concepts
+- Advanced features
+- Troubleshooting
+
+### Basic Concepts
+
+The fundamental concepts you need to understand:
+
+#### Headers and Structure
+
+Markdown documents use headers to create structure. Headers are created using `#` symbols.
+
+#### Links and References
+
+Internal links can reference other documents:
+
+- [Back to README](README.md)
+- [See API documentation](api.md)
+
+### Advanced Features
+
+For advanced users, we provide:
+
+1. Custom extensions
+2. Plugin support
+3. Theme customization
+
+## Configuration
+
+Configuration options are stored in `config.yaml`:
+
+```yaml
+server:
+  port: 8080
+  host: localhost
+```
+
+## Troubleshooting
+
+If you encounter issues:
+
+1. Check the [README](README.md) first
+2. Review [common issues](CONTRIBUTING.md)
+3. Contact support
+
+## Next Steps
+
+After reading this guide, check out:
+
+- [API Reference](api.md)
+- [Contributing Guidelines](CONTRIBUTING.md)

test/solidlsp/markdown/__init__.py

@@ -0,0 +1 @@
+"""Tests for markdown language server functionality."""

test/solidlsp/markdown/test_markdown_basic.py

@@ -0,0 +1,65 @@
+"""
+Basic integration tests for the markdown language server functionality.
+
+These tests validate the functionality of the language server APIs
+like request_document_symbols using the markdown test repository.
+"""
+
+import pytest
+
+from solidlsp import SolidLanguageServer
+from solidlsp.ls_config import Language
+
+
+@pytest.mark.markdown
+class TestMarkdownLanguageServerBasics:
+    """Test basic functionality of the markdown language server."""
+
+    @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True)
+    def test_markdown_language_server_initialization(self, language_server: SolidLanguageServer) -> None:
+        """Test that markdown language server can be initialized successfully."""
+        assert language_server is not None
+        assert language_server.language == Language.MARKDOWN
+
+    @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True)
+    def test_markdown_request_document_symbols(self, language_server: SolidLanguageServer) -> None:
+        """Test request_document_symbols for markdown files."""
+        # Test getting symbols from README.md
+        all_symbols, root_symbols = language_server.request_document_symbols("README.md", include_body=False)
+
+        # Extract heading symbols (LSP Symbol Kind 15 is String, but marksman uses kind 15 for headings)
+        # Note: Different markdown LSPs may use different symbol kinds for headings
+        # Marksman typically uses kind 15 (String) for markdown headings
+        heading_names = [symbol["name"] for symbol in all_symbols]
+
+        # Should detect headings from README.md
+        assert "Test Repository" in heading_names or len(all_symbols) > 0, "Should find at least one heading"
+
+    @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True)
+    def test_markdown_request_symbols_from_guide(self, language_server: SolidLanguageServer) -> None:
+        """Test symbol detection in guide.md file."""
+        all_symbols, root_symbols = language_server.request_document_symbols("guide.md", include_body=False)
+
+        # At least some headings should be found
+        assert len(all_symbols) > 0, f"Should find headings in guide.md, found {len(all_symbols)}"
+
+    @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True)
+    def test_markdown_request_symbols_from_api(self, language_server: SolidLanguageServer) -> None:
+        """Test symbol detection in api.md file."""
+        all_symbols, root_symbols = language_server.request_document_symbols("api.md", include_body=False)
+
+        # Should detect headings from api.md
+        assert len(all_symbols) > 0, f"Should find headings in api.md, found {len(all_symbols)}"
+
+    @pytest.mark.parametrize("language_server", [Language.MARKDOWN], indirect=True)
+    def test_markdown_request_document_symbols_with_body(self, language_server: SolidLanguageServer) -> None:
+        """Test request_document_symbols with body extraction."""
+        # Test with include_body=True
+        all_symbols, root_symbols = language_server.request_document_symbols("README.md", include_body=True)
+
+        # Should have found some symbols
+        assert len(all_symbols) > 0, "Should find symbols in README.md"
+
+        # Note: Not all markdown LSPs provide body information for symbols
+        # This test is more lenient and just verifies the API works
+        assert all_symbols is not None, "Should return symbols even if body extraction is limited"