diff --git a/.serena/memories/adding_new_language_support_guide.md b/.serena/memories/adding_new_language_support_guide.md new file mode 100644 index 00000000..a106ff8d --- /dev/null +++ b/.serena/memories/adding_new_language_support_guide.md @@ -0,0 +1,393 @@ +# Adding New Language Support to Serena + +This guide explains how to add support for a new programming language to Serena, using the C# implementation as a reference. + +## Overview + +Adding a new language involves: +1. **Language Server Implementation** - Creating a language-specific server class +2. **Language Registration** - Adding the language to enums and configurations +3. **Test Repository** - Creating a minimal test project +4. **Test Suite** - Writing comprehensive tests +5. **Runtime Dependencies** - Configuring automatic language server downloads + +## Step 1: Language Server Implementation + +### 1.1 Create Language Server Class + +Create a new file in `src/solidlsp/language_servers/` (e.g., `new_language_server.py`): + +```python +from solidlsp.ls import SolidLanguageServer +from solidlsp.ls_config import LanguageServerConfig +from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo + +class NewLanguageServer(SolidLanguageServer): + """ + Language server implementation for NewLanguage. + """ + + def __init__(self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str): + # Determine language server command + cmd = self._get_language_server_command() + + super().__init__( + config, + logger, + repository_root_path, + ProcessLaunchInfo(cmd=cmd, cwd=repository_root_path), + "new_language", # Language ID for LSP + ) + + def _get_language_server_command(self) -> list[str]: + """Get the command to start the language server.""" + # Example: return ["new-language-server", "--stdio"] + pass + + @override + def is_ignored_dirname(self, dirname: str) -> bool: + """Define language-specific directories to ignore.""" + return super().is_ignored_dirname(dirname) or dirname in ["build", "dist", "target"] +``` + +### 1.2 Language Server Discovery and Installation + +For languages requiring automatic installation, implement download logic similar to C#: + +```python +@classmethod +def _ensure_server_installed(cls, logger: LanguageServerLogger) -> str: + """Ensure language server is installed and return path.""" + # Check system installation first + system_server = shutil.which("new-language-server") + if system_server: + return system_server + + # Download and install if needed + server_path = cls._download_and_install_server(logger) + return server_path + +def _download_and_install_server(cls, logger: LanguageServerLogger) -> str: + """Download and install the language server.""" + # Implementation specific to your language server + pass +``` + +### 1.3 LSP Initialization + +Override initialization methods if needed: + +```python +def _get_initialize_params(self) -> InitializeParams: + """Return language-specific initialization parameters.""" + return { + "processId": os.getpid(), + "rootUri": PathUtils.path_to_uri(self.repository_root_path), + "capabilities": { + # Language-specific capabilities + } + } + +def _start_server(self): + """Start the language server with custom handlers.""" + # Set up notification handlers + self.server.on_notification("window/logMessage", self._handle_log_message) + + # Start server and initialize + self.server.start() + init_response = self.server.send.initialize(self._get_initialize_params()) + self.server.notify.initialized({}) +``` + +## Step 2: Language Registration + +### 2.1 Add to Language Enum + +In `src/solidlsp/ls_config.py`, add your language to the `Language` enum: + +```python +class Language(str, Enum): + # Existing languages... + NEW_LANGUAGE = "new_language" + + def get_source_fn_matcher(self) -> FilenameMatcher: + match self: + # Existing cases... + case self.NEW_LANGUAGE: + return FilenameMatcher("*.newlang", "*.nl") # File extensions +``` + +### 2.2 Update Language Server Factory + +In `src/solidlsp/ls.py`, add your language to the `create` method: + +```python +@classmethod +def create(cls, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str) -> "SolidLanguageServer": + match config.code_language: + # Existing cases... + case Language.NEW_LANGUAGE: + from solidlsp.language_servers.new_language_server import NewLanguageServer + return NewLanguageServer(config, logger, repository_root_path) +``` + +## Step 3: Test Repository + +### 3.1 Create Test Project + +Create a minimal project in `test/resources/repos/new_language/test_repo/`: + +``` +test/resources/repos/new_language/test_repo/ +├── main.newlang # Main source file +├── lib/ +│ └── helper.newlang # Additional source for testing +├── project.toml # Project configuration (if applicable) +└── .gitignore # Ignore build artifacts +``` + +### 3.2 Example Source Files + +Create meaningful source files that demonstrate: +- **Classes/Types** - For symbol testing +- **Functions/Methods** - For reference finding +- **Imports/Dependencies** - For cross-file operations +- **Nested Structures** - For hierarchical symbol testing + +Example `main.newlang`: +``` +import lib.helper + +class Calculator { + func add(a: Int, b: Int) -> Int { + return a + b + } + + func subtract(a: Int, b: Int) -> Int { + return helper.subtract(a, b) // Reference to imported function + } +} + +class Program { + func main() { + let calc = Calculator() + let result = calc.add(5, 3) // Reference to add method + print(result) + } +} +``` + +## Step 4: Test Suite + +### 4.1 Create Test File + +Create `test/solidlsp/new_language/test_new_language_basic.py`: + +```python +import os +import pytest + +from solidlsp import SolidLanguageServer +from solidlsp.ls_config import Language +from solidlsp.ls_utils import SymbolUtils + +@pytest.mark.new_language +class TestNewLanguageServer: + @pytest.mark.parametrize("language_server", [Language.NEW_LANGUAGE], indirect=True) + def test_find_symbol(self, language_server: SolidLanguageServer) -> None: + """Test finding symbols in the full symbol tree.""" + symbols = language_server.request_full_symbol_tree() + assert SymbolUtils.symbol_tree_contains_name(symbols, "Calculator"), "Calculator class not found" + assert SymbolUtils.symbol_tree_contains_name(symbols, "add"), "add method not found" + assert SymbolUtils.symbol_tree_contains_name(symbols, "Program"), "Program class not found" + + @pytest.mark.parametrize("language_server", [Language.NEW_LANGUAGE], indirect=True) + def test_get_document_symbols(self, language_server: SolidLanguageServer) -> None: + """Test getting document symbols.""" + file_path = "main.newlang" + symbols = language_server.request_document_symbols(file_path) + + assert len(symbols) > 0 + + # Check for expected classes + class_names = [s.get("name") for s in symbols if s.get("kind") == 5] # 5 = class + assert "Calculator" in class_names + assert "Program" in class_names + + @pytest.mark.parametrize("language_server", [Language.NEW_LANGUAGE], indirect=True) + def test_find_references(self, language_server: SolidLanguageServer) -> None: + """Test finding references to symbols.""" + file_path = "main.newlang" + symbols = language_server.request_document_symbols(file_path) + + # Find the 'add' method + add_symbol = None + for sym in symbols: + if sym.get("name") == "add": + add_symbol = sym + break + + assert add_symbol is not None, "Could not find 'add' method" + + # Test finding references + sel_start = add_symbol["selectionRange"]["start"] + refs = language_server.request_references(file_path, sel_start["line"], sel_start["character"]) + + # Should find reference in main() method + assert len(refs) > 0, "No references found for 'add' method" + + @pytest.mark.parametrize("language_server", [Language.NEW_LANGUAGE], indirect=True) + def test_cross_file_references(self, language_server: SolidLanguageServer) -> None: + """Test finding references across multiple files.""" + # Test imports and cross-file symbol usage + file_path = "lib/helper.newlang" + symbols = language_server.request_document_symbols(file_path) + + # Verify symbols exist in helper file + assert len(symbols) > 0 + + # Test that main.newlang references helper functions + main_file = "main.newlang" + main_symbols = language_server.request_document_symbols(main_file) + assert len(main_symbols) > 0 +``` + +### 4.2 Test Utilities and Edge Cases + +Add additional test methods for: +- **Nested namespaces/modules** +- **Error handling** - Invalid syntax, missing files +- **Performance** - Large files, many symbols +- **Language-specific features** - Generics, interfaces, etc. + +## Step 5: Runtime Dependencies (Optional) + +If your language server needs automatic installation, add runtime dependencies to the language server class: + +```python +from solidlsp.language_servers.common import RuntimeDependency + +RUNTIME_DEPENDENCIES = [ + RuntimeDependency( + id="NewLanguageServer", + platform_id="linux-x64", + package_name="new-language-server", + package_version="1.0.0", + url="https://github.com/new-lang/server/releases/download/v1.0.0/server-linux.tar.gz", + archive_type="tar.gz", + binary_name="new-language-server", + extract_path="bin/" + ), + # Add other platforms as needed +] +``` + +## Step 6: Testing + +### 6.1 Run Language-Specific Tests + +```bash +# Run only your language tests +pytest -m new_language + +# Run with verbose output +pytest -m new_language -v + +# Run specific test file +pytest test/solidlsp/new_language/test_new_language_basic.py +``` + +### 6.2 Integration Testing + +Test the full integration: + +```python +# Test that language server can be created and started +from test.conftest import create_ls +from solidlsp.ls_config import Language + +def test_integration(): + ls = create_ls(Language.NEW_LANGUAGE) + ls.start() + try: + # Test basic operations + symbols = ls.request_full_symbol_tree() + assert len(symbols) > 0 + finally: + ls.stop() +``` + +## Step 7: Documentation and Configuration + +### 7.1 Update pytest.ini + +Add your language marker to `pytest.ini`: + +```ini +[tool.pytest.ini_options] +markers = [ + "new_language: marks tests as requiring NewLanguage language server" +] +``` + +### 7.2 CI/CD Configuration + +Update GitHub Actions workflows to include your language in CI testing. + +### 7.3 Documentation + +Update: +- **README.md** - Add language to supported languages list +- **CHANGELOG.md** - Document the new language support +- **Language-specific docs** - Installation requirements, known issues + +## Common Patterns and Best Practices + +### Error Handling + +```python +def _start_server(self): + try: + self.server.start() + init_response = self.server.send.initialize(self._get_initialize_params()) + except Exception as e: + raise LanguageServerException(f"Failed to start NewLanguage server: {e}") from e +``` + +### Performance Optimization + +```python +@override +def is_ignored_dirname(self, dirname: str) -> bool: + """Optimize by ignoring build/cache directories.""" + ignored_dirs = {"node_modules", "build", "dist", "target", ".cache"} + return super().is_ignored_dirname(dirname) or dirname in ignored_dirs +``` + +### Language-Specific Configuration + +```python +def _get_initialize_params(self) -> InitializeParams: + """Configure for optimal NewLanguage support.""" + return { + "processId": os.getpid(), + "rootUri": PathUtils.path_to_uri(self.repository_root_path), + "initializationOptions": { + # Language-specific options + "new_language": { + "enable_formatting": True, + "enable_diagnostics": True, + } + }, + "capabilities": { + "textDocument": { + "documentSymbol": {"hierarchicalDocumentSymbolSupport": True}, + "references": {"dynamicRegistration": True}, + "definition": {"dynamicRegistration": True}, + } + } + } +``` + +This comprehensive guide should enable you to add robust support for any new programming language to Serena, following the established patterns and ensuring thorough testing. \ No newline at end of file diff --git a/.serena/memories/current_solid_lsp_architecture.md b/.serena/memories/current_solid_lsp_architecture.md deleted file mode 100644 index 90a60ad9..00000000 --- a/.serena/memories/current_solid_lsp_architecture.md +++ /dev/null @@ -1,123 +0,0 @@ -# Current Architecture: Solid-LSP Implementation - -## Overview - -Solid-LSP (`src/solidlsp/`) is Serena's current language server integration layer, designed as a **simplified, deadlock-resistant** replacement for the problematic multilspy architecture. It enables **single-process operation** with MCP servers while maintaining all semantic code analysis capabilities. - -## Current System Status - -- **Default Implementation**: Solid-LSP is the only language server implementation (multilspy completely removed) -- **Process Isolation**: **Disabled by default** (`USE_PROCESS_ISOLATION = False`) -- **MCP Integration**: Runs safely in the same process as MCP server -- **Performance**: Lower latency and resource usage than process-isolated multilspy - -## Core Architecture - -### Primary Components - -#### 1. **`SolidLanguageServer`** (`src/solidlsp/ls.py`) -- **1,634 lines**: Main language server interface -- **Enhanced Methods**: Same semantic capabilities as multilspy but with improved async handling -- **Lifecycle Control**: `start()`, `stop()`, `is_running()`, `language_server()` property -- **Clean Async Patterns**: No coroutine leakage between async contexts - -#### 2. **`SolidLanguageServerHandler`** (`src/solidlsp/ls_handler.py`) -- **512 lines**: Simplified LSP protocol handler -- **Direct Process Management**: More straightforward than multilspy's complex orchestration -- **Resource Management**: Better cleanup and process termination - -#### 3. **Protocol Layer** (`src/solidlsp/lsp_protocol_handler/`) -- **Simplified Protocol**: Direct LSP communication without excessive abstraction -- **Types and Constants**: LSP type definitions and protocol constants -- **Request Handling**: Streamlined request/response pattern - -### Language Server Support - -Solid-LSP maintains **identical language support** to multilspy: - -#### Primary Languages (Directly Supported) -- **Python**: Pyright Language Server (`src/solidlsp/language_servers/pyright_language_server/`) -- **Java**: Eclipse JDTLS (`src/solidlsp/language_servers/eclipse_jdtls/`) -- **TypeScript/JavaScript**: TypeScript Language Server (`src/solidlsp/language_servers/typescript_language_server/`) - -#### Additional Languages (Full Support) -- **C#**: OmniSharp (`src/solidlsp/language_servers/omnisharp/`) -- **Rust**: Rust-Analyzer (`src/solidlsp/language_servers/rust_analyzer/`) -- **Go**: Gopls (`src/solidlsp/language_servers/gopls/`) -- **Ruby**: Solargraph (`src/solidlsp/language_servers/solargraph/`) -- **C++**: Clangd (`src/solidlsp/language_servers/clangd_language_server/`) -- **Dart**: Dart Language Server (`src/solidlsp/language_servers/dart_language_server/`) -- **PHP**: Intelephense (`src/solidlsp/language_servers/intelephense/`) -- **Kotlin**: Kotlin Language Server (`src/solidlsp/language_servers/kotlin_language_server/`) - -## Key Architectural Improvements - -### 1. **Simplified Async Patterns** -- **Clean Boundaries**: Proper async context management prevents MCP server contamination -- **No Coroutine Leakage**: Eliminates the unawaited coroutine warnings from multilspy -- **Direct Operations**: Fewer abstraction layers where async context could be corrupted - -### 2. **Single Process Safety** -- **Safe MCP Integration**: Designed to work within MCP server process without deadlocks -- **Eliminated IPC Overhead**: Direct method calls instead of inter-process communication -- **Resource Efficiency**: Lower memory usage and faster startup times - -### 3. **Enhanced Process Control** -- **Direct Process Management**: `_start_server_process()`, `_start_server()` methods -- **Better State Management**: `_server_context` attribute for enhanced control -- **Improved Cleanup**: More reliable resource management and process termination - -## Integration Points - -### MCP Server Integration -Current implementation in `src/serena/mcp.py:590`: -```python -if not USE_PROCESS_ISOLATION: - mcp_factory = SerenaMCPFactorySingleProcess(context=context, project=project_file) -else: - mcp_factory = SerenaMCPFactoryWithProcessIsolation(context=context, project=project_file) -``` - -### Agent Integration -Language server creation in `src/serena/agent.py:642-689`: -```python -def create_ls_for_project(...) -> SolidLanguageServer: - # Creates LanguageServerConfig with project settings - # Returns SolidLanguageServer.create() instance -``` - -## Configuration and Settings - -### Core Configuration -- **`USE_PROCESS_ISOLATION = False`**: Process isolation disabled by default -- **Language Detection**: Automatic based on project composition -- **Timeout Settings**: Configurable language server timeouts -- **LSP Communication Tracing**: Optional debugging support - -### Project-Level Settings -- **Ignored Paths**: Respects project configuration and gitignore -- **Language Selection**: Automatic or manual language server selection -- **Timeout Configuration**: Per-project timeout settings - -## Performance Benefits - -### vs. Multilspy + Process Isolation -1. **Lower Latency**: Direct method calls instead of IPC -2. **Reduced Memory**: Single process instead of multiple processes -3. **Faster Startup**: No process creation overhead -4. **Simpler Debugging**: All components in same process with unified stack traces - -### Stability Improvements -1. **No Asyncio Deadlocks**: Clean async boundaries prevent contamination -2. **Reliable Operations**: `find_symbol` and other tools work consistently -3. **Resource Management**: Better cleanup prevents resource leaks -4. **Error Handling**: Simplified error propagation and handling - -## Current Operational State - -- **Production Ready**: Default implementation for all Serena deployments -- **Fully Tested**: All semantic tools (`find_symbol`, `replace_symbol_body`, etc.) working reliably -- **MCP Compatible**: Stable operation with Claude Desktop and other MCP clients -- **Cross-Platform**: Works on all supported operating systems - -This architecture represents the successful resolution of the multilspy asyncio contamination issues while maintaining full semantic code analysis capabilities and improving overall performance. \ No newline at end of file diff --git a/.serena/memories/multilspy_historical_context.md b/.serena/memories/multilspy_historical_context.md deleted file mode 100644 index 7f73a588..00000000 --- a/.serena/memories/multilspy_historical_context.md +++ /dev/null @@ -1,73 +0,0 @@ -# Historical Context: Multilspy Era and Migration to Solid-LSP - -## What multilspy Was - -Multilspy was Serena's original language server integration layer that provided semantic code analysis through Language Server Protocol (LSP). It was located in `src/multilspy/` and included: - -### Core Components (No Longer Present) -- **`multilspy.LanguageServer`**: Main language server interface with async methods like `request_full_symbol_tree()` -- **`LanguageServerHandler`**: Complex async task management and protocol handling -- **`lsp_protocol_handler`**: Separate protocol abstraction layer -- **Process Management**: Complex threading and async orchestration - -### Supported Languages -Multilspy supported the same languages now supported by solid-lsp: -- Python (Pyright), Java (Eclipse JDTLS), TypeScript/JavaScript, C#, Rust, Go, Ruby, C++, Dart, PHP, Kotlin - -## Why We Moved Away from Multilspy - -### 1. **Asyncio Contamination Issues** -- **Root Problem**: MCP server runs its own asyncio event loop, while multilspy created additional asyncio loops -- **Coroutine Leakage**: `multilspy.LanguageServer.request_full_symbol_tree()` coroutines leaked into MCP server context but were never awaited -- **Evidence**: MCP server logs showed: `RuntimeWarning: coroutine 'LanguageServer.request_full_symbol_tree' was never awaited` -- **Result**: Accumulated unawaited coroutines caused resource exhaustion and apparent "hangs" - -### 2. **Dual Event Loop Interference** -- **MCP Server**: Asyncio loop A for handling requests -- **SerenaAgent**: Asyncio loop B for language servers + loop C for dashboard -- **Conflict**: Multiple asyncio contexts caused contamination and deadlocks - -### 3. **Complex Architecture** -- **Over-abstraction**: Multiple layers of protocol handling increased complexity -- **Threading Issues**: Complex async task management made debugging difficult -- **Resource Overhead**: Separate protocol handler processes added overhead - -### 4. **MCP-Specific Problems** -- **Only in MCP environments**: Issues didn't occur in direct script execution -- **Mandatory Process Isolation**: Required separate processes to prevent asyncio contamination -- **Performance Penalty**: IPC overhead was necessary for stability - -## The Process Isolation Workaround - -Before solid-lsp, the **only way** to use multilspy with MCP was through process isolation: - -```python -# Old architecture (when multilspy was still present) -if USE_SOLID_LSP: - mcp_factory = SerenaMCPFactorySingleProcess(context=context, project=project_file) -else: - # multilspy required process isolation to prevent asyncio contamination - mcp_factory = SerenaMCPFactoryWithProcessIsolation(context=context, project=project_file) -``` - -### Why Process Isolation Was Mandatory for Multilspy -- **Complete Separation**: MCP server, SerenaAgent, and language servers in different processes -- **IPC Overhead**: All communication through inter-process communication -- **Resource Cost**: Higher memory usage and startup time -- **Debugging Complexity**: Distributed architecture made troubleshooting harder - -## Timeline of Changes - -1. **Pre-solid-lsp**: Multilspy was the only option, required process isolation for MCP -2. **Solid-lsp Introduction**: New implementation designed to eliminate asyncio issues -3. **Current State**: Multilspy completely removed, solid-lsp is the only implementation -4. **Process Isolation**: Now disabled by default (`USE_PROCESS_ISOLATION = False`) since it's no longer needed - -## Key Lessons Learned - -- **Asyncio Complexity**: Multiple event loops in the same application are extremely difficult to manage correctly -- **MCP Integration Challenges**: MCP server's async nature requires careful consideration of async boundaries -- **Simplicity Benefits**: Removing abstraction layers often improves stability and performance -- **Process Isolation Trade-offs**: While effective for isolation, the performance cost is significant when not necessary - -This migration represents a successful architectural evolution from a complex, problematic system to a simpler, more reliable one. \ No newline at end of file diff --git a/.serena/memories/serena_core_concepts_and_architecture.md b/.serena/memories/serena_core_concepts_and_architecture.md new file mode 100644 index 00000000..49edd2aa --- /dev/null +++ b/.serena/memories/serena_core_concepts_and_architecture.md @@ -0,0 +1,175 @@ +# Serena Core Concepts and Architecture + +## High-Level Architecture + +Serena is built around a dual-layer architecture: + +1. **SerenaAgent** - The main orchestrator that manages projects, tools, and user interactions +2. **SolidLanguageServer** - A unified wrapper around Language Server Protocol (LSP) implementations + +## Core Components + +### 1. SerenaAgent (`src/serena/agent.py`) + +The central coordinator that: +- Manages active projects and their configurations +- Coordinates between different tools and contexts +- Handles language server lifecycle +- Manages memory persistence +- Provides MCP (Model Context Protocol) server interface + +Key responsibilities: +- **Project Management** - Activating, switching between projects +- **Tool Registry** - Loading and managing available tools based on context/mode +- **Language Server Integration** - Starting/stopping language servers per project +- **Memory Management** - Persistent storage of project knowledge +- **Task Execution** - Coordinating complex multi-step operations + +### 2. SolidLanguageServer (`src/solidlsp/ls.py`) + +A unified abstraction over multiple language servers that provides: +- **Language-agnostic interface** for symbol operations +- **Caching layer** for performance optimization +- **Error handling and recovery** for unreliable language servers +- **Uniform API** regardless of underlying LSP implementation + +Core capabilities: +- Symbol discovery and navigation +- Code completion and hover information +- Find references and definitions +- Document and workspace symbol search +- File watching and change notifications + +### 3. Tool System (`src/serena/tools/`) + +Modular tool architecture with several categories: + +#### File Tools (`file_tools.py`) +- File system operations (read, write, list directories) +- Text search and pattern matching +- Regex-based replacements + +#### Symbol Tools (`symbol_tools.py`) +- Language-aware symbol finding and navigation +- Symbol body replacement and insertion +- Reference finding across codebase + +#### Memory Tools (`memory_tools.py`) +- Project knowledge persistence +- Memory retrieval and management +- Onboarding information storage + +#### Configuration Tools (`config_tools.py`) +- Project activation and switching +- Mode and context management +- Tool inclusion/exclusion + +### 4. Configuration System (`src/serena/config/`) + +Multi-layered configuration supporting: +- **Contexts** - Define available tools and their behavior +- **Modes** - Specify operational patterns (interactive, editing, etc.) +- **Projects** - Per-project settings and language server configs +- **Tool Sets** - Grouped tool collections for different use cases + +## Language Server Integration + +### Language Support Model + +Each supported language has: +1. **Language Server Implementation** (`src/solidlsp/language_servers/`) +2. **Runtime Dependencies** - Managed downloads of language servers +3. **Test Repository** (`test/resources/repos//`) +4. **Test Suite** (`test/solidlsp//`) + +### Language Server Lifecycle + +1. **Discovery** - Find language servers or download them automatically +2. **Initialization** - Start server process and perform LSP handshake +3. **Project Setup** - Open workspace and configure language-specific settings +4. **Operation** - Handle requests/responses with caching and error recovery +5. **Shutdown** - Clean shutdown of server processes + +### Supported Languages + +Current language support includes: +- **C#** - Microsoft.CodeAnalysis.LanguageServer (.NET 9) +- **Python** - Pyright or Jedi +- **TypeScript/JavaScript** - TypeScript Language Server +- **Rust** - rust-analyzer +- **Go** - gopls +- **Java** - Eclipse JDT Language Server +- **Kotlin** - Kotlin Language Server +- **PHP** - Intelephense +- **Ruby** - Solargraph +- **Clojure** - clojure-lsp +- **Elixir** - ElixirLS +- **Dart** - Dart Language Server +- **C/C++** - clangd +- **Terraform** - terraform-ls + +## Memory and Knowledge Management + +### Memory System +- **Markdown-based storage** in `.serena/memories/` directory +- **Contextual retrieval** - memories loaded based on relevance +- **Project-specific** knowledge persistence +- **Onboarding support** - guided setup for new projects + +### Knowledge Categories +- **Project Structure** - Directory layouts, build systems +- **Architecture Patterns** - How the codebase is organized +- **Development Workflows** - Testing, building, deployment +- **Domain Knowledge** - Business logic and requirements + +## MCP Server Interface + +Serena exposes its functionality through Model Context Protocol: +- **Tool Discovery** - AI agents can enumerate available tools +- **Context-Aware Operations** - Tools behave based on active project/mode +- **Stateful Sessions** - Maintains project state across interactions +- **Error Handling** - Graceful degradation when tools fail + +## Error Handling and Resilience + +### Language Server Reliability +- **Timeout Management** - Configurable timeouts for LSP requests +- **Process Recovery** - Automatic restart of crashed language servers +- **Fallback Behavior** - Graceful degradation when LSP unavailable +- **Caching Strategy** - Reduces impact of server failures + +### Project Activation Safety +- **Validation** - Verify project structure before activation +- **Error Isolation** - Project failures don't affect other projects +- **Recovery Mechanisms** - Automatic cleanup and retry logic + +## Performance Considerations + +### Caching Strategy +- **Symbol Cache** - In-memory caching of expensive symbol operations +- **File System Cache** - Reduced disk I/O for repeated operations +- **Language Server Cache** - Persistent cache across sessions + +### Resource Management +- **Language Server Pooling** - Reuse servers across projects when possible +- **Memory Management** - Automatic cleanup of unused resources +- **Background Operations** - Async operations don't block user interactions + +## Extension Points + +### Adding New Languages +1. Implement language server class in `src/solidlsp/language_servers/` +2. Add runtime dependencies configuration +3. Create test repository and test suite +4. Update language enumeration and configuration + +### Adding New Tools +1. Inherit from `Tool` base class in `tools_base.py` +2. Implement required methods and parameter validation +3. Register tool in appropriate tool registry +4. Add to context/mode configurations as needed + +### Custom Contexts and Modes +- Define new contexts in YAML configuration files +- Specify tool sets and operational patterns +- Configure for specific development workflows \ No newline at end of file diff --git a/.serena/memories/serena_project_structure.md b/.serena/memories/serena_project_structure.md deleted file mode 100644 index 4475b0b1..00000000 --- a/.serena/memories/serena_project_structure.md +++ /dev/null @@ -1,173 +0,0 @@ -# Serena Project Structure Overview - -## Top-Level Organization - -Serena is organized into several key directories, each serving distinct architectural purposes: - -``` -serena/ -├── src/ # Main source code -│ ├── interprompt/ # Template and prompt management system -│ ├── serena/ # Core Serena functionality -│ └── solidlsp/ # Language server integration layer -├── tests/ # Test suites -├── docs/ # Documentation -├── prompts/ # Prompt templates and configurations -├── contexts/ # Context definitions (desktop-app, agent, ide-assistant) -├── modes/ # Mode definitions (planning, editing, interactive, one-shot) -└── pyproject.toml # Project configuration and dependencies -``` - -## Core Source Structure (`src/`) - -### 1. **Serena Core (`src/serena/`)** -The heart of the Serena system containing: - -#### **Agent and Core Logic** -- **`agent.py`**: Main `SerenaAgent` class, tool implementations, project management -- **`mcp.py`**: Model Context Protocol server implementation and MCP factories -- **`config.py`**: Configuration system (contexts, modes, registered configurations) -- **`constants.py`**: System constants including `USE_PROCESS_ISOLATION = False` - -#### **Specialized Components** -- **`dashboard.py`**: Web dashboard for monitoring and logging (`MemoryLogHandler`, `SerenaDashboardAPI`) -- **`symbol.py`**: Symbol management (`SymbolLocation`, `Symbol`, `SymbolManager`) -- **`text_utils.py`**: Text processing utilities for search and file operations -- **`gui_log_viewer.py`**: GUI log window implementation - -#### **Integration Layers** -- **`agno.py`**: Agno framework integration (`SerenaAgnoToolkit`, `SerenaAgnoAgentProvider`) -- **`process_isolated_agent.py`**: Process isolation infrastructure (available but unused by default) -- **`prompt_factory.py`**: Prompt generation and management - -#### **Utility Modules (`src/serena/util/`)** -- **`file_system.py`**: File scanning, gitignore parsing (`GitignoreParser`, `scan_directory`) -- **`git.py`**: Git operations and status checking -- **`shell.py`**: Shell command execution (`execute_shell_command`) -- **`thread.py`**: Threading utilities with timeout support -- **`inspection.py`**: Code inspection and language detection utilities - -### 2. **Solid-LSP (`src/solidlsp/`)** -Current language server integration layer: - -#### **Core Language Server Components** -- **`ls.py`**: Main `SolidLanguageServer` class (1,634 lines) -- **`ls_handler.py`**: `SolidLanguageServerHandler` for protocol management (512 lines) -- **`ls_request.py`**: Request handling (`LanguageServerRequest`) -- **`ls_types.py`**: LSP type definitions -- **`ls_utils.py`**: Utility classes (`TextUtils`, `PathUtils`, `FileUtils`, `SymbolUtils`) - -#### **Protocol Handling (`src/solidlsp/lsp_protocol_handler/`)** -- **`server.py`**: LSP protocol server implementation -- **`lsp_requests.py`**: LSP request and notification classes -- **`lsp_types.py`**: Complete LSP type system definitions -- **`lsp_constants.py`**: LSP protocol constants - -#### **Language Server Implementations (`src/solidlsp/language_servers/`)** -Each language has its own subdirectory with specific implementations: -- **`pyright_language_server/`**: Python support via Pyright -- **`eclipse_jdtls/`**: Java support via Eclipse JDTLS -- **`typescript_language_server/`**: TypeScript/JavaScript support -- **`omnisharp/`**: C# support via OmniSharp -- **`rust_analyzer/`**: Rust support via Rust-Analyzer -- **`gopls/`**: Go support via Gopls -- **`solargraph/`**: Ruby support via Solargraph -- **`clangd_language_server/`**: C++ support via Clangd -- **`dart_language_server/`**: Dart support -- **`intelephense/`**: PHP support via Intelephense -- **`kotlin_language_server/`**: Kotlin support - -### 3. **Interprompt (`src/interprompt/`)** -Template and prompt management system: - -- **`multilang_prompt.py`**: Multi-language prompt templates (`MultiLangPromptTemplate`) -- **`jinja_template.py`**: Jinja2 template integration (`JinjaTemplate`) -- **`prompt_factory.py`**: Prompt factory base classes -- **`util/class_decorators.py`**: Utility decorators like `@singleton` - -## Configuration and Templates - -### **Prompt Templates (`prompts/`)** -Organized by context and functionality: -- **Context-specific prompts**: Different prompt sets for different execution contexts -- **Tool-specific prompts**: Specialized prompts for specific tools and operations -- **Multi-language support**: Prompts available in multiple languages when applicable - -### **Contexts (`contexts/`)** -Execution environment definitions: -- **`desktop-app.yml`**: Desktop application context -- **`agent.yml`**: Agent-specific context -- **`ide-assistant.yml`**: IDE assistant context -- **Custom contexts**: Support for user-defined contexts - -### **Modes (`modes/`)** -Behavior mode definitions: -- **`planning.yml`**: Planning mode behavior -- **`editing.yml`**: Editing mode behavior -- **`interactive.yml`**: Interactive mode behavior -- **`one-shot.yml`**: One-shot execution mode - -## Key Architectural Patterns - -### 1. **Four-Layer Configuration Hierarchy** -1. **Global**: `serena_config.yml` -2. **CLI Arguments**: Runtime overrides -3. **Project**: `.serena/project.yml` -4. **Active Modes**: Runtime behavior modification - -### 2. **Tool System Architecture** -- **Base Classes**: `ToolInterface`, `Tool` in `agent.py` -- **Marker Interfaces**: `ToolMarkerCanEdit`, `ToolMarkerDoesNotRequireActiveProject` -- **Tool Registry**: `ToolRegistry` with automatic tool discovery via `_iter_tool_classes()` -- **Tool Categories**: Semantic tools, file operations, project management, meta-tools - -### 3. **Integration Patterns** -- **MCP Server**: Primary integration via `SerenaMCPFactory` classes -- **Agno Agent**: Model-agnostic integration via `SerenaAgnoToolkit` -- **Process Isolation**: Optional isolation via `ProcessIsolatedSerenaAgent` - -### 4. **Memory Management** -- **Project Memories**: `.serena/memories/` directory for project-specific information -- **Memory Tools**: `WriteMemoryTool`, `ReadMemoryTool`, `ListMemoriesTool` -- **Memory Managers**: `MemoriesManager`, `MemoriesManagerMDFilesInProject` - -## Dependencies and Build System - -### **Development Tools** -- **`uv`**: Dependency management and virtual environment -- **`poe`**: Task orchestration (defined in `pyproject.toml`) -- **Essential Commands**: `uv run poe lint`, `uv run poe format`, `uv run poe type-check`, `uv run poe test` - -### **Key Dependencies** -- **Language Server Protocol**: LSP implementations for each supported language -- **MCP (Model Context Protocol)**: For integration with Claude Desktop and other MCP clients -- **Agno Framework**: For model-agnostic agent implementations -- **AsyncIO**: For concurrent operations (carefully managed to avoid contamination) - -## Project State Management - -### **Project Configuration** -- **Global Config**: `serena_config.yml` in user home directory -- **Project Config**: `.serena/project.yml` in project root -- **Auto-generation**: Automatic creation of default configurations when missing - -### **Language Detection** -- **Automatic**: Based on file composition analysis -- **Manual Override**: Via project configuration -- **Multi-language**: Support for projects with multiple languages - -## Runtime Architecture - -### **Default Operation Mode** -- **Single Process**: MCP server, agent, and language servers in same process -- **Direct Communication**: No IPC overhead -- **Solid-LSP**: Only language server implementation -- **Process Isolation**: Disabled by default (`USE_PROCESS_ISOLATION = False`) - -### **Semantic Tool Integration** -- **Symbol-based Operations**: `find_symbol`, `replace_symbol_body`, `insert_after_symbol` -- **Regex-based Operations**: `replace_regex` for fine-grained edits -- **File Operations**: `read_file`, `create_text_file`, `list_dir` -- **Project Operations**: `search_for_pattern`, `get_symbols_overview` - -This structure reflects Serena's evolution from a complex, multi-process system to a streamlined, single-process architecture that maintains all semantic capabilities while improving performance and reliability. \ No newline at end of file diff --git a/.serena/memories/serena_repository_structure.md b/.serena/memories/serena_repository_structure.md new file mode 100644 index 00000000..69f173de --- /dev/null +++ b/.serena/memories/serena_repository_structure.md @@ -0,0 +1,113 @@ +# Serena Repository Structure + +## Overview +Serena is a multi-language code assistant that combines two main components: +1. **Serena Core** - The main agent framework with tools and MCP server +2. **SolidLSP** - A unified Language Server Protocol wrapper for multiple programming languages + +## Top-Level Structure + +``` +serena/ +├── src/ # Main source code +│ ├── serena/ # Serena agent framework +│ ├── solidlsp/ # LSP wrapper library +│ └── interprompt/ # Multi-language prompt templates +├── test/ # Test suites +│ ├── serena/ # Serena agent tests +│ ├── solidlsp/ # Language server tests +│ └── resources/repos/ # Test repositories for each language +├── scripts/ # Build and utility scripts +├── resources/ # Static resources and configurations +├── pyproject.toml # Python project configuration +├── README.md # Project documentation +└── CHANGELOG.md # Version history +``` + +## Source Code Organization + +### Serena Core (`src/serena/`) +- **`agent.py`** - Main SerenaAgent class that orchestrates everything +- **`tools/`** - MCP tools for file operations, symbols, memory, etc. + - `file_tools.py` - File system operations (read, write, search) + - `symbol_tools.py` - Symbol-based code operations (find, edit) + - `memory_tools.py` - Knowledge persistence and retrieval + - `config_tools.py` - Project and mode management + - `workflow_tools.py` - Onboarding and meta-operations +- **`config/`** - Configuration management + - `serena_config.py` - Main configuration classes + - `context_mode.py` - Context and mode definitions +- **`util/`** - Utility modules +- **`mcp.py`** - MCP server implementation +- **`cli.py`** - Command-line interface + +### SolidLSP (`src/solidlsp/`) +- **`ls.py`** - Main SolidLanguageServer class +- **`language_servers/`** - Language-specific implementations + - `csharp_language_server.py` - C# (Microsoft.CodeAnalysis.LanguageServer) + - `python_server.py` - Python (Pyright) + - `typescript_language_server.py` - TypeScript + - `rust_analyzer.py` - Rust + - `gopls.py` - Go + - And many more... +- **`ls_config.py`** - Language server configuration +- **`ls_types.py`** - LSP type definitions +- **`ls_utils.py`** - Utilities for working with LSP data + +### Interprompt (`src/interprompt/`) +- Multi-language prompt template system +- Jinja2-based templating with language fallbacks + +## Test Structure + +### Language Server Tests (`test/solidlsp/`) +Each language has its own test directory: +``` +test/solidlsp/ +├── csharp/ +│ └── test_csharp_basic.py +├── python/ +│ └── test_python_basic.py +├── typescript/ +│ └── test_typescript_basic.py +└── ... +``` + +### Test Resources (`test/resources/repos/`) +Contains minimal test projects for each language: +``` +test/resources/repos/ +├── csharp/test_repo/ +│ ├── serena.sln +│ ├── TestProject.csproj +│ ├── Program.cs +│ └── Models/Person.cs +├── python/test_repo/ +├── typescript/test_repo/ +└── ... +``` + +### Test Infrastructure +- **`test/conftest.py`** - Shared test fixtures and utilities +- **`create_ls()`** function - Creates language server instances for testing +- **`language_server` fixture** - Parametrized fixture for multi-language tests + +## Key Configuration Files + +- **`pyproject.toml`** - Python dependencies, build config, and tool settings +- **`.serena/`** directories - Project-specific Serena configuration and memories +- **`CLAUDE.md`** - Instructions for AI assistants working on the project + +## Dependencies Management + +The project uses modern Python tooling: +- **uv** for fast dependency resolution and virtual environments +- **pytest** for testing with language-specific markers (`@pytest.mark.csharp`) +- **ruff** for linting and formatting +- **mypy** for type checking + +## Build and Development + +- **Docker support** - Full containerized development environment +- **GitHub Actions** - CI/CD with language server testing +- **Development scripts** in `scripts/` directory \ No newline at end of file diff --git a/.serena/memories/supported_language_servers.md b/.serena/memories/supported_language_servers.md deleted file mode 100644 index e1455281..00000000 --- a/.serena/memories/supported_language_servers.md +++ /dev/null @@ -1,97 +0,0 @@ -# Supported Language Servers in Solid-LSP - -Serena currently supports multiple programming languages through language servers via the solid-lsp architecture located in `src/solidlsp/language_servers/`. - -## Primary Languages (Directly Supported) - -### Python -- **Language Server**: Pyright Language Server -- **Location**: `src/solidlsp/language_servers/pyright_language_server/` -- **Features**: Full semantic analysis, type checking, symbol navigation -- **Previously**: Also supported Jedi (now deprecated in favor of Pyright) - -### Java -- **Language Server**: Eclipse JDTLS (Java Development Tools Language Server) -- **Location**: `src/solidlsp/language_servers/eclipse_jdtls/` -- **Features**: Full Java language support, Maven/Gradle integration -- **Note**: Startup can be slow, especially on initial launch - -### TypeScript/JavaScript -- **Language Server**: TypeScript Language Server -- **Location**: `src/solidlsp/language_servers/typescript_language_server/` -- **Features**: TypeScript and JavaScript support, type checking, IntelliSense - -## Additional Supported Languages - -### C# -- **Language Server**: OmniSharp -- **Location**: `src/solidlsp/language_servers/omnisharp/` -- **Features**: Full C# language support, .NET integration - -### Rust -- **Language Server**: Rust-Analyzer -- **Location**: `src/solidlsp/language_servers/rust_analyzer/` -- **Features**: Rust language support, cargo integration - -### Go -- **Language Server**: Gopls -- **Location**: `src/solidlsp/language_servers/gopls/` -- **Features**: Go language support, module system integration - -### Ruby -- **Language Server**: Solargraph -- **Location**: `src/solidlsp/language_servers/solargraph/` -- **Features**: Ruby language support, gem integration - -### C++ -- **Language Server**: Clangd -- **Location**: `src/solidlsp/language_servers/clangd_language_server/` -- **Features**: C++ language support, CMake integration - -### Dart -- **Language Server**: Dart Language Server -- **Location**: `src/solidlsp/language_servers/dart_language_server/` -- **Features**: Dart and Flutter support - -### PHP -- **Language Server**: Intelephense -- **Location**: `src/solidlsp/language_servers/intelephense/` -- **Features**: PHP language support, Composer integration - -### Kotlin -- **Language Server**: Kotlin Language Server -- **Location**: `src/solidlsp/language_servers/kotlin_language_server/` -- **Features**: Kotlin language support, Gradle integration - -## Language Server Protocol (LSP) Integration - -All language servers utilize the Language Server Protocol (LSP) to provide: -- **Semantic Analysis**: Symbol definitions, references, and relationships -- **Code Navigation**: Go-to-definition, find-references, symbol search -- **Error Detection**: Syntax and semantic error reporting -- **Code Completion**: IntelliSense-style code completion -- **Documentation**: Hover information and signature help - -## Architecture Integration - -### Solid-LSP Framework -- **Unified Interface**: All language servers use the same `SolidLanguageServer` interface -- **Configuration**: `LanguageServerConfig` handles language-specific settings -- **Lifecycle Management**: Start, stop, and restart capabilities for all servers -- **Process Management**: Direct process control for each language server - -### Project Integration -- **Automatic Detection**: Language selection based on project file composition -- **Configuration**: Project-specific language server settings via `.serena/project.yml` -- **Ignored Paths**: Respects gitignore and custom ignore patterns -- **Timeout Settings**: Configurable per-language server timeouts - -## Adding New Language Servers - -The architecture supports adding new language servers by: -1. **Creating Language Server Class**: Implement language-specific server in `src/solidlsp/language_servers/` -2. **Configuration**: Add language detection and configuration rules -3. **Integration**: Register with the solid-lsp framework -4. **Testing**: Ensure semantic tools work correctly with the new language server - -The system is designed to be extensible while maintaining consistent behavior across all supported languages. \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index ae5217f5..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1,66 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Essential Commands - -This project uses `uv` for dependency management and `poe` for task orchestration: - -- **Linting**: `uv run poe lint` (only allowed command for linting) -- **Formatting**: `uv run poe format` (only allowed command for formatting) -- **Type checking**: `uv run poe type-check` (only allowed command for type checking) -- **Testing**: `uv run poe test [args]` (preferred) or `uv run pytest [args]` - -**Important**: Always run `format`, `type-check` and `test` at the end of tasks to ensure code quality. Fix any issues and re-run until they pass. - -## Architecture Overview - -Serena is a powerful coding agent toolkit that turns LLMs into fully-featured agents working directly on codebases through semantic code retrieval and editing tools. - -### Core Components - -- **src/serena/**: Main package containing core functionality - - `mcp.py`: Model Context Protocol server implementation - - `agno.py`: Agno framework integration for model-agnostic agents - - `agent.py`: Core agent implementation with semantic tools - - **llm/**: LLM integration modules - - **util/**: Utility functions and helpers - -- **src/multilspy/**: Language server integration layer - - **language_servers/**: Language-specific implementations - - Python (pyright/jedi), Java (Eclipse JDTLS), TypeScript/JavaScript - - C#, Rust, Go, Ruby, C++, PHP support - - Provides semantic code analysis through Language Server Protocol (LSP) - -- **src/interprompt/**: Template and prompt management system - -### Key Design Principles - -- **Semantic Code Understanding**: Uses language servers (LSP) for symbol-level code analysis rather than text-based approaches -- **Multiple Integration Methods**: Can be used as MCP server, Agno agent, or integrated into custom frameworks -- **Language Agnostic**: Supports multiple programming languages through language server adapters -- **Context and Mode System**: Configurable behavior for different environments (desktop-app, ide-assistant, agent) - -### Integration Patterns - -- **MCP Server**: Primary integration method for Claude Desktop and other MCP clients -- **Agno Agent**: Model-agnostic agent framework for any LLM with GUI support -- **Framework Adapter**: Tools can be adapted to any agent framework (example: SerenaAgnoToolkit) - -### Configuration System - -Four-layer configuration hierarchy: -1. `serena_config.yml` - Global settings -2. CLI arguments - Client-specific overrides -3. `.serena/project.yml` - Project-specific settings -4. Active modes - Runtime behavior modification - -The codebase implements sophisticated semantic code operations through language servers, enabling precise symbol-level editing and code understanding that goes beyond simple text manipulation. - -## Working on Serena's Code - -**Important**: When working on this codebase, remember that you are using Serena's own tools to improve Serena itself. This means: - -- You can use Serena's semantic tools (`find_symbol`, `replace_symbol_body`, `search_for_pattern`, etc.) to analyze and edit Serena's own source code -- When asked to "perform a task on Serena", you're being asked to modify/improve the current Serena codebase -- You have access to the full power of Serena's semantic code understanding to work on Serena's code \ No newline at end of file