Merge branch 'main' of github.com:oraios/serena

.github/workflows/pytest.yml

@@ -121,13 +121,15 @@ jobs:
             sudo apt-get install -y ccls
           elif [[ "${{ runner.os }}" == "macOS" ]]; then
             brew install ccls
+          elif [[ "${{ runner.os }}" == "Windows" ]]; then
+            choco install ccls -y
           fi
-          # Windows: ccls requires building from source with MSYS2/LLVM, skipped in CI
           # Verify installation
           if command -v ccls &> /dev/null; then
             echo "ccls installed: $(ccls --version 2>&1 | head -1)"
           else
-            echo "ccls not available on this platform (expected on Windows)"
+            echo "ERROR: ccls installation failed"
+            exit 1
           fi
       - name: Setup Java (for JVM based languages)
         uses: actions/setup-java@v4

docs/01-about/020_programming-languages.md

@@ -33,6 +33,7 @@ Some languages require additional installations or setup steps, as noted.
 * **C/C++**  
   Default: clangd. Optional alternate: ccls (experimental, opt-in). 
   For best results, provide a `compile_commands.json` at the repository root.
+  See the [C/C++ Setup Guide](../03-special-guides/cpp_setup) for details.
 * **Clojure**
 * **Dart**
 * **Elixir**  

docs/03-special-guides/cpp_setup.md

@@ -0,0 +1,100 @@
+# C/C++ Setup Guide
+
+This guide explains how to prepare a C/C++ project so that Serena can provide reliable code intelligence via clangd or ccls language servers.
+This is only necessary if you use the language server variant of Serena, for users of the Serena JetBrains plugin no setup is required
+and the limitations described below do not apply.
+
+---
+
+## General
+
+Serena supports two C/C++ language servers, clangd (default) and ccls.
+Both have their pros and cons and require a properly configured `compile_commands.json` 
+for cross-file reference finding, see below for details.
+
+Your project must have a `compile_commands.json` file at the repository root. 
+This file is essential for correct parsing and cross-file reference finding.
+
+
+## compile_commands.json Requirements
+
+For reliable cross-file reference finding with clangd, your `compile_commands.json` must:
+
+1. **Include proper C++ standard flags** (e.g., `-std=c++17`)
+2. **Include all necessary include paths** (`-I` flags)
+
+---
+
+### With clangd
+
+Serena automatically downloads and manages clangd. Since clangd does not properly work with relative paths in `compile_commands.json`,
+Serena will detect them and transform them into absolute paths automatically (writing a new `compile_commands.json` file), if needed.
+
+#### Customizing the Compilation Database Location
+
+By default, Serena creates the transformed compilation database at `.serena/compile_commands.json`. 
+You can customize this location via project settings:
+
+```yaml
+# .serena/project.yml
+language_servers:
+  cpp:
+    compile_commands_dir: custom/rel/path (defaults to .serena)
+```
+
+### With ccls
+
+ccls requires manual installation and configuration. It may perform better in some situations.
+
+#### Installation
+
+**Linux:**
+```bash
+# Ubuntu/Debian (22.04+)
+sudo apt-get install ccls
+
+# Fedora/RHEL
+sudo dnf install ccls
+
+# Arch Linux
+sudo pacman -S ccls
+```
+
+**macOS:**
+```bash
+brew install ccls
+```
+
+**Windows:**
+
+```bash
+choco install ccls
+```
+
+#### Configuration
+
+After installing ccls, configure Serena to use it via project settings (in `.serena/project.yml`)
+by adding `cpp_ccls` to the `languages` list. Replace `cpp` with `cpp_ccls` if you already have the `cpp` entry.
+
+ccls can handle relative paths in `compile_commands.json`, so no transformation is necessary
+and no transformed `compile_commands.json` file will be created.
+
+---
+
+## Known Limitations
+
+### Files Created After Server Initialization
+
+Both clangd and ccls have a fundamental limitation: 
+**files created by external mechanisms after the language server starts are not automatically indexed**.
+
+Cross-file references to newly created files will not work unless the new file is at some point opened by the language server (for example, by a symbol lookup in it), or until `compile_commands.json` is updated and 
+the language server is restarted.
+
+---
+
+## Reference
+
+- Clangd official documentation: https://clangd.llvm.org/
+- Clangd project setup: https://clangd.llvm.org/installation#project-setup
+- CCLS repository: https://github.com/MaskRay/ccls

src/solidlsp/language_servers/ccls_language_server.py

@@ -1,12 +1,8 @@
 """
-Provides C/C++ specific instantiation of the LanguageServer class using ccls.
-
 This is an alternative to clangd for large C++ codebases where ccls may perform
 better for indexing and navigation. Requires ccls to be installed and available
 on PATH, or configured via ls_specific_settings with key "ls_path".
 
-For best results, ensure a compile_commands.json exists at the repository root.
-
 Installation
 ------------
 ccls must be installed manually as there are no prebuilt binaries available for
@@ -23,10 +19,9 @@ direct download. Install using your system package manager:
 - Homebrew: ``brew install ccls``
 
 **Windows:**
-- MSYS2 (MinGW): Build from source using MSYS2 toolchain
+- Chocolatey: ``choco install ccls``
-- No native prebuilt binaries available; must build from source
 
-For build-from-source instructions (required on Windows), see:
+For alternative installation methods and build-from-source instructions, see:
 https://github.com/MaskRay/ccls/wiki/Build
 
 Official documentation:
@@ -51,7 +46,7 @@ from solidlsp.settings import SolidLSPSettings
 log = logging.getLogger(__name__)
 
 
-class CclsLanguageServer(SolidLanguageServer):
+class CCLS(SolidLanguageServer):
     """
     C/C++ language server implementation using ccls.
 
@@ -89,7 +84,7 @@ class CclsLanguageServer(SolidLanguageServer):
                     "  Linux (Fedora/RHEL):   sudo dnf install ccls\n"
                     "  Linux (Arch):          sudo pacman -S ccls\n"
                     "  macOS (Homebrew):      brew install ccls\n"
-                    "  Windows:               Build from source (see wiki)\n\n"
+                    "  Windows:               choco install ccls\n\n"
                     "For build instructions and more details, see:\n"
                     "  https://github.com/MaskRay/ccls/wiki/Build"
                 )

src/solidlsp/language_servers/clangd_language_server.py

@@ -1,14 +1,11 @@
-"""
+import json
-Provides C/C++ specific instantiation of the LanguageServer class. Contains various configurations and settings specific to C/C++.
-"""
-
 import logging
 import os
 import pathlib
 import threading
 from typing import Any, cast
 
-from solidlsp.ls import LanguageServerDependencyProvider, LanguageServerDependencyProviderSinglePath, SolidLanguageServer
+from solidlsp.ls import LanguageServerDependencyProvider, LanguageServerDependencyProviderSinglePath, ProcessLaunchInfo, SolidLanguageServer
 from solidlsp.ls_config import LanguageServerConfig
 from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
 from solidlsp.settings import SolidLSPSettings
@@ -35,6 +32,89 @@ class ClangdLanguageServer(SolidLanguageServer):
         self.initialize_searcher_command_available = threading.Event()
         self.resolve_main_method_available = threading.Event()
 
+    def _prepare_compile_commands(self) -> str | None:
+        """
+        Prepare clangd compilation database with absolute directory paths.
+
+        Clangd requires absolute directory paths in compile_commands.json for correct
+        cross-file reference finding. This method reads the compile_commands.json,
+        converts relative directory paths to absolute paths, and writes a transformed
+        compilation database to the serena managed directory.
+
+        The transformed file is persisted in .serena/serena_compile_commands.json
+        (or a configurable directory via ls_specific_settings) and is not deleted
+        on cleanup. This allows clangd to use the absolute-path version without
+        modifying the user's original compile_commands.json.
+
+        Returns the path to the serena directory containing the transformed database,
+        or None if no transformation was needed.
+        """
+        compile_db_path = os.path.join(self.repository_root_path, "compile_commands.json")
+
+        if not os.path.exists(compile_db_path):
+            # No compile_commands.json, nothing to do
+            return None
+
+        try:
+            with open(compile_db_path, encoding="utf-8") as f:
+                compile_commands = json.load(f)
+
+            if not compile_commands:
+                return None
+
+            # Check if any entries have relative directory paths
+            has_relative = False
+            for entry in compile_commands:
+                directory = entry.get("directory", "")
+                if directory and not os.path.isabs(directory):
+                    has_relative = True
+                    # Convert to absolute path
+                    entry["directory"] = os.path.abspath(os.path.join(self.repository_root_path, directory))
+
+            if not has_relative:
+                # No relative paths found, no need to create transformed database
+                return None
+
+            # Get the target directory from ls_specific_settings, default to .serena
+            cpp_settings: dict[str, Any] = self._custom_settings or {}
+            compile_commands_rel_dir = cpp_settings.get("compile_commands_dir", ".serena")
+            compile_commands_dir = os.path.join(self.repository_root_path, compile_commands_rel_dir)
+            os.makedirs(compile_commands_dir, exist_ok=True)
+
+            # Write the transformed compile_commands.json
+            # clangd looks for compile_commands.json in the --compile-commands-dir
+            compile_commands_path = os.path.join(compile_commands_dir, "compile_commands.json")
+            with open(compile_commands_path, "w", encoding="utf-8") as f:
+                json.dump(compile_commands, f, indent=2)
+
+            # Track the directory for --compile-commands-dir
+
+            log.info(f"Created serena compilation database with absolute paths at {compile_commands_path}")
+            return compile_commands_dir
+
+        except (OSError, json.JSONDecodeError) as e:
+            log.warning(f"Failed to prepare compile_commands.json: {e}")
+            return None
+
+    def _create_process_launch_info(self) -> ProcessLaunchInfo:
+        """
+        Override to add --compile-commands-dir argument if we created a serena compilation database.
+        """
+        # First, ensure the serena compile commands database is prepared
+        compile_commands_dir = self._prepare_compile_commands()
+
+        # Get the default launch info from parent
+        launch_info = super()._create_process_launch_info()
+
+        # If we created a serena compilation database, add --compile-commands-dir to the command
+        if compile_commands_dir:
+            # Insert --compile-commands-dir after the executable path
+            cmd = launch_info.cmd
+            assert isinstance(cmd, list)
+            launch_info.cmd = [cmd[0], f"--compile-commands-dir={compile_commands_dir}"] + cmd[1:]
+
+        return launch_info
+
     def _create_dependency_provider(self) -> LanguageServerDependencyProvider:
         return self.DependencyProvider(self._custom_settings, self._ls_resources_dir)
 
@@ -116,8 +196,10 @@ class ClangdLanguageServer(SolidLanguageServer):
                 os.chmod(clangd_executable_path, 0o755)
             return clangd_executable_path
 
-        def _create_launch_command(self, core_path: str) -> list[str] | str:
+        def _create_launch_command(self, core_path: str) -> list[str]:
-            return [core_path]
+            # --background-index enables clangd to index all files in the project,
+            # which is required for finding cross-file references
+            return [core_path, "--background-index"]
 
     @staticmethod
     def _get_initialize_params(repository_absolute_path: str) -> InitializeParams:
@@ -132,6 +214,11 @@ class ClangdLanguageServer(SolidLanguageServer):
                     "synchronization": {"didSave": True, "dynamicRegistration": True},
                     "completion": {"dynamicRegistration": True, "completionItem": {"snippetSupport": True}},
                     "definition": {"dynamicRegistration": True},
+                    "references": {"dynamicRegistration": True},
+                    "documentSymbol": {
+                        "dynamicRegistration": True,
+                        "hierarchicalDocumentSymbolSupport": True,
+                    },
                 },
                 "workspace": {"workspaceFolders": True, "didChangeConfiguration": {"dynamicRegistration": True}},
             },
@@ -160,6 +247,7 @@ class ClangdLanguageServer(SolidLanguageServer):
             await lsp.request_references(...)
             # Shutdown the LanguageServer on exit from scope
         # LanguageServer has been shutdown
+        ```
         """
 
         def register_capability_handler(params: Any) -> None:
@@ -213,10 +301,15 @@ class ClangdLanguageServer(SolidLanguageServer):
         }
 
         self.server.notify.initialized({})
-
+        # set ready flag, clangd sends no meaningful notification when ready
-        # set ready flag
         # TODO This defeats the purpose of the event; we should wait for the server to actually be ready
         self.server_ready.set()
 
         # wait for server to be ready
         self.server_ready.wait()
+
+    def _shutdown(self, timeout: float = 5.0) -> None:
+        """Shutdown the clangd language server."""
+        # The serena compilation database persists in .serena/ for reuse
+        # Call parent shutdown
+        super()._shutdown(timeout=timeout)

src/solidlsp/ls_config.py

@@ -307,9 +307,9 @@ class Language(str, Enum):
 
                 return ClangdLanguageServer
             case self.CPP_CCLS:
-                from solidlsp.language_servers.ccls_language_server import CclsLanguageServer
+                from solidlsp.language_servers.ccls_language_server import CCLS
 
-                return CclsLanguageServer
+                return CCLS
             case self.PHP:
                 from solidlsp.language_servers.intelephense import Intelephense
 

test/resources/repos/cpp/test_repo/compile_commands.json

@@ -1,12 +1,12 @@
 [
   {
     "directory": ".",
-    "command": "g++ -I . -c a.cpp",
+    "command": "g++ -std=c++17 -I . -c a.cpp",
     "file": "a.cpp"
   },
   {
     "directory": ".",
-    "command": "g++ -I . -c b.cpp",
+    "command": "g++ -std=c++17 -I . -c b.cpp",
     "file": "b.cpp"
   }
 ]

test/solidlsp/cpp/test_cpp_basic.py

@@ -7,8 +7,8 @@ server is not available.
 """
 
 import os
+import pathlib
 import shutil
-from typing import cast
 
 import pytest
 
@@ -17,18 +17,11 @@ from solidlsp.ls_config import Language
 from solidlsp.ls_utils import SymbolUtils
 
 
-def _clangd_available() -> bool:
-    return shutil.which("clangd") is not None
-
-
 def _ccls_available() -> bool:
     return shutil.which("ccls") is not None
 
 
-# Build parametrize list based on availability
+_cpp_servers: list[Language] = [Language.CPP]
-_cpp_servers: list[Language] = []
-if _clangd_available():
-    _cpp_servers.append(Language.CPP)
 if _ccls_available():
     _cpp_servers.append(Language.CPP_CCLS)
 
@@ -70,9 +63,9 @@ class TestCppLanguageServer:
         assert add_symbol is not None, "Could not find 'add' function symbol in b.cpp"
 
         sel_start = add_symbol["selectionRange"]["start"]
-        refs = language_server.request_references(file_path, sel_start["line"], sel_start["character"] + 1)
+        refs = language_server.request_references(file_path, sel_start["line"], sel_start["character"])
-        ref_files = cast(list[str], [ref.get("relativePath", "") for ref in refs])
+        ref_files = [ref.get("relativePath", "") for ref in refs]
-        assert any("a.cpp" in ref_file for ref_file in ref_files), "Should find reference in a.cpp"
+        assert any("a.cpp" in ref_file for ref_file in ref_files), f"Should find reference in a.cpp, {refs=}"
 
         # Verify second call returns same results (stability check)
         def _ref_key(ref: dict) -> tuple:
@@ -88,5 +81,64 @@ class TestCppLanguageServer:
                 e.get("character", -1),
             )
 
-        refs2 = language_server.request_references(file_path, sel_start["line"], sel_start["character"] + 1)
+        refs2 = language_server.request_references(file_path, sel_start["line"], sel_start["character"])
         assert sorted(map(_ref_key, refs2)) == sorted(map(_ref_key, refs)), "Reference results should be stable across calls"
+
+    @pytest.mark.parametrize("language_server", _cpp_servers, indirect=True)
+    @pytest.mark.xfail(
+        strict=True,
+        reason=("Both clangd and ccls do not support cross-file references for newly created files that were never opened by the LS."),
+    )
+    def test_find_references_in_newly_written_file(self, language_server: SolidLanguageServer) -> None:
+        # Create a new file that references the 'add' function from b.cpp
+        new_file_path = os.path.join("temp_new_file.cpp")
+        new_file_abs_path = os.path.join(language_server.repository_root_path, new_file_path)
+
+        try:
+            # Write the new file with a reference to add()
+            with open(new_file_abs_path, "w", encoding="utf-8") as f:
+                f.write(
+                    """
+#include "b.hpp"
+
+int use_add() {
+    int result = add(5, 3);
+    return result;
+}
+"""
+                )
+
+            # Open the new file so clangd knows about it
+            with language_server.open_file(new_file_path):
+                # Request document symbols to ensure the file is fully loaded by clangd
+                new_file_symbols = language_server.request_document_symbols(new_file_path).get_all_symbols_and_roots()
+                assert new_file_symbols, "New file should have symbols"
+
+            # Verify the file stays in open_file_buffers after the context exits
+            uri = pathlib.Path(new_file_abs_path).as_uri()
+            assert uri in language_server.open_file_buffers, "File should remain in open_file_buffers"
+
+            # Find the 'add' symbol in b.cpp
+            b_file_path = os.path.join("b.cpp")
+            symbols = language_server.request_document_symbols(b_file_path).get_all_symbols_and_roots()
+            symbol_list = symbols[0] if symbols and isinstance(symbols[0], list) else symbols
+            add_symbol = None
+            for sym in symbol_list:
+                if sym.get("name") == "add":
+                    add_symbol = sym
+                    break
+            assert add_symbol is not None, "Could not find 'add' function symbol in b.cpp"
+
+            # Request references for 'add'
+            sel_start = add_symbol["selectionRange"]["start"]
+            refs = language_server.request_references(b_file_path, sel_start["line"], sel_start["character"])
+            ref_files = [ref.get("relativePath", "") for ref in refs]
+
+            # Should find reference in the newly written file
+            assert any(
+                "temp_new_file.cpp" in ref_file for ref_file in ref_files
+            ), f"Should find reference in newly written temp_new_file.cpp, {ref_files=}"
+        finally:
+            # Clean up the new file
+            if os.path.exists(new_file_abs_path):
+                os.remove(new_file_abs_path)