From 1946d9c19595c46139226147a5a8ddb68747d45f Mon Sep 17 00:00:00 2001 From: MaxMiksa Date: Mon, 12 Jan 2026 16:56:02 -0500 Subject: [PATCH 01/29] SolidLSP: open file before TS rename Ensure didOpen/didClose around textDocument/rename and add regression test. --- src/solidlsp/ls.py | 3 +- test/solidlsp/test_rename_didopen.py | 42 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 test/solidlsp/test_rename_didopen.py diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index be66e36c..900d7578 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -1856,7 +1856,8 @@ class SolidLanguageServer(ABC): newName=new_name, ) - return self.server.send.rename(params) + with self.open_file(relative_file_path): + return self.server.send.rename(params) def apply_text_edits_to_file(self, relative_path: str, edits: list[ls_types.TextEdit]) -> None: """ diff --git a/test/solidlsp/test_rename_didopen.py b/test/solidlsp/test_rename_didopen.py new file mode 100644 index 00000000..735e4642 --- /dev/null +++ b/test/solidlsp/test_rename_didopen.py @@ -0,0 +1,42 @@ +from unittest.mock import MagicMock + +from solidlsp.ls import SolidLanguageServer + + +class DummyLanguageServer(SolidLanguageServer): + def _start_server(self) -> None: + raise AssertionError("Not used in this test") + + +def test_request_rename_symbol_edit_opens_file_before_rename(tmp_path) -> None: + (tmp_path / "index.ts").write_text("export const x = 1;\n", encoding="utf-8") + + events: list[str] = [] + + notify = MagicMock() + notify.did_open_text_document.side_effect = lambda *_args, **_kwargs: events.append("didOpen") + notify.did_close_text_document.side_effect = lambda *_args, **_kwargs: events.append("didClose") + + send = MagicMock() + send.rename.side_effect = lambda *_args, **_kwargs: events.append("rename") + + server = MagicMock() + server.notify = notify + server.send = send + + language_server = object.__new__(DummyLanguageServer) + language_server.repository_root_path = str(tmp_path) + language_server.server_started = True + language_server.open_file_buffers = {} + language_server._encoding = "utf-8" + language_server.language_id = "typescript" + language_server.server = server + + result = language_server.request_rename_symbol_edit( + relative_file_path="index.ts", + line=0, + column=0, + new_name="y", + ) + assert result is None + assert events == ["didOpen", "rename", "didClose"] From 0deaa923ec1a90ff96e79127b8850e417e02bbfc Mon Sep 17 00:00:00 2001 From: zen010101 <60574100+zen010101@users.noreply.github.com> Date: Sat, 17 Jan 2026 23:16:07 +0800 Subject: [PATCH 02/29] feat(pascal): Add auto-update for Pascal Language Server - Automatically download pasls from zen010101/pascal-language-server releases - Check for updates every 24 hours with version comparison - SHA256 checksum verification for security - Atomic installation with rollback on failure - Tarfile path traversal protection - Handle nested archive directory structure CI improvements: - Fix FPC installation for Ubuntu/macOS/Windows - Add proper FPC verification step that fails on actual errors Tests: - 33 unit tests for version comparison, checksum, tarfile safety - 8 integration tests with actual pasls functionality --- .github/workflows/pytest.yml | 63 +- .../language_servers/pascal_server.py | 577 +++++++++++++++++- .../pascal/test_pascal_auto_update.py | 255 ++++++++ 3 files changed, 874 insertions(+), 21 deletions(-) create mode 100644 test/solidlsp/pascal/test_pascal_auto_update.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0d654c52..e292d5d7 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -375,9 +375,15 @@ jobs: sudo apt-get install -y fpc fpc-source # Set environment variables for pasls echo "PP=/usr/bin/fpc" >> $GITHUB_ENV - # Find FPC source directory (version may vary) - FPCDIR=$(ls -d /usr/share/fpcsrc/*/ 2>/dev/null | head -1 || echo "/usr/share/fpcsrc") + # Find FPC source directory (version may vary) - remove trailing slash + FPCDIR=$(ls -d /usr/share/fpcsrc/*/ 2>/dev/null | head -1 | sed 's:/$::') + if [[ -z "$FPCDIR" ]]; then + FPCDIR="/usr/share/fpcsrc" + fi echo "FPCDIR=$FPCDIR" >> $GITHUB_ENV + echo "=== FPC source directory structure ===" + ls -la "$FPCDIR" || echo "FPCDIR not found" + ls -la "$FPCDIR/rtl" 2>/dev/null || echo "rtl subdirectory not found" elif [[ "${{ runner.os }}" == "macOS" ]]; then brew install fpc # Download FPC source from SourceForge (fpc-src-laz cask is incompatible with ARM64) @@ -386,9 +392,21 @@ jobs: mkdir -p "$HOME/fpcsrc" tar -xzf fpc-source.tar.gz -C "$HOME/fpcsrc" rm fpc-source.tar.gz - # Set environment variables for pasls + # Check extracted directory structure (might be nested) + echo "=== Extracted FPC source structure ===" + ls -la "$HOME/fpcsrc" + # Find the actual FPC source root (contains rtl, packages, etc.) + if [[ -d "$HOME/fpcsrc/fpc-${FPC_VERSION}/rtl" ]]; then + FPCDIR="$HOME/fpcsrc/fpc-${FPC_VERSION}" + elif [[ -d "$HOME/fpcsrc/fpc-${FPC_VERSION}/fpc-${FPC_VERSION}/rtl" ]]; then + FPCDIR="$HOME/fpcsrc/fpc-${FPC_VERSION}/fpc-${FPC_VERSION}" + else + FPCDIR="$HOME/fpcsrc/fpc-${FPC_VERSION}" + fi echo "PP=$(which fpc)" >> $GITHUB_ENV - echo "FPCDIR=$HOME/fpcsrc/fpc-${FPC_VERSION}" >> $GITHUB_ENV + echo "FPCDIR=$FPCDIR" >> $GITHUB_ENV + echo "=== FPC source directory ===" + ls -la "$FPCDIR" || echo "FPCDIR not found" elif [[ "${{ runner.os }}" == "Windows" ]]; then FPC_VERSION="3.2.2" # Download freepascal-ootb (includes FPC compiler) @@ -417,9 +435,44 @@ jobs: - name: Verify FPC installation shell: bash run: | + echo "=== Environment variables ===" echo "PP=$PP" echo "FPCDIR=$FPCDIR" - fpc -v || echo "FPC not in PATH, using PP directly" + + # Create a simple test program + if [[ "${{ runner.os }}" == "Windows" ]]; then + TEST_PAS="$TEMP/fpc_test.pas" + TEST_OUT="$TEMP/fpc_test" + else + TEST_PAS="/tmp/fpc_test.pas" + TEST_OUT="/tmp/fpc_test" + fi + echo "program fpc_test; begin writeln('FPC works'); end." > "$TEST_PAS" + + # Compile using PP (the compiler we actually use in tests) + echo "=== Compiling test program with PP=$PP ===" + if [[ -n "$PP" ]]; then + "$PP" "$TEST_PAS" -o"$TEST_OUT" 2>&1 + else + echo "ERROR: PP environment variable is not set" + exit 1 + fi + + # Verify output binary exists + if [[ -f "$TEST_OUT" ]] || [[ -f "${TEST_OUT}.exe" ]]; then + echo "FPC compilation test PASSED" + else + echo "ERROR: FPC compilation failed - no output binary at $TEST_OUT" + exit 1 + fi + + # Verify FPCDIR exists (required for pasls) + if [[ -d "$FPCDIR" ]]; then + echo "FPCDIR exists: $FPCDIR" + else + echo "ERROR: FPCDIR does not exist: $FPCDIR" + exit 1 + fi - name: Cache language servers id: cache-language-servers uses: actions/cache@v3 diff --git a/src/solidlsp/language_servers/pascal_server.py b/src/solidlsp/language_servers/pascal_server.py index bb1c9e13..e7d872cf 100644 --- a/src/solidlsp/language_servers/pascal_server.py +++ b/src/solidlsp/language_servers/pascal_server.py @@ -4,22 +4,39 @@ Contains various configurations and settings specific to Pascal and Free Pascal. pasls installation strategy: 1. Use existing pasls from PATH -2. Download prebuilt binary from GitHub releases +2. Download prebuilt binary from GitHub releases (auto-updated) Supported platforms for binary download: - linux-x64, linux-arm64 - osx-x64, osx-arm64 - win-x64 +Auto-update features: +- Checks for updates every 24 hours via GitHub API +- SHA256 checksum verification before installation +- Atomic update with rollback on failure +- Windows file locking detection + You can pass the following entries in ls_specific_settings["pascal"]: - (reserved for future use) """ +from __future__ import annotations + +import hashlib +import json import logging import os import pathlib +import platform import shutil +import tarfile import threading +import time +import urllib.error +import urllib.request +import uuid +import zipfile from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection, quote_windows_path from solidlsp.ls import SolidLanguageServer @@ -37,8 +54,18 @@ class PascalLanguageServer(SolidLanguageServer): Contains various configurations and settings specific to Free Pascal and Lazarus. """ - PASLS_VERSION = "0.1.0" - PASLS_RELEASES_URL = "https://github.com/zen010101/pascal-language-server/releases/download" + # URL configuration + PASLS_RELEASES_URL = "https://github.com/zen010101/pascal-language-server/releases/latest/download" + PASLS_API_URL = "https://api.github.com/repos/zen010101/pascal-language-server/releases/latest" + + # Update check interval (seconds) + UPDATE_CHECK_INTERVAL = 86400 # 24 hours + + # Metadata directory name + META_DIR = ".meta" + + # Network timeout (seconds) + NETWORK_TIMEOUT = 10 def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings): """ @@ -56,10 +83,472 @@ class PascalLanguageServer(SolidLanguageServer): self.server_ready = threading.Event() self.completions_available_event = threading.Event() + # ============== Metadata Directory Management ============== + + @classmethod + def _meta_dir(cls, pasls_dir: str) -> str: + """Get metadata directory path, create if not exists.""" + meta_path = os.path.join(pasls_dir, cls.META_DIR) + os.makedirs(meta_path, exist_ok=True) + return meta_path + + @classmethod + def _meta_file(cls, pasls_dir: str, filename: str) -> str: + """Get metadata file path.""" + return os.path.join(cls._meta_dir(pasls_dir), filename) + + # ============== Version Management ============== + + @staticmethod + def _normalize_version(version: str | None) -> str: + """Normalize version string by removing 'v' prefix and whitespace.""" + if not version: + return "" + return version.strip().lstrip("vV") + + @classmethod + def _is_newer_version(cls, latest: str | None, local: str | None) -> bool: + """Compare versions, return True if latest is newer than local.""" + if not latest: + return False + if not local: + return True + + latest_norm = cls._normalize_version(latest) + local_norm = cls._normalize_version(local) + + if not latest_norm: + return False + if not local_norm: + return True + + try: + + def parse_version(v: str) -> list[int]: + parts = [] + for part in v.split("."): + num = "" + for c in part: + if c.isdigit(): + num += c + else: + break + parts.append(int(num) if num else 0) + return parts + + latest_parts = parse_version(latest_norm) + local_parts = parse_version(local_norm) + + # Pad to same length + max_len = max(len(latest_parts), len(local_parts)) + latest_parts.extend([0] * (max_len - len(latest_parts))) + local_parts.extend([0] * (max_len - len(local_parts))) + + return latest_parts > local_parts + except Exception: + log.warning(f"Failed to parse versions for comparison: {latest_norm} vs {local_norm}") + return False + + @classmethod + def _get_latest_version(cls) -> str | None: + """Get latest version from GitHub API, return None on failure.""" + try: + headers = {"Accept": "application/vnd.github.v3+json", "User-Agent": "Serena-LSP"} + # Support GITHUB_TOKEN for CI environments with rate limits + github_token = os.environ.get("GITHUB_TOKEN") + if github_token: + headers["Authorization"] = f"token {github_token}" + + req = urllib.request.Request(cls.PASLS_API_URL, headers=headers) + with urllib.request.urlopen(req, timeout=cls.NETWORK_TIMEOUT) as response: + data = json.loads(response.read().decode()) + return data.get("tag_name") + except Exception as e: + log.debug(f"Failed to get latest pasls version: {type(e).__name__}: {e}") + return None + + @classmethod + def _get_local_version(cls, pasls_dir: str) -> str | None: + """Read local version file.""" + version_file = cls._meta_file(pasls_dir, "version") + if os.path.exists(version_file): + try: + with open(version_file, encoding="utf-8") as f: + return f.read().strip() + except OSError: + return None + return None + + @classmethod + def _save_local_version(cls, pasls_dir: str, version: str) -> None: + """Save version to local file.""" + version_file = cls._meta_file(pasls_dir, "version") + try: + with open(version_file, "w", encoding="utf-8") as f: + f.write(version) + except OSError as e: + log.warning(f"Failed to save version file: {e}") + + # ============== Update Check Timing ============== + + @classmethod + def _should_check_update(cls, pasls_dir: str) -> bool: + """Check if we should query for updates (more than 24 hours since last check).""" + last_check_file = cls._meta_file(pasls_dir, "last_check") + if not os.path.exists(last_check_file): + return True + try: + with open(last_check_file, encoding="utf-8") as f: + last_check = float(f.read().strip()) + return (time.time() - last_check) > cls.UPDATE_CHECK_INTERVAL + except (OSError, ValueError): + return True + + @classmethod + def _update_last_check(cls, pasls_dir: str) -> None: + """Update last check timestamp.""" + last_check_file = cls._meta_file(pasls_dir, "last_check") + try: + with open(last_check_file, "w", encoding="utf-8") as f: + f.write(str(time.time())) + except OSError as e: + log.warning(f"Failed to update last check time: {e}") + + # ============== SHA256 Checksum ============== + + @classmethod + def _get_checksums(cls) -> dict[str, str] | None: + """Download checksums file from GitHub, return {filename: sha256} dict.""" + checksums_url = f"{cls.PASLS_RELEASES_URL}/checksums.sha256" + try: + req = urllib.request.Request(checksums_url, headers={"User-Agent": "Serena-LSP"}) + with urllib.request.urlopen(req, timeout=cls.NETWORK_TIMEOUT) as response: + content = response.read().decode("utf-8") + checksums = {} + for line in content.strip().split("\n"): + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) >= 2: + sha256 = parts[0] + filename = parts[1].lstrip("*") # Remove possible * prefix + checksums[filename] = sha256 + return checksums + except Exception as e: + log.warning(f"Failed to get checksums: {type(e).__name__}: {e}") + return None + + @staticmethod + def _calculate_sha256(file_path: str) -> str: + """Calculate SHA256 checksum of a file.""" + sha256_hash = hashlib.sha256() + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + sha256_hash.update(chunk) + return sha256_hash.hexdigest() + + @classmethod + def _verify_checksum(cls, file_path: str, expected_sha256: str) -> bool: + """Verify file checksum.""" + try: + actual_sha256 = cls._calculate_sha256(file_path) + if actual_sha256.lower() == expected_sha256.lower(): + log.debug(f"Checksum verified: {file_path}") + return True + else: + log.error(f"Checksum mismatch for {file_path}: expected {expected_sha256}, got {actual_sha256}") + return False + except Exception as e: + log.error(f"Failed to verify checksum: {e}") + return False + + # ============== Windows File Locking ============== + + @staticmethod + def _is_file_locked(file_path: str) -> bool: + """Check if file is locked (Windows).""" + if platform.system() != "Windows": + return False + + if not os.path.exists(file_path): + return False + + try: + with open(file_path, "a"): + pass + return False + except (OSError, PermissionError): + return True + + @classmethod + def _safe_remove(cls, file_path: str) -> bool: + """Safely remove file, handle Windows file locking.""" + if not os.path.exists(file_path): + return True + + if platform.system() == "Windows" and cls._is_file_locked(file_path): + temp_name = f"{file_path}.old.{uuid.uuid4().hex[:8]}" + try: + os.rename(file_path, temp_name) + log.info(f"File locked, renamed to: {temp_name}") + cls._mark_for_cleanup(os.path.dirname(file_path), temp_name) + return True + except PermissionError: + log.warning(f"Cannot remove/rename locked file: {file_path}") + return False + else: + try: + os.remove(file_path) + return True + except OSError as e: + log.warning(f"Failed to remove file {file_path}: {e}") + return False + + @classmethod + def _mark_for_cleanup(cls, pasls_dir: str, file_path: str) -> None: + """Mark file for later cleanup.""" + cleanup_file = cls._meta_file(pasls_dir, "cleanup_list") + try: + with open(cleanup_file, "a", encoding="utf-8") as f: + f.write(file_path + "\n") + except OSError: + pass + + @classmethod + def _cleanup_old_files(cls, pasls_dir: str) -> None: + """Clean up old files marked for deletion.""" + cleanup_file = cls._meta_file(pasls_dir, "cleanup_list") + if not os.path.exists(cleanup_file): + return + + try: + with open(cleanup_file, encoding="utf-8") as f: + files = [line.strip() for line in f if line.strip()] + + remaining = [] + for file_path in files: + if os.path.exists(file_path): + try: + os.remove(file_path) + log.debug(f"Cleaned up old file: {file_path}") + except OSError: + remaining.append(file_path) + + if remaining: + with open(cleanup_file, "w", encoding="utf-8") as f: + f.write("\n".join(remaining) + "\n") + else: + os.remove(cleanup_file) + except OSError: + pass + + # ============== Download and Atomic Update ============== + + @classmethod + def _download_archive(cls, url: str, target_path: str) -> bool: + """Download archive to specified path.""" + try: + os.makedirs(os.path.dirname(target_path), exist_ok=True) + req = urllib.request.Request(url, headers={"User-Agent": "Serena-LSP"}) + with urllib.request.urlopen(req, timeout=60) as response: + with open(target_path, "wb") as f: + while True: + chunk = response.read(8192) + if not chunk: + break + f.write(chunk) + return True + except Exception as e: + log.error(f"Failed to download {url}: {type(e).__name__}: {e}") + return False + + @classmethod + def _is_safe_tar_member(cls, member: tarfile.TarInfo, target_dir: str) -> bool: + """Check if tar member is safe (prevent path traversal attack).""" + # Check for .. in path components + if ".." in member.name.split("/") or ".." in member.name.split("\\"): + return False + + # Check extracted path is within target directory + abs_target = os.path.abspath(target_dir) + abs_member = os.path.abspath(os.path.join(target_dir, member.name)) + + return abs_member.startswith(abs_target + os.sep) or abs_member == abs_target + + @classmethod + def _extract_archive(cls, archive_path: str, target_dir: str, archive_type: str) -> bool: + """Safely extract archive to specified directory.""" + try: + os.makedirs(target_dir, exist_ok=True) + + if archive_type == "gztar": + with tarfile.open(archive_path, "r:gz") as tar: + for member in tar.getmembers(): + if not cls._is_safe_tar_member(member, target_dir): + log.error(f"Unsafe tar member detected (path traversal): {member.name}") + return False + tar.extractall(target_dir) + + elif archive_type == "zip": + with zipfile.ZipFile(archive_path, "r") as zip_ref: + for name in zip_ref.namelist(): + if ".." in name.split("/") or ".." in name.split("\\"): + log.error(f"Unsafe zip member detected (path traversal): {name}") + return False + abs_target = os.path.abspath(target_dir) + abs_member = os.path.abspath(os.path.join(target_dir, name)) + if not (abs_member.startswith(abs_target + os.sep) or abs_member == abs_target): + log.error(f"Unsafe zip member detected (path traversal): {name}") + return False + zip_ref.extractall(target_dir) + + else: + log.error(f"Unsupported archive type: {archive_type}") + return False + + # Handle nested directory: if extraction created a single subdirectory, + # move its contents up to target_dir (common with GitHub release archives) + cls._flatten_single_subdir(target_dir) + + return True + except Exception as e: + log.error(f"Failed to extract archive: {type(e).__name__}: {e}") + return False + + @classmethod + def _flatten_single_subdir(cls, target_dir: str) -> None: + """If target_dir contains only a single subdirectory, move its contents up.""" + entries = os.listdir(target_dir) + if len(entries) == 1: + subdir = os.path.join(target_dir, entries[0]) + if os.path.isdir(subdir): + # Move all contents from subdir to target_dir + for item in os.listdir(subdir): + src = os.path.join(subdir, item) + dst = os.path.join(target_dir, item) + shutil.move(src, dst) + # Remove the now-empty subdirectory + os.rmdir(subdir) + + @classmethod + def _get_archive_filename(cls, dep: RuntimeDependency) -> str: + """Get archive filename from URL.""" + assert dep.url is not None, "RuntimeDependency.url must be set" + return dep.url.split("/")[-1] + + @classmethod + def _atomic_install(cls, pasls_dir: str, deps: RuntimeDependencyCollection, checksums: dict[str, str] | None) -> bool: + """Atomic update: download -> verify checksum -> extract -> replace.""" + temp_dir = pasls_dir + ".tmp" + backup_dir = pasls_dir + ".backup" + temp_archive_dir = os.path.join(os.path.expanduser("~"), "solidlsp_tmp") + + try: + dep = deps.get_single_dep_for_current_platform() + assert dep.url is not None, "RuntimeDependency.url must be set" + assert dep.archive_type is not None, "RuntimeDependency.archive_type must be set" + + archive_filename = cls._get_archive_filename(dep) + archive_path = os.path.join(temp_archive_dir, archive_filename) + + # 1. Clean up any existing temp directory + if os.path.exists(temp_dir): + shutil.rmtree(temp_dir) + os.makedirs(temp_archive_dir, exist_ok=True) + + # 2. Download archive + log.info(f"Downloading pasls archive: {archive_filename}") + if not cls._download_archive(dep.url, archive_path): + log.error("Failed to download pasls archive") + return False + + # 3. Verify SHA256 checksum (critical security step, before extraction) + if checksums: + expected_sha256 = checksums.get(archive_filename) + if expected_sha256: + log.info(f"Verifying SHA256 checksum for {archive_filename}...") + if not cls._verify_checksum(archive_path, expected_sha256): + log.error(f"SHA256 checksum verification FAILED for {archive_filename}") + log.error("Aborting installation due to checksum mismatch - possible security issue!") + try: + os.remove(archive_path) + except OSError: + pass + return False + log.info("SHA256 checksum verified successfully") + else: + log.warning(f"No checksum found for {archive_filename} in checksums file") + else: + log.warning("No checksums available - skipping verification (not recommended for production)") + + # 4. Extract to temp directory + os.makedirs(temp_dir, exist_ok=True) + log.info("Extracting archive to temporary directory...") + if not cls._extract_archive(archive_path, temp_dir, dep.archive_type): + log.error("Failed to extract archive") + return False + + # 5. Set execute permission + binary_path = deps.binary_path(temp_dir) + if os.path.exists(binary_path): + try: + os.chmod(binary_path, 0o755) + except OSError: + pass # May fail on Windows + + # 6. Backup old version + if os.path.exists(pasls_dir): + if os.path.exists(backup_dir): + shutil.rmtree(backup_dir) + shutil.move(pasls_dir, backup_dir) + + # 7. Replace with new version + shutil.move(temp_dir, pasls_dir) + + # 8. Restore meta directory from backup (preserves version info, last_check, etc.) + if os.path.exists(backup_dir): + backup_meta = os.path.join(backup_dir, cls.META_DIR) + if os.path.exists(backup_meta): + target_meta = os.path.join(pasls_dir, cls.META_DIR) + if not os.path.exists(target_meta): + shutil.copytree(backup_meta, target_meta) + + # 9. Clean up downloaded archive and temp directory + try: + os.remove(archive_path) + os.rmdir(temp_archive_dir) + except OSError: + pass + + log.info("pasls installation completed successfully") + return True + + except Exception as e: + log.error(f"Installation failed: {e}") + + # Rollback + if os.path.exists(backup_dir) and not os.path.exists(pasls_dir): + try: + shutil.move(backup_dir, pasls_dir) + log.info("Rolled back to previous version") + except Exception as rollback_error: + log.error(f"Rollback failed: {rollback_error}") + + # Clean up temp directory + if os.path.exists(temp_dir): + try: + shutil.rmtree(temp_dir) + except Exception: + pass + + return False + @classmethod def _setup_runtime_dependencies(cls, solidlsp_settings: SolidLSPSettings) -> str: """ Setup runtime dependencies for Pascal Language Server (pasls). + Automatically checks for updates every 24 hours with security verification. Returns: str: The command to start the pasls server @@ -71,13 +560,21 @@ class PascalLanguageServer(SolidLanguageServer): log.info(f"Found pasls in PATH: {pasls_in_path}") return quote_windows_path(pasls_in_path) - # Use RuntimeDependencyCollection for download + pasls_dir = cls.ls_resources_dir(solidlsp_settings) + os.makedirs(pasls_dir, exist_ok=True) + + # Clean up old files from previous sessions + cls._cleanup_old_files(pasls_dir) + + # Use RuntimeDependencyCollection for platform detection + # Asset names follow zen010101/pascal-language-server release convention: + # pasls-{cpu_arch}-{os}.{ext} where cpu_arch is x86_64/aarch64/i386 deps = RuntimeDependencyCollection( [ RuntimeDependency( id="PascalLanguageServer", description="Pascal Language Server for Linux (x64)", - url=f"{cls.PASLS_RELEASES_URL}/v{cls.PASLS_VERSION}/pasls-linux-x64.tar.gz", + url=f"{cls.PASLS_RELEASES_URL}/pasls-x86_64-linux.tar.gz", platform_id="linux-x64", archive_type="gztar", binary_name="pasls", @@ -85,7 +582,7 @@ class PascalLanguageServer(SolidLanguageServer): RuntimeDependency( id="PascalLanguageServer", description="Pascal Language Server for Linux (arm64)", - url=f"{cls.PASLS_RELEASES_URL}/v{cls.PASLS_VERSION}/pasls-linux-arm64.tar.gz", + url=f"{cls.PASLS_RELEASES_URL}/pasls-aarch64-linux.tar.gz", platform_id="linux-arm64", archive_type="gztar", binary_name="pasls", @@ -93,23 +590,23 @@ class PascalLanguageServer(SolidLanguageServer): RuntimeDependency( id="PascalLanguageServer", description="Pascal Language Server for macOS (x64)", - url=f"{cls.PASLS_RELEASES_URL}/v{cls.PASLS_VERSION}/pasls-darwin-x64.tar.gz", + url=f"{cls.PASLS_RELEASES_URL}/pasls-x86_64-darwin.zip", platform_id="osx-x64", - archive_type="gztar", + archive_type="zip", binary_name="pasls", ), RuntimeDependency( id="PascalLanguageServer", description="Pascal Language Server for macOS (arm64)", - url=f"{cls.PASLS_RELEASES_URL}/v{cls.PASLS_VERSION}/pasls-darwin-arm64.tar.gz", + url=f"{cls.PASLS_RELEASES_URL}/pasls-aarch64-darwin.zip", platform_id="osx-arm64", - archive_type="gztar", + archive_type="zip", binary_name="pasls", ), RuntimeDependency( id="PascalLanguageServer", description="Pascal Language Server for Windows (x64)", - url=f"{cls.PASLS_RELEASES_URL}/v{cls.PASLS_VERSION}/pasls-win32-x64.zip", + url=f"{cls.PASLS_RELEASES_URL}/pasls-x86_64-win64.zip", platform_id="win-x64", archive_type="zip", binary_name="pasls.exe", @@ -117,17 +614,65 @@ class PascalLanguageServer(SolidLanguageServer): ] ) - pasls_dir = cls.ls_resources_dir(solidlsp_settings) pasls_executable_path = deps.binary_path(pasls_dir) + # Determine if download is needed + need_download = False + latest_version = None + checksums = None + if not os.path.exists(pasls_executable_path): - log.info(f"Downloading pasls to {pasls_dir}...") - deps.install(pasls_dir) + # First install + log.info("pasls not found, will download...") + need_download = True + latest_version = cls._get_latest_version() + checksums = cls._get_checksums() + elif cls._should_check_update(pasls_dir): + # Check for updates + log.debug("Checking for pasls updates...") + latest_version = cls._get_latest_version() + local_version = cls._get_local_version(pasls_dir) + + if cls._is_newer_version(latest_version, local_version): + log.info(f"New pasls version available: {latest_version} (current: {local_version})") + + # Check Windows file locking + if cls._is_file_locked(pasls_executable_path): + log.warning("Cannot update pasls: file is in use. Will retry next time.") + else: + need_download = True + checksums = cls._get_checksums() + else: + log.debug(f"pasls is up to date: {local_version}") + + if need_download: + if cls._atomic_install(pasls_dir, deps, checksums): + # Update metadata after successful installation + if latest_version: + cls._save_local_version(pasls_dir, latest_version) + else: + # API failed but download succeeded, record placeholder version + cls._save_local_version(pasls_dir, "unknown") + cls._update_last_check(pasls_dir) + else: + # Installation failed, use existing version if available + if not os.path.exists(pasls_executable_path): + raise RuntimeError("Failed to install pasls and no local version available") + log.warning("Update failed, using existing version") + + # Update check time even if no update (avoid frequent checks) + if not need_download and cls._should_check_update(pasls_dir): + cls._update_last_check(pasls_dir) assert os.path.exists(pasls_executable_path), f"pasls executable not found at {pasls_executable_path}" - os.chmod(pasls_executable_path, 0o755) - log.info(f"Using pasls at: {pasls_executable_path}") + # Ensure execute permission + try: + os.chmod(pasls_executable_path, 0o755) + except OSError: + pass # May fail on Windows, ignore + + log.info(f"Using pasls at: {pasls_executable_path}") return quote_windows_path(pasls_executable_path) @staticmethod diff --git a/test/solidlsp/pascal/test_pascal_auto_update.py b/test/solidlsp/pascal/test_pascal_auto_update.py new file mode 100644 index 00000000..2b6e2235 --- /dev/null +++ b/test/solidlsp/pascal/test_pascal_auto_update.py @@ -0,0 +1,255 @@ +""" +Unit tests for the Pascal language server auto-update functionality. + +These tests validate the version comparison, checksum verification, +and other helper methods without requiring network access or the +actual Pascal language server. +""" + +from __future__ import annotations + +import hashlib +import os +import tarfile +import tempfile +import time + +import pytest + +from solidlsp.language_servers.pascal_server import PascalLanguageServer + +pytestmark = [pytest.mark.pascal] + + +class TestVersionNormalization: + """Test version string normalization.""" + + def test_normalize_version_with_v_prefix(self) -> None: + """Test that 'v' prefix is stripped.""" + assert PascalLanguageServer._normalize_version("v1.0.0") == "1.0.0" + + def test_normalize_version_with_capital_v_prefix(self) -> None: + """Test that 'V' prefix is stripped.""" + assert PascalLanguageServer._normalize_version("V1.0.0") == "1.0.0" + + def test_normalize_version_without_prefix(self) -> None: + """Test version without prefix is unchanged.""" + assert PascalLanguageServer._normalize_version("1.0.0") == "1.0.0" + + def test_normalize_version_with_whitespace(self) -> None: + """Test that whitespace is stripped.""" + assert PascalLanguageServer._normalize_version(" v1.0.0 ") == "1.0.0" + + def test_normalize_version_empty(self) -> None: + """Test empty version returns empty string.""" + assert PascalLanguageServer._normalize_version("") == "" + + def test_normalize_version_none(self) -> None: + """Test None returns empty string.""" + assert PascalLanguageServer._normalize_version(None) == "" + + +class TestVersionComparison: + """Test version comparison logic.""" + + def test_newer_version_major(self) -> None: + """Test detection of newer major version.""" + assert PascalLanguageServer._is_newer_version("v2.0.0", "v1.0.0") is True + + def test_newer_version_minor(self) -> None: + """Test detection of newer minor version.""" + assert PascalLanguageServer._is_newer_version("v1.1.0", "v1.0.0") is True + + def test_newer_version_patch(self) -> None: + """Test detection of newer patch version.""" + assert PascalLanguageServer._is_newer_version("v1.0.1", "v1.0.0") is True + + def test_same_version(self) -> None: + """Test same version returns False.""" + assert PascalLanguageServer._is_newer_version("v1.0.0", "v1.0.0") is False + + def test_older_version(self) -> None: + """Test older version returns False.""" + assert PascalLanguageServer._is_newer_version("v1.0.0", "v2.0.0") is False + + def test_latest_none_returns_false(self) -> None: + """Test None latest version returns False.""" + assert PascalLanguageServer._is_newer_version(None, "v1.0.0") is False + + def test_local_none_returns_true(self) -> None: + """Test None local version returns True (first install).""" + assert PascalLanguageServer._is_newer_version("v1.0.0", None) is True + + def test_both_none_returns_false(self) -> None: + """Test both None returns False.""" + assert PascalLanguageServer._is_newer_version(None, None) is False + + def test_version_with_different_lengths(self) -> None: + """Test versions with different number of parts.""" + assert PascalLanguageServer._is_newer_version("v1.0.1", "v1.0") is True + assert PascalLanguageServer._is_newer_version("v1.0", "v1.0.1") is False + + def test_version_with_prerelease(self) -> None: + """Test versions with prerelease suffixes.""" + # Prerelease suffix is ignored, only numeric parts are compared + assert PascalLanguageServer._is_newer_version("v1.1.0-beta", "v1.0.0") is True + + +class TestSHA256Checksum: + """Test SHA256 checksum calculation and verification.""" + + def test_calculate_sha256(self) -> None: + """Test SHA256 calculation for a known content.""" + with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f: + f.write(b"test content") + temp_path = f.name + + try: + result = PascalLanguageServer._calculate_sha256(temp_path) + expected = hashlib.sha256(b"test content").hexdigest() + assert result == expected + finally: + os.unlink(temp_path) + + def test_verify_checksum_correct(self) -> None: + """Test checksum verification with correct checksum.""" + with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f: + f.write(b"test content") + temp_path = f.name + + try: + expected = hashlib.sha256(b"test content").hexdigest() + assert PascalLanguageServer._verify_checksum(temp_path, expected) is True + finally: + os.unlink(temp_path) + + def test_verify_checksum_incorrect(self) -> None: + """Test checksum verification with incorrect checksum.""" + with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f: + f.write(b"test content") + temp_path = f.name + + try: + wrong_checksum = "0" * 64 + assert PascalLanguageServer._verify_checksum(temp_path, wrong_checksum) is False + finally: + os.unlink(temp_path) + + def test_verify_checksum_case_insensitive(self) -> None: + """Test checksum verification is case insensitive.""" + with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f: + f.write(b"test content") + temp_path = f.name + + try: + expected = hashlib.sha256(b"test content").hexdigest().upper() + assert PascalLanguageServer._verify_checksum(temp_path, expected) is True + finally: + os.unlink(temp_path) + + +class TestTarfileSafety: + """Test tarfile path traversal protection.""" + + def test_safe_tar_member_normal_path(self) -> None: + """Test normal path is considered safe.""" + member = tarfile.TarInfo(name="pasls") + assert PascalLanguageServer._is_safe_tar_member(member, "/tmp/target") is True + + def test_safe_tar_member_nested_path(self) -> None: + """Test nested path is considered safe.""" + member = tarfile.TarInfo(name="subdir/pasls") + assert PascalLanguageServer._is_safe_tar_member(member, "/tmp/target") is True + + def test_unsafe_tar_member_path_traversal(self) -> None: + """Test path traversal is detected.""" + member = tarfile.TarInfo(name="../etc/passwd") + assert PascalLanguageServer._is_safe_tar_member(member, "/tmp/target") is False + + def test_unsafe_tar_member_hidden_traversal(self) -> None: + """Test hidden path traversal in nested path.""" + member = tarfile.TarInfo(name="subdir/../../etc/passwd") + assert PascalLanguageServer._is_safe_tar_member(member, "/tmp/target") is False + + def test_safe_tar_member_similar_name(self) -> None: + """Test path containing '..' in filename (not directory) is safe.""" + member = tarfile.TarInfo(name="file..name") + assert PascalLanguageServer._is_safe_tar_member(member, "/tmp/target") is True + + +class TestMetadataManagement: + """Test metadata directory and file management.""" + + def test_meta_dir_creates_directory(self) -> None: + """Test _meta_dir creates directory if not exists.""" + with tempfile.TemporaryDirectory() as temp_dir: + meta_path = PascalLanguageServer._meta_dir(temp_dir) + assert os.path.exists(meta_path) + assert meta_path == os.path.join(temp_dir, PascalLanguageServer.META_DIR) + + def test_meta_file_returns_correct_path(self) -> None: + """Test _meta_file returns correct path.""" + with tempfile.TemporaryDirectory() as temp_dir: + meta_file = PascalLanguageServer._meta_file(temp_dir, "version") + expected = os.path.join(temp_dir, PascalLanguageServer.META_DIR, "version") + assert meta_file == expected + + +class TestUpdateCheckTiming: + """Test update check timing logic.""" + + def test_should_check_update_no_last_check(self) -> None: + """Test should check when no last_check file exists.""" + with tempfile.TemporaryDirectory() as temp_dir: + assert PascalLanguageServer._should_check_update(temp_dir) is True + + def test_should_check_update_recent_check(self) -> None: + """Test should not check when recently checked.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create meta dir and last_check file with current time + meta_dir = PascalLanguageServer._meta_dir(temp_dir) + last_check_file = os.path.join(meta_dir, "last_check") + with open(last_check_file, "w") as f: + f.write(str(time.time())) + + assert PascalLanguageServer._should_check_update(temp_dir) is False + + def test_should_check_update_old_check(self) -> None: + """Test should check when last check was > 24 hours ago.""" + with tempfile.TemporaryDirectory() as temp_dir: + # Create meta dir and last_check file with old time + meta_dir = PascalLanguageServer._meta_dir(temp_dir) + last_check_file = os.path.join(meta_dir, "last_check") + old_time = time.time() - (PascalLanguageServer.UPDATE_CHECK_INTERVAL + 3600) + with open(last_check_file, "w") as f: + f.write(str(old_time)) + + assert PascalLanguageServer._should_check_update(temp_dir) is True + + def test_update_last_check_creates_file(self) -> None: + """Test _update_last_check creates timestamp file.""" + with tempfile.TemporaryDirectory() as temp_dir: + PascalLanguageServer._update_last_check(temp_dir) + last_check_file = PascalLanguageServer._meta_file(temp_dir, "last_check") + assert os.path.exists(last_check_file) + + with open(last_check_file) as f: + timestamp = float(f.read().strip()) + assert abs(timestamp - time.time()) < 5 # within 5 seconds + + +class TestVersionPersistence: + """Test local version persistence.""" + + def test_save_and_get_local_version(self) -> None: + """Test saving and retrieving local version.""" + with tempfile.TemporaryDirectory() as temp_dir: + PascalLanguageServer._save_local_version(temp_dir, "v1.0.0") + version = PascalLanguageServer._get_local_version(temp_dir) + assert version == "v1.0.0" + + def test_get_local_version_not_exists(self) -> None: + """Test getting version when file doesn't exist.""" + with tempfile.TemporaryDirectory() as temp_dir: + version = PascalLanguageServer._get_local_version(temp_dir) + assert version is None From 53524062d80b5e2535adfb358af6b45593927227 Mon Sep 17 00:00:00 2001 From: zen010101 <60574100+zen010101@users.noreply.github.com> Date: Sun, 18 Jan 2026 13:43:23 +0800 Subject: [PATCH 03/29] feat(pascal): Support ls_specific_settings for Pascal LSP Add support for configuring Pascal Language Server via ls_specific_settings: - pp: Path to FPC compiler driver (must be fpc.exe, not ppc386.exe) - fpcdir: Path to FPC source directory - lazarusdir: Path to Lazarus directory - fpc_target: Override target OS (FPCTARGET env var) - fpc_target_cpu: Override target CPU (FPCTARGETCPU env var) These settings are passed as environment variables to pasls, enabling CodeTools to properly configure target platform and locate source files. This fixes hover/navigation issues with complex include directives like those in mORMot2 project. --- .../language_servers/pascal_server.py | 64 ++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/src/solidlsp/language_servers/pascal_server.py b/src/solidlsp/language_servers/pascal_server.py index bb1c9e13..2fe9ad46 100644 --- a/src/solidlsp/language_servers/pascal_server.py +++ b/src/solidlsp/language_servers/pascal_server.py @@ -12,7 +12,26 @@ Supported platforms for binary download: - win-x64 You can pass the following entries in ls_specific_settings["pascal"]: -- (reserved for future use) + +Environment variables (recommended for CodeTools configuration): +- pp: Path to FPC compiler driver, must be "fpc.exe" (e.g., "D:/laz32/fpc/bin/i386-win32/fpc.exe"). + Do NOT use backend compilers like ppc386.exe or ppcx64.exe - CodeTools queries fpc.exe for + configuration (fpc -iV, fpc -iTO, etc.). This is the most important setting for hover/navigation. +- fpcdir: Path to FPC source directory (e.g., "D:/laz32/fpcsrc"). Helps CodeTools locate + standard library sources for better navigation. +- lazarusdir: Path to Lazarus directory (e.g., "D:/laz32/lazarus"). Required for Lazarus + projects using LCL and other Lazarus components. + +Target platform overrides (use only if pp setting is not sufficient): +- fpc_target: Override target OS (e.g., "Win32", "Win64", "Linux"). Sets FPCTARGET env var. +- fpc_target_cpu: Override target CPU (e.g., "i386", "x86_64", "aarch64"). Sets FPCTARGETCPU. + +Example configuration in ~/.serena/serena_config.yml: + ls_specific_settings: + pascal: + pp: "D:/laz32/fpc/bin/i386-win32/fpc.exe" + fpcdir: "D:/laz32/fpcsrc" + lazarusdir: "D:/laz32/lazarus" """ import logging @@ -46,10 +65,51 @@ class PascalLanguageServer(SolidLanguageServer): Use LanguageServer.create() instead. """ pasls_executable_path = self._setup_runtime_dependencies(solidlsp_settings) + + # Build environment variables for pasls + # These control CodeTools' configuration and target platform settings + proc_env: dict[str, str] = {} + + # Read from ls_specific_settings["pascal"] + from solidlsp.ls_config import Language + + pascal_settings = solidlsp_settings.get_ls_specific_settings(Language.PASCAL) + + # pp: Path to FPC compiler driver (must be fpc.exe, NOT ppc386.exe/ppcx64.exe) + # CodeTools queries fpc.exe for configuration via "fpc -iV", "fpc -iTO", etc. + pp = pascal_settings.get("pp", "") + if pp: + proc_env["PP"] = pp + log.info(f"Setting PP={pp} from ls_specific_settings") + + # fpcdir: Path to FPC source directory (e.g., "D:/laz32/fpcsrc") + fpcdir = pascal_settings.get("fpcdir", "") + if fpcdir: + proc_env["FPCDIR"] = fpcdir + log.info(f"Setting FPCDIR={fpcdir} from ls_specific_settings") + + # lazarusdir: Path to Lazarus directory (e.g., "D:/laz32/lazarus") + lazarusdir = pascal_settings.get("lazarusdir", "") + if lazarusdir: + proc_env["LAZARUSDIR"] = lazarusdir + log.info(f"Setting LAZARUSDIR={lazarusdir} from ls_specific_settings") + + # fpc_target: Override target OS (e.g., "Win32", "Win64", "Linux") + fpc_target = pascal_settings.get("fpc_target", "") + if fpc_target: + proc_env["FPCTARGET"] = fpc_target + log.info(f"Setting FPCTARGET={fpc_target} from ls_specific_settings") + + # fpc_target_cpu: Override target CPU (e.g., "i386", "x86_64", "aarch64") + fpc_target_cpu = pascal_settings.get("fpc_target_cpu", "") + if fpc_target_cpu: + proc_env["FPCTARGETCPU"] = fpc_target_cpu + log.info(f"Setting FPCTARGETCPU={fpc_target_cpu} from ls_specific_settings") + super().__init__( config, repository_root_path, - ProcessLaunchInfo(cmd=pasls_executable_path, cwd=repository_root_path), + ProcessLaunchInfo(cmd=pasls_executable_path, cwd=repository_root_path, env=proc_env), "pascal", solidlsp_settings, ) From 4d679dc99c7ac27fc4060d584cae6687a3239760 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 18 Jan 2026 18:29:50 +0100 Subject: [PATCH 04/29] LSPFileBuffer: Make content hash computation lazy #951 Fix content hash computation ignoring file encoding and hard-coding utf-8 --- src/solidlsp/ls.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 396ba533..945b67a2 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -59,31 +59,25 @@ class ReferenceInSymbol: character: int -@dataclasses.dataclass class LSPFileBuffer: """ This class is used to store the contents of an open LSP file in memory. """ - # uri of the file - uri: str + def __init__(self, uri: str, contents: str, encoding: str, version: int, language_id: str, ref_count: int) -> None: + self.uri = uri + self.contents = contents + self.version = version + self.language_id = language_id + self.ref_count = ref_count + self.encoding = encoding + self._content_hash: str | None = None - # The contents of the file - contents: str - - # The version of the file - version: int - - # The language id of the file - language_id: str - - # reference count of the file - ref_count: int - - content_hash: str = "" - - def __post_init__(self) -> None: - self.content_hash = hashlib.md5(self.contents.encode("utf-8")).hexdigest() + @property + def content_hash(self) -> str: + if self._content_hash is None: + self._content_hash = hashlib.md5(self.contents.encode(self.encoding)).hexdigest() + return self._content_hash def split_lines(self) -> list[str]: """Splits the contents of the file into lines.""" @@ -590,7 +584,9 @@ class SolidLanguageServer(ABC): version = 0 language_id = self._get_language_id_for_file(relative_file_path) - self.open_file_buffers[uri] = LSPFileBuffer(uri, contents, version, language_id, 1) + self.open_file_buffers[uri] = LSPFileBuffer( + uri=uri, contents=contents, encoding=self._encoding, version=version, language_id=language_id, ref_count=1 + ) self.server.notify.did_open_text_document( { From f97e1bb99a78f5b630706bcd0425cad09297a5d3 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 18 Jan 2026 19:06:43 +0100 Subject: [PATCH 05/29] Force black below v26, because of pathspec compatibility --- pyproject.toml | 2 +- uv.lock | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cd634260..d6c4b373 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ text = "MIT" [project.optional-dependencies] dev = [ - "black[jupyter]>=23.7.0", + "black[jupyter]>=23.7.0, <26", # black 26 is incompatible with our pathspec version "jinja2", # In version 1.0.4 we get a NoneType error related to some config conversion (yml_analytics is None and should be a list) "mypy>=1.16.1", diff --git a/uv.lock b/uv.lock index c8bbfb14..1b96f636 100644 --- a/uv.lock +++ b/uv.lock @@ -627,7 +627,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/2e/d4fcb2978f826358b673f779f78fa8a32ee37df11920dc2bb5589cbeecef/greenlet-3.2.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:784ae58bba89fa1fa5733d170d42486580cab9decda3484779f4759345b29822", size = 270219, upload-time = "2025-06-05T16:10:10.414Z" }, { url = "https://files.pythonhosted.org/packages/16/24/929f853e0202130e4fe163bc1d05a671ce8dcd604f790e14896adac43a52/greenlet-3.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0921ac4ea42a5315d3446120ad48f90c3a6b9bb93dd9b3cf4e4d84a66e42de83", size = 630383, upload-time = "2025-06-05T16:38:51.785Z" }, { url = "https://files.pythonhosted.org/packages/d1/b2/0320715eb61ae70c25ceca2f1d5ae620477d246692d9cc284c13242ec31c/greenlet-3.2.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d2971d93bb99e05f8c2c0c2f4aa9484a18d98c4c3bd3c62b65b7e6ae33dfcfaf", size = 642422, upload-time = "2025-06-05T16:41:35.259Z" }, - { url = "https://files.pythonhosted.org/packages/bd/49/445fd1a210f4747fedf77615d941444349c6a3a4a1135bba9701337cd966/greenlet-3.2.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c667c0bf9d406b77a15c924ef3285e1e05250948001220368e039b6aa5b5034b", size = 638375, upload-time = "2025-06-05T16:48:18.235Z" }, { url = "https://files.pythonhosted.org/packages/7e/c8/ca19760cf6eae75fa8dc32b487e963d863b3ee04a7637da77b616703bc37/greenlet-3.2.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:592c12fb1165be74592f5de0d70f82bc5ba552ac44800d632214b76089945147", size = 637627, upload-time = "2025-06-05T16:13:02.858Z" }, { url = "https://files.pythonhosted.org/packages/65/89/77acf9e3da38e9bcfca881e43b02ed467c1dedc387021fc4d9bd9928afb8/greenlet-3.2.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29e184536ba333003540790ba29829ac14bb645514fbd7e32af331e8202a62a5", size = 585502, upload-time = "2025-06-05T16:12:49.642Z" }, { url = "https://files.pythonhosted.org/packages/97/c6/ae244d7c95b23b7130136e07a9cc5aadd60d59b5951180dc7dc7e8edaba7/greenlet-3.2.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:93c0bb79844a367782ec4f429d07589417052e621aa39a5ac1fb99c5aa308edc", size = 1114498, upload-time = "2025-06-05T16:36:46.598Z" }, @@ -1990,7 +1989,7 @@ requires-dist = [ { name = "agno", marker = "extra == 'agno'", specifier = ">=2.2.1" }, { name = "anthropic", specifier = ">=0.54.0" }, { name = "beautifulsoup4", specifier = ">=4.14.2" }, - { name = "black", extras = ["jupyter"], marker = "extra == 'dev'", specifier = ">=23.7.0" }, + { name = "black", extras = ["jupyter"], marker = "extra == 'dev'", specifier = ">=23.7.0,<26" }, { name = "docstring-parser", specifier = ">=0.16" }, { name = "dotenv", specifier = ">=0.9.9" }, { name = "flask", specifier = ">=3.0.0" }, From ed46e3f3d2afb91a67c76f13915a9c801295df78 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sun, 18 Jan 2026 22:56:05 +0100 Subject: [PATCH 06/29] Improve workflow description when using the JetBrains plugin Resolves #942 --- docs/02-usage/025_jetbrains_plugin.md | 36 +++++++++++++++++++-------- docs/02-usage/040_workflow.md | 5 ++++ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/docs/02-usage/025_jetbrains_plugin.md b/docs/02-usage/025_jetbrains_plugin.md index a2176d8a..9d46778e 100644 --- a/docs/02-usage/025_jetbrains_plugin.md +++ b/docs/02-usage/025_jetbrains_plugin.md @@ -17,7 +17,21 @@ especially for users of JetBrains IDEs. **Purchasing the JetBrains Plugin supports the Serena project.** The proceeds from plugin sales allow us to dedicate more resources to further developing and improving Serena. -## Configuring Serena + +## Advantages of the JetBrains Plugin + +There are multiple features that are only available when using the JetBrains plugin: + +* **External library indexing**: Dependencies and libraries are fully indexed and accessible to Serena +* **No additional setup**: No need to download or configure separate language servers +* **Enhanced performance**: Faster tool execution thanks to optimized IDE integration +* **Multi-language excellence**: First-class support for polyglot projects with multiple languages and frameworks +* **Enhanced retrieval capabilities**: The plugin supports additional retrieval tools for type hierarchy information as well as fast and reliable documentation/type signature retrieval + +We are also working on additional features like a `move_symbol` tool and debugging-related capabilities that +will be available exclusively through the JetBrains plugin. + +## Configuring Serena to Use the JetBrains Plugin After installing the plugin, you need to configure Serena to use it. @@ -44,18 +58,20 @@ you will see `Languages: Using JetBrains backend` in the configuration overview. You will also notice that your client will use the JetBrains-specific tools like `jet_brains_find_symbol` and others like it. +## Workflow -## Advantages of the JetBrains Plugin +Having installed the plugin in your IDE and having configured Serena to use the JetBrains backend, +the general workflow is simple: +1. Open the project you want to work on in your JetBrains IDE +2. Open the project's root folder as a project in Serena (see [Project Creation](project-creation-indexing) and [Project Activation](project-activation)) +3. Start using Serena tools as usual -There are multiple features that are only available when using the JetBrains plugin: +Note that the folder that is open in your IDE and the project's root folder must match. -* **External library indexing**: Dependencies and libraries are fully indexed and accessible to Serena -* **No additional setup**: No need to download or configure separate language servers -* **Enhanced performance**: Faster tool execution thanks to optimized IDE integration -* **Multi-language excellence**: First-class support for polyglot projects with multiple languages and frameworks - -We are also working on additional features like a `move_symbol` tool and debugging-related capabilities that -will be available exclusively through the JetBrains plugin. +:::{tip} +If you need to work on multiple projects in the same agent session, create a monorepo folder +containing all the projects and open that folder in both Serena and your IDE. +::: ## Usage with Other Editors diff --git a/docs/02-usage/040_workflow.md b/docs/02-usage/040_workflow.md index c6b916b8..91d27d3d 100644 --- a/docs/02-usage/040_workflow.md +++ b/docs/02-usage/040_workflow.md @@ -15,6 +15,8 @@ setting up a project with Serena typically involves the following steps: (project-creation-indexing)= ## Project Creation & Indexing +Project creation is the process of defining fundamental project settings that are relevant to Serena's operation. + You can create a project either * implicitly, by just activating a directory as a project while already in a conversation; this will use default settings for your project (skip to the next section). * explicitly, using the project creation command, or @@ -72,6 +74,7 @@ While in the project directory, run this command: Indexing has to be called only once. During regular usage, Serena will automatically update the index whenever files change. +(project-activation)= ## Project Activation Project activation makes Serena aware of the project you want to work with. @@ -89,6 +92,8 @@ You can either choose to do this * when the MCP server starts, by passing the project path or name as a command-line argument (e.g. when using a single-project mode like `ide` or `claude-code`): `--project ` +When working with the JetBrains plugin, be sure to have the same project folder open as a project in your IDE, +i.e. the folder that is activated in Serena should correspond to the root folder of the project in your IDE. ## Onboarding & Memories From 4b64867a52463dbdde88432f03c02b72675a8394 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 19 Jan 2026 00:01:30 +0100 Subject: [PATCH 07/29] SerenaAgent.create_system_prompt: Fix use of exposed tools instead of active tools Improve handling of active tools in general Fixes #954 --- src/serena/agent.py | 48 ++++++++++++++++------------------ src/serena/tools/tools_base.py | 2 +- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index b3f205ec..6a5c7feb 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -52,7 +52,11 @@ class AvailableTools: :param tools: the list of available tools """ self.tools = tools - self.tool_names = [tool.get_name_from_cls() for tool in tools] + self.tool_names = sorted([tool.get_name_from_cls() for tool in tools]) + """ + the list of available tool names, sorted alphabetically + """ + self._tool_name_set = set(self.tool_names) self.tool_marker_names = set() for marker_class in iter_subclasses(ToolMarker): for tool in tools: @@ -62,6 +66,9 @@ class AvailableTools: def __len__(self) -> int: return len(self.tools) + def contains_tool_name(self, tool_name: str) -> bool: + return tool_name in self._tool_name_set + class ToolSet: """ @@ -289,7 +296,7 @@ class SerenaAgent: self._modes = modes # determine the subset of active tools (depending on active modes and the active project) - self._active_tools: dict[type[Tool], Tool] = {} + self._active_tools: AvailableTools = self._exposed_tools self._update_active_tools() # activate a project configuration (if provided or if there is only a single project available) @@ -491,12 +498,13 @@ class SerenaAgent: return template.render(available_tools=self._exposed_tools.tool_names, available_markers=self._exposed_tools.tool_marker_names) def create_system_prompt(self) -> str: - available_markers = self._exposed_tools.tool_marker_names - log.info("Generating system prompt with available_tools=(see exposed tools), available_markers=%s", available_markers) + available_tools = self._active_tools + available_markers = available_tools.tool_marker_names + log.info("Generating system prompt with available_tools=(see active tools), available_markers=%s", available_markers) system_prompt = self.prompt_factory.create_system_prompt( context_system_prompt=self._format_prompt(self._context.prompt), mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self._modes], - available_tools=self._exposed_tools.tool_names, + available_tools=available_tools.tool_names, available_markers=available_markers, ) @@ -522,13 +530,10 @@ class SerenaAgent: if self._active_project.project_config.read_only: tool_set = tool_set.without_editing_tools() - self._active_tools = { - tool_class: tool_instance - for tool_class, tool_instance in self._all_tools.items() - if tool_set.includes_name(tool_instance.get_name()) - } + tools = [t for t in self._all_tools.values() if tool_set.includes_name(t.get_name())] + self._active_tools = AvailableTools(tools) - log.info(f"Active tools ({len(self._active_tools)}): {', '.join(self.get_active_tool_names())}") + log.info(f"Active tools ({len(self._active_tools)}): {', '.join(self._active_tools.tool_names)}") def issue_task( self, task: Callable[[], T], name: str | None = None, logged: bool = True, timeout: float | None = None @@ -617,27 +622,18 @@ class SerenaAgent: self._activate_project(project_instance) return project_instance - def get_active_tool_classes(self) -> list[type["Tool"]]: - """ - :return: the list of active tool classes for the current project - """ - return list(self._active_tools.keys()) - def get_active_tool_names(self) -> list[str]: """ - :return: the list of names of the active tools for the current project + :return: the list of names of the active tools for the current project, sorted alphabetically """ - return sorted([tool.get_name_from_cls() for tool in self.get_active_tool_classes()]) + return self._active_tools.tool_names - def tool_is_active(self, tool_class: type["Tool"] | str) -> bool: + def tool_is_active(self, tool_name: str) -> bool: """ - :param tool_class: the class or name of the tool to check + :param tool_class: the name of the tool to check :return: True if the tool is active, False otherwise """ - if isinstance(tool_class, str): - return tool_class in self.get_active_tool_names() - else: - return tool_class in self.get_active_tool_classes() + return self._active_tools.contains_tool_name(tool_name) def get_current_config_overview(self) -> str: """ @@ -726,7 +722,7 @@ class SerenaAgent: return self._all_tools[tool_class] # type: ignore def print_tool_overview(self) -> None: - ToolRegistry().print_tool_overview(self._active_tools.values()) + ToolRegistry().print_tool_overview(self._active_tools.tools) def __del__(self) -> None: self.shutdown() diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index de4f0c39..e28629a1 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -230,7 +230,7 @@ class Tool(Component): return result def is_active(self) -> bool: - return self.agent.tool_is_active(self.__class__) + return self.agent.tool_is_active(self.get_name()) def apply_ex(self, log_call: bool = True, catch_exceptions: bool = True, mcp_ctx: Context | None = None, **kwargs) -> str: # type: ignore """ From abef88c4828cb330ef5dea49e1c3f5cb34682dcd Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 19 Jan 2026 18:33:43 +0100 Subject: [PATCH 08/29] SerenaConfig: Rename gui_log_window_enabled to gui_log_window This removes the discrepancy with the configuration file (where it was always gui_log_window) --- scripts/profile_tool_call.py | 2 +- src/serena/agent.py | 4 ++-- src/serena/cli.py | 2 +- src/serena/config/serena_config.py | 20 +++++++------------- src/serena/mcp.py | 2 +- test/serena/test_serena_agent.py | 2 +- test/serena/test_tool_parameter_types.py | 2 +- 7 files changed, 14 insertions(+), 20 deletions(-) diff --git a/scripts/profile_tool_call.py b/scripts/profile_tool_call.py index 8fe922a5..1ee06081 100644 --- a/scripts/profile_tool_call.py +++ b/scripts/profile_tool_call.py @@ -25,7 +25,7 @@ if __name__ == "__main__": serena_config = SerenaConfig.from_config_file() serena_config.log_level = logging.INFO - serena_config.gui_log_window_enabled = False + serena_config.gui_log_window = False serena_config.web_dashboard = False agent = SerenaAgent(str(project_path), serena_config=serena_config) diff --git a/src/serena/agent.py b/src/serena/agent.py index 6a5c7feb..dbeb27e8 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -216,7 +216,7 @@ class SerenaAgent: # open GUI log window if enabled self._gui_log_viewer: Optional["GuiLogViewer"] = None - if self.serena_config.gui_log_window_enabled: + if self.serena_config.gui_log_window: log.info("Opening GUI window") if platform.system() == "Darwin": log.warning("GUI log window is not supported on macOS") @@ -269,7 +269,7 @@ class SerenaAgent: if ( self.serena_config.web_dashboard and not self.serena_config.web_dashboard_open_on_launch - and not self.serena_config.gui_log_window_enabled + and not self.serena_config.gui_log_window ): tool_inclusion_definitions.append(ToolInclusionDefinition(included_optional_tools=[OpenDashboardTool.get_name_from_cls()])) tool_inclusion_definitions.append(self.serena_config) diff --git a/src/serena/cli.py b/src/serena/cli.py index 40d0fc92..9ba0a267 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -746,7 +746,7 @@ class ProjectCommands(AutoRegisteringGroup): try: # Create SerenaAgent with dashboard disabled log.info("Creating SerenaAgent with disabled dashboard...") - config = SerenaConfig(gui_log_window_enabled=False, web_dashboard=False) + config = SerenaConfig(gui_log_window=False, web_dashboard=False) agent = SerenaAgent(project=project_path, serena_config=config) log.info("SerenaAgent created successfully") diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index f449a64f..130f23b4 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -403,8 +403,10 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): For testing purposes, it can also be instantiated directly with the desired parameters. """ + # *** fields that are mapped directly to/from the configuration file (DO NOT RENAME) *** + projects: list[RegisteredProject] = field(default_factory=list) - gui_log_window_enabled: bool = False + gui_log_window: bool = False log_level: int = logging.INFO trace_lsp_communication: bool = False web_dashboard: bool = True @@ -446,12 +448,6 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): # *** static members *** CONFIG_FILE = "serena_config.yml" - CONFIG_FILE_RENAMED_KEYS = { - "gui_log_window_enabled": "gui_log_window", - } - """ - maps attributes to the keys used in the configuration file, in case they differ - """ # *** methods *** @@ -539,11 +535,10 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): instance.projects.append(project) def get_value_or_default(field_name: str) -> Any: - key = cls.CONFIG_FILE_RENAMED_KEYS.get(field_name, field_name) nonlocal num_migrations - if key not in loaded_commented_yaml: + if field_name not in loaded_commented_yaml: num_migrations += 1 - return loaded_commented_yaml.get(key, get_dataclass_default(SerenaConfig, field_name)) + return loaded_commented_yaml.get(field_name, get_dataclass_default(SerenaConfig, field_name)) # determine language backend language_backend = get_dataclass_default(SerenaConfig, "language_backend") @@ -560,7 +555,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): instance.language_backend = language_backend # set other configuration parameters (primitive types) - instance.gui_log_window_enabled = get_value_or_default("gui_log_window_enabled") + instance.gui_log_window = get_value_or_default("gui_log_window") instance.web_dashboard_listen_address = get_value_or_default("web_dashboard_listen_address") instance.log_level = loaded_commented_yaml.get("log_level", loaded_commented_yaml.get("gui_log_level", logging.INFO)) instance.web_dashboard = get_value_or_default("web_dashboard") @@ -690,8 +685,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): field_name = field_info.name if field_name in ["projects", "language_backend"] or field_name.startswith("_"): continue - yaml_key = SerenaConfig.CONFIG_FILE_RENAMED_KEYS.get(field_name, field_name) - loaded_original_yaml[yaml_key] = getattr(self, field_name) + loaded_original_yaml[field_name] = getattr(self, field_name) # convert project objects into list of paths loaded_original_yaml["projects"] = sorted({str(project.project_root) for project in self.projects}) diff --git a/src/serena/mcp.py b/src/serena/mcp.py index 2e885cf9..b0367717 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -305,7 +305,7 @@ class SerenaMCPFactory: if enable_web_dashboard is not None: config.web_dashboard = enable_web_dashboard if enable_gui_log_window is not None: - config.gui_log_window_enabled = enable_gui_log_window + config.gui_log_window = enable_gui_log_window if open_web_dashboard is not None: config.web_dashboard_open_on_launch = open_web_dashboard if log_level is not None: diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index a030256f..6e7a0792 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -55,7 +55,7 @@ def serena_config(): ) test_projects.append(RegisteredProject.from_project_instance(project)) - config = SerenaConfig(gui_log_window_enabled=False, web_dashboard=False, log_level=logging.ERROR) + config = SerenaConfig(gui_log_window=False, web_dashboard=False, log_level=logging.ERROR) config.projects = test_projects return config diff --git a/test/serena/test_tool_parameter_types.py b/test/serena/test_tool_parameter_types.py index 4fdf0dc8..9899a5bb 100644 --- a/test/serena/test_tool_parameter_types.py +++ b/test/serena/test_tool_parameter_types.py @@ -13,7 +13,7 @@ def test_all_tool_parameters_have_type(context): For every tool exposed by Serena, ensure that the generated Open‑AI schema contains a ``type`` entry for each parameter. """ - cfg = SerenaConfig(gui_log_window_enabled=False, web_dashboard=False, log_level=logging.ERROR) + cfg = SerenaConfig(gui_log_window=False, web_dashboard=False, log_level=logging.ERROR) registry = ToolRegistry() cfg.included_optional_tools = tuple(registry.get_tool_names_optional()) factory = SerenaMCPFactory(context=context) From 0c43e60810940283ce4ee1ba1404eabcc030a865 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 19 Jan 2026 19:23:21 +0100 Subject: [PATCH 09/29] SerenaConfig: Unify handling of file-mapped configuration entries Fixes 'fixed_tools' not being loaded --- src/serena/config/serena_config.py | 69 +++++++++++++++--------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 130f23b4..7bfbb3e8 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -5,7 +5,7 @@ The Serena Model Context Protocol (MCP) Server import dataclasses import os import shutil -from collections.abc import Sequence +from collections.abc import Iterator, Sequence from copy import deepcopy from dataclasses import dataclass, field from datetime import datetime @@ -448,6 +448,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): # *** static members *** CONFIG_FILE = "serena_config.yml" + CONFIG_FIELDS_WITH_TYPE_CONVERSION = {"projects", "language_backend"} # *** methods *** @@ -455,6 +456,15 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): def config_file_path(self) -> str | None: return self._config_file_path + def _iter_config_file_mapped_fields_without_type_conversion(self) -> Iterator[str]: + for field_info in dataclasses.fields(self): + field_name = field_info.name + if field_name.startswith("_"): + continue + if field_name in self.CONFIG_FIELDS_WITH_TYPE_CONVERSION: + continue + yield field_name + def _tostring_includes(self) -> list[str]: return ["config_file_path"] @@ -509,14 +519,23 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): # create the configuration instance instance = cls(_loaded_commented_yaml=loaded_commented_yaml, _config_file_path=config_file_path) + num_migrations = 0 + + def get_value_or_default(field_name: str) -> Any: + nonlocal num_migrations + if field_name not in loaded_commented_yaml: + num_migrations += 1 + return loaded_commented_yaml.get(field_name, get_dataclass_default(SerenaConfig, field_name)) + + # transfer regular fields that do not require type conversion + for field_name in instance._iter_config_file_mapped_fields_without_type_conversion(): + assert hasattr(instance, field_name) + setattr(instance, field_name, get_value_or_default(field_name)) # read projects if "projects" not in loaded_commented_yaml: raise SerenaConfigError("`projects` key not found in Serena configuration. Please update your `serena_config.yml` file.") - - # load list of known projects instance.projects = [] - num_migrations = 0 for path in loaded_commented_yaml["projects"]: path = Path(path).resolve() if not path.exists() or (path.is_dir() and not (path / ProjectConfig.rel_path_to_project_yml()).exists()): @@ -534,12 +553,6 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): ) instance.projects.append(project) - def get_value_or_default(field_name: str) -> Any: - nonlocal num_migrations - if field_name not in loaded_commented_yaml: - num_migrations += 1 - return loaded_commented_yaml.get(field_name, get_dataclass_default(SerenaConfig, field_name)) - # determine language backend language_backend = get_dataclass_default(SerenaConfig, "language_backend") if "language_backend" in loaded_commented_yaml: @@ -554,20 +567,12 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): del loaded_commented_yaml["jetbrains"] instance.language_backend = language_backend - # set other configuration parameters (primitive types) - instance.gui_log_window = get_value_or_default("gui_log_window") - instance.web_dashboard_listen_address = get_value_or_default("web_dashboard_listen_address") - instance.log_level = loaded_commented_yaml.get("log_level", loaded_commented_yaml.get("gui_log_level", logging.INFO)) - instance.web_dashboard = get_value_or_default("web_dashboard") - instance.web_dashboard_open_on_launch = get_value_or_default("web_dashboard_open_on_launch") - instance.jetbrains_plugin_server_address = get_value_or_default("jetbrains_plugin_server_address") - instance.tool_timeout = get_value_or_default("tool_timeout") - instance.trace_lsp_communication = get_value_or_default("trace_lsp_communication") - instance.excluded_tools = get_value_or_default("excluded_tools") - instance.included_optional_tools = get_value_or_default("included_optional_tools") - instance.token_count_estimator = get_value_or_default("token_count_estimator") - instance.default_max_tool_answer_chars = get_value_or_default("default_max_tool_answer_chars") - instance.ls_specific_settings = get_value_or_default("ls_specific_settings") + # migrate deprecated "gui_log_level" field if necessary + if "gui_log_level" in loaded_commented_yaml: + num_migrations += 1 + if "log_level" not in loaded_commented_yaml: + instance.log_level = loaded_commented_yaml["gui_log_level"] + del loaded_commented_yaml["gui_log_level"] # re-save the configuration file if any migrations were performed if num_migrations > 0: @@ -677,23 +682,19 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): assert self._loaded_commented_yaml is not None, "Cannot save configuration without loaded YAML" - loaded_original_yaml = deepcopy(self._loaded_commented_yaml) + commented_yaml = deepcopy(self._loaded_commented_yaml) # update fields with current values - # iterate over all dataclass fields - for field_info in dataclasses.fields(self): - field_name = field_info.name - if field_name in ["projects", "language_backend"] or field_name.startswith("_"): - continue - loaded_original_yaml[field_name] = getattr(self, field_name) + for field_name in self._iter_config_file_mapped_fields_without_type_conversion(): + commented_yaml[field_name] = getattr(self, field_name) # convert project objects into list of paths - loaded_original_yaml["projects"] = sorted({str(project.project_root) for project in self.projects}) + commented_yaml["projects"] = sorted({str(project.project_root) for project in self.projects}) # convert language backend to string - loaded_original_yaml["language_backend"] = self.language_backend.value + commented_yaml["language_backend"] = self.language_backend.value - save_yaml(self.config_file_path, loaded_original_yaml, preserve_comments=True) + save_yaml(self.config_file_path, commented_yaml, preserve_comments=True) def propagate_settings(self) -> None: """ From 4a44cba6c818303660d031ad8d9b9e876e8bb92f Mon Sep 17 00:00:00 2001 From: Torben Leth Date: Mon, 19 Jan 2026 21:43:16 +0100 Subject: [PATCH 10/29] feat(al): normalize AL symbol names by stripping object type and ID metadata AL Language Server returns symbol names with full object format like 'Table 50000 "TEST Customer"', but symbol names should be pure without metadata - uniformly across all languages. This follows the same pattern as Java LS which strips type information from names. Metadata (object type, ID) is available via the hover LSP method when using include_info=True in find_symbol. Changes: - Add extract_al_display_name() to parse AL object names - Override request_document_symbols() to normalize names at source - Update tests to expect normalized names --- .../language_servers/al_language_server.py | 72 +++++++- test/solidlsp/al/test_al_basic.py | 159 +++++++++++++----- 2 files changed, 192 insertions(+), 39 deletions(-) diff --git a/src/solidlsp/language_servers/al_language_server.py b/src/solidlsp/language_servers/al_language_server.py index d6052a2a..4a304b21 100644 --- a/src/solidlsp/language_servers/al_language_server.py +++ b/src/solidlsp/language_servers/al_language_server.py @@ -4,6 +4,7 @@ import logging import os import pathlib import platform +import re import stat import time import zipfile @@ -13,7 +14,7 @@ import requests from overrides import override from solidlsp.language_servers.common import quote_windows_path -from solidlsp.ls import SolidLanguageServer +from solidlsp.ls import DocumentSymbols, LSPFileBuffer, SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_types import SymbolKind, UnifiedSymbolInformation from solidlsp.lsp_protocol_handler.lsp_types import Definition, DefinitionParams, LocationLink @@ -22,6 +23,45 @@ from solidlsp.settings import SolidLSPSettings log = logging.getLogger(__name__) +# Regex pattern to match AL object names like: +# - 'Table 50000 "TEST Customer"' -> captures 'TEST Customer' +# - 'Codeunit 50000 CustomerMgt' -> captures 'CustomerMgt' +# - 'Interface IPaymentProcessor' -> captures 'IPaymentProcessor' +# - 'Enum 50000 CustomerType' -> captures 'CustomerType' +# Pattern: [] (|) +_AL_OBJECT_NAME_PATTERN = re.compile( + r"^(?:Table|Page|Codeunit|Enum|Interface|Report|Query|XMLPort|PermissionSet|" + r"PermissionSetExtension|Profile|PageExtension|TableExtension|EnumExtension|" + r"PageCustomization|ReportExtension|ControlAddin|DotNetPackage)" # Object type + r"(?:\s+\d+)?" # Optional object ID + r"\s+" # Required space before name + r'(?:"([^"]+)"|(\S+))$' # Quoted name (group 1) or unquoted identifier (group 2) +) + + +def extract_al_display_name(full_name: str) -> str: + """ + Extract the display name from an AL symbol's full name. + + AL Language Server returns symbol names in format: + - 'Table 50000 "TEST Customer"' -> 'TEST Customer' + - 'Codeunit 50000 CustomerMgt' -> 'CustomerMgt' + - 'Interface IPaymentProcessor' -> 'IPaymentProcessor' + - 'fields' -> 'fields' (non-AL-object symbols pass through unchanged) + + Args: + full_name: The full symbol name as returned by AL Language Server + + Returns: + The extracted display name for matching, or the original name if not an AL object + + """ + match = _AL_OBJECT_NAME_PATTERN.match(full_name) + if match: + # Return quoted name (group 1) or unquoted name (group 2) + return match.group(1) or match.group(2) or full_name + return full_name + class ALLanguageServer(SolidLanguageServer): """ @@ -957,3 +997,33 @@ class ALLanguageServer(SolidLanguageServer): except Exception as e: log.warning(f"Failed to set active workspace: {e}") # Non-critical error, continue operation + + @override + def request_document_symbols(self, relative_file_path: str, file_buffer: LSPFileBuffer | None = None) -> DocumentSymbols: + """ + Override to normalize AL symbol names by stripping object type and ID metadata. + + AL Language Server returns symbol names with full object format like + 'Table 50000 "TEST Customer"', but symbol names should be pure without metadata. + This follows the same pattern as Java LS which strips type information from names. + + Metadata (object type, ID) is available via the hover LSP method when using + include_info=True in find_symbol. + """ + # Get symbols from parent implementation + document_symbols = super().request_document_symbols(relative_file_path, file_buffer=file_buffer) + + # Normalize names by stripping AL object metadata + def normalize_name(symbol: UnifiedSymbolInformation) -> None: + symbol["name"] = extract_al_display_name(symbol["name"]) + + # Process children recursively + if symbol.get("children"): + for child in symbol["children"]: + normalize_name(child) + + # Apply to all root symbols + for sym in document_symbols.root_symbols: + normalize_name(sym) + + return document_symbols diff --git a/test/solidlsp/al/test_al_basic.py b/test/solidlsp/al/test_al_basic.py index 8b4fb7a7..bbcdff76 100644 --- a/test/solidlsp/al/test_al_basic.py +++ b/test/solidlsp/al/test_al_basic.py @@ -2,7 +2,9 @@ import os import pytest +from serena.symbol import LanguageServerSymbol from solidlsp import SolidLanguageServer +from solidlsp.language_servers.al_language_server import extract_al_display_name from solidlsp.ls_config import Language from solidlsp.ls_utils import SymbolUtils from test.conftest import language_tests_enabled @@ -10,37 +12,127 @@ from test.conftest import language_tests_enabled pytestmark = [pytest.mark.al, pytest.mark.skipif(not language_tests_enabled(Language.AL), reason="AL tests are disabled")] +class TestExtractALDisplayName: + """Tests for the extract_al_display_name function.""" + + def test_table_with_quoted_name(self) -> None: + """Test extraction from Table with quoted name.""" + assert extract_al_display_name('Table 50000 "TEST Customer"') == "TEST Customer" + + def test_page_with_quoted_name(self) -> None: + """Test extraction from Page with quoted name.""" + assert extract_al_display_name('Page 50001 "TEST Customer Card"') == "TEST Customer Card" + + def test_codeunit_unquoted(self) -> None: + """Test extraction from Codeunit with unquoted name.""" + assert extract_al_display_name("Codeunit 50000 CustomerMgt") == "CustomerMgt" + + def test_enum_unquoted(self) -> None: + """Test extraction from Enum with unquoted name.""" + assert extract_al_display_name("Enum 50000 CustomerType") == "CustomerType" + + def test_interface_no_id(self) -> None: + """Test extraction from Interface (no ID).""" + assert extract_al_display_name("Interface IPaymentProcessor") == "IPaymentProcessor" + + def test_table_extension(self) -> None: + """Test extraction from TableExtension.""" + assert extract_al_display_name('TableExtension 50000 "Ext Customer"') == "Ext Customer" + + def test_page_extension(self) -> None: + """Test extraction from PageExtension.""" + assert extract_al_display_name('PageExtension 50000 "My Page Ext"') == "My Page Ext" + + def test_non_al_object_unchanged(self) -> None: + """Test that non-AL-object names pass through unchanged.""" + assert extract_al_display_name("fields") == "fields" + assert extract_al_display_name("CreateCustomer") == "CreateCustomer" + assert extract_al_display_name("Name") == "Name" + + def test_report_with_quoted_name(self) -> None: + """Test extraction from Report.""" + assert extract_al_display_name('Report 50000 "Sales Invoice"') == "Sales Invoice" + + def test_query_unquoted(self) -> None: + """Test extraction from Query.""" + assert extract_al_display_name("Query 50000 CustomerQuery") == "CustomerQuery" + + @pytest.mark.al class TestALLanguageServer: + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_symbol_names_are_normalized(self, language_server: SolidLanguageServer) -> None: + """Test that AL symbol names are normalized (metadata stripped).""" + file_path = os.path.join("src", "Tables", "Customer.Table.al") + symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() + + _all_symbols, root_symbols = symbols + customer_table = None + for sym in root_symbols: + if sym.get("name") == "TEST Customer": + customer_table = sym + break + + assert customer_table is not None, "Could not find 'TEST Customer' table symbol (name should be normalized)" + # Name should be just "TEST Customer", not "Table 50000 'TEST Customer'" + assert customer_table["name"] == "TEST Customer", f"Expected normalized name 'TEST Customer', got '{customer_table['name']}'" + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_find_symbol_exact_match(self, language_server: SolidLanguageServer) -> None: + """Test that find_symbol can match AL symbols by normalized name without substring_matching.""" + file_path = os.path.join("src", "Tables", "Customer.Table.al") + symbols = language_server.request_document_symbols(file_path) + + # Find symbols that match 'TEST Customer' using LanguageServerSymbol.find() + for root in symbols.root_symbols: + ls_symbol = LanguageServerSymbol(root) + matches = ls_symbol.find("TEST Customer", substring_matching=False) + if matches: + assert len(matches) >= 1, "Should find at least one match for 'TEST Customer'" + assert matches[0].name == "TEST Customer", f"Expected 'TEST Customer', got '{matches[0].name}'" + return + + pytest.fail("Could not find 'TEST Customer' symbol by exact name match") + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_find_codeunit_exact_match(self, language_server: SolidLanguageServer) -> None: + """Test finding a codeunit by its normalized name.""" + file_path = os.path.join("src", "Codeunits", "CustomerMgt.Codeunit.al") + symbols = language_server.request_document_symbols(file_path) + + for root in symbols.root_symbols: + ls_symbol = LanguageServerSymbol(root) + matches = ls_symbol.find("CustomerMgt", substring_matching=False) + if matches: + assert len(matches) >= 1 + assert matches[0].name == "CustomerMgt" + return + + pytest.fail("Could not find 'CustomerMgt' symbol by exact name match") + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) def test_find_symbol(self, language_server: SolidLanguageServer) -> None: - """Test that AL Language Server can find symbols in the test repository.""" + """Test that AL Language Server can find symbols in the test repository with normalized names.""" symbols = language_server.request_full_symbol_tree() - # Check for table symbols - AL returns full object names like 'Table 50000 "TEST Customer"' - assert SymbolUtils.symbol_tree_contains_name(symbols, 'Table 50000 "TEST Customer"'), "TEST Customer table not found in symbol tree" + # Check for table symbols - names should be normalized (no "Table 50000" prefix) + assert SymbolUtils.symbol_tree_contains_name(symbols, "TEST Customer"), "TEST Customer table not found in symbol tree" # Check for page symbols - assert SymbolUtils.symbol_tree_contains_name( - symbols, 'Page 50001 "TEST Customer Card"' - ), "TEST Customer Card page not found in symbol tree" - assert SymbolUtils.symbol_tree_contains_name( - symbols, 'Page 50002 "TEST Customer List"' - ), "TEST Customer List page not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "TEST Customer Card"), "TEST Customer Card page not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "TEST Customer List"), "TEST Customer List page not found in symbol tree" # Check for codeunit symbols - assert SymbolUtils.symbol_tree_contains_name(symbols, "Codeunit 50000 CustomerMgt"), "CustomerMgt codeunit not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "CustomerMgt"), "CustomerMgt codeunit not found in symbol tree" assert SymbolUtils.symbol_tree_contains_name( - symbols, "Codeunit 50001 PaymentProcessorImpl" + symbols, "PaymentProcessorImpl" ), "PaymentProcessorImpl codeunit not found in symbol tree" # Check for enum symbol - assert SymbolUtils.symbol_tree_contains_name(symbols, "Enum 50000 CustomerType"), "CustomerType enum not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "CustomerType"), "CustomerType enum not found in symbol tree" # Check for interface symbol - assert SymbolUtils.symbol_tree_contains_name( - symbols, "Interface IPaymentProcessor" - ), "IPaymentProcessor interface not found in symbol tree" + assert SymbolUtils.symbol_tree_contains_name(symbols, "IPaymentProcessor"), "IPaymentProcessor interface not found in symbol tree" @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) def test_find_table_fields(self, language_server: SolidLanguageServer) -> None: @@ -52,7 +144,7 @@ class TestALLanguageServer: customer_table = None _all_symbols, root_symbols = symbols for sym in root_symbols: - if "TEST Customer" in sym.get("name", ""): + if sym.get("name") == "TEST Customer": customer_table = sym break @@ -81,11 +173,11 @@ class TestALLanguageServer: file_path = os.path.join("src", "Codeunits", "CustomerMgt.Codeunit.al") symbols = language_server.request_document_symbols(file_path).get_all_symbols_and_roots() - # Find the codeunit symbol - AL returns 'Codeunit 50000 CustomerMgt' + # Find the codeunit symbol - name should be normalized to 'CustomerMgt' codeunit_symbol = None _all_symbols, root_symbols = symbols for sym in root_symbols: - if "CustomerMgt" in sym.get("name", ""): + if sym.get("name") == "CustomerMgt": codeunit_symbol = sym break @@ -95,7 +187,6 @@ class TestALLanguageServer: if "children" in codeunit_symbol: procedure_names = [child.get("name", "") for child in codeunit_symbol.get("children", [])] assert any("CreateCustomer" in name for name in procedure_names), "CreateCustomer procedure not found" - # Note: UpdateCustomerBalance doesn't exist in our test repo, check for actual procedures assert any("TestNoSeries" in name for name in procedure_names), "TestNoSeries procedure not found" @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) @@ -105,11 +196,11 @@ class TestALLanguageServer: table_file = os.path.join("src", "Tables", "Customer.Table.al") symbols = language_server.request_document_symbols(table_file).get_all_symbols_and_roots() - # Find the Customer table symbol + # Find the Customer table symbol (name is normalized) customer_symbol = None _all_symbols, root_symbols = symbols for sym in root_symbols: - if "TEST Customer" in sym.get("name", ""): + if sym.get("name") == "TEST Customer": customer_symbol = sym break @@ -133,29 +224,21 @@ class TestALLanguageServer: # Get all symbols to verify cross-file visibility symbols = language_server.request_full_symbol_tree() - # Count how many AL-specific symbols we found - al_symbols = [] + # Count how many AL object symbols we found (names are now normalized) + al_object_names = [] - def collect_symbols(syms): + def collect_symbols(syms: list) -> None: for sym in syms: if isinstance(sym, dict): name = sym.get("name", "") - # Look for AL object names (Table, Page, Codeunit, etc.) - if any(keyword in name for keyword in ["Table", "Page", "Codeunit", "Enum", "Interface"]): - al_symbols.append(name) + # These are normalized names now, so just collect them + al_object_names.append(name) if "children" in sym: collect_symbols(sym["children"]) collect_symbols(symbols) - # We should find symbols from multiple files - assert len(al_symbols) >= 5, f"Expected at least 5 AL object symbols, found {len(al_symbols)}: {al_symbols}" - - # Verify we have symbols from different AL object types - has_table = any("Table" in s for s in al_symbols) - has_page = any("Page" in s for s in al_symbols) - has_codeunit = any("Codeunit" in s for s in al_symbols) - - assert has_table, f"No Table symbols found in: {al_symbols}" - assert has_page, f"No Page symbols found in: {al_symbols}" - assert has_codeunit, f"No Codeunit symbols found in: {al_symbols}" + # We should find expected normalized names + assert "TEST Customer" in al_object_names, f"TEST Customer not found in: {al_object_names}" + assert "CustomerMgt" in al_object_names, f"CustomerMgt not found in: {al_object_names}" + assert "CustomerType" in al_object_names, f"CustomerType not found in: {al_object_names}" From 10ba2d890bf6627698a387b29a811f106bb27ad9 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 19 Jan 2026 23:05:08 +0100 Subject: [PATCH 11/29] Fix logging error (kind is not necessarily an int) --- src/serena/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index 9ba0a267..2fed9a95 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -806,7 +806,7 @@ class ProjectCommands(AutoRegisteringGroup): symbol_name = selected_symbol.get("name_path", "unknown") symbol_kind = selected_symbol.get("kind", "unknown") - log.info("Using symbol for testing: %s (kind: %d)", symbol_name, symbol_kind) + log.info("Using symbol for testing: %s (kind: %s)", symbol_name, symbol_kind) # Test 2: FindSymbolTool log.info("Testing FindSymbolTool for symbol: %s", symbol_name) From 93294a2b30fdf0fe4bfdbb6c245f9427b6ccf08c Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 19 Jan 2026 23:06:18 +0100 Subject: [PATCH 12/29] Fix healh-check assuming non-compact symbol overview Fixes #958 --- src/serena/cli.py | 3 +-- src/serena/tools/symbol_tools.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index 2fed9a95..2a8c4358 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -779,8 +779,7 @@ class ProjectCommands(AutoRegisteringGroup): # Test 1: Get symbols overview log.info("Testing GetSymbolsOverviewTool on file: %s", target_file) - overview_result = agent.execute_task(lambda: overview_tool.apply(target_file)) - overview_data = json.loads(overview_result) + overview_data = agent.execute_task(lambda: overview_tool.get_symbol_overview(target_file)) log.info("GetSymbolsOverviewTool returned %d symbols", len(overview_data)) if not overview_data: diff --git a/src/serena/tools/symbol_tools.py b/src/serena/tools/symbol_tools.py index 2c0ec882..ae74bbfd 100644 --- a/src/serena/tools/symbol_tools.py +++ b/src/serena/tools/symbol_tools.py @@ -64,20 +64,28 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead): Don't adjust unless there is really no other way to get the content required for the task. :return: a JSON object containing symbols grouped by kind in a compact format. """ + result = self.get_symbol_overview(relative_path, depth=depth) + compact_result = self._transform_symbols_to_compact_format(result) + result_json_str = self._to_json(compact_result) + return self._limit_length(result_json_str, max_answer_chars) + + def get_symbol_overview(self, relative_path: str, depth: int = 0) -> list[dict]: + """ + :param relative_path: relative path to a source file + :param depth: the depth up to which descendants shall be retrieved + :return: a list of symbol dictionaries representing the symbol overview of the file + """ symbol_retriever = self.create_language_server_symbol_retriever() - file_path = os.path.join(self.project.project_root, relative_path) # The symbol overview is capable of working with both files and directories, # but we want to ensure that the user provides a file path. + file_path = os.path.join(self.project.project_root, relative_path) if not os.path.exists(file_path): raise FileNotFoundError(f"File or directory {relative_path} does not exist in the project.") if os.path.isdir(file_path): raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ") - result = symbol_retriever.get_symbol_overview(relative_path, depth=depth)[relative_path] - # Transform to compact format - compact_result = self._transform_symbols_to_compact_format(result) - result_json_str = self._to_json(compact_result) - return self._limit_length(result_json_str, max_answer_chars) + + return symbol_retriever.get_symbol_overview(relative_path, depth=depth)[relative_path] @staticmethod def _transform_symbols_to_compact_format(symbols: list[dict[str, Any]]) -> dict[str, list]: From 18ab2cea9ae2e9ae5598941ffedab365bbf8c7b3 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Mon, 19 Jan 2026 23:21:00 +0100 Subject: [PATCH 13/29] Improve docstring explaining overrides in ls_specific_settings [skip ci] --- src/solidlsp/language_servers/csharp_language_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solidlsp/language_servers/csharp_language_server.py b/src/solidlsp/language_servers/csharp_language_server.py index 24b39a38..d0025f16 100644 --- a/src/solidlsp/language_servers/csharp_language_server.py +++ b/src/solidlsp/language_servers/csharp_language_server.py @@ -191,7 +191,7 @@ class CSharpLanguageServer(SolidLanguageServer): Provides C# specific instantiation of the LanguageServer class using `Microsoft.CodeAnalysis.LanguageServer`, the official Roslyn-based language server from Microsoft. - You can pass a list of runtime dependency overrides in ls_specific_settings["csharp"]. This is a list of + You can pass a list of runtime dependency overrides in ls_specific_settings["csharp"]["runtime_dependencies"]. This is a list of dicts, each containing at least the "id" key, and optionally "platform_id" to uniquely identify the dependency to override. For example, to override the URL of the .NET runtime on windows-x64, add the entry: From 7af2fe3e2bf788ac2666a42b4184cbb2c6e1c133 Mon Sep 17 00:00:00 2001 From: Torben Leth Date: Tue, 20 Jan 2026 00:24:21 +0100 Subject: [PATCH 14/29] feat(al): inject full object name into hover and add path normalization - Override request_hover() to inject cached original names (e.g. 'Table 50000 "TEST Customer"') into hover responses, preserving object type and ID metadata that the AL LSP doesn't include - Add _normalize_path() for consistent cache keys across Windows/Unix path formats - Add comprehensive tests for hover injection and path normalization --- .../language_servers/al_language_server.py | 51 ++- test/solidlsp/al/test_al_basic.py | 314 ++++++++++++++++++ 2 files changed, 363 insertions(+), 2 deletions(-) diff --git a/src/solidlsp/language_servers/al_language_server.py b/src/solidlsp/language_servers/al_language_server.py index 4a304b21..bb66437c 100644 --- a/src/solidlsp/language_servers/al_language_server.py +++ b/src/solidlsp/language_servers/al_language_server.py @@ -13,6 +13,7 @@ from pathlib import Path import requests from overrides import override +from solidlsp import ls_types from solidlsp.language_servers.common import quote_windows_path from solidlsp.ls import DocumentSymbols, LSPFileBuffer, SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig @@ -109,6 +110,14 @@ class ALLanguageServer(SolidLanguageServer): super().__init__(config, repository_root_path, ProcessLaunchInfo(cmd=cmd, cwd=repository_root_path), "al", solidlsp_settings) + # Cache mapping (file_path, line, char) -> original_full_name for hover injection + self._al_original_names: dict[tuple[str, int, int], str] = {} + + @staticmethod + def _normalize_path(path: str) -> str: + """Normalize file path for consistent cache key usage across platforms.""" + return path.replace("\\", "/") + @classmethod def _download_al_extension(cls, url: str, target_dir: str) -> bool: """ @@ -1013,9 +1022,21 @@ class ALLanguageServer(SolidLanguageServer): # Get symbols from parent implementation document_symbols = super().request_document_symbols(relative_file_path, file_buffer=file_buffer) - # Normalize names by stripping AL object metadata + # Normalize names by stripping AL object metadata, storing originals for hover def normalize_name(symbol: UnifiedSymbolInformation) -> None: - symbol["name"] = extract_al_display_name(symbol["name"]) + original_name = symbol["name"] + normalized_name = extract_al_display_name(original_name) + + # Store original name if it was normalized (for hover injection) + if original_name != normalized_name: + sel_range = symbol.get("selectionRange", {}) + start = sel_range.get("start", {}) + line = start.get("line", 0) + char = start.get("character", 0) + normalized_path = self._normalize_path(relative_file_path) + self._al_original_names[(normalized_path, line, char)] = original_name + + symbol["name"] = normalized_name # Process children recursively if symbol.get("children"): @@ -1027,3 +1048,29 @@ class ALLanguageServer(SolidLanguageServer): normalize_name(sym) return document_symbols + + @override + def request_hover(self, relative_file_path: str, line: int, column: int) -> ls_types.Hover | None: + """ + Override to inject original AL object name (with type and ID) into hover responses. + + When hovering over a symbol whose name was normalized, we prepend the original + full name (e.g., 'Table 50000 "TEST Customer"') to the hover content. + """ + hover = super().request_hover(relative_file_path, line, column) + + if hover is None: + return None + + # Check if we have an original name for this position (using normalized path for consistency) + normalized_path = self._normalize_path(relative_file_path) + original_name = self._al_original_names.get((normalized_path, line, column)) + + if original_name and "contents" in hover: + contents = hover["contents"] + if isinstance(contents, dict) and "value" in contents: + # Prepend the original full name to the hover content + prefix = f"**{original_name}**\n\n---\n\n" + contents["value"] = prefix + contents["value"] + + return hover diff --git a/test/solidlsp/al/test_al_basic.py b/test/solidlsp/al/test_al_basic.py index bbcdff76..d103aba2 100644 --- a/test/solidlsp/al/test_al_basic.py +++ b/test/solidlsp/al/test_al_basic.py @@ -242,3 +242,317 @@ class TestALLanguageServer: assert "TEST Customer" in al_object_names, f"TEST Customer not found in: {al_object_names}" assert "CustomerMgt" in al_object_names, f"CustomerMgt not found in: {al_object_names}" assert "CustomerType" in al_object_names, f"CustomerType not found in: {al_object_names}" + + +@pytest.mark.al +class TestALHoverInjection: + """Tests for hover injection of original AL object names with type and ID.""" + + def _get_symbol_hover(self, language_server: SolidLanguageServer, file_path: str, symbol_name: str) -> tuple[dict | None, str | None]: + """Helper to get hover info for a symbol by name. + + Returns (hover_info, hover_value) tuple. + """ + symbols = language_server.request_document_symbols(file_path) + for sym in symbols.root_symbols: + if sym.get("name") == symbol_name: + sel_range = sym.get("selectionRange", {}) + start = sel_range.get("start", {}) + line = start.get("line", 0) + char = start.get("character", 0) + hover = language_server.request_hover(file_path, line, char) + if hover and "contents" in hover: + return hover, hover["contents"].get("value", "") + return hover, None + return None, None + + def _get_child_symbol_hover( + self, language_server: SolidLanguageServer, file_path: str, parent_name: str, child_name_contains: str + ) -> tuple[dict | None, str | None]: + """Helper to get hover info for a child symbol. + + Returns (hover_info, hover_value) tuple. + """ + symbols = language_server.request_document_symbols(file_path) + for sym in symbols.root_symbols: + if sym.get("name") == parent_name: + for child in sym.get("children", []): + if child_name_contains in child.get("name", ""): + sel_range = child.get("selectionRange", {}) + start = sel_range.get("start", {}) + line = start.get("line", 0) + char = start.get("character", 0) + hover = language_server.request_hover(file_path, line, char) + if hover and "contents" in hover: + return hover, hover["contents"].get("value", "") + return hover, None + return None, None + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_table_injects_full_name(self, language_server: SolidLanguageServer) -> None: + """Test that hovering over a Table symbol shows the full object name with ID.""" + file_path = os.path.join("src", "Tables", "Customer.Table.al") + hover, value = self._get_symbol_hover(language_server, file_path, "TEST Customer") + + assert hover is not None, "Hover should return a result for Table symbol" + assert value is not None, "Hover should have content" + assert '**Table 50000 "TEST Customer"**' in value, f"Hover should contain full Table name with ID. Got: {value[:200]}" + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_page_injects_full_name(self, language_server: SolidLanguageServer) -> None: + """Test that hovering over a Page symbol shows the full object name with ID.""" + file_path = os.path.join("src", "Pages", "CustomerCard.Page.al") + hover, value = self._get_symbol_hover(language_server, file_path, "TEST Customer Card") + + assert hover is not None, "Hover should return a result for Page symbol" + assert value is not None, "Hover should have content" + assert '**Page 50001 "TEST Customer Card"**' in value, f"Hover should contain full Page name with ID. Got: {value[:200]}" + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_codeunit_injects_full_name(self, language_server: SolidLanguageServer) -> None: + """Test that hovering over a Codeunit symbol shows the full object name with ID.""" + file_path = os.path.join("src", "Codeunits", "CustomerMgt.Codeunit.al") + hover, value = self._get_symbol_hover(language_server, file_path, "CustomerMgt") + + assert hover is not None, "Hover should return a result for Codeunit symbol" + assert value is not None, "Hover should have content" + assert "**Codeunit 50000 CustomerMgt**" in value, f"Hover should contain full Codeunit name with ID. Got: {value[:200]}" + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_enum_injects_full_name(self, language_server: SolidLanguageServer) -> None: + """Test that hovering over an Enum symbol shows the full object name with ID.""" + file_path = os.path.join("src", "Enums", "CustomerType.Enum.al") + hover, value = self._get_symbol_hover(language_server, file_path, "CustomerType") + + assert hover is not None, "Hover should return a result for Enum symbol" + assert value is not None, "Hover should have content" + assert "**Enum 50000 CustomerType**" in value, f"Hover should contain full Enum name with ID. Got: {value[:200]}" + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_interface_injects_full_name(self, language_server: SolidLanguageServer) -> None: + """Test that hovering over an Interface symbol shows the full object name (no ID for interfaces).""" + file_path = os.path.join("src", "Interfaces", "IPaymentProcessor.Interface.al") + hover, value = self._get_symbol_hover(language_server, file_path, "IPaymentProcessor") + + assert hover is not None, "Hover should return a result for Interface symbol" + assert value is not None, "Hover should have content" + assert "**Interface IPaymentProcessor**" in value, f"Hover should contain full Interface name. Got: {value[:200]}" + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_procedure_no_injection(self, language_server: SolidLanguageServer) -> None: + """Test that hovering over a procedure does NOT inject object name (procedures are not normalized).""" + file_path = os.path.join("src", "Codeunits", "CustomerMgt.Codeunit.al") + hover, value = self._get_child_symbol_hover(language_server, file_path, "CustomerMgt", "CreateCustomer") + + assert hover is not None, "Hover should return a result for procedure" + assert value is not None, "Hover should have content" + # Procedure hover should NOT start with ** (no injection) + assert not value.startswith("**"), f"Procedure hover should not have injected name. Got: {value[:200]}" + # But should contain procedure info + assert "CreateCustomer" in value, f"Hover should contain procedure name. Got: {value[:200]}" + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_field_no_injection(self, language_server: SolidLanguageServer) -> None: + """Test that hovering over a field does NOT inject object name (fields are not normalized).""" + file_path = os.path.join("src", "Tables", "Customer.Table.al") + symbols = language_server.request_document_symbols(file_path) + + # Navigate to a field: Table -> fields -> specific field + for sym in symbols.root_symbols: + if sym.get("name") == "TEST Customer": + for child in sym.get("children", []): + if child.get("name") == "fields": + for field in child.get("children", []): + if "Name" in field.get("name", ""): + sel_range = field.get("selectionRange", {}) + start = sel_range.get("start", {}) + line = start.get("line", 0) + char = start.get("character", 0) + hover = language_server.request_hover(file_path, line, char) + + assert hover is not None, "Hover should return a result for field" + value = hover.get("contents", {}).get("value", "") + # Field hover should NOT start with ** (no injection) + assert not value.startswith("**"), f"Field hover should not have injected name. Got: {value[:200]}" + return + + pytest.fail("Could not find a field to test hover on") + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_multiple_objects_correct_injection(self, language_server: SolidLanguageServer) -> None: + """Test that multiple AL objects each get their correct full name injected.""" + test_cases = [ + (os.path.join("src", "Tables", "Customer.Table.al"), "TEST Customer", 'Table 50000 "TEST Customer"'), + (os.path.join("src", "Codeunits", "CustomerMgt.Codeunit.al"), "CustomerMgt", "Codeunit 50000 CustomerMgt"), + (os.path.join("src", "Enums", "CustomerType.Enum.al"), "CustomerType", "Enum 50000 CustomerType"), + ] + + for file_path, symbol_name, expected_full_name in test_cases: + hover, value = self._get_symbol_hover(language_server, file_path, symbol_name) + + assert hover is not None, f"Hover should return a result for {symbol_name}" + assert value is not None, f"Hover should have content for {symbol_name}" + assert ( + f"**{expected_full_name}**" in value + ), f"Hover for {symbol_name} should contain '{expected_full_name}'. Got: {value[:200]}" + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_contains_separator_after_injection(self, language_server: SolidLanguageServer) -> None: + """Test that injected hover has a separator between injected name and original content.""" + file_path = os.path.join("src", "Tables", "Customer.Table.al") + hover, value = self._get_symbol_hover(language_server, file_path, "TEST Customer") + + assert hover is not None, "Hover should return a result" + assert value is not None, "Hover should have content" + # Should have the separator after the bold name + assert "---" in value, f"Hover should contain separator. Got: {value[:300]}" + # The separator should come after the injected name + bold_end = value.find("**", 2) # Find closing ** + separator_pos = value.find("---") + assert separator_pos > bold_end, "Separator should come after the injected name" + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_preserves_original_content(self, language_server: SolidLanguageServer) -> None: + """Test that the original hover content is preserved after the injected name.""" + file_path = os.path.join("src", "Tables", "Customer.Table.al") + hover, value = self._get_symbol_hover(language_server, file_path, "TEST Customer") + + assert hover is not None, "Hover should return a result" + assert value is not None, "Hover should have content" + # Original AL hover content should still be present (the table structure) + assert "```al" in value, f"Hover should contain original AL code block. Got: {value[:500]}" + assert 'Table "TEST Customer"' in value, f"Hover should contain original table definition. Got: {value[:500]}" + + +@pytest.mark.al +class TestALPathNormalization: + """Tests for path normalization in hover injection cache.""" + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_with_forward_slash_path(self, language_server: SolidLanguageServer) -> None: + """Test that hover injection works with forward slash paths.""" + file_path = "src/Tables/Customer.Table.al" + symbols = language_server.request_document_symbols(file_path) + + for sym in symbols.root_symbols: + if sym.get("name") == "TEST Customer": + sel_range = sym.get("selectionRange", {}) + start = sel_range.get("start", {}) + line = start.get("line", 0) + char = start.get("character", 0) + + hover = language_server.request_hover(file_path, line, char) + assert hover is not None, "Hover should return a result" + value = hover.get("contents", {}).get("value", "") + assert '**Table 50000 "TEST Customer"**' in value, f"Hover should have injection. Got: {value[:200]}" + return + + pytest.fail("Could not find TEST Customer symbol") + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_with_backslash_path(self, language_server: SolidLanguageServer) -> None: + """Test that hover injection works with backslash paths (Windows style).""" + file_path = "src\\Tables\\Customer.Table.al" + symbols = language_server.request_document_symbols(file_path) + + for sym in symbols.root_symbols: + if sym.get("name") == "TEST Customer": + sel_range = sym.get("selectionRange", {}) + start = sel_range.get("start", {}) + line = start.get("line", 0) + char = start.get("character", 0) + + hover = language_server.request_hover(file_path, line, char) + assert hover is not None, "Hover should return a result" + value = hover.get("contents", {}).get("value", "") + assert '**Table 50000 "TEST Customer"**' in value, f"Hover should have injection. Got: {value[:200]}" + return + + pytest.fail("Could not find TEST Customer symbol") + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_with_mixed_path_formats_symbols_backslash_hover_forward(self, language_server: SolidLanguageServer) -> None: + """Test hover works when symbols requested with backslash but hover with forward slash.""" + file_path_backslash = "src\\Tables\\Customer.Table.al" + file_path_forward = "src/Tables/Customer.Table.al" + + # Request symbols with backslash path + symbols = language_server.request_document_symbols(file_path_backslash) + + for sym in symbols.root_symbols: + if sym.get("name") == "TEST Customer": + sel_range = sym.get("selectionRange", {}) + start = sel_range.get("start", {}) + line = start.get("line", 0) + char = start.get("character", 0) + + # Request hover with forward slash path (different format) + hover = language_server.request_hover(file_path_forward, line, char) + assert hover is not None, "Hover should return a result" + value = hover.get("contents", {}).get("value", "") + assert ( + '**Table 50000 "TEST Customer"**' in value + ), f"Hover injection should work with mixed path formats. Got: {value[:200]}" + return + + pytest.fail("Could not find TEST Customer symbol") + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_with_mixed_path_formats_symbols_forward_hover_backslash(self, language_server: SolidLanguageServer) -> None: + """Test hover works when symbols requested with forward slash but hover with backslash.""" + file_path_forward = "src/Tables/Customer.Table.al" + file_path_backslash = "src\\Tables\\Customer.Table.al" + + # Request symbols with forward slash path + symbols = language_server.request_document_symbols(file_path_forward) + + for sym in symbols.root_symbols: + if sym.get("name") == "TEST Customer": + sel_range = sym.get("selectionRange", {}) + start = sel_range.get("start", {}) + line = start.get("line", 0) + char = start.get("character", 0) + + # Request hover with backslash path (different format) + hover = language_server.request_hover(file_path_backslash, line, char) + assert hover is not None, "Hover should return a result" + value = hover.get("contents", {}).get("value", "") + assert ( + '**Table 50000 "TEST Customer"**' in value + ), f"Hover injection should work with mixed path formats. Got: {value[:200]}" + return + + pytest.fail("Could not find TEST Customer symbol") + + @pytest.mark.parametrize("language_server", [Language.AL], indirect=True) + def test_hover_caching_multiple_files_different_path_formats(self, language_server: SolidLanguageServer) -> None: + """Test that hover injection cache works correctly across multiple files with different path formats.""" + test_cases = [ + ("src/Tables/Customer.Table.al", "src\\Tables\\Customer.Table.al", "TEST Customer", 'Table 50000 "TEST Customer"'), + ( + "src\\Codeunits\\CustomerMgt.Codeunit.al", + "src/Codeunits/CustomerMgt.Codeunit.al", + "CustomerMgt", + "Codeunit 50000 CustomerMgt", + ), + ] + + for symbols_path, hover_path, symbol_name, expected_injection in test_cases: + # Request symbols with one path format + symbols = language_server.request_document_symbols(symbols_path) + + for sym in symbols.root_symbols: + if sym.get("name") == symbol_name: + sel_range = sym.get("selectionRange", {}) + start = sel_range.get("start", {}) + line = start.get("line", 0) + char = start.get("character", 0) + + # Request hover with different path format + hover = language_server.request_hover(hover_path, line, char) + assert hover is not None, f"Hover should return a result for {symbol_name}" + value = hover.get("contents", {}).get("value", "") + assert ( + f"**{expected_injection}**" in value + ), f"Hover for {symbol_name} should have injection with mixed paths. Got: {value[:200]}" + break From b9f82774ce3a73b4d37acc41ad5fed96044a004b Mon Sep 17 00:00:00 2001 From: Torben Leth Date: Tue, 20 Jan 2026 01:08:50 +0100 Subject: [PATCH 15/29] fix(al): guard against missing position in name cache storage Only store the original name in the cache if we have valid position data. This prevents false cache matches at position (0, 0) when selectionRange is missing from a symbol. Addresses PR #960 review comment. --- .../language_servers/al_language_server.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/solidlsp/language_servers/al_language_server.py b/src/solidlsp/language_servers/al_language_server.py index bb66437c..c2bad1e2 100644 --- a/src/solidlsp/language_servers/al_language_server.py +++ b/src/solidlsp/language_servers/al_language_server.py @@ -1028,13 +1028,16 @@ class ALLanguageServer(SolidLanguageServer): normalized_name = extract_al_display_name(original_name) # Store original name if it was normalized (for hover injection) + # Only store if we have valid position data to avoid false matches at (0, 0) if original_name != normalized_name: - sel_range = symbol.get("selectionRange", {}) - start = sel_range.get("start", {}) - line = start.get("line", 0) - char = start.get("character", 0) - normalized_path = self._normalize_path(relative_file_path) - self._al_original_names[(normalized_path, line, char)] = original_name + sel_range = symbol.get("selectionRange") + if sel_range: + start = sel_range.get("start") + if start and "line" in start and "character" in start: + line = start["line"] + char = start["character"] + normalized_path = self._normalize_path(relative_file_path) + self._al_original_names[(normalized_path, line, char)] = original_name symbol["name"] = normalized_name From d05d8b765fca84199d22640492725fc4ca0c95b3 Mon Sep 17 00:00:00 2001 From: Torben Leth Date: Tue, 20 Jan 2026 01:15:48 +0100 Subject: [PATCH 16/29] refactor(al): move extract_al_display_name into ALLanguageServer class Move the AL name extraction pattern and function into the class as a static method since they are only used by the language server. Addresses PR #960 review comment. --- .../language_servers/al_language_server.py | 80 +++++++++---------- test/solidlsp/al/test_al_basic.py | 28 +++---- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/solidlsp/language_servers/al_language_server.py b/src/solidlsp/language_servers/al_language_server.py index c2bad1e2..30ccbcaa 100644 --- a/src/solidlsp/language_servers/al_language_server.py +++ b/src/solidlsp/language_servers/al_language_server.py @@ -24,45 +24,6 @@ from solidlsp.settings import SolidLSPSettings log = logging.getLogger(__name__) -# Regex pattern to match AL object names like: -# - 'Table 50000 "TEST Customer"' -> captures 'TEST Customer' -# - 'Codeunit 50000 CustomerMgt' -> captures 'CustomerMgt' -# - 'Interface IPaymentProcessor' -> captures 'IPaymentProcessor' -# - 'Enum 50000 CustomerType' -> captures 'CustomerType' -# Pattern: [] (|) -_AL_OBJECT_NAME_PATTERN = re.compile( - r"^(?:Table|Page|Codeunit|Enum|Interface|Report|Query|XMLPort|PermissionSet|" - r"PermissionSetExtension|Profile|PageExtension|TableExtension|EnumExtension|" - r"PageCustomization|ReportExtension|ControlAddin|DotNetPackage)" # Object type - r"(?:\s+\d+)?" # Optional object ID - r"\s+" # Required space before name - r'(?:"([^"]+)"|(\S+))$' # Quoted name (group 1) or unquoted identifier (group 2) -) - - -def extract_al_display_name(full_name: str) -> str: - """ - Extract the display name from an AL symbol's full name. - - AL Language Server returns symbol names in format: - - 'Table 50000 "TEST Customer"' -> 'TEST Customer' - - 'Codeunit 50000 CustomerMgt' -> 'CustomerMgt' - - 'Interface IPaymentProcessor' -> 'IPaymentProcessor' - - 'fields' -> 'fields' (non-AL-object symbols pass through unchanged) - - Args: - full_name: The full symbol name as returned by AL Language Server - - Returns: - The extracted display name for matching, or the original name if not an AL object - - """ - match = _AL_OBJECT_NAME_PATTERN.match(full_name) - if match: - # Return quoted name (group 1) or unquoted name (group 2) - return match.group(1) or match.group(2) or full_name - return full_name - class ALLanguageServer(SolidLanguageServer): """ @@ -79,6 +40,45 @@ class ALLanguageServer(SolidLanguageServer): - File opening requirement before symbol retrieval """ + # Regex pattern to match AL object names like: + # - 'Table 50000 "TEST Customer"' -> captures 'TEST Customer' + # - 'Codeunit 50000 CustomerMgt' -> captures 'CustomerMgt' + # - 'Interface IPaymentProcessor' -> captures 'IPaymentProcessor' + # - 'Enum 50000 CustomerType' -> captures 'CustomerType' + # Pattern: [] (|) + _AL_OBJECT_NAME_PATTERN = re.compile( + r"^(?:Table|Page|Codeunit|Enum|Interface|Report|Query|XMLPort|PermissionSet|" + r"PermissionSetExtension|Profile|PageExtension|TableExtension|EnumExtension|" + r"PageCustomization|ReportExtension|ControlAddin|DotNetPackage)" # Object type + r"(?:\s+\d+)?" # Optional object ID + r"\s+" # Required space before name + r'(?:"([^"]+)"|(\S+))$' # Quoted name (group 1) or unquoted identifier (group 2) + ) + + @staticmethod + def _extract_al_display_name(full_name: str) -> str: + """ + Extract the display name from an AL symbol's full name. + + AL Language Server returns symbol names in format: + - 'Table 50000 "TEST Customer"' -> 'TEST Customer' + - 'Codeunit 50000 CustomerMgt' -> 'CustomerMgt' + - 'Interface IPaymentProcessor' -> 'IPaymentProcessor' + - 'fields' -> 'fields' (non-AL-object symbols pass through unchanged) + + Args: + full_name: The full symbol name as returned by AL Language Server + + Returns: + The extracted display name for matching, or the original name if not an AL object + + """ + match = ALLanguageServer._AL_OBJECT_NAME_PATTERN.match(full_name) + if match: + # Return quoted name (group 1) or unquoted name (group 2) + return match.group(1) or match.group(2) or full_name + return full_name + def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings): """ Initialize the AL Language Server. @@ -1025,7 +1025,7 @@ class ALLanguageServer(SolidLanguageServer): # Normalize names by stripping AL object metadata, storing originals for hover def normalize_name(symbol: UnifiedSymbolInformation) -> None: original_name = symbol["name"] - normalized_name = extract_al_display_name(original_name) + normalized_name = self._extract_al_display_name(original_name) # Store original name if it was normalized (for hover injection) # Only store if we have valid position data to avoid false matches at (0, 0) diff --git a/test/solidlsp/al/test_al_basic.py b/test/solidlsp/al/test_al_basic.py index d103aba2..51bae0b7 100644 --- a/test/solidlsp/al/test_al_basic.py +++ b/test/solidlsp/al/test_al_basic.py @@ -4,7 +4,7 @@ import pytest from serena.symbol import LanguageServerSymbol from solidlsp import SolidLanguageServer -from solidlsp.language_servers.al_language_server import extract_al_display_name +from solidlsp.language_servers.al_language_server import ALLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import SymbolUtils from test.conftest import language_tests_enabled @@ -13,49 +13,49 @@ pytestmark = [pytest.mark.al, pytest.mark.skipif(not language_tests_enabled(Lang class TestExtractALDisplayName: - """Tests for the extract_al_display_name function.""" + """Tests for the ALLanguageServer._extract_al_display_name method.""" def test_table_with_quoted_name(self) -> None: """Test extraction from Table with quoted name.""" - assert extract_al_display_name('Table 50000 "TEST Customer"') == "TEST Customer" + assert ALLanguageServer._extract_al_display_name('Table 50000 "TEST Customer"') == "TEST Customer" def test_page_with_quoted_name(self) -> None: """Test extraction from Page with quoted name.""" - assert extract_al_display_name('Page 50001 "TEST Customer Card"') == "TEST Customer Card" + assert ALLanguageServer._extract_al_display_name('Page 50001 "TEST Customer Card"') == "TEST Customer Card" def test_codeunit_unquoted(self) -> None: """Test extraction from Codeunit with unquoted name.""" - assert extract_al_display_name("Codeunit 50000 CustomerMgt") == "CustomerMgt" + assert ALLanguageServer._extract_al_display_name("Codeunit 50000 CustomerMgt") == "CustomerMgt" def test_enum_unquoted(self) -> None: """Test extraction from Enum with unquoted name.""" - assert extract_al_display_name("Enum 50000 CustomerType") == "CustomerType" + assert ALLanguageServer._extract_al_display_name("Enum 50000 CustomerType") == "CustomerType" def test_interface_no_id(self) -> None: """Test extraction from Interface (no ID).""" - assert extract_al_display_name("Interface IPaymentProcessor") == "IPaymentProcessor" + assert ALLanguageServer._extract_al_display_name("Interface IPaymentProcessor") == "IPaymentProcessor" def test_table_extension(self) -> None: """Test extraction from TableExtension.""" - assert extract_al_display_name('TableExtension 50000 "Ext Customer"') == "Ext Customer" + assert ALLanguageServer._extract_al_display_name('TableExtension 50000 "Ext Customer"') == "Ext Customer" def test_page_extension(self) -> None: """Test extraction from PageExtension.""" - assert extract_al_display_name('PageExtension 50000 "My Page Ext"') == "My Page Ext" + assert ALLanguageServer._extract_al_display_name('PageExtension 50000 "My Page Ext"') == "My Page Ext" def test_non_al_object_unchanged(self) -> None: """Test that non-AL-object names pass through unchanged.""" - assert extract_al_display_name("fields") == "fields" - assert extract_al_display_name("CreateCustomer") == "CreateCustomer" - assert extract_al_display_name("Name") == "Name" + assert ALLanguageServer._extract_al_display_name("fields") == "fields" + assert ALLanguageServer._extract_al_display_name("CreateCustomer") == "CreateCustomer" + assert ALLanguageServer._extract_al_display_name("Name") == "Name" def test_report_with_quoted_name(self) -> None: """Test extraction from Report.""" - assert extract_al_display_name('Report 50000 "Sales Invoice"') == "Sales Invoice" + assert ALLanguageServer._extract_al_display_name('Report 50000 "Sales Invoice"') == "Sales Invoice" def test_query_unquoted(self) -> None: """Test extraction from Query.""" - assert extract_al_display_name("Query 50000 CustomerQuery") == "CustomerQuery" + assert ALLanguageServer._extract_al_display_name("Query 50000 CustomerQuery") == "CustomerQuery" @pytest.mark.al From 5ec66d81e2772426c2774547a586f6dd30b79501 Mon Sep 17 00:00:00 2001 From: Torben Leth Date: Tue, 20 Jan 2026 02:09:29 +0100 Subject: [PATCH 17/29] fix(al): normalize paths at method entry for cross-platform compatibility Move path normalization to the start of request_document_symbols and request_hover methods. This ensures backslash paths work on Linux/macOS where backslash is a literal character, not a path separator. --- .../language_servers/al_language_server.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/solidlsp/language_servers/al_language_server.py b/src/solidlsp/language_servers/al_language_server.py index 30ccbcaa..5dead90d 100644 --- a/src/solidlsp/language_servers/al_language_server.py +++ b/src/solidlsp/language_servers/al_language_server.py @@ -1019,6 +1019,9 @@ class ALLanguageServer(SolidLanguageServer): Metadata (object type, ID) is available via the hover LSP method when using include_info=True in find_symbol. """ + # Normalize path separators for cross-platform compatibility (backslash → forward slash) + relative_file_path = self._normalize_path(relative_file_path) + # Get symbols from parent implementation document_symbols = super().request_document_symbols(relative_file_path, file_buffer=file_buffer) @@ -1036,8 +1039,7 @@ class ALLanguageServer(SolidLanguageServer): if start and "line" in start and "character" in start: line = start["line"] char = start["character"] - normalized_path = self._normalize_path(relative_file_path) - self._al_original_names[(normalized_path, line, char)] = original_name + self._al_original_names[(relative_file_path, line, char)] = original_name symbol["name"] = normalized_name @@ -1060,14 +1062,16 @@ class ALLanguageServer(SolidLanguageServer): When hovering over a symbol whose name was normalized, we prepend the original full name (e.g., 'Table 50000 "TEST Customer"') to the hover content. """ + # Normalize path separators for cross-platform compatibility (backslash → forward slash) + relative_file_path = self._normalize_path(relative_file_path) + hover = super().request_hover(relative_file_path, line, column) if hover is None: return None - # Check if we have an original name for this position (using normalized path for consistency) - normalized_path = self._normalize_path(relative_file_path) - original_name = self._al_original_names.get((normalized_path, line, column)) + # Check if we have an original name for this position + original_name = self._al_original_names.get((relative_file_path, line, column)) if original_name and "contents" in hover: contents = hover["contents"] From 3d610551270829021445e89db4f3f002e02fc297 Mon Sep 17 00:00:00 2001 From: Yinan Zhang Date: Wed, 21 Jan 2026 11:35:10 +0800 Subject: [PATCH 18/29] fix: handle HTTPError in JetBrainsPluginClient initialization When the JetBrains plugin service returns HTTP 503 error, the client now gracefully handles it by setting _plugin_version and _project_root to None, allowing the port scanning logic to continue trying other ports. Fixes #962 --- src/serena/tools/jetbrains_plugin_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/serena/tools/jetbrains_plugin_client.py b/src/serena/tools/jetbrains_plugin_client.py index 028fa66d..d7517711 100644 --- a/src/serena/tools/jetbrains_plugin_client.py +++ b/src/serena/tools/jetbrains_plugin_client.py @@ -93,7 +93,7 @@ class JetBrainsPluginClient(ToStringMixin): status_response: PluginStatusDTO = cast(jb.PluginStatusDTO, self._make_request("GET", "/status")) self._project_root = status_response["project_root"] self._plugin_version = Version(status_response["plugin_version"]) - except ConnectionError: + except (ConnectionError, requests.exceptions.HTTPError): self._project_root = None self._plugin_version = None From b0a1546bb6cfad1ee54711c8dda99c15423de4c8 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 21 Jan 2026 11:14:51 +0700 Subject: [PATCH 19/29] Finalized setup instructions for Antigravity Serena works in it now --- docs/02-usage/030_clients.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/02-usage/030_clients.md b/docs/02-usage/030_clients.md index 2f3d291a..7b90fac8 100644 --- a/docs/02-usage/030_clients.md +++ b/docs/02-usage/030_clients.md @@ -269,13 +269,6 @@ Then make sure to configure the working directory to be the project root. ## Antigravity -:::{warning} -At the time of writing (12/2025), Antigravity does not seem to work with Serena due to schema validation issues -which are beyond our control. -The client starts Serena but then crashes with `[internal] marshal message: string field contains invalid UTF-8`. -Nevertheless, we provide a configuration that should work once the issue is resolved. -::: - Add this configuration: ```json @@ -296,6 +289,12 @@ Add this configuration: } ``` +You will have to prompt Antigravity's agent to "Activate the current project using serena's activation tool" after starting Antigravity in the project directory (once in the first chat enough, all other chat sessions will continue using the same Serena session). + + +Unlike VSCode, Antigravity does not currently support including the working directory in the MCP configuration. +Also, the current client will be shown as `none` in Serena's dashboard (Antigravity currently does not fully support the MCP specifications). This is not a problem, all tools will work as expected. + ## Other Clients For other clients, follow the [general instructions](#clients-general-instructions) above to set up Serena as an MCP server. From 1d766bea2afe3fcd7cc5083400b104c7bf3df8c7 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 21 Jan 2026 12:39:48 +0100 Subject: [PATCH 20/29] JetBrainsPluginClient: Improve error handling when retrieving service status #962 --- src/serena/tools/jetbrains_plugin_client.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/serena/tools/jetbrains_plugin_client.py b/src/serena/tools/jetbrains_plugin_client.py index d7517711..08e3289f 100644 --- a/src/serena/tools/jetbrains_plugin_client.py +++ b/src/serena/tools/jetbrains_plugin_client.py @@ -87,15 +87,16 @@ class JetBrainsPluginClient(ToStringMixin): self._session.headers.update({"Content-Type": "application/json", "Accept": "application/json"}) # connect and obtain status - self._project_root: str | None - self._plugin_version: Version | None + self._project_root: str | None = None + self._plugin_version: Version | None = None try: status_response: PluginStatusDTO = cast(jb.PluginStatusDTO, self._make_request("GET", "/status")) self._project_root = status_response["project_root"] self._plugin_version = Version(status_response["plugin_version"]) - except (ConnectionError, requests.exceptions.HTTPError): - self._project_root = None - self._plugin_version = None + except ConnectionError: # expected if no server is running at the port + pass + except Exception as e: + log.warning("Failed to obtain status from JetBrains plugin service at port %d: %s", e, port, exc_info=True) @property def _base_url(self) -> str: From fe0cc7bb0c8ddca1ce86fea1b72e020ed09a4539 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 22 Jan 2026 00:45:14 +0100 Subject: [PATCH 21/29] Avoid circular import issue --- src/serena/tools/tools_base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/serena/tools/tools_base.py b/src/serena/tools/tools_base.py index e28629a1..9fb1a393 100644 --- a/src/serena/tools/tools_base.py +++ b/src/serena/tools/tools_base.py @@ -14,7 +14,6 @@ from sensai.util.string import dict_string from serena.project import MemoriesManager, Project from serena.prompt_factory import PromptFactory -from serena.symbol import LanguageServerSymbolRetriever from serena.util.class_decorators import singleton from serena.util.inspection import iter_subclasses from solidlsp.ls_exceptions import SolidLSPException @@ -22,6 +21,7 @@ from solidlsp.ls_exceptions import SolidLSPException if TYPE_CHECKING: from serena.agent import SerenaAgent from serena.code_editor import CodeEditor + from serena.symbol import LanguageServerSymbolRetriever log = logging.getLogger(__name__) T = TypeVar("T") @@ -46,7 +46,9 @@ class Component(ABC): def memories_manager(self) -> "MemoriesManager": return self.project.memories_manager - def create_language_server_symbol_retriever(self) -> LanguageServerSymbolRetriever: + def create_language_server_symbol_retriever(self) -> "LanguageServerSymbolRetriever": + from serena.symbol import LanguageServerSymbolRetriever + if not self.agent.is_using_language_server(): raise Exception("Cannot create LanguageServerSymbolRetriever; agent is not in language server mode.") language_server_manager = self.agent.get_language_server_manager_or_raise() From c2e795fce4ecabd5a5db32811d8ce0e0e799fc61 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 22 Jan 2026 00:00:08 +0100 Subject: [PATCH 22/29] Ensure memory-efficient high-level document symbol cache #944 * Introduce SymbolBody representation (compact), from which the body can be reconstructed, rendering body computation lazy * Introduce SymbolBodyFactory for the creation of such instances, ensuring that instances created from the same factory share the same file content, allowing the in-memory and on-disk caches to have a significantly smaller memory footprint * Change type of "body" key in UnifiedSymbolRepresentation to use SymbolBody instead of str * Adjust all usages of the "body" key --- src/serena/symbol.py | 11 +- src/solidlsp/ls.py | 109 ++++++++++++------ src/solidlsp/ls_types.py | 14 ++- test/solidlsp/bash/test_bash_basic.py | 2 +- test/solidlsp/dart/test_dart_basic.py | 3 +- .../elixir/test_elixir_symbol_retrieval.py | 2 +- .../erlang/test_erlang_symbol_retrieval.py | 2 +- test/solidlsp/python/test_symbol_retrieval.py | 3 +- .../ruby/test_ruby_symbol_retrieval.py | 2 +- test/solidlsp/toml/test_toml_basic.py | 6 +- test/solidlsp/toml/test_toml_edge_cases.py | 2 +- .../solidlsp/vue/test_vue_symbol_retrieval.py | 2 +- test/solidlsp/yaml_ls/test_yaml_basic.py | 6 +- 13 files changed, 108 insertions(+), 56 deletions(-) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index ad807a95..3cec48da 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -300,7 +300,11 @@ class LanguageServerSymbol(Symbol, ToStringMixin): @property def body(self) -> str | None: - return self.symbol_root.get("body") + body = self.symbol_root.get("body") + if body is None: + return None + else: + return body.get_text() def get_name_path(self) -> str: """ @@ -418,9 +422,10 @@ class LanguageServerSymbol(Symbol, ToStringMixin): result["body_location"] = {"start_line": body_start_line, "end_line": body_end_line} if include_body: - if self.body is None: + body = self.body + if body is None: log.warning("Requested body for symbol, but it is not present. The symbol might have been loaded with include_body=False.") - result["body"] = self.body + result["body"] = body if child_inclusion_predicate is None: child_inclusion_predicate = lambda s: True diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index 578d338a..116f31fc 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -84,6 +84,58 @@ class LSPFileBuffer: return self.contents.split("\n") +class SymbolBody: + """ + Representation of the body of a symbol, which allows the extraction of the symbol's text + from the lines of the file it is defined in. + + Instances that share the same lines buffer are memory-efficient, + using only 4 integers and a reference to the lines buffer from which the text can be extracted, + i.e. a core representation of only about 40 bytes per body. + """ + + def __init__(self, lines: list[str], start_line: int, start_col: int, end_line: int, end_col: int) -> None: + self._lines = lines + self._start_line = start_line + self._start_col = start_col + self._end_line = end_line + self._end_col = end_col + + def get_text(self) -> str: + # extract relevant lines + symbol_body = "\n".join(self._lines[self._start_line : self._end_line + 1]) + + # remove leading indentation + symbol_body = symbol_body[self._start_col :] + + # TODO: handle end_col properly (this was never implemented) + + return symbol_body + + +class SymbolBodyFactory: + """ + A factory for the creation of SymbolBody instances from symbols dictionaries. + Instances created from the same factory instance are memory-efficient, as they share + the same lines buffer. + """ + + def __init__(self, file_buffer: LSPFileBuffer): + self._lines = file_buffer.split_lines() + + def create_symbol_body(self, symbol: GenericDocumentSymbol) -> SymbolBody: + existing_body = symbol.get("body", None) + if existing_body and isinstance(existing_body, SymbolBody): + return existing_body + + assert "location" in symbol + start_line = symbol["location"]["range"]["start"]["line"] # type: ignore + end_line = symbol["location"]["range"]["end"]["line"] # type: ignore + start_col = symbol["location"]["range"]["start"]["character"] # type: ignore + end_col = symbol["location"]["range"]["end"]["character"] # type: ignore + return SymbolBody(self._lines, start_line, start_col, end_line, end_col) + + class DocumentSymbols: # IMPORTANT: Instances of this class are persisted in the high-level document symbol cache @@ -206,7 +258,7 @@ class SolidLanguageServer(ABC): """ RAW_DOCUMENT_SYMBOL_CACHE_FILENAME = "raw_document_symbols.pkl" RAW_DOCUMENT_SYMBOL_CACHE_FILENAME_LEGACY_FALLBACK = "document_symbols_cache_v23-06-25.pkl" - DOCUMENT_SYMBOL_CACHE_VERSION = 3 + DOCUMENT_SYMBOL_CACHE_VERSION = 4 DOCUMENT_SYMBOL_CACHE_FILENAME = "document_symbols.pkl" # To be overridden and extended by subclasses @@ -1106,7 +1158,7 @@ class SolidLanguageServer(ABC): assert isinstance(root_symbols, list), f"Unexpected response from Language Server: {root_symbols}" log.debug("Received %d root symbols for %s from the language server", len(root_symbols), relative_file_path) - file_lines = file_data.split_lines() + body_factory = SymbolBodyFactory(file_data) def convert_to_unified_symbol(original_symbol_dict: GenericDocumentSymbol) -> ls_types.UnifiedSymbolInformation: """ @@ -1137,8 +1189,7 @@ class SolidLanguageServer(ABC): if "relativePath" not in location: location["relativePath"] = relative_file_path # type: ignore - if "body" not in item: - item["body"] = self.retrieve_symbol_body(item, file_lines=file_lines) + item["body"] = self.create_symbol_body(item, factory=body_factory) # handle missing selectionRange if "selectionRange" not in item: @@ -1475,32 +1526,17 @@ class SolidLanguageServer(ABC): return ls_types.SignatureHelp(**response) # type: ignore - def retrieve_symbol_body( + def create_symbol_body( self, symbol: ls_types.UnifiedSymbolInformation | LSPTypes.SymbolInformation, - file_lines: list[str] | None = None, - file_buffer: LSPFileBuffer | None = None, - ) -> str: - """ - Load the body of the given symbol. If the body is already contained in the symbol, just return it. - """ - existing_body = symbol.get("body", None) - if existing_body: - return str(existing_body) + factory: SymbolBodyFactory | None = None, + ) -> SymbolBody: + if factory is None: + assert "relativePath" in symbol["location"] + with self._open_file_context(symbol["location"]["relativePath"]) as f: # type: ignore + factory = SymbolBodyFactory(f) - assert "location" in symbol - symbol_start_line = symbol["location"]["range"]["start"]["line"] - symbol_end_line = symbol["location"]["range"]["end"]["line"] - assert "relativePath" in symbol["location"] - if file_lines is None: - with self._open_file_context(symbol["location"]["relativePath"], file_buffer) as f: # type: ignore - file_lines = f.split_lines() - symbol_body = "\n".join(file_lines[symbol_start_line : symbol_end_line + 1]) - - # remove leading indentation - symbol_start_column = symbol["location"]["range"]["start"]["character"] # type: ignore - symbol_body = symbol_body[symbol_start_column:] - return symbol_body + return factory.create_symbol_body(symbol) def request_referencing_symbols( self, @@ -1549,8 +1585,12 @@ class SolidLanguageServer(ABC): ref_col = ref["range"]["start"]["character"] with self.open_file(ref_path) as file_data: + body_factory = SymbolBodyFactory(file_data) + # Get the containing symbol for this reference - containing_symbol = self.request_containing_symbol(ref_path, ref_line, ref_col, include_body=include_body) + containing_symbol = self.request_containing_symbol( + ref_path, ref_line, ref_col, include_body=include_body, body_factory=body_factory + ) if containing_symbol is None: # TODO: HORRIBLE HACK! I don't know how to do it better for now... # THIS IS BOUND TO BREAK IN MANY CASES! IT IS ALSO SPECIFIC TO PYTHON! @@ -1588,11 +1628,6 @@ class SolidLanguageServer(ABC): ) name = os.path.splitext(os.path.basename(ref_path))[0] - if include_body: - body = self.retrieve_full_file_content(ref_path) - else: - body = "" - containing_symbol = ls_types.UnifiedSymbolInformation( kind=ls_types.SymbolKind.File, range=fileRange, @@ -1600,8 +1635,11 @@ class SolidLanguageServer(ABC): location=location, name=name, children=[], - body=body, ) + + if include_body: + containing_symbol["body"] = self.create_symbol_body(containing_symbol, factory=body_factory) + if containing_symbol is None or (not include_file_symbols and containing_symbol["kind"] == ls_types.SymbolKind.File): continue @@ -1648,6 +1686,7 @@ class SolidLanguageServer(ABC): column: int | None = None, strict: bool = False, include_body: bool = False, + body_factory: SymbolBodyFactory | None = None, ) -> ls_types.UnifiedSymbolInformation | None: """ Finds the first symbol containing the position for the given file. @@ -1747,7 +1786,7 @@ class SolidLanguageServer(ABC): # Return the one with the greatest starting position (i.e. the innermost container). containing_symbol = max(containing_symbols, key=lambda s: s["location"]["range"]["start"]["line"]) if include_body: - containing_symbol["body"] = self.retrieve_symbol_body(containing_symbol) + containing_symbol["body"] = self.create_symbol_body(containing_symbol, factory=body_factory) return containing_symbol else: return None diff --git a/src/solidlsp/ls_types.py b/src/solidlsp/ls_types.py index 06e6ae9b..0eafcfea 100644 --- a/src/solidlsp/ls_types.py +++ b/src/solidlsp/ls_types.py @@ -5,12 +5,16 @@ Defines wrapper objects around the types returned by LSP to ensure decoupling be from __future__ import annotations from enum import Enum, IntEnum -from typing import NotRequired, Union +from typing import TYPE_CHECKING, NotRequired, Union from typing_extensions import TypedDict from solidlsp.lsp_protocol_handler.lsp_types import DiagnosticSeverity +if TYPE_CHECKING: + from .ls import SymbolBody + + URI = str DocumentUri = str Uint = int @@ -187,8 +191,12 @@ class SymbolTag(IntEnum): class UnifiedSymbolInformation(TypedDict): - """Represents information about programming constructs like variables, classes, + """ + Represents information about programming constructs like variables, classes, interfaces etc. + + This is a unifying extension of `lsp_types.SymbolInformation` and `lsp_types.DocumentSymbol`, + with added fields for SolidLSP/Serena use. """ deprecated: NotRequired[bool] @@ -234,7 +242,7 @@ class UnifiedSymbolInformation(TypedDict): """ The range that should be selected and revealed when this symbol is being picked, e.g the name of a function. Must be contained by the `range`. """ - body: NotRequired[str] + body: NotRequired["SymbolBody"] """ The body of the symbol. """ children: list[UnifiedSymbolInformation] diff --git a/test/solidlsp/bash/test_bash_basic.py b/test/solidlsp/bash/test_bash_basic.py index 1f2aa995..a3ead08f 100644 --- a/test/solidlsp/bash/test_bash_basic.py +++ b/test/solidlsp/bash/test_bash_basic.py @@ -50,7 +50,7 @@ class TestBashLanguageServerBasics: assert greet_user_symbol is not None, "Should find greet_user function" if "body" in greet_user_symbol: - body = greet_user_symbol["body"] + body = greet_user_symbol["body"].get_text() assert "function greet_user()" in body, "Function body should contain function definition" assert "case" in body.lower(), "Function body should contain case statement" diff --git a/test/solidlsp/dart/test_dart_basic.py b/test/solidlsp/dart/test_dart_basic.py index 6ea1163a..f469e8cf 100644 --- a/test/solidlsp/dart/test_dart_basic.py +++ b/test/solidlsp/dart/test_dart_basic.py @@ -212,7 +212,8 @@ class TestDartLanguageServer: assert containing_symbol["name"] == "add" assert containing_symbol["kind"] == SymbolKind.Method if "body" in containing_symbol: - assert "add" in containing_symbol["body"] or "final result" in containing_symbol["body"] + body = containing_symbol["body"].get_text() + assert "add" in body or "final result" in body @pytest.mark.parametrize("language_server", [Language.DART], indirect=True) def test_request_containing_symbol_class(self, language_server: SolidLanguageServer) -> None: diff --git a/test/solidlsp/elixir/test_elixir_symbol_retrieval.py b/test/solidlsp/elixir/test_elixir_symbol_retrieval.py index df5869f7..89a3127f 100644 --- a/test/solidlsp/elixir/test_elixir_symbol_retrieval.py +++ b/test/solidlsp/elixir/test_elixir_symbol_retrieval.py @@ -53,7 +53,7 @@ class TestElixirLanguageServerSymbols: assert containing_symbol["name"] == "def create_user(pid, id, name, email, roles \\\\ [])" assert containing_symbol["kind"] == SymbolKind.Method or containing_symbol["kind"] == SymbolKind.Function if "body" in containing_symbol: - assert "def create_user" in containing_symbol["body"] + assert "def create_user" in containing_symbol["body"].get_text() @pytest.mark.parametrize("language_server", [Language.ELIXIR], indirect=True) def test_request_containing_symbol_module(self, language_server: SolidLanguageServer) -> None: diff --git a/test/solidlsp/erlang/test_erlang_symbol_retrieval.py b/test/solidlsp/erlang/test_erlang_symbol_retrieval.py index f423698f..d63c0206 100644 --- a/test/solidlsp/erlang/test_erlang_symbol_retrieval.py +++ b/test/solidlsp/erlang/test_erlang_symbol_retrieval.py @@ -52,7 +52,7 @@ class TestErlangLanguageServerSymbols: assert "create_user" in containing_symbol["name"] assert containing_symbol["kind"] == SymbolKind.Method or containing_symbol["kind"] == SymbolKind.Function if "body" in containing_symbol: - assert "create_user" in containing_symbol["body"] + assert "create_user" in containing_symbol["body"].get_text() @pytest.mark.parametrize("language_server", [Language.ERLANG], indirect=True) def test_request_containing_symbol_module(self, language_server: SolidLanguageServer) -> None: diff --git a/test/solidlsp/python/test_symbol_retrieval.py b/test/solidlsp/python/test_symbol_retrieval.py index 6875bc89..ff98bd70 100644 --- a/test/solidlsp/python/test_symbol_retrieval.py +++ b/test/solidlsp/python/test_symbol_retrieval.py @@ -34,7 +34,7 @@ class TestLanguageServerSymbols: assert containing_symbol["name"] == "create_user" assert containing_symbol["kind"] == SymbolKind.Method if "body" in containing_symbol: - assert containing_symbol["body"].strip().startswith("def create_user(self") + assert containing_symbol["body"].get_text().strip().startswith("def create_user(self") @pytest.mark.parametrize("language_server", [Language.PYTHON], indirect=True) def test_references_to_variables(self, language_server: SolidLanguageServer) -> None: @@ -463,7 +463,6 @@ class TestLanguageServerSymbols: for ref in language_server.request_referencing_symbols(file_path, 4, 6, include_imports=False, include_file_symbols=True) ] assert {ref["kind"] for ref in references_to_typing} == {SymbolKind.File} - assert {ref["body"] for ref in references_to_typing} == {""} # now include bodies references_to_typing = [ diff --git a/test/solidlsp/ruby/test_ruby_symbol_retrieval.py b/test/solidlsp/ruby/test_ruby_symbol_retrieval.py index ae089830..55efcf02 100644 --- a/test/solidlsp/ruby/test_ruby_symbol_retrieval.py +++ b/test/solidlsp/ruby/test_ruby_symbol_retrieval.py @@ -53,7 +53,7 @@ class TestRubyLanguageServerSymbols: # Verify body content if available if "body" in containing_symbol: - body = containing_symbol["body"] + body = containing_symbol["body"].get_text() assert "def create_user" in body, "Method body should contain method definition" assert len(body.strip()) > 0, "Method body should not be empty" diff --git a/test/solidlsp/toml/test_toml_basic.py b/test/solidlsp/toml/test_toml_basic.py index db4ddcf6..b217aef8 100644 --- a/test/solidlsp/toml/test_toml_basic.py +++ b/test/solidlsp/toml/test_toml_basic.py @@ -137,7 +137,7 @@ class TestTomlLanguageServerBasics: # Check that body exists and contains expected content # Note: Taplo includes the section header in the body assert "body" in package_symbol, "'package' symbol should have body" - package_body = package_symbol["body"] + package_body = package_symbol["body"].get_text() assert 'name = "test_project"' in package_body, "Body should contain 'name' field" assert 'version = "0.1.0"' in package_body, "Body should contain 'version' field" assert 'edition = "2021"' in package_body, "Body should contain 'edition' field" @@ -146,7 +146,7 @@ class TestTomlLanguageServerBasics: deps_symbol = next((s for s in all_symbols if s.get("name") == "dependencies"), None) assert deps_symbol is not None, "Should find 'dependencies' symbol" assert "body" in deps_symbol, "'dependencies' symbol should have body" - deps_body = deps_symbol["body"] + deps_body = deps_symbol["body"].get_text() assert "serde" in deps_body, "Body should contain serde dependency" assert "tokio" in deps_body, "Body should contain tokio dependency" @@ -160,7 +160,7 @@ class TestTomlLanguageServerBasics: ) assert features_symbol is not None, "Should find top-level 'features' table symbol" assert "body" in features_symbol, "'features' symbol should have body" - features_body = features_symbol["body"] + features_body = features_symbol["body"].get_text() assert "default" in features_body, "Body should contain 'default' feature" @pytest.mark.parametrize("language_server", [Language.TOML], indirect=True) diff --git a/test/solidlsp/toml/test_toml_edge_cases.py b/test/solidlsp/toml/test_toml_edge_cases.py index 3f651682..faea142e 100644 --- a/test/solidlsp/toml/test_toml_edge_cases.py +++ b/test/solidlsp/toml/test_toml_edge_cases.py @@ -152,7 +152,7 @@ class TestTomlEdgeCases: assert endpoint_symbol is not None if "body" in endpoint_symbol: - body = endpoint_symbol["body"] + body = endpoint_symbol["body"].get_text() # Body should contain the inline table syntax assert "url" in body or "version" in body, f"Body should contain inline table contents, got: {body}" diff --git a/test/solidlsp/vue/test_vue_symbol_retrieval.py b/test/solidlsp/vue/test_vue_symbol_retrieval.py index b055fc68..7d33f88f 100644 --- a/test/solidlsp/vue/test_vue_symbol_retrieval.py +++ b/test/solidlsp/vue/test_vue_symbol_retrieval.py @@ -43,7 +43,7 @@ class TestVueSymbolRetrieval: # Verify the body is included if available if "body" in containing_symbol: - assert "handleDigit" in containing_symbol["body"], "Function body should contain function name" + assert "handleDigit" in containing_symbol["body"].get_text(), "Function body should contain function name" @pytest.mark.parametrize("language_server", [Language.VUE], indirect=True) def test_request_containing_symbol_computed_property(self, language_server: SolidLanguageServer) -> None: diff --git a/test/solidlsp/yaml_ls/test_yaml_basic.py b/test/solidlsp/yaml_ls/test_yaml_basic.py index b1eafc83..ded7f01a 100644 --- a/test/solidlsp/yaml_ls/test_yaml_basic.py +++ b/test/solidlsp/yaml_ls/test_yaml_basic.py @@ -124,7 +124,7 @@ class TestYAMLLanguageServerBasics: # Check that body exists and contains expected content assert "body" in app_symbol, "'app' symbol should have body" - app_body = app_symbol["body"] + app_body = app_symbol["body"].get_text() assert "app:" in app_body, "Body should start with 'app:'" assert "name: test-application" in app_body, "Body should contain 'name' field" assert "version: 1.0.0" in app_body, "Body should contain 'version' field" @@ -135,13 +135,13 @@ class TestYAMLLanguageServerBasics: name_symbols = [s for s in all_symbols if s.get("name") == "name" and "body" in s] assert len(name_symbols) > 0, "Should find 'name' symbols with bodies" # At least one should contain "test-application" - assert any("test-application" in s["body"] for s in name_symbols), "Should find name with test-application" + assert any("test-application" in s["body"].get_text() for s in name_symbols), "Should find name with test-application" # Find the database symbol and check its body database_symbol = next((s for s in all_symbols if s.get("name") == "database"), None) assert database_symbol is not None, "Should find 'database' symbol" assert "body" in database_symbol, "'database' symbol should have body" - db_body = database_symbol["body"] + db_body = database_symbol["body"].get_text() assert "database:" in db_body, "Body should start with 'database:'" assert "host: localhost" in db_body, "Body should contain host configuration" assert "port: 5432" in db_body, "Body should contain port configuration" From 0f3e23c4e8abd67baa7c59329b74e47232f8978f Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 21 Jan 2026 12:40:36 +0100 Subject: [PATCH 23/29] cli: "project create" now saves the created project to SerenaConfig Resolves #868 --- src/serena/cli.py | 14 ++++++++++++-- src/serena/config/serena_config.py | 18 +++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index 2a8c4358..580815fe 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -18,7 +18,7 @@ from tqdm import tqdm from serena.agent import SerenaAgent from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import LanguageBackend, ProjectConfig, SerenaConfig, SerenaPaths +from serena.config.serena_config import LanguageBackend, ProjectConfig, SerenaConfig, SerenaPaths, RegisteredProject from serena.constants import ( DEFAULT_CONTEXT, DEFAULT_MODES, @@ -521,7 +521,8 @@ class ProjectCommands(AutoRegisteringGroup): :raises FileExistsError: If project.yml already exists :raises ValueError: If an unsupported language is specified """ - yml_path = os.path.join(project_path, ProjectConfig.rel_path_to_project_yml()) + project_root = Path(project_path).resolve() + yml_path = ProjectConfig.path_to_project_yml(project_root) if os.path.exists(yml_path): raise FileExistsError(f"Project file {yml_path} already exists.") @@ -540,6 +541,15 @@ class ProjectCommands(AutoRegisteringGroup): yml_path = ProjectConfig.path_to_project_yml(project_path) languages_str = ", ".join([lang.value for lang in generated_conf.languages]) if generated_conf.languages else "N/A" click.echo(f"Generated project with languages {{{languages_str}}} at {yml_path}.") + + # add to SerenaConfig's list of registered projects + serena_config = SerenaConfig.from_config_file() + if serena_config.get_registered_project(str(project_root)) is None: + registered_project = RegisteredProject(str(project_root), generated_conf) + serena_config.projects.append(registered_project) + click.echo(f"Saving project registration to {serena_config}") + serena_config.save() + return generated_conf @staticmethod diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 7bfbb3e8..9523cf05 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -613,14 +613,14 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): def project_names(self) -> list[str]: return sorted(project.project_config.project_name for project in self.projects) - def get_project(self, project_root_or_name: str) -> Optional["Project"]: + def get_registered_project(self, project_root_or_name: str) -> Optional[RegisteredProject]: # look for project by name project_candidates = [] for project in self.projects: if project.project_config.project_name == project_root_or_name: project_candidates.append(project) if len(project_candidates) == 1: - return project_candidates[0].get_project_instance() + return project_candidates[0] elif len(project_candidates) > 1: raise ValueError( f"Multiple projects found with name '{project_root_or_name}'. Please activate it by location instead. " @@ -630,13 +630,21 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): if os.path.isdir(project_root_or_name): for project in self.projects: if project.matches_root_path(project_root_or_name): - return project.get_project_instance() + return project return None + def get_project(self, project_root_or_name: str) -> Optional["Project"]: + registered_project = self.get_registered_project(project_root_or_name) + if registered_project is None: + return None + else: + return registered_project.get_project_instance() + def add_project_from_path(self, project_root: Path | str) -> "Project": """ - Add a project to the Serena configuration from a given path. Will raise a FileExistsError if a - project already exists at the path. + Add a new project to the Serena configuration from a given path, auto-generating the project + with defaults if it does not exist. + Will raise a FileExistsError if a project already exists at the path. :param project_root: the path to the project to add :return: the project that was added From 1d0e38be853d9c68ecdc50b6d86cd01bdb7de3c3 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 22 Jan 2026 13:46:10 +0100 Subject: [PATCH 24/29] cli: Remove deprecated indexing command --- src/serena/cli.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index 580815fe..345c58c3 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -18,7 +18,7 @@ from tqdm import tqdm from serena.agent import SerenaAgent from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import LanguageBackend, ProjectConfig, SerenaConfig, SerenaPaths, RegisteredProject +from serena.config.serena_config import LanguageBackend, ProjectConfig, RegisteredProject, SerenaConfig, SerenaPaths from serena.constants import ( DEFAULT_CONTEXT, DEFAULT_MODES, @@ -614,17 +614,6 @@ class ProjectCommands(AutoRegisteringGroup): ProjectCommands._index_project(project, log_level, timeout=timeout) - @staticmethod - @click.command( - "index-deprecated", help="Deprecated alias for 'serena project index'.", context_settings={"max_content_width": _MAX_CONTENT_WIDTH} - ) - @click.argument("project", type=click.Path(exists=True), default=os.getcwd(), required=False) - @click.option("--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), default="WARNING") - @click.option("--timeout", type=float, default=10, help="Timeout for indexing a single file.") - def index_deprecated(project: str, log_level: str, timeout: float) -> None: - click.echo("Deprecated! Use `serena project index` instead.") - ProjectCommands._index_project(project, log_level, timeout=timeout) - @staticmethod def _index_project(project: str, log_level: str, timeout: float) -> None: lvl = logging.getLevelNamesMapping()[log_level.upper()] @@ -1034,7 +1023,6 @@ prompts = PromptCommands() # Expose toplevel commands for the same reason top_level = TopLevelCommands() start_mcp_server = top_level.start_mcp_server -index_project = project.index_deprecated # needed for the help script to work - register all subcommands to the top-level group for subgroup in (mode, context, project, config, tools, prompts): From 1ec5a731a3c373442d6e4015b35b441f9e8636fc Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 22 Jan 2026 14:29:28 +0100 Subject: [PATCH 25/29] cli: Allow "project index" with project name (not just path), refactoring --- src/serena/cli.py | 45 +++++++++++------------- src/serena/config/serena_config.py | 33 +++++++++++++++-- test/serena/test_cli_project_commands.py | 6 ++-- 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/src/serena/cli.py b/src/serena/cli.py index 345c58c3..fba9ccdc 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -510,16 +510,16 @@ class ProjectCommands(AutoRegisteringGroup): ) @staticmethod - def _create_project(project_path: str, name: str | None, language: tuple[str, ...]) -> ProjectConfig: + def _create_project(project_path: str, name: str | None, language: tuple[str, ...]) -> RegisteredProject: """ Helper method to create a project configuration file. :param project_path: Path to the project directory :param name: Optional project name (defaults to directory name if not specified) :param language: Tuple of language names - :return: The generated ProjectConfig instance :raises FileExistsError: If project.yml already exists :raises ValueError: If an unsupported language is specified + :return: the RegisteredProject instance """ project_root = Path(project_path).resolve() yml_path = ProjectConfig.path_to_project_yml(project_root) @@ -544,13 +544,12 @@ class ProjectCommands(AutoRegisteringGroup): # add to SerenaConfig's list of registered projects serena_config = SerenaConfig.from_config_file() - if serena_config.get_registered_project(str(project_root)) is None: + registered_project = serena_config.get_registered_project(str(project_root)) + if registered_project is None: registered_project = RegisteredProject(str(project_root), generated_conf) - serena_config.projects.append(registered_project) - click.echo(f"Saving project registration to {serena_config}") - serena_config.save() + serena_config.add_registered_project(registered_project) - return generated_conf + return registered_project @staticmethod @click.command("create", help="Create a new Serena project configuration.", context_settings={"max_content_width": _MAX_CONTENT_WIDTH}) @@ -569,10 +568,10 @@ class ProjectCommands(AutoRegisteringGroup): @click.option("--timeout", type=float, default=10, help="Timeout for indexing a single file (only used if --index is set).") def create(project_path: str, name: str | None, language: tuple[str, ...], index: bool, log_level: str, timeout: float) -> None: try: - ProjectCommands._create_project(project_path, name, language) + registered_project = ProjectCommands._create_project(project_path, name, language) if index: click.echo("Indexing project...") - ProjectCommands._index_project(project_path, log_level, timeout=timeout) + ProjectCommands._index_project(registered_project, log_level, timeout=timeout) except FileExistsError as e: raise click.ClickException(f"Project already exists: {e}\nUse 'serena project index' to index an existing project.") except ValueError as e: @@ -584,7 +583,7 @@ class ProjectCommands(AutoRegisteringGroup): help="Index a project by saving symbols to the LSP cache. Auto-creates project.yml if it doesn't exist.", context_settings={"max_content_width": _MAX_CONTENT_WIDTH}, ) - @click.argument("project", type=click.Path(exists=True), default=os.getcwd(), required=False) + @click.argument("project", type=PROJECT_TYPE, default=os.getcwd(), required=False) @click.option("--name", type=str, default=None, help="Project name (only used if auto-creating project.yml).") @click.option( "--language", @@ -600,32 +599,30 @@ class ProjectCommands(AutoRegisteringGroup): ) @click.option("--timeout", type=float, default=10, help="Timeout for indexing a single file.") def index(project: str, name: str | None, language: tuple[str, ...], log_level: str, timeout: float) -> None: - # Check if project.yml exists, if not auto-create it - yml_path = os.path.join(project, ProjectConfig.rel_path_to_project_yml()) - if not os.path.exists(yml_path): - click.echo(f"Project configuration not found at {yml_path}. Auto-creating...") + serena_config = SerenaConfig.from_config_file() + registered_project = serena_config.get_registered_project(project, autoregister=True) + if registered_project is None: + # Project not found; auto-create it + click.echo(f"No existing project found for '{project}'. Attempting auto-creation ...") try: - ProjectCommands._create_project(project, name, language) - except FileExistsError: - # Race condition - file was created between check and creation - pass - except ValueError as e: + registered_project = ProjectCommands._create_project(project, name, language) + except Exception as e: raise click.ClickException(str(e)) - ProjectCommands._index_project(project, log_level, timeout=timeout) + ProjectCommands._index_project(registered_project, log_level, timeout=timeout) @staticmethod - def _index_project(project: str, log_level: str, timeout: float) -> None: + def _index_project(registered_project: RegisteredProject, log_level: str, timeout: float) -> None: lvl = logging.getLevelNamesMapping()[log_level.upper()] logging.configure(level=lvl) serena_config = SerenaConfig.from_config_file() - proj = Project.load(os.path.abspath(project)) - click.echo(f"Indexing symbols in project {project} …") + proj = registered_project.get_project_instance() + click.echo(f"Indexing symbols in {proj} …") ls_mgr = proj.create_language_server_manager( log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings ) try: - log_file = os.path.join(project, ".serena", "logs", "indexing.txt") + log_file = os.path.join(proj.project_root, ".serena", "logs", "indexing.txt") files = proj.gather_source_files() diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 9523cf05..bde964b9 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -374,6 +374,14 @@ class RegisteredProject(ToStringMixin): project_instance=project_instance, ) + @classmethod + def from_project_root(cls, project_root: str | Path) -> "RegisteredProject": + project_config = ProjectConfig.load(project_root) + return RegisteredProject( + project_root=str(project_root), + project_config=project_config, + ) + def matches_root_path(self, path: str | Path) -> bool: """ Check if the given path matches the project root path. @@ -613,7 +621,12 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): def project_names(self) -> list[str]: return sorted(project.project_config.project_name for project in self.projects) - def get_registered_project(self, project_root_or_name: str) -> Optional[RegisteredProject]: + def get_registered_project(self, project_root_or_name: str, autoregister: bool = False) -> Optional[RegisteredProject]: + """ + :param project_root_or_name: path to the project root or the name of the project + :param autoregister: whether to register the project if it exists but is not registered yet + :return: the registered project, or None if not found + """ # look for project by name project_candidates = [] for project in self.projects: @@ -631,6 +644,14 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): for project in self.projects: if project.matches_root_path(project_root_or_name): return project + # no registered project found; auto-register if project configuration exists + if autoregister: + config_path = ProjectConfig.path_to_project_yml(project_root_or_name) + if os.path.isfile(config_path): + registered_project = RegisteredProject.from_project_root(project_root_or_name) + self.add_registered_project(registered_project) + return registered_project + # nothing found return None def get_project(self, project_root_or_name: str) -> Optional["Project"]: @@ -640,6 +661,13 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): else: return registered_project.get_project_instance() + def add_registered_project(self, registered_project: RegisteredProject) -> None: + """ + Adds a registered project, saving the configuration file. + """ + self.projects.append(registered_project) + self.save() + def add_project_from_path(self, project_root: Path | str) -> "Project": """ Add a new project to the Serena configuration from a given path, auto-generating the project @@ -666,8 +694,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): project_config = ProjectConfig.load(project_root, autogenerate=True) new_project = Project(project_root=str(project_root), project_config=project_config, is_newly_created=True) - self.projects.append(RegisteredProject.from_project_instance(new_project)) - self.save() + self.add_registered_project(RegisteredProject.from_project_instance(new_project)) return new_project diff --git a/test/serena/test_cli_project_commands.py b/test/serena/test_cli_project_commands.py index 893299e6..1052d403 100644 --- a/test/serena/test_cli_project_commands.py +++ b/test/serena/test_cli_project_commands.py @@ -236,20 +236,20 @@ class TestProjectCreateHelper: def test_create_project_helper_returns_config(self, temp_project_dir): """Test that _create_project returns a ProjectConfig with explicit language.""" - config = ProjectCommands._create_project(temp_project_dir, "test-project", ("python",)) + config = ProjectCommands._create_project(temp_project_dir, "test-project", ("python",)).project_config assert isinstance(config, ProjectConfig) assert config.project_name == "test-project" def test_create_project_helper_with_auto_detect(self, temp_project_dir_with_python_file): """Test _create_project with auto-detected language.""" - config = ProjectCommands._create_project(temp_project_dir_with_python_file, "my-project", ()) + config = ProjectCommands._create_project(temp_project_dir_with_python_file, "my-project", ()).project_config assert isinstance(config, ProjectConfig) assert config.project_name == "my-project" assert len(config.languages) >= 1 def test_create_project_helper_with_languages(self, temp_project_dir): """Test _create_project with language specification.""" - config = ProjectCommands._create_project(temp_project_dir, None, ("python", "typescript")) + config = ProjectCommands._create_project(temp_project_dir, None, ("python", "typescript")).project_config assert isinstance(config, ProjectConfig) assert len(config.languages) >= 1 From dabf14f1f6289e3a69f51a6cf2fb1679746b15cd Mon Sep 17 00:00:00 2001 From: zen010101 <60574100+zen010101@users.noreply.github.com> Date: Fri, 23 Jan 2026 16:49:07 +0800 Subject: [PATCH 26/29] docs: Add Pascal language server configuration documentation Document pasls settings including: - Language server installation (PATH vs auto-download) - Auto-update behavior (24h interval, SHA256 verification, atomic updates) - Configuration via environment variables (PP, FPCDIR, LAZARUSDIR, etc.) --- docs/02-usage/050_configuration.md | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docs/02-usage/050_configuration.md b/docs/02-usage/050_configuration.md index f164d567..6d0831e3 100644 --- a/docs/02-usage/050_configuration.md +++ b/docs/02-usage/050_configuration.md @@ -215,6 +215,45 @@ Notes: - `GOFLAGS` (from the environment you start Serena in) may also affect the Go build context. Prefer `buildFlags` for tags. - Build context changes are only picked up when `gopls` starts. After changing `gopls_settings` (or relevant env vars like `GOFLAGS`), restart the Serena process (or server) that hosts the Go language server, or use your client's "Restart language server" action if it causes `gopls` to restart. +#### Pascal (`pasls`) + +Serena uses [pasls](https://github.com/genericptr/pascal-language-server) (Pascal Language Server) for Pascal/Free Pascal support. + +**Language Server Installation:** + +1. If `pasls` is found in your system PATH, Serena uses it directly +2. Otherwise, Serena automatically downloads a prebuilt binary from GitHub releases + +Supported platforms for automatic download: Linux (x64, arm64), macOS (x64, arm64), Windows (x64). + +**Auto-Update:** + +Serena automatically checks for pasls updates every 24 hours. Updates include: +- SHA256 checksum verification before installation +- Atomic update with rollback on failure +- Windows file locking detection (defers update if pasls is in use) + +**Configuration via Environment Variables:** + +pasls reads compiler paths from environment variables. Set these before starting Serena: + +| Variable | Description | +| -------------- | -------------------------------------------------------- | +| `PP` | Path to FPC compiler executable | +| `FPCDIR` | Path to FPC source directory | +| `LAZARUSDIR` | Path to Lazarus directory (only needed for LCL projects) | +| `FPCTARGET` | Target OS (e.g., `linux`, `win64`, `darwin`) | +| `FPCTARGETCPU` | Target CPU (e.g., `x86_64`, `aarch64`, `i386`) | + +Example (bash): + +```bash +export PP=/usr/bin/fpc +export FPCDIR=/usr/share/fpcsrc +export LAZARUSDIR=/usr/share/lazarus +serena start-mcp-server --project /path/to/pascal-project +``` + ### Custom Prompts All of Serena's prompts can be fully customized. From 5502e5246a88b0e5a88cd955d58063b2a2616e02 Mon Sep 17 00:00:00 2001 From: zen010101 <60574100+zen010101@users.noreply.github.com> Date: Sat, 24 Jan 2026 05:45:15 +0800 Subject: [PATCH 27/29] chore: retrigger CI From ab3787c8e3058980f8d13460248137f260d77368 Mon Sep 17 00:00:00 2001 From: zen010101 <60574100+zen010101@users.noreply.github.com> Date: Sat, 24 Jan 2026 07:06:59 +0800 Subject: [PATCH 28/29] docs: Update Pascal docs to focus on ls_specific_settings Changed documentation to describe ls_specific_settings configuration instead of environment variables, as suggested by reviewer. --- docs/02-usage/050_configuration.md | 36 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/docs/02-usage/050_configuration.md b/docs/02-usage/050_configuration.md index 6d0831e3..716d541a 100644 --- a/docs/02-usage/050_configuration.md +++ b/docs/02-usage/050_configuration.md @@ -233,27 +233,33 @@ Serena automatically checks for pasls updates every 24 hours. Updates include: - Atomic update with rollback on failure - Windows file locking detection (defers update if pasls is in use) -**Configuration via Environment Variables:** +**Configuration:** -pasls reads compiler paths from environment variables. Set these before starting Serena: +Configure pasls via `ls_specific_settings.pascal` in `serena_config.yml`: -| Variable | Description | -| -------------- | -------------------------------------------------------- | -| `PP` | Path to FPC compiler executable | -| `FPCDIR` | Path to FPC source directory | -| `LAZARUSDIR` | Path to Lazarus directory (only needed for LCL projects) | -| `FPCTARGET` | Target OS (e.g., `linux`, `win64`, `darwin`) | -| `FPCTARGETCPU` | Target CPU (e.g., `x86_64`, `aarch64`, `i386`) | +| Setting | Description | +| ---------------- | --------------------------------------------------------------------------- | +| `pp` | Path to FPC compiler driver (must be `fpc` or `fpc.exe`, not `ppc386.exe`) | +| `fpcdir` | Path to FPC source directory | +| `lazarusdir` | Path to Lazarus directory (required for LCL projects) | +| `fpc_target` | Target OS override (e.g., `Win32`, `Win64`, `Linux`) | +| `fpc_target_cpu` | Target CPU override (e.g., `i386`, `x86_64`, `aarch64`) | -Example (bash): +Example configuration: -```bash -export PP=/usr/bin/fpc -export FPCDIR=/usr/share/fpcsrc -export LAZARUSDIR=/usr/share/lazarus -serena start-mcp-server --project /path/to/pascal-project +```yaml +ls_specific_settings: + pascal: + pp: "D:/laz32/fpc/bin/i386-win32/fpc.exe" + fpcdir: "D:/laz32/fpcsrc" + lazarusdir: "D:/laz32/lazarus" ``` +Notes: +- The `pp` setting is the most important for hover and navigation to work correctly. +- Use the FPC compiler driver (`fpc`/`fpc.exe`), not backend compilers like `ppc386.exe`. +- These settings are passed as environment variables to the pasls process. + ### Custom Prompts All of Serena's prompts can be fully customized. From 0fbc2b5f82ba52cc964c73694eb26cc7c5bf1f77 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Sat, 24 Jan 2026 02:11:41 +0100 Subject: [PATCH 29/29] Adjust name path matching logic to consider overload indices at all positions in the name path (now matching the implementation in the JetBrains plugin) Resolves #935 --- src/serena/symbol.py | 103 ++++++++++++++++++++----------------- test/serena/test_symbol.py | 78 +++++++++++++++++++++++----- 2 files changed, 121 insertions(+), 60 deletions(-) diff --git a/src/serena/symbol.py b/src/serena/symbol.py index 3cec48da..e91856c1 100644 --- a/src/serena/symbol.py +++ b/src/serena/symbol.py @@ -115,6 +115,18 @@ class Symbol(ToStringMixin, ABC): """ +class NamePathComponent: + def __init__(self, name: str, overload_idx: int | None = None) -> None: + self.name = name + self.overload_idx = overload_idx + + def __repr__(self) -> str: + if self.overload_idx is not None: + return f"{self.name}[{self.overload_idx}]" + else: + return self.name + + class NamePathMatcher(ToStringMixin): """ Matches name paths of symbols against search patterns. @@ -131,6 +143,29 @@ class NamePathMatcher(ToStringMixin): Append an index `[i]` to match a specific overload only, e.g. "MyClass/my_method[1]". """ + class PatternComponent(NamePathComponent): + @classmethod + def from_string(cls, component_str: str) -> Self: + overload_idx = None + if component_str.endswith("]") and "[" in component_str: + bracket_idx = component_str.rfind("[") + index_part = component_str[bracket_idx + 1 : -1] + if index_part.isdigit(): + component_str = component_str[:bracket_idx] + overload_idx = int(index_part) + return cls(name=component_str, overload_idx=overload_idx) + + def matches(self, name_path_component: NamePathComponent, substring_matching: bool) -> bool: + if substring_matching: + if self.name not in name_path_component.name: + return False + else: + if self.name != name_path_component.name: + return False + if self.overload_idx is not None and self.overload_idx != name_path_component.overload_idx: + return False + return True + def __init__(self, name_path_pattern: str, substring_matching: bool) -> None: """ :param name_path_pattern: the name path expression to match against @@ -140,51 +175,32 @@ class NamePathMatcher(ToStringMixin): self._expr = name_path_pattern self._substring_matching = substring_matching self._is_absolute_pattern = name_path_pattern.startswith(NAME_PATH_SEP) - self._pattern_parts = name_path_pattern.lstrip(NAME_PATH_SEP).rstrip(NAME_PATH_SEP).split(NAME_PATH_SEP) - - # extract overload index "[idx]" if present at end of last part - self._overload_idx: int | None = None - last_part = self._pattern_parts[-1] - if last_part.endswith("]") and "[" in last_part: - bracket_idx = last_part.rfind("[") - index_part = last_part[bracket_idx + 1 : -1] - if index_part.isdigit(): - self._pattern_parts[-1] = last_part[:bracket_idx] - self._overload_idx = int(index_part) + self._components = [ + self.PatternComponent.from_string(x) for x in name_path_pattern.lstrip(NAME_PATH_SEP).rstrip(NAME_PATH_SEP).split(NAME_PATH_SEP) + ] def _tostring_includes(self) -> list[str]: return ["_expr"] def matches_ls_symbol(self, symbol: "LanguageServerSymbol") -> bool: - return self.matches_components(symbol.get_name_path_parts(), symbol.overload_idx) + return self.matches_reversed_components(symbol.iter_name_path_components_reversed()) - def matches_components(self, symbol_name_path_parts: list[str], overload_idx: int | None) -> bool: - # filtering based on ancestors - if len(self._pattern_parts) > len(symbol_name_path_parts): - # can't possibly match if pattern has more parts than symbol - return False - if self._is_absolute_pattern and len(self._pattern_parts) != len(symbol_name_path_parts): - # for absolute patterns, the number of parts must match exactly - return False - if symbol_name_path_parts[-len(self._pattern_parts) : -1] != self._pattern_parts[:-1]: - # ancestors must match - return False - - # matching the last part of the symbol name - name_to_match = self._pattern_parts[-1] - symbol_name = symbol_name_path_parts[-1] - if self._substring_matching: - if name_to_match not in symbol_name: + def matches_reversed_components(self, components_reversed: Iterator[NamePathComponent]) -> bool: + for i, pattern_component in enumerate(reversed(self._components)): + try: + symbol_component = next(components_reversed) + except StopIteration: return False - else: - if name_to_match != symbol_name: + use_substring_matching = self._substring_matching and (i == 0) + if not pattern_component.matches(symbol_component, use_substring_matching): return False - - # check for matching overload index - if self._overload_idx is not None: - if overload_idx != self._overload_idx: + if self._is_absolute_pattern: + # ensure that there are no more components in the symbol + try: + next(components_reversed) return False - + except StopIteration: + pass return True @@ -311,18 +327,13 @@ class LanguageServerSymbol(Symbol, ToStringMixin): Get the name path of the symbol, e.g. "class/method/inner_function" or "class/method[1]" (overloaded method with identifying index). """ - name_path = NAME_PATH_SEP.join(self.get_name_path_parts()) - if "overload_idx" in self.symbol_root: - name_path += f"[{self.symbol_root['overload_idx']}]" + name_path = NAME_PATH_SEP.join(reversed([str(x) for x in self.iter_name_path_components_reversed()])) return name_path - def get_name_path_parts(self) -> list[str]: - """ - Get the parts of the name path of the symbol (e.g. ["class", "method", "inner_function"]). - """ - ancestors_within_file = list(self.iter_ancestors(up_to_symbol_kind=SymbolKind.File)) - ancestors_within_file.reverse() - return [a.name for a in ancestors_within_file] + [self.name] + def iter_name_path_components_reversed(self) -> Iterator[NamePathComponent]: + yield NamePathComponent(self.name, self.overload_idx) + for ancestor in self.iter_ancestors(up_to_symbol_kind=SymbolKind.File): + yield NamePathComponent(ancestor.name, ancestor.overload_idx) def iter_children(self) -> Iterator[Self]: for c in self.symbol_root["children"]: diff --git a/test/serena/test_symbol.py b/test/serena/test_symbol.py index 77558ecd..2e5009de 100644 --- a/test/serena/test_symbol.py +++ b/test/serena/test_symbol.py @@ -1,6 +1,6 @@ import pytest -from serena.symbol import LanguageServerSymbolRetriever, NamePathMatcher +from serena.symbol import LanguageServerSymbolRetriever, NamePathComponent, NamePathMatcher from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language @@ -9,17 +9,17 @@ class TestSymbolNameMatching: def _create_assertion_error_message( self, name_path_pattern: str, - symbol_name_path_parts: list[str], + name_path_components: list[NamePathComponent], is_substring_match: bool, expected_result: bool, actual_result: bool, ) -> str: """Helper to create a detailed error message for assertions.""" - qnp_repr = "/".join(symbol_name_path_parts) + qnp_repr = "/".join(map(str, name_path_components)) return ( f"Pattern '{name_path_pattern}' (substring: {is_substring_match}) vs " - f"Qualname parts {symbol_name_path_parts} (as '{qnp_repr}'). " + f"Name path components {name_path_components} (as '{qnp_repr}'). " f"Expected: {expected_result}, Got: {actual_result}" ) @@ -79,7 +79,8 @@ class TestSymbolNameMatching: ) def test_match_simple_name(self, name_path_pattern, symbol_name_path_parts, is_substring_match, expected): """Tests matching for simple names (no '/' in pattern).""" - result = NamePathMatcher(name_path_pattern, is_substring_match).matches_components(symbol_name_path_parts, None) + symbol_name_path_components = [NamePathComponent(part) for part in symbol_name_path_parts] + result = NamePathMatcher(name_path_pattern, is_substring_match).matches_reversed_components(reversed(symbol_name_path_components)) error_msg = self._create_assertion_error_message(name_path_pattern, symbol_name_path_parts, is_substring_match, expected, result) assert result == expected, error_msg @@ -146,6 +147,13 @@ class TestSymbolNameMatching: ), pytest.param("/bar/foo", ["bar"], False, False, id="A: '/bar/foo' does not match ['bar'] (pattern longer)"), pytest.param("/bar/foo", ["baz", "foo"], False, False, id="A: '/bar/foo' does not match ['baz', 'foo'] (first part mismatch)"), + pytest.param( + "/bar/foo", + ["baz", "bar", "foo"], + False, + False, + id="A: '/bar/foo' does not match ['baz', 'bar', 'foo'] (only suffix match for abs pattern)", + ), pytest.param("/bar/foo", ["bar", "baz"], False, False, id="A: '/bar/foo' does not match ['bar', 'baz'] (last part mismatch)"), # Substring matches (is_substring_match=True) pytest.param("/bar/foo", ["bar", "foobar"], True, True, id="A: '/bar/foo' matches ['bar', 'foobar'] as substring"), @@ -159,24 +167,66 @@ class TestSymbolNameMatching: ) def test_match_name_path_pattern_path_len_2(self, name_path_pattern, symbol_name_path_parts, is_substring_match, expected): """Tests matching for qualified names (e.g. 'module/class/func').""" - result = NamePathMatcher(name_path_pattern, is_substring_match).matches_components(symbol_name_path_parts, None) + symbol_name_path_components = [NamePathComponent(part) for part in symbol_name_path_parts] + result = NamePathMatcher(name_path_pattern, is_substring_match).matches_reversed_components(reversed(symbol_name_path_components)) error_msg = self._create_assertion_error_message(name_path_pattern, symbol_name_path_parts, is_substring_match, expected, result) assert result == expected, error_msg @pytest.mark.parametrize( - "name_path_pattern, symbol_name_path_parts, symbol_overload_idx, expected", + "name_path_pattern, symbol_name_path_components, expected", [ - pytest.param("bar/foo", ["bar", "foo"], 0, True, id="R: 'bar/foo' matches ['bar', 'foo'] with overload_index=0"), - pytest.param("bar/foo", ["bar", "foo"], 1, True, id="R: 'bar/foo' matches ['bar', 'foo'] with overload_index=1"), - pytest.param("bar/foo[0]", ["bar", "foo"], 0, True, id="R: 'bar/foo[0]' matches ['bar', 'foo'] with overload_index=0"), - pytest.param("bar/foo[1]", ["bar", "foo"], 0, False, id="R: 'bar/foo[1]' does not match ['bar', 'foo'] with overload_index=0"), + pytest.param( + "bar/foo", + [NamePathComponent("bar"), NamePathComponent("foo", 0)], + True, + id="R: 'bar/foo' matches ['bar', 'foo'] with overload_index=0", + ), + pytest.param( + "bar/foo", + [NamePathComponent("bar"), NamePathComponent("foo", 1)], + True, + id="R: 'bar/foo' matches ['bar', 'foo'] with overload_index=1", + ), + pytest.param( + "bar/foo[0]", + [NamePathComponent("bar"), NamePathComponent("foo", 0)], + True, + id="R: 'bar/foo[0]' matches ['bar', 'foo'] with overload_index=0", + ), + pytest.param( + "bar/foo[1]", + [NamePathComponent("bar"), NamePathComponent("foo", 0)], + False, + id="R: 'bar/foo[1]' does not match ['bar', 'foo'] with overload_index=0", + ), + pytest.param( + "bar/foo", [NamePathComponent("bar", 0), NamePathComponent("foo")], True, id="R: 'bar/foo' matches ['bar[0]', 'foo']" + ), + pytest.param( + "bar/foo", [NamePathComponent("bar", 0), NamePathComponent("foo", 1)], True, id="R: 'bar/foo' matches ['bar[0]', 'foo[1]']" + ), + pytest.param( + "bar[0]/foo", [NamePathComponent("bar", 0), NamePathComponent("foo")], True, id="R: 'bar[0]/foo' matches ['bar[0]', 'foo']" + ), + pytest.param( + "bar[0]/foo[1]", + [NamePathComponent("bar", 0), NamePathComponent("foo", 1)], + True, + id="R: 'bar[0]/foo[1]' matches ['bar[0]', 'foo[1]']", + ), + pytest.param( + "bar[0]/foo[1]", + [NamePathComponent("bar", 1), NamePathComponent("foo", 0)], + False, + id="R: 'bar[0]/foo[1]' does not match ['bar[1]', 'foo[0]']", + ), ], ) - def test_match_name_path_pattern_with_overload_idx(self, name_path_pattern, symbol_name_path_parts, symbol_overload_idx, expected): + def test_match_name_path_pattern_with_overload_idx(self, name_path_pattern, symbol_name_path_components, expected): """Tests matching for qualified names (e.g. 'module/class/func').""" matcher = NamePathMatcher(name_path_pattern, False) - result = matcher.matches_components(symbol_name_path_parts, symbol_overload_idx) - error_msg = self._create_assertion_error_message(name_path_pattern, symbol_name_path_parts, False, expected, result) + result = matcher.matches_reversed_components(reversed(symbol_name_path_components)) + error_msg = self._create_assertion_error_message(name_path_pattern, symbol_name_path_components, False, expected, result) assert result == expected, error_msg