From 01434796212bb1856e7e418a3d854710bf544e52 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Sun, 28 Jun 2026 16:27:07 +0200 Subject: [PATCH] Tests: centralized test enabling logic, better parallelization in CI Removing some heavy toolchains from windows/macos run (they will be tested only in the job based off the maximal docker image) --- .github/workflows/pytest.yml | 254 ++++++++++++------ test/conftest.py | 194 ++++++++++--- test/serena/test_serena_agent.py | 9 +- test/serena/test_symbol_editing.py | 5 +- test/solidlsp/ansible/test_ansible_basic.py | 4 +- test/solidlsp/bsl/test_bsl_basic.py | 3 +- test/solidlsp/bsl/test_bsl_diagnostics.py | 5 +- test/solidlsp/crystal/test_crystal_basic.py | 4 +- .../crystal/test_crystal_diagnostics.py | 5 +- test/solidlsp/elixir/test_elixir_basic.py | 5 +- .../elixir/test_elixir_ignored_dirs.py | 6 +- .../elixir/test_elixir_integration.py | 5 +- .../elixir/test_elixir_symbol_retrieval.py | 5 +- test/solidlsp/elm/test_elm_basic.py | 5 +- test/solidlsp/erlang/test_erlang_basic.py | 5 +- .../erlang/test_erlang_ignored_dirs.py | 6 +- .../erlang/test_erlang_symbol_retrieval.py | 5 +- test/solidlsp/fsharp/test_fsharp_basic.py | 17 +- test/solidlsp/haskell/test_haskell_basic.py | 6 +- test/solidlsp/julia/test_julia_basic.py | 5 +- test/solidlsp/kotlin/test_kotlin_basic.py | 5 +- test/solidlsp/lean4/test_lean4_basic.py | 3 + test/solidlsp/lean4/test_lean4_diagnostics.py | 3 + test/solidlsp/matlab/test_matlab_basic.py | 18 +- test/solidlsp/nix/test_nix_basic.py | 4 +- test/solidlsp/ocaml/test_cross_file_refs.py | 2 + test/solidlsp/ocaml/test_ocaml_basic.py | 4 +- test/solidlsp/ocaml/test_ocaml_diagnostics.py | 5 +- test/solidlsp/perl/test_perl_basic.py | 5 +- test/solidlsp/r/test_r_basic.py | 4 +- test/solidlsp/rego/test_rego_basic.py | 5 +- test/solidlsp/swift/test_swift_basic.py | 9 +- .../systemverilog/test_systemverilog_basic.py | 7 +- .../test_systemverilog_diagnostics.py | 5 + .../terraform/test_terraform_basic.py | 6 +- 35 files changed, 408 insertions(+), 230 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 6b822f51..3aa59854 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -15,13 +15,37 @@ concurrency: jobs: cpu: - name: Tests on ${{ matrix.os }} + name: ${{ matrix.batch }} (${{ matrix.os }}) runs-on: ${{ matrix.os }} + # Marker groups are defined once here and combined in the "Compute marker expression" step. + # The catch-all batch is derived as the negation of all named groups, so adding a language to + # any group automatically removes it from catch-all -- the partition can never desync, and no + # test can fall through to "no batch". To add/rebalance: edit a MARKERS_* var, and (for a new + # group) add a case branch below + the batch name to the matrix list. + env: + MARKERS_JVM: "java or csharp or fsharp or kotlin or scala or groovy or clojure or bsl" + MARKERS_NATIVE: "cpp or rust or go or zig or pascal" + MARKERS_SWIFT: "swift" # split out: swiftly toolchain install is the slowest setup, isolate + cache it + MARKERS_WEB: "typescript or svelte or vue or angular or html or scss or json or solidity or elm" + MARKERS_PYTHON: "python" + MARKERS_SCRIPTING: "ruby or php or perl or lua or luau or bash or powershell or elixir or erlang or dart or haxe" + MARKERS_OTHER_LANGS: "terraform or rego or ansible or yaml or toml or markdown or latex or nix or crystal or cue or fortran or ada or matlab or systemverilog or hlsl or msl or al or ocaml or haskell or lean4 or julia or r" strategy: fail-fast: false matrix: os: [windows-latest, macos-latest] python-version: ["3.11"] + # Test execution is split into parallel batches by pytest marker. Each batch installs ONLY + # the toolchains its languages need -- steps below are gated with `if: matrix.batch == ...` + # -- so e.g. Swift/Lean are set up only in the batch that runs their tests, not in all 7. + # Node/Python/uv stay in every batch (cheap, and needed for the venv + auto-installed LSes). + # "catch-all" carries the unmarked core-framework tests in test/serena and needs no language + # toolchains. Rebalance once per-batch timings show up. + batch: [jvm, native, swift, web, python, scripting, other-langs, catch-all] + # Swift only runs on macOS (disabled on Windows via the central guard), so skip windows x swift. + exclude: + - os: windows-latest + batch: swift steps: - uses: actions/checkout@v4 - name: Free disk space @@ -41,6 +65,7 @@ jobs: with: python-version: "${{ matrix.python-version }}" - uses: actions/setup-go@v5 + if: matrix.batch == 'native' with: go-version: ">=1.17.0" - name: Set up Node.js @@ -54,8 +79,10 @@ jobs: mkdir -p $HOME/.cache/go-build mkdir -p $HOME/go/bin - name: Install uv - shell: bash - run: curl -LsSf https://astral.sh/uv/install.sh | sh + # Use the official action (not `curl | sh`): it adds uv to GITHUB_PATH (the real OS PATH), + # so the Python language server's subprocess `shutil.which("uv"/"uvx")` finds it on Windows. + # The bash installer only puts uv on the Git-Bash MSYS PATH, invisible to Windows processes. + uses: astral-sh/setup-uv@v5 - name: Cache uv virtualenv id: cache-uv uses: actions/cache@v3 @@ -75,12 +102,15 @@ jobs: shell: bash run: uv pip list - name: Check formatting + # Lint is batch-independent; run it only once per OS instead of in every batch. + if: matrix.batch == 'catch-all' shell: bash run: uv run poe lint # Add Go bin directory to PATH for this workflow # GITHUB_PATH is a special file that GitHub Actions uses to modify PATH # Writing to this file adds the directory to the PATH for subsequent steps - name: Cache Go binaries + if: matrix.batch == 'native' id: cache-go-binaries uses: actions/cache@v3 with: @@ -89,7 +119,7 @@ jobs: ~/.cache/go-build key: go-binaries-${{ runner.os }}-gopls-latest - name: Install gopls - if: steps.cache-go-binaries.outputs.cache-hit != 'true' + if: matrix.batch == 'native' && steps.cache-go-binaries.outputs.cache-hit != 'true' shell: bash run: go install golang.org/x/tools/gopls@latest # Erlang currently not tested in CI, random hangings on macos, always hangs on ubuntu @@ -107,10 +137,12 @@ jobs: # # Ensure erlang_ls is in PATH # echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Install clojure tools + if: matrix.batch == 'jvm' uses: DeLaGuardo/setup-clojure@13.4 with: cli: latest - name: Install ccls (C/C++ Language Server) + if: matrix.batch == 'native' shell: bash run: | if [[ "${{ runner.os }}" == "Linux" ]]; then @@ -129,18 +161,22 @@ jobs: exit 1 fi - name: Setup Java (for JVM based languages) + if: matrix.batch == 'jvm' uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '21' - name: Setup .NET SDK (for F# and C# languages) + if: matrix.batch == 'jvm' uses: actions/setup-dotnet@v4 with: dotnet-version: '10.0.x' - name: List .NET runtimes + if: matrix.batch == 'jvm' shell: bash run: dotnet --list-runtimes - name: Install Terraform + if: matrix.batch == 'other-langs' uses: hashicorp/setup-terraform@v3 with: terraform_version: "1.5.0" @@ -151,25 +187,46 @@ jobs: # Installation of swift with the action screws with installation of ruby on macOS for some reason # We can try again when version 3 of the action is released, where they will also use swiftly # Until then, we use custom code to install swift. Sourcekit-lsp is installed automatically with swift + - name: Cache Swift toolchain (swiftly) + if: runner.os == 'macOS' && matrix.batch == 'swift' + id: cache-swiftly + uses: actions/cache@v3 + with: + # macOS keeps the toolchain (incl. sourcekit-lsp) under ~/Library/Developer/Toolchains, + # NOT under ~/.swiftly -- both must be cached or the toolchain is lost on restore. + path: | + ~/.swiftly + ~/Library/Developer/Toolchains + key: swiftly-${{ runner.os }}-6.1.2-v2 - name: Install Swift with swiftly (macOS) - if: runner.os == 'macOS' + if: runner.os == 'macOS' && matrix.batch == 'swift' run: | - echo "=== Installing swiftly on macOS ===" - curl -O https://download.swift.org/swiftly/darwin/swiftly.pkg && \ - installer -pkg swiftly.pkg -target CurrentUserHomeDirectory && \ - ~/.swiftly/bin/swiftly init --quiet-shell-followup && \ - . "${SWIFTLY_HOME_DIR:-$HOME/.swiftly}/env.sh" && \ + set -e + # Install swiftly itself only if its binary isn't already present (e.g. restored from cache). + if [ ! -x "$HOME/.swiftly/bin/swiftly" ]; then + echo "=== Installing swiftly ===" + curl -O https://download.swift.org/swiftly/darwin/swiftly.pkg + installer -pkg swiftly.pkg -target CurrentUserHomeDirectory + "$HOME/.swiftly/bin/swiftly" init --quiet-shell-followup + fi + . "${SWIFTLY_HOME_DIR:-$HOME/.swiftly}/env.sh" hash -r - swiftly install --use 6.1.2 - swiftly use 6.1.2 - echo "~/.swiftly/bin" >> $GITHUB_PATH - echo "Swiftly installed successfully" - # Verify sourcekit-lsp is working before proceeding - echo "=== Verifying sourcekit-lsp installation ===" - which sourcekit-lsp || echo "Warning: sourcekit-lsp not found in PATH" - sourcekit-lsp --help || echo "Warning: sourcekit-lsp not responding" + # Always run install: it downloads on a cache miss and is a fast no-op when the toolchain is + # genuinely present from cache. No `--use` -- that triggers the interactive .swift-version prompt. + swiftly install 6.1.2 + # Locate sourcekit-lsp directly (macOS toolchains live under ~/Library/Developer/Toolchains) + # and put its directory on PATH -- avoids `swiftly use`, which prompts/hangs in CI. + SK="$(find "$HOME/Library/Developer/Toolchains" "$HOME/.swiftly" -name sourcekit-lsp 2>/dev/null | head -1)" + if [ -z "$SK" ]; then + echo "ERROR: sourcekit-lsp not found after Swift install. Searched:" + ls -la "$HOME/Library/Developer/Toolchains" 2>/dev/null || echo " (no ~/Library/Developer/Toolchains)" + ls -la "$HOME/.swiftly" 2>/dev/null || echo " (no ~/.swiftly)" + exit 1 + fi + echo "sourcekit-lsp: $SK" + dirname "$SK" >> "$GITHUB_PATH" - name: Install Swift with swiftly (Ubuntu) - if: runner.os == 'Linux' + if: runner.os == 'Linux' && matrix.batch == 'swift' run: | echo "=== Installing swiftly on Ubuntu ===" # Install dependencies BEFORE Swift to avoid exit code 1 @@ -190,38 +247,48 @@ jobs: which sourcekit-lsp || echo "Warning: sourcekit-lsp not found in PATH" sourcekit-lsp --help || echo "Warning: sourcekit-lsp not responding" - name: Install Ruby + if: matrix.batch == 'scripting' uses: ruby/setup-ruby@v1 with: ruby-version: '3.4' - name: Install Ruby language server + if: matrix.batch == 'scripting' shell: bash run: gem install ruby-lsp - name: Install Zig + if: matrix.batch == 'native' uses: mlugg/setup-zig@v2 with: version: 0.14.1 + - name: Cache ZLS (Zig Language Server) + if: matrix.batch == 'native' + id: cache-zls + uses: actions/cache@v3 + with: + path: ~/.zls + key: zls-${{ runner.os }}-0.14.0 - name: Install ZLS (Zig Language Server) + if: matrix.batch == 'native' shell: bash run: | - if [[ "${{ runner.os }}" == "Linux" ]]; then - wget https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-linux.tar.xz - tar -xf zls-x86_64-linux.tar.xz - sudo mv zls /usr/local/bin/ - rm zls-x86_64-linux.tar.xz - elif [[ "${{ runner.os }}" == "macOS" ]]; then - wget https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-macos.tar.xz - tar -xf zls-x86_64-macos.tar.xz - sudo mv zls /usr/local/bin/ - rm zls-x86_64-macos.tar.xz - elif [[ "${{ runner.os }}" == "Windows" ]]; then - curl -L -o zls.zip https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-windows.zip - unzip -o zls.zip - mkdir -p "$HOME/bin" - mv zls.exe "$HOME/bin/" - echo "$HOME/bin" >> $GITHUB_PATH - rm zls.zip + ZLS_DIR="$HOME/.zls" + if [[ "${{ steps.cache-zls.outputs.cache-hit }}" != "true" ]]; then + mkdir -p "$ZLS_DIR" + if [[ "${{ runner.os }}" == "Linux" ]]; then + wget -O /tmp/zls.tar.xz https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-linux.tar.xz + tar -xf /tmp/zls.tar.xz -C "$ZLS_DIR" && rm /tmp/zls.tar.xz + elif [[ "${{ runner.os }}" == "macOS" ]]; then + wget -O /tmp/zls.tar.xz https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-macos.tar.xz + tar -xf /tmp/zls.tar.xz -C "$ZLS_DIR" && rm /tmp/zls.tar.xz + elif [[ "${{ runner.os }}" == "Windows" ]]; then + curl -L -o zls.zip https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-windows.zip + unzip -o zls.zip -d "$ZLS_DIR" && rm zls.zip + fi + chmod +x "$ZLS_DIR/zls" 2>/dev/null || true fi + echo "$ZLS_DIR" >> "$GITHUB_PATH" - name: Install verible-verilog-ls (SystemVerilog Language Server) + if: matrix.batch == 'other-langs' shell: bash run: | VERIBLE_VERSION="v0.0-4051-g9fdb4057" @@ -252,6 +319,7 @@ jobs: echo "WARNING: verible-verilog-ls not found in PATH" fi - name: Install Lua Language Server + if: matrix.batch == 'scripting' shell: bash run: | LUA_LS_VERSION="3.15.0" @@ -297,18 +365,22 @@ jobs: rm lua-ls.zip fi - name: Install ansible-core and ansible-lint (for Ansible language server tests) + if: matrix.batch == 'other-langs' shell: bash run: uv run pip install ansible-core ansible-lint - name: Install Haxe + if: matrix.batch == 'scripting' # The Haxe LS binary (server.js) is auto-downloaded and runs on Node.js, # but it delegates to the Haxe compiler for code analysis at runtime. uses: krdlab/setup-haxe@v2 with: haxe-version: 4.3.7 - name: Install Elm + if: matrix.batch == 'web' shell: bash run: npm install -g elm@0.19.1-6 - name: Install Regal (Rego Language Server) + if: matrix.batch == 'other-langs' shell: bash run: | REGAL_VERSION="0.39.0" @@ -335,35 +407,39 @@ jobs: mv regal.exe "$HOME/bin/" echo "$HOME/bin" >> $GITHUB_PATH fi + - name: Cache Free Pascal Compiler + if: matrix.batch == 'native' + id: cache-fpc + uses: actions/cache@v3 + with: + path: | + ~/fpc + ~/fpcsrc + key: fpc-${{ runner.os }}-3.2.2 - name: Install Free Pascal Compiler + if: matrix.batch == 'native' shell: bash run: | + # Only the downloads are gated on the cache; the PP/FPCDIR/PATH detection always runs + # (it works the same on freshly-downloaded or cache-restored ~/fpc + ~/fpcsrc). + FPC_CACHE_HIT="${{ steps.cache-fpc.outputs.cache-hit }}" + FPC_VERSION="3.2.2" if [[ "${{ runner.os }}" == "Linux" ]]; then sudo apt-get update sudo apt-get install -y fpc fpc-source - # Set environment variables for pasls echo "PP=/usr/bin/fpc" >> $GITHUB_ENV - # Find FPC source directory (version may vary) - remove trailing slash FPCDIR=$(ls -d /usr/share/fpcsrc/*/ 2>/dev/null | head -1 | sed 's:/$::') - if [[ -z "$FPCDIR" ]]; then - FPCDIR="/usr/share/fpcsrc" - fi + [[ -z "$FPCDIR" ]] && FPCDIR="/usr/share/fpcsrc" echo "FPCDIR=$FPCDIR" >> $GITHUB_ENV - echo "=== FPC source directory structure ===" - ls -la "$FPCDIR" || echo "FPCDIR not found" - ls -la "$FPCDIR/rtl" 2>/dev/null || echo "rtl subdirectory not found" elif [[ "${{ runner.os }}" == "macOS" ]]; then - brew install fpc - # Download FPC source from SourceForge (fpc-src-laz cask is incompatible with ARM64) - FPC_VERSION="3.2.2" - curl -L -o fpc-source.tar.gz "https://sourceforge.net/projects/freepascal/files/Source/${FPC_VERSION}/fpc-${FPC_VERSION}.source.tar.gz/download" - mkdir -p "$HOME/fpcsrc" - tar -xzf fpc-source.tar.gz -C "$HOME/fpcsrc" - rm fpc-source.tar.gz - # Check extracted directory structure (might be nested) - echo "=== Extracted FPC source structure ===" - ls -la "$HOME/fpcsrc" - # Find the actual FPC source root (contains rtl, packages, etc.) + command -v fpc >/dev/null || brew install fpc + if [[ "$FPC_CACHE_HIT" != "true" ]]; then + # FPC source from SourceForge (fpc-src-laz cask is incompatible with ARM64) + curl -L -o fpc-source.tar.gz "https://sourceforge.net/projects/freepascal/files/Source/${FPC_VERSION}/fpc-${FPC_VERSION}.source.tar.gz/download" + mkdir -p "$HOME/fpcsrc" + tar -xzf fpc-source.tar.gz -C "$HOME/fpcsrc" + rm fpc-source.tar.gz + fi if [[ -d "$HOME/fpcsrc/fpc-${FPC_VERSION}/rtl" ]]; then FPCDIR="$HOME/fpcsrc/fpc-${FPC_VERSION}" elif [[ -d "$HOME/fpcsrc/fpc-${FPC_VERSION}/fpc-${FPC_VERSION}/rtl" ]]; then @@ -373,34 +449,26 @@ jobs: fi echo "PP=$(which fpc)" >> $GITHUB_ENV echo "FPCDIR=$FPCDIR" >> $GITHUB_ENV - echo "=== FPC source directory ===" - ls -la "$FPCDIR" || echo "FPCDIR not found" elif [[ "${{ runner.os }}" == "Windows" ]]; then - FPC_VERSION="3.2.2" - # Download freepascal-ootb (includes FPC compiler) - curl -L -o fpc-ootb.zip https://github.com/fredvs/freepascal-ootb/releases/download/${FPC_VERSION}/fpc-ootb-322-x86_64-win64.zip - mkdir -p "$HOME/fpc" - unzip -q fpc-ootb.zip -d "$HOME/fpc" - rm fpc-ootb.zip - # Download FPC source from SourceForge (fpc-ootb only has compiled units, not source) - curl -L -o fpc-source.zip "https://sourceforge.net/projects/freepascal/files/Source/${FPC_VERSION}/fpc-${FPC_VERSION}.source.zip/download" - mkdir -p "$HOME/fpcsrc" - unzip -q fpc-source.zip -d "$HOME/fpcsrc" - rm fpc-source.zip - # Find fpc executable (fpc-ootb uses fpc-ootb.exe as the compiler) - echo "=== FPC directory structure ===" - find "$HOME/fpc" -name "*.exe" -type f 2>/dev/null | head -10 + if [[ "$FPC_CACHE_HIT" != "true" ]]; then + # freepascal-ootb (compiler) + source (ootb ships compiled units only, not source) + curl -L -o fpc-ootb.zip https://github.com/fredvs/freepascal-ootb/releases/download/${FPC_VERSION}/fpc-ootb-322-x86_64-win64.zip + mkdir -p "$HOME/fpc" + unzip -q fpc-ootb.zip -d "$HOME/fpc" + rm fpc-ootb.zip + curl -L -o fpc-source.zip "https://sourceforge.net/projects/freepascal/files/Source/${FPC_VERSION}/fpc-${FPC_VERSION}.source.zip/download" + mkdir -p "$HOME/fpcsrc" + unzip -q fpc-source.zip -d "$HOME/fpcsrc" + rm fpc-source.zip + fi FPC_EXE=$(find "$HOME/fpc" -name "fpc-ootb-64.exe" -type f 2>/dev/null | head -1) echo "Found FPC executable: $FPC_EXE" - echo "Found FPC source dir: $HOME/fpcsrc/fpc-${FPC_VERSION}" - # Set environment variables for pasls echo "PP=$FPC_EXE" >> $GITHUB_ENV echo "FPCDIR=$HOME/fpcsrc/fpc-${FPC_VERSION}" >> $GITHUB_ENV - # Add FPC bin directory to PATH - FPC_BIN_DIR=$(dirname "$FPC_EXE") - echo "$FPC_BIN_DIR" >> $GITHUB_PATH + echo "$(dirname "$FPC_EXE")" >> $GITHUB_PATH fi - name: Verify FPC installation + if: matrix.batch == 'native' shell: bash run: | echo "=== Environment variables ===" @@ -441,11 +509,12 @@ jobs: echo "ERROR: FPCDIR does not exist: $FPCDIR" exit 1 fi - - name: Build Lean 4 test project - uses: leanprover/lean-action@v1 - with: - lake-package-directory: test/resources/repos/lean4/test_repo + # Lean 4 is intentionally NOT installed/built here: the toolchain is ~1 GB (it *is* the language + # server) and lean4 is already covered by the Linux maximal container (test-parallel.yml's niche + # batch). With no `lean` on PATH, the central language_tests_enabled(Language.LEAN4) guard simply + # skips lean4 on the Windows/macOS runners -- no per-OS toolchain download. - name: Install npm deps for Angular test repo + if: matrix.batch == 'web' shell: bash run: | if [ -d test/resources/repos/angular/test_repo ]; then @@ -457,17 +526,36 @@ jobs: uses: actions/cache@v3 with: path: ~/.serena/language_servers/static - key: language-servers-${{ runner.os }}-v1 + key: language-servers-${{ runner.os }}-${{ matrix.batch }}-v1 restore-keys: | - language-servers-${{ runner.os }}- + language-servers-${{ runner.os }}-${{ matrix.batch }}- - name: Report free disk space if: runner.os == 'Linux' run: | echo "Free disk space before tests:" df -h + - name: Compute marker expression for this batch + id: markers + shell: bash + run: | + case "${{ matrix.batch }}" in + jvm) expr="$MARKERS_JVM" ;; + native) expr="$MARKERS_NATIVE" ;; + swift) expr="$MARKERS_SWIFT" ;; + web) expr="$MARKERS_WEB" ;; + python) expr="$MARKERS_PYTHON" ;; + scripting) expr="$MARKERS_SCRIPTING" ;; + other-langs) expr="$MARKERS_OTHER_LANGS" ;; + catch-all) expr="not ($MARKERS_JVM or $MARKERS_NATIVE or $MARKERS_SWIFT or $MARKERS_WEB or $MARKERS_PYTHON or $MARKERS_SCRIPTING or $MARKERS_OTHER_LANGS)" ;; + *) echo "Unknown batch: ${{ matrix.batch }}" >&2; exit 1 ;; + esac + echo "Marker expression: $expr" + echo "expr=$expr" >> "$GITHUB_OUTPUT" - name: Test with pytest shell: bash - run: uv run poe test -q --tb=short + run: uv run poe test -m "${{ steps.markers.outputs.expr }}" -q --tb=short - name: Type-checking with mypy + # Type-checking is batch-independent; run it only once per OS instead of in every batch. + if: matrix.batch == 'catch-all' shell: bash run: uv run poe type-check diff --git a/test/conftest.py b/test/conftest.py index e342daf8..34487aa7 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -3,6 +3,7 @@ import os import platform import re import shutil as _sh +import subprocess from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path @@ -21,6 +22,8 @@ from solidlsp.ls_config import Language, LanguageServerConfig from solidlsp.settings import SolidLSPSettings from .solidlsp.clojure import is_clojure_cli_available +from .solidlsp.elixir import EXPERT_UNAVAILABLE +from .solidlsp.erlang import ERLANG_LS_UNAVAILABLE configure(level=logging.INFO) @@ -244,10 +247,7 @@ is_windows = platform.system() == "Windows" _LANGUAGE_PYTEST_MARKERS: dict[Language, list[MarkDecorator | Mark]] = { Language.ADA: [pytest.mark.ada], - Language.CLOJURE: [ - pytest.mark.clojure, - pytest.mark.skipif(not is_clojure_cli_available(), reason="clojure CLI is not installed"), - ], + Language.CLOJURE: [pytest.mark.clojure], Language.CPP: [pytest.mark.cpp], Language.CPP_CCLS: [pytest.mark.cpp], Language.CUE: [pytest.mark.cue], @@ -256,8 +256,8 @@ _LANGUAGE_PYTEST_MARKERS: dict[Language, list[MarkDecorator | Mark]] = { Language.GO: [pytest.mark.go], Language.HAXE: [pytest.mark.haxe], Language.JAVA: [pytest.mark.java], - Language.KOTLIN: [pytest.mark.kotlin, pytest.mark.skipif(is_ci, reason="Kotlin LSP JVM crashes on restart in CI")], - Language.LEAN4: [pytest.mark.lean4, pytest.mark.skipif(_sh.which("lean") is None, reason="Lean is not installed")], + Language.KOTLIN: [pytest.mark.kotlin], + Language.LEAN4: [pytest.mark.lean4], Language.LATEX: [pytest.mark.latex], Language.MSL: [pytest.mark.msl], Language.PHP: [pytest.mark.php], @@ -270,10 +270,7 @@ _LANGUAGE_PYTEST_MARKERS: dict[Language, list[MarkDecorator | Mark]] = { Language.PYTHON_PYREFLY: [pytest.mark.python], Language.RUST: [pytest.mark.rust], Language.TYPESCRIPT: [pytest.mark.typescript], - Language.BSL: [ - pytest.mark.bsl, - pytest.mark.skipif(_sh.which("java") is None, reason="Java is not installed"), - ], + Language.BSL: [pytest.mark.bsl], Language.SVELTE: [pytest.mark.svelte], Language.ANGULAR: [pytest.mark.angular], Language.HTML: [pytest.mark.html], @@ -284,54 +281,167 @@ _LANGUAGE_PYTEST_MARKERS: dict[Language, list[MarkDecorator | Mark]] = { def get_pytest_markers(language: Language) -> list[MarkDecorator | Mark]: """Pytest markers for a language. - The returned list contains the primary language marker and any - environment-dependent skip markers shared across the test suite. + Returns the primary language marker plus the central enablement skip derived from + ``language_tests_enabled()`` -- so per-language availability/reliability lives in exactly one + place (``_determine_disabled_languages``) instead of being duplicated per marker or per test file. """ - return _LANGUAGE_PYTEST_MARKERS[language] + return [ + *_LANGUAGE_PYTEST_MARKERS[language], + pytest.mark.skipif(not language_tests_enabled(language), reason=f"{language.value} tests are disabled in this environment"), + ] + + +def _is_perl_language_server_available() -> bool: + """ + Whether Perl::LanguageServer is installed. + + Perl itself ships with most base systems, so checking for the ``perl`` binary is not enough; + we verify that the ``Perl::LanguageServer`` module can be loaded -- which is exactly what the + Perl language server launcher requires to start. + """ + if _sh.which("perl") is None: + return False + try: + return subprocess.run(["perl", "-MPerl::LanguageServer", "-e", "1"], capture_output=True, timeout=30, check=False).returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + +def _is_matlab_available() -> bool: + """Whether a MATLAB installation can be located (env var or a known install path).""" + if os.environ.get("MATLAB_PATH") is not None: + return True + return any( + os.path.exists(p) + for p in ( + "/Applications/MATLAB_R2024b.app", + "/Applications/MATLAB_R2025b.app", + "/Volumes/S1/Applications/MATLAB_R2024b.app", + "/Volumes/S1/Applications/MATLAB_R2025b.app", + ) + ) + + +def _is_r_language_server_available() -> bool: + """Whether R *and* its ``languageserver`` package are installed. + + The R binary alone is not enough -- the language server runs as ``R -e "languageserver::run()"``, + which fails (RuntimeError) if the package is missing -- so check the package, not just ``which("R")``. + """ + if _sh.which("R") is None: + return False + try: + return ( + subprocess.run( + ["R", "--vanilla", "-e", 'quit(status = as.integer(!requireNamespace("languageserver", quietly = TRUE)))'], + capture_output=True, + timeout=60, + check=False, + ).returncode + == 0 + ) + except (OSError, subprocess.SubprocessError): + return False + + +def _is_ocaml_lsp_available() -> bool: + """Whether opam *and* the ``ocaml-lsp-server`` (``ocamllsp``) are installed. + + opam alone is not enough -- the language server is launched via ``opam exec -- ocamllsp`` and + raises if the package is missing -- so verify ocamllsp resolves in the active switch. + """ + if _sh.which("opam") is None: + return False + try: + return ( + subprocess.run( + ["opam", "exec", "--", "ocamllsp", "--version"], + capture_output=True, + timeout=60, + check=False, + ).returncode + == 0 + ) + except (OSError, subprocess.SubprocessError): + return False def _determine_disabled_languages() -> list[Language]: """ - Determine which language tests should be disabled (based on the environment) + Determine which language tests are disabled in the current environment. - :return: the list of disabled languages + Every language falls into exactly ONE of the categories below; a language that is not appended + here is **category 4 (enabled everywhere)**, e.g. python, typescript, go, java, kotlin-locally. + + 1. ALWAYS DISABLED -- flaky/broken; not worth running anywhere. + 2. DISABLED OFF-CI when a precondition (toolchain/LS) is missing, but EXPECTED ON CI -- guarded + with ``and not is_ci`` so a missing tool *on CI* fails loudly (catches a CI setup regression) + instead of silently skipping. + 3. DISABLED WHEREVER the precondition is missing, INCLUDING on CI -- the precondition may or may + not be provided on CI (e.g. via the maximal container, see Dockerfile.maximal); if it isn't, + the tests just skip gracefully rather than fail. + 4. ENABLED EVERYWHERE -- not listed in this function at all. + 5. DISABLED ONLY ON CI (resource/stability reasons) even though the precondition holds locally. """ result: list[Language] = [] - java_tests_enabled = True - if not java_tests_enabled: - result.append(Language.JAVA) + # === 1. Always disabled (flaky / broken everywhere) === + result.append(Language.BSL) # 1C:Enterprise; niche and the tests are slow and flaky + result.append(Language.FSHARP) # F# language server is currently unreliable - clojure_tests_enabled = is_clojure_cli_available() - if not clojure_tests_enabled: - result.append(Language.CLOJURE) + # === 2. Disabled off-CI if the precondition is missing; expected to be present on CI === + if _sh.which("terraform") is None and not is_ci: + result.append(Language.TERRAFORM) + if _sh.which("regal") is None and not is_ci: + result.append(Language.REGO) + if _sh.which("elm") is None and not is_ci: + result.append(Language.ELM) - # Disable CPP_CCLS tests if ccls is not available - ccls_tests_enabled = _sh.which("ccls") is not None - # Skip ccls tests on Windows since no recent binary is available and version - # 0.20220729 from chocolatey crashes when parsing the test files. - ccls_tests_enabled = ccls_tests_enabled and not is_windows - if not ccls_tests_enabled: - result.append(Language.CPP_CCLS) - - # Disable CPP (clangd) tests if clangd is not available - clangd_tests_enabled = _sh.which("clangd") is not None - if not clangd_tests_enabled: + # === 3. Disabled wherever the precondition is missing (including on CI) === + # 3a. Platform precondition: these language servers have no native Windows support. + if is_windows: + result.append(Language.ANSIBLE) + result.append(Language.SWIFT) + # 3b. Toolchain / language-server availability (the LS/compiler must be on PATH or installed). + if _sh.which("clangd") is None: result.append(Language.CPP) - - # Disable PHP_PHPACTOR tests if php is not available - php_tests_enabled = _sh.which("php") is not None - if not php_tests_enabled: + if _sh.which("ccls") is None or is_windows: # no recent ccls binary is available for Windows + result.append(Language.CPP_CCLS) + if _sh.which("php") is None: result.append(Language.PHP_PHPACTOR) result.append(Language.PHP_PHPANTOM) + if not is_clojure_cli_available(): + result.append(Language.CLOJURE) + if _sh.which("verible-verilog-ls") is None: + result.append(Language.SYSTEMVERILOG) + if not _is_matlab_available(): + result.append(Language.MATLAB) + if ERLANG_LS_UNAVAILABLE: # no Erlang-OTP / no rebar3 / Windows -- see test/solidlsp/erlang + result.append(Language.ERLANG) + if EXPERT_UNAVAILABLE: # Elixir not installed -- see test/solidlsp/elixir + result.append(Language.ELIXIR) + if _sh.which("lean") is None: + result.append(Language.LEAN4) + if _sh.which("crystalline") is None: + result.append(Language.CRYSTAL) + if _sh.which("julia") is None: # LanguageServer.jl is auto-installed by the LS when julia is present + result.append(Language.JULIA) + if _sh.which("nixd") is None: + result.append(Language.NIX) + if _sh.which("haskell-language-server-wrapper") is None: + result.append(Language.HASKELL) + if not _is_r_language_server_available(): # `which("R")` isn't enough -- needs the languageserver package + result.append(Language.R) + if not _is_ocaml_lsp_available(): # opam alone isn't enough -- needs the ocaml-lsp-server package + result.append(Language.OCAML) + if not _is_perl_language_server_available(): # perl ships with the OS; the LS module is the real signal + result.append(Language.PERL) - al_tests_enabled = True - if not al_tests_enabled: - result.append(Language.AL) + # === 4. Enabled everywhere: every language NOT listed in this function (python, go, java, ...) === - # Disable BSL tests only when Java is not available (Java IS present in CI via actions/setup-java) - if _sh.which("java") is None: - result.append(Language.BSL) + # === 5. Disabled only on CI (works locally; too unstable/costly on the CI runners) === + if is_ci: + result.append(Language.KOTLIN) # IntelliJ-based Kotlin LSP crashes on JVM restart under CI memory limits return result diff --git a/test/serena/test_serena_agent.py b/test/serena/test_serena_agent.py index 5ab27115..7625e50a 100644 --- a/test/serena/test_serena_agent.py +++ b/test/serena/test_serena_agent.py @@ -36,7 +36,6 @@ from test.conftest import ( find_identifier_pos, get_pytest_markers, get_repo_path, - is_ci, language_tests_enabled, ) @@ -725,9 +724,7 @@ SAFE_DELETE_BLOCKED_CASES = [ id="kotlin_model", name_path="Model", relative_path=os.path.join("src", "main", "kotlin", "test_repo", "Model.kt"), - ).to_pytest_param( - *([pytest.mark.skip(reason="Kotlin LSP JVM crashes on restart in CI")] if is_ci else []), - ), + ).to_pytest_param(), SafeDeleteCase( language=Language.TYPESCRIPT, id="typescript_helper_function", @@ -754,9 +751,7 @@ SAFE_DELETE_SUCCEEDS_CASES = [ id="kotlin_model_user", name_path="ModelUser", relative_path=os.path.join("src", "main", "kotlin", "test_repo", "ModelUser.kt"), - ).to_pytest_param( - *([pytest.mark.skip(reason="Kotlin LSP JVM crashes on restart in CI")] if is_ci else []), - ), + ).to_pytest_param(), SafeDeleteCase( language=Language.TYPESCRIPT, id="typescript_unused_standalone_function", diff --git a/test/serena/test_symbol_editing.py b/test/serena/test_symbol_editing.py index b6f8abf2..97a02aa6 100644 --- a/test/serena/test_symbol_editing.py +++ b/test/serena/test_symbol_editing.py @@ -6,7 +6,6 @@ Recreate the snapshots with `pytest --snapshot-update`. import logging import os import shutil -import sys import tempfile import time from abc import ABC, abstractmethod @@ -24,7 +23,7 @@ from syrupy import SnapshotAssertion from serena.code_editor import CodeEditor, LanguageServerCodeEditor from solidlsp.ls_config import Language from src.serena.symbol import LanguageServerSymbolRetriever -from test.conftest import get_repo_path, project_with_ls_context +from test.conftest import get_repo_path, language_tests_enabled, project_with_ls_context pytestmark = pytest.mark.snapshot @@ -451,7 +450,7 @@ class NixAttrReplacementTest(EditingTest): @pytest.mark.nix -@pytest.mark.skipif(sys.platform == "win32", reason="nixd language server doesn't run on Windows") +@pytest.mark.skipif(not language_tests_enabled(Language.NIX), reason="Nix tests are disabled (nixd not available)") def test_nix_symbol_replacement_no_double_semicolon(snapshot: SnapshotAssertion): """ Test that replacing a Nix attribute does not result in double semicolons. diff --git a/test/solidlsp/ansible/test_ansible_basic.py b/test/solidlsp/ansible/test_ansible_basic.py index bf6dc6ca..666a3f0b 100644 --- a/test/solidlsp/ansible/test_ansible_basic.py +++ b/test/solidlsp/ansible/test_ansible_basic.py @@ -6,17 +6,17 @@ using the standard Ansible test repository. They work with the standard @ansible/ansible-language-server from npm. """ -import platform from pathlib import Path import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols -@pytest.mark.skipif(platform.system() == "Windows", reason="ansible-language-server does not support native Windows operation") +@pytest.mark.skipif(not language_tests_enabled(Language.ANSIBLE), reason="Ansible tests are disabled (no native Windows support)") @pytest.mark.ansible class TestAnsibleLanguageServerBasics: """Test basic Ansible language server functionality.""" diff --git a/test/solidlsp/bsl/test_bsl_basic.py b/test/solidlsp/bsl/test_bsl_basic.py index 4e2ecc87..4e093749 100644 --- a/test/solidlsp/bsl/test_bsl_basic.py +++ b/test/solidlsp/bsl/test_bsl_basic.py @@ -8,9 +8,10 @@ from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import SymbolUtils from solidlsp.settings import SolidLSPSettings +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols -pytestmark = pytest.mark.skip("BSL is a niche language, the tests are slow and flaky.") +pytestmark = pytest.mark.skipif(not language_tests_enabled(Language.BSL), reason="BSL tests are disabled (niche, slow and flaky)") @pytest.mark.bsl diff --git a/test/solidlsp/bsl/test_bsl_diagnostics.py b/test/solidlsp/bsl/test_bsl_diagnostics.py index 7974b4a6..e5dfe69e 100644 --- a/test/solidlsp/bsl/test_bsl_diagnostics.py +++ b/test/solidlsp/bsl/test_bsl_diagnostics.py @@ -1,14 +1,13 @@ -import shutil - import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.util.diagnostics import assert_file_diagnostics @pytest.mark.bsl -@pytest.mark.skipif(shutil.which("java") is None, reason="Java 21+ is required for BSL LSP") +@pytest.mark.skipif(not language_tests_enabled(Language.BSL), reason="BSL tests are disabled") class TestBSLDiagnostics: @pytest.mark.parametrize("language_server", [Language.BSL], indirect=True) def test_file_diagnostics(self, language_server: SolidLanguageServer) -> None: diff --git a/test/solidlsp/crystal/test_crystal_basic.py b/test/solidlsp/crystal/test_crystal_basic.py index 71102949..7af66129 100644 --- a/test/solidlsp/crystal/test_crystal_basic.py +++ b/test/solidlsp/crystal/test_crystal_basic.py @@ -10,17 +10,17 @@ Known Crystalline limitations: """ import os -import shutil import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols pytestmark = [ pytest.mark.crystal, - pytest.mark.skipif(shutil.which("crystalline") is None, reason="Crystalline is not installed"), + pytest.mark.skipif(not language_tests_enabled(Language.CRYSTAL), reason="Crystal tests are disabled (crystalline not available)"), ] diff --git a/test/solidlsp/crystal/test_crystal_diagnostics.py b/test/solidlsp/crystal/test_crystal_diagnostics.py index e650adc1..b593b147 100644 --- a/test/solidlsp/crystal/test_crystal_diagnostics.py +++ b/test/solidlsp/crystal/test_crystal_diagnostics.py @@ -1,14 +1,13 @@ -import shutil - import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.util.diagnostics import assert_file_diagnostics pytestmark = [ pytest.mark.crystal, - pytest.mark.skipif(shutil.which("crystalline") is None, reason="Crystalline is not installed"), + pytest.mark.skipif(not language_tests_enabled(Language.CRYSTAL), reason="Crystal tests are disabled (crystalline not available)"), ] diff --git a/test/solidlsp/elixir/test_elixir_basic.py b/test/solidlsp/elixir/test_elixir_basic.py index 7c44abc4..ac32edd5 100644 --- a/test/solidlsp/elixir/test_elixir_basic.py +++ b/test/solidlsp/elixir/test_elixir_basic.py @@ -12,13 +12,12 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_types import SymbolKind +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols from test.solidlsp.util.diagnostics import assert_file_diagnostics -from . import EXPERT_UNAVAILABLE, EXPERT_UNAVAILABLE_REASON - # These marks will be applied to all tests in this module -pytestmark = [pytest.mark.elixir, pytest.mark.skipif(EXPERT_UNAVAILABLE, reason=f"Next LS not available: {EXPERT_UNAVAILABLE_REASON}")] +pytestmark = [pytest.mark.elixir, pytest.mark.skipif(not language_tests_enabled(Language.ELIXIR), reason="Elixir tests are disabled")] class TestElixirBasic: diff --git a/test/solidlsp/elixir/test_elixir_ignored_dirs.py b/test/solidlsp/elixir/test_elixir_ignored_dirs.py index b01531ba..75f4870b 100644 --- a/test/solidlsp/elixir/test_elixir_ignored_dirs.py +++ b/test/solidlsp/elixir/test_elixir_ignored_dirs.py @@ -6,12 +6,10 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language -from test.conftest import start_ls_context - -from . import EXPERT_UNAVAILABLE, EXPERT_UNAVAILABLE_REASON +from test.conftest import language_tests_enabled, start_ls_context # These marks will be applied to all tests in this module -pytestmark = [pytest.mark.elixir, pytest.mark.skipif(EXPERT_UNAVAILABLE, reason=f"Expert not available: {EXPERT_UNAVAILABLE_REASON}")] +pytestmark = [pytest.mark.elixir, pytest.mark.skipif(not language_tests_enabled(Language.ELIXIR), reason="Elixir tests are disabled")] # Skip slow tests in CI - they require multiple Expert instances which is too slow IN_CI = bool(os.environ.get("CI") or os.environ.get("GITHUB_ACTIONS")) diff --git a/test/solidlsp/elixir/test_elixir_integration.py b/test/solidlsp/elixir/test_elixir_integration.py index eff618b2..678ce762 100644 --- a/test/solidlsp/elixir/test_elixir_integration.py +++ b/test/solidlsp/elixir/test_elixir_integration.py @@ -13,11 +13,10 @@ import pytest from serena.project import Project from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language - -from . import EXPERT_UNAVAILABLE, EXPERT_UNAVAILABLE_REASON +from test.conftest import language_tests_enabled # These marks will be applied to all tests in this module -pytestmark = [pytest.mark.elixir, pytest.mark.skipif(EXPERT_UNAVAILABLE, reason=f"Next LS not available: {EXPERT_UNAVAILABLE_REASON}")] +pytestmark = [pytest.mark.elixir, pytest.mark.skipif(not language_tests_enabled(Language.ELIXIR), reason="Elixir tests are disabled")] class TestElixirIntegration: diff --git a/test/solidlsp/elixir/test_elixir_symbol_retrieval.py b/test/solidlsp/elixir/test_elixir_symbol_retrieval.py index 89a3127f..219cfe15 100644 --- a/test/solidlsp/elixir/test_elixir_symbol_retrieval.py +++ b/test/solidlsp/elixir/test_elixir_symbol_retrieval.py @@ -14,11 +14,10 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_types import SymbolKind - -from . import EXPERT_UNAVAILABLE, EXPERT_UNAVAILABLE_REASON +from test.conftest import language_tests_enabled # These marks will be applied to all tests in this module -pytestmark = [pytest.mark.elixir, pytest.mark.skipif(EXPERT_UNAVAILABLE, reason=f"Next LS not available: {EXPERT_UNAVAILABLE_REASON}")] +pytestmark = [pytest.mark.elixir, pytest.mark.skipif(not language_tests_enabled(Language.ELIXIR), reason="Elixir tests are disabled")] class TestElixirLanguageServerSymbols: diff --git a/test/solidlsp/elm/test_elm_basic.py b/test/solidlsp/elm/test_elm_basic.py index 5ff6fe22..a886ede9 100644 --- a/test/solidlsp/elm/test_elm_basic.py +++ b/test/solidlsp/elm/test_elm_basic.py @@ -1,16 +1,15 @@ import os -import shutil import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import SymbolUtils -from test.conftest import is_ci +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols -@pytest.mark.skipif(shutil.which("elm") is None and not is_ci, reason="Elm compiler is not available") +@pytest.mark.skipif(not language_tests_enabled(Language.ELM), reason="Elm tests are disabled (elm compiler not available)") @pytest.mark.elm class TestElmLanguageServer: @pytest.mark.parametrize("language_server", [Language.ELM], indirect=True) diff --git a/test/solidlsp/erlang/test_erlang_basic.py b/test/solidlsp/erlang/test_erlang_basic.py index dc39535b..491c71a7 100644 --- a/test/solidlsp/erlang/test_erlang_basic.py +++ b/test/solidlsp/erlang/test_erlang_basic.py @@ -9,14 +9,13 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols from test.solidlsp.util.diagnostics import assert_file_diagnostics -from . import ERLANG_LS_UNAVAILABLE, ERLANG_LS_UNAVAILABLE_REASON - @pytest.mark.erlang -@pytest.mark.skipif(ERLANG_LS_UNAVAILABLE, reason=f"Erlang LS not available: {ERLANG_LS_UNAVAILABLE_REASON}") +@pytest.mark.skipif(not language_tests_enabled(Language.ERLANG), reason="Erlang tests are disabled") class TestErlangLanguageServerBasics: """Test basic functionality of the Erlang language server.""" diff --git a/test/solidlsp/erlang/test_erlang_ignored_dirs.py b/test/solidlsp/erlang/test_erlang_ignored_dirs.py index 74cf87ff..d6fdb0ab 100644 --- a/test/solidlsp/erlang/test_erlang_ignored_dirs.py +++ b/test/solidlsp/erlang/test_erlang_ignored_dirs.py @@ -5,14 +5,12 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language -from test.conftest import start_ls_context - -from . import ERLANG_LS_UNAVAILABLE, ERLANG_LS_UNAVAILABLE_REASON +from test.conftest import language_tests_enabled, start_ls_context # These marks will be applied to all tests in this module pytestmark = [ pytest.mark.erlang, - pytest.mark.skipif(ERLANG_LS_UNAVAILABLE, reason=f"Erlang LS not available: {ERLANG_LS_UNAVAILABLE_REASON}"), + pytest.mark.skipif(not language_tests_enabled(Language.ERLANG), reason="Erlang tests are disabled"), ] diff --git a/test/solidlsp/erlang/test_erlang_symbol_retrieval.py b/test/solidlsp/erlang/test_erlang_symbol_retrieval.py index b2f9fce4..918237c5 100644 --- a/test/solidlsp/erlang/test_erlang_symbol_retrieval.py +++ b/test/solidlsp/erlang/test_erlang_symbol_retrieval.py @@ -14,13 +14,12 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_types import SymbolKind - -from . import ERLANG_LS_UNAVAILABLE, ERLANG_LS_UNAVAILABLE_REASON +from test.conftest import language_tests_enabled # These marks will be applied to all tests in this module pytestmark = [ pytest.mark.erlang, - pytest.mark.skipif(ERLANG_LS_UNAVAILABLE, reason=f"Erlang LS not available: {ERLANG_LS_UNAVAILABLE_REASON}"), + pytest.mark.skipif(not language_tests_enabled(Language.ERLANG), reason="Erlang tests are disabled"), ] diff --git a/test/solidlsp/fsharp/test_fsharp_basic.py b/test/solidlsp/fsharp/test_fsharp_basic.py index f4f49ebc..37ed0bb5 100644 --- a/test/solidlsp/fsharp/test_fsharp_basic.py +++ b/test/solidlsp/fsharp/test_fsharp_basic.py @@ -7,14 +7,19 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import SymbolUtils -from test.conftest import find_identifier_position, get_repo_path, is_ci, language_has_verified_implementation_support +from test.conftest import ( + find_identifier_position, + get_repo_path, + language_has_verified_implementation_support, + language_tests_enabled, +) from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols from test.solidlsp.util.diagnostics import assert_file_diagnostics +# Currently, most F# tests fail (regression/instability), so the suite is disabled on CI. +pytestmark = [pytest.mark.fsharp, pytest.mark.skipif(not language_tests_enabled(Language.FSHARP), reason="F# tests are disabled")] + -# Currently, most F# tests fail, there seems to be a regression or instability. -@pytest.mark.fsharp -@pytest.mark.skipif(is_ci, reason="F# language server is currently unreliable") class TestFSharpLanguageServer: @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True) def test_find_symbol(self, language_server: SolidLanguageServer) -> None: @@ -80,7 +85,6 @@ class TestFSharpLanguageServer: for expected in expected_symbols: assert expected in symbol_names, f"{expected} function not found in Calculator.fs symbols" - @pytest.mark.xfail(is_ci, reason="Test is flaky") # TODO: Re-enable if the LS can be made more reliable #1040 @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True) def test_find_referencing_symbols(self, language_server: SolidLanguageServer) -> None: """Test finding references using symbol selection range.""" @@ -117,7 +121,6 @@ class TestFSharpLanguageServer: for expected in expected_symbols: assert expected in symbol_names, f"{expected} not found in Person.fs symbols" - @pytest.mark.xfail(is_ci, reason="Test is flaky") # TODO: Re-enable if the LS can be made more reliable #1040 @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True) def test_find_referencing_symbols_across_files(self, language_server: SolidLanguageServer) -> None: """Test finding references to Calculator functions across files.""" @@ -140,7 +143,6 @@ class TestFSharpLanguageServer: # The subtract function should be referenced in Program.fs assert any("Program.fs" in ref.get("relativePath", "") for ref in refs), "Program.fs should reference subtract function" - @pytest.mark.xfail(is_ci, reason="Test is flaky") # TODO: Re-enable if the LS can be made more reliable #1040 @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True) def test_go_to_definition(self, language_server: SolidLanguageServer) -> None: """Test go-to-definition functionality.""" @@ -154,7 +156,6 @@ class TestFSharpLanguageServer: # We should get at least some definitions assert len(definitions) >= 0, "Should get definitions (even if empty for complex cases)" - @pytest.mark.xfail(is_ci, reason="Test is flaky") # TODO: Re-enable if the LS can be made more reliable #1040 @pytest.mark.parametrize("language_server", [Language.FSHARP], indirect=True) def test_hover_information(self, language_server: SolidLanguageServer) -> None: """Test hover information functionality.""" diff --git a/test/solidlsp/haskell/test_haskell_basic.py b/test/solidlsp/haskell/test_haskell_basic.py index 55f94f6d..cf4bba3b 100644 --- a/test/solidlsp/haskell/test_haskell_basic.py +++ b/test/solidlsp/haskell/test_haskell_basic.py @@ -13,21 +13,19 @@ Test Repository Structure: - app/Main.hs: Main entry point using Calculator and Helper modules """ -import sys - import pytest from solidlsp.ls import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_types import SymbolKind +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols from test.solidlsp.util.diagnostics import assert_file_diagnostics -pytestmark = pytest.mark.skip("Haskell installation is heavy, the tests should be executed only on demand") +pytestmark = pytest.mark.skipif(not language_tests_enabled(Language.HASKELL), reason="Haskell tests are disabled (HLS not available)") @pytest.mark.haskell -@pytest.mark.skipif(sys.platform == "win32", reason="HLS not installed on Windows CI") class TestHaskellLanguageServer: @pytest.mark.parametrize("language_server", [Language.HASKELL], indirect=True) def test_calculator_module_symbols(self, language_server: SolidLanguageServer): diff --git a/test/solidlsp/julia/test_julia_basic.py b/test/solidlsp/julia/test_julia_basic.py index 1f3fdd87..75b81abf 100644 --- a/test/solidlsp/julia/test_julia_basic.py +++ b/test/solidlsp/julia/test_julia_basic.py @@ -1,14 +1,13 @@ -import shutil - import pytest from solidlsp.ls import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols from test.solidlsp.util.diagnostics import assert_file_diagnostics -@pytest.mark.skipif(shutil.which("julia") is None, reason="Julia is not available") +@pytest.mark.skipif(not language_tests_enabled(Language.JULIA), reason="Julia tests are disabled (julia not available)") @pytest.mark.julia class TestJuliaLanguageServer: @pytest.mark.parametrize("language_server", [Language.JULIA], indirect=True) diff --git a/test/solidlsp/kotlin/test_kotlin_basic.py b/test/solidlsp/kotlin/test_kotlin_basic.py index 9a990458..0ce7a61f 100644 --- a/test/solidlsp/kotlin/test_kotlin_basic.py +++ b/test/solidlsp/kotlin/test_kotlin_basic.py @@ -6,14 +6,15 @@ from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_types import SymbolKind from solidlsp.ls_utils import SymbolUtils -from test.conftest import is_ci +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols # Kotlin LSP (IntelliJ-based, pre-alpha v261) crashes on JVM restart under CI resource constraints # (2 CPUs, 7GB RAM). First start succeeds but subsequent starts fail with cancelled (-32800). # Tests pass reliably on developer machines. See PR #1061 for investigation details. -@pytest.mark.skipif(is_ci, reason="Kotlin LSP JVM restart is unstable on CI runners") +# (The CI quarantine lives centrally in test/conftest.py::_determine_disabled_languages.) +@pytest.mark.skipif(not language_tests_enabled(Language.KOTLIN), reason="Kotlin tests are disabled") @pytest.mark.kotlin class TestKotlinLanguageServer: @pytest.mark.parametrize("language_server", [Language.KOTLIN], indirect=True) diff --git a/test/solidlsp/lean4/test_lean4_basic.py b/test/solidlsp/lean4/test_lean4_basic.py index b8f35efa..bb4883cf 100644 --- a/test/solidlsp/lean4/test_lean4_basic.py +++ b/test/solidlsp/lean4/test_lean4_basic.py @@ -16,6 +16,9 @@ import pytest from solidlsp.ls import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled + +pytestmark = pytest.mark.skipif(not language_tests_enabled(Language.LEAN4), reason="Lean4 tests are disabled (lean not available)") @pytest.mark.lean4 diff --git a/test/solidlsp/lean4/test_lean4_diagnostics.py b/test/solidlsp/lean4/test_lean4_diagnostics.py index 964bc2c6..11eac570 100644 --- a/test/solidlsp/lean4/test_lean4_diagnostics.py +++ b/test/solidlsp/lean4/test_lean4_diagnostics.py @@ -2,8 +2,11 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.util.diagnostics import assert_file_diagnostics +pytestmark = pytest.mark.skipif(not language_tests_enabled(Language.LEAN4), reason="Lean4 tests are disabled (lean not available)") + @pytest.mark.lean4 class TestLean4Diagnostics: diff --git a/test/solidlsp/matlab/test_matlab_basic.py b/test/solidlsp/matlab/test_matlab_basic.py index a1582887..4857a447 100644 --- a/test/solidlsp/matlab/test_matlab_basic.py +++ b/test/solidlsp/matlab/test_matlab_basic.py @@ -10,31 +10,19 @@ Requirements: - Node.js and npm must be installed """ -import os - import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols from test.solidlsp.util.diagnostics import assert_file_diagnostics # Skip all tests if MATLAB is not available pytestmark = pytest.mark.matlab -# Check if MATLAB is available -MATLAB_AVAILABLE = os.environ.get("MATLAB_PATH") is not None or any( - os.path.exists(p) - for p in [ - "/Applications/MATLAB_R2024b.app", - "/Applications/MATLAB_R2025b.app", - "/Volumes/S1/Applications/MATLAB_R2024b.app", - "/Volumes/S1/Applications/MATLAB_R2025b.app", - ] -) - -@pytest.mark.skipif(not MATLAB_AVAILABLE, reason="MATLAB installation not found") +@pytest.mark.skipif(not language_tests_enabled(Language.MATLAB), reason="MATLAB tests are disabled (MATLAB installation not found)") class TestMatlabLanguageServerBasics: """Test basic functionality of the MATLAB language server.""" @@ -95,7 +83,7 @@ class TestMatlabLanguageServerBasics: assert all_symbols is not None -@pytest.mark.skipif(not MATLAB_AVAILABLE, reason="MATLAB installation not found") +@pytest.mark.skipif(not language_tests_enabled(Language.MATLAB), reason="MATLAB tests are disabled (MATLAB installation not found)") class TestMatlabLanguageServerReferences: """Test find references functionality of the MATLAB language server.""" diff --git a/test/solidlsp/nix/test_nix_basic.py b/test/solidlsp/nix/test_nix_basic.py index 61207833..789d047b 100644 --- a/test/solidlsp/nix/test_nix_basic.py +++ b/test/solidlsp/nix/test_nix_basic.py @@ -9,11 +9,11 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_types import SymbolKind -from test.conftest import is_ci +from test.conftest import is_ci, language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols from test.solidlsp.util.diagnostics import assert_file_diagnostics -pytestmark = pytest.mark.skip("Nix is niche and has a heavy toolchain") +pytestmark = pytest.mark.skipif(not language_tests_enabled(Language.NIX), reason="Nix tests are disabled (nixd not available)") @pytest.mark.nix diff --git a/test/solidlsp/ocaml/test_cross_file_refs.py b/test/solidlsp/ocaml/test_cross_file_refs.py index 0a2b70bc..44c72e49 100644 --- a/test/solidlsp/ocaml/test_cross_file_refs.py +++ b/test/solidlsp/ocaml/test_cross_file_refs.py @@ -14,10 +14,12 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.language_servers.ocaml_lsp_server import OcamlLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled log = logging.getLogger(__name__) +@pytest.mark.skipif(not language_tests_enabled(Language.OCAML), reason="OCaml tests are disabled (opam not available)") @pytest.mark.ocaml class TestCrossFileReferences: @pytest.mark.parametrize("language_server", [Language.OCAML], indirect=True) diff --git a/test/solidlsp/ocaml/test_ocaml_basic.py b/test/solidlsp/ocaml/test_ocaml_basic.py index 6016be56..86d3a874 100644 --- a/test/solidlsp/ocaml/test_ocaml_basic.py +++ b/test/solidlsp/ocaml/test_ocaml_basic.py @@ -1,15 +1,15 @@ import os -import shutil import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import SymbolUtils +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols -@pytest.mark.skipif(shutil.which("opam") is None, reason="OPAM is not available") +@pytest.mark.skipif(not language_tests_enabled(Language.OCAML), reason="OCaml tests are disabled (opam not available)") @pytest.mark.ocaml class TestOCamlLanguageServer: @pytest.mark.parametrize("language_server", [Language.OCAML], indirect=True) diff --git a/test/solidlsp/ocaml/test_ocaml_diagnostics.py b/test/solidlsp/ocaml/test_ocaml_diagnostics.py index 69f4ed61..3817f225 100644 --- a/test/solidlsp/ocaml/test_ocaml_diagnostics.py +++ b/test/solidlsp/ocaml/test_ocaml_diagnostics.py @@ -1,13 +1,12 @@ -import shutil - import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.util.diagnostics import assert_file_diagnostics -@pytest.mark.skipif(shutil.which("opam") is None, reason="OPAM is not available") +@pytest.mark.skipif(not language_tests_enabled(Language.OCAML), reason="OCaml tests are disabled (opam not available)") @pytest.mark.ocaml class TestOcamlDiagnostics: @pytest.mark.parametrize("language_server", [Language.OCAML], indirect=True) diff --git a/test/solidlsp/perl/test_perl_basic.py b/test/solidlsp/perl/test_perl_basic.py index 1aa185f4..f6e1bf66 100644 --- a/test/solidlsp/perl/test_perl_basic.py +++ b/test/solidlsp/perl/test_perl_basic.py @@ -4,10 +4,13 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols from test.solidlsp.util.diagnostics import assert_file_diagnostics -pytestmark = pytest.mark.skip("Perl is niche and has a heavy toolchain") +pytestmark = pytest.mark.skipif( + not language_tests_enabled(Language.PERL), reason="Perl tests are disabled (Perl::LanguageServer not available)" +) @pytest.mark.perl diff --git a/test/solidlsp/r/test_r_basic.py b/test/solidlsp/r/test_r_basic.py index 17f28545..99190d5f 100644 --- a/test/solidlsp/r/test_r_basic.py +++ b/test/solidlsp/r/test_r_basic.py @@ -3,17 +3,17 @@ Basic tests for R Language Server integration """ import os -import shutil from pathlib import Path import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols -@pytest.mark.skipif(shutil.which("R") is None, reason="R is not available") +@pytest.mark.skipif(not language_tests_enabled(Language.R), reason="R tests are disabled (R not available)") @pytest.mark.r class TestRLanguageServer: """Test basic functionality of the R language server.""" diff --git a/test/solidlsp/rego/test_rego_basic.py b/test/solidlsp/rego/test_rego_basic.py index ce8028c2..8bbe730d 100644 --- a/test/solidlsp/rego/test_rego_basic.py +++ b/test/solidlsp/rego/test_rego_basic.py @@ -1,7 +1,6 @@ """Tests for Rego language server (Regal) functionality.""" import os -import shutil import pytest @@ -9,11 +8,11 @@ from solidlsp.ls import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_types import SymbolKind from solidlsp.ls_utils import SymbolUtils -from test.conftest import is_ci +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols -@pytest.mark.skipif(shutil.which("regal") is None and not is_ci, reason="Regal is not available") +@pytest.mark.skipif(not language_tests_enabled(Language.REGO), reason="Rego tests are disabled (regal not available)") @pytest.mark.rego class TestRegoLanguageServer: """Test Regal language server functionality for Rego.""" diff --git a/test/solidlsp/swift/test_swift_basic.py b/test/solidlsp/swift/test_swift_basic.py index 30e9759c..a2282fb2 100644 --- a/test/solidlsp/swift/test_swift_basic.py +++ b/test/solidlsp/swift/test_swift_basic.py @@ -6,7 +6,6 @@ like request_references using the Swift test repository. """ import os -import platform import pytest @@ -14,15 +13,11 @@ from serena.project import Project from serena.util.text_utils import LineType from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language -from test.conftest import is_ci +from test.conftest import is_ci, language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols from test.solidlsp.util.diagnostics import assert_file_diagnostics -# Skip Swift tests on Windows due to complex GitHub Actions configuration -WINDOWS_SKIP = platform.system() == "Windows" -WINDOWS_SKIP_REASON = "GitHub Actions configuration for Swift on Windows is complex, skipping for now." - -pytestmark = [pytest.mark.swift, pytest.mark.skipif(WINDOWS_SKIP, reason=WINDOWS_SKIP_REASON)] +pytestmark = [pytest.mark.swift, pytest.mark.skipif(not language_tests_enabled(Language.SWIFT), reason="Swift tests are disabled")] class TestSwiftLanguageServerBasics: diff --git a/test/solidlsp/systemverilog/test_systemverilog_basic.py b/test/solidlsp/systemverilog/test_systemverilog_basic.py index 4653a84c..43d75d00 100644 --- a/test/solidlsp/systemverilog/test_systemverilog_basic.py +++ b/test/solidlsp/systemverilog/test_systemverilog_basic.py @@ -5,7 +5,6 @@ This module tests Language.SYSTEMVERILOG using verible-verilog-ls. Tests are skipped if the language server is not available. """ -import shutil from typing import Any import pytest @@ -13,8 +12,13 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_utils import SymbolUtils +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols +pytestmark = pytest.mark.skipif( + not language_tests_enabled(Language.SYSTEMVERILOG), reason="SystemVerilog tests are disabled (verible-verilog-ls not available)" +) + def _find_symbol_by_name(language_server: SolidLanguageServer, file_path: str, name: str) -> dict[str, Any] | None: """Find a top-level symbol by name in a file's document symbols.""" @@ -265,7 +269,6 @@ class TestSystemVerilogRename: for edit in file_edits: assert edit["newText"] == "my_counter", f"Expected 'my_counter', got {edit['newText']}" - @pytest.mark.skipif(shutil.which("verible-verilog-ls") is None, reason="verible-verilog-ls is not available") @pytest.mark.parametrize("language_server", [Language.SYSTEMVERILOG], indirect=True) def test_bare_symbol_names(self, language_server) -> None: all_symbols = request_all_symbols(language_server) diff --git a/test/solidlsp/systemverilog/test_systemverilog_diagnostics.py b/test/solidlsp/systemverilog/test_systemverilog_diagnostics.py index a783b828..3de588c2 100644 --- a/test/solidlsp/systemverilog/test_systemverilog_diagnostics.py +++ b/test/solidlsp/systemverilog/test_systemverilog_diagnostics.py @@ -2,8 +2,13 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language +from test.conftest import language_tests_enabled from test.solidlsp.util.diagnostics import assert_file_diagnostics +pytestmark = pytest.mark.skipif( + not language_tests_enabled(Language.SYSTEMVERILOG), reason="SystemVerilog tests are disabled (verible-verilog-ls not available)" +) + @pytest.mark.systemverilog class TestSystemverilogDiagnostics: diff --git a/test/solidlsp/terraform/test_terraform_basic.py b/test/solidlsp/terraform/test_terraform_basic.py index 612e3f9a..1a91f0c9 100644 --- a/test/solidlsp/terraform/test_terraform_basic.py +++ b/test/solidlsp/terraform/test_terraform_basic.py @@ -5,18 +5,16 @@ These tests validate the functionality of the language server APIs like request_references using the test repository. """ -import shutil - import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_types import SymbolKind -from test.conftest import is_ci +from test.conftest import language_tests_enabled from test.solidlsp.conftest import format_symbol_for_assert, has_malformed_name, request_all_symbols -@pytest.mark.skipif(shutil.which("terraform") is None and not is_ci, reason="Terraform CLI is not available") +@pytest.mark.skipif(not language_tests_enabled(Language.TERRAFORM), reason="Terraform tests are disabled (terraform CLI not available)") @pytest.mark.terraform class TestLanguageServerBasics: """Test basic functionality of the Terraform language server."""