Tests: centralized test enabling logic, better parallelization in CI

Removing some heavy toolchains from windows/macos run (they will be tested only in the job based off the maximal docker image)

.github/workflows/pytest.yml

@@ -15,13 +15,37 @@ concurrency:
 
 jobs:
   cpu:
-    name: Tests on ${{ matrix.os }}
+    name: ${{ matrix.batch }} (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
+    # Marker groups are defined once here and combined in the "Compute marker expression" step.
+    # The catch-all batch is derived as the negation of all named groups, so adding a language to
+    # any group automatically removes it from catch-all -- the partition can never desync, and no
+    # test can fall through to "no batch". To add/rebalance: edit a MARKERS_* var, and (for a new
+    # group) add a case branch below + the batch name to the matrix list.
+    env:
+      MARKERS_JVM: "java or csharp or fsharp or kotlin or scala or groovy or clojure or bsl"
+      MARKERS_NATIVE: "cpp or rust or go or zig or pascal"
+      MARKERS_SWIFT: "swift"  # split out: swiftly toolchain install is the slowest setup, isolate + cache it
+      MARKERS_WEB: "typescript or svelte or vue or angular or html or scss or json or solidity or elm"
+      MARKERS_PYTHON: "python"
+      MARKERS_SCRIPTING: "ruby or php or perl or lua or luau or bash or powershell or elixir or erlang or dart or haxe"
+      MARKERS_OTHER_LANGS: "terraform or rego or ansible or yaml or toml or markdown or latex or nix or crystal or cue or fortran or ada or matlab or systemverilog or hlsl or msl or al or ocaml or haskell or lean4 or julia or r"
     strategy:
       fail-fast: false
       matrix:
         os: [windows-latest, macos-latest]
         python-version: ["3.11"]
+        # Test execution is split into parallel batches by pytest marker. Each batch installs ONLY
+        # the toolchains its languages need -- steps below are gated with `if: matrix.batch == ...`
+        # -- so e.g. Swift/Lean are set up only in the batch that runs their tests, not in all 7.
+        # Node/Python/uv stay in every batch (cheap, and needed for the venv + auto-installed LSes).
+        # "catch-all" carries the unmarked core-framework tests in test/serena and needs no language
+        # toolchains. Rebalance once per-batch timings show up.
+        batch: [jvm, native, swift, web, python, scripting, other-langs, catch-all]
+        # Swift only runs on macOS (disabled on Windows via the central guard), so skip windows x swift.
+        exclude:
+          - os: windows-latest
+            batch: swift
     steps:
       - uses: actions/checkout@v4
       - name: Free disk space
@@ -41,6 +65,7 @@ jobs:
         with:
           python-version: "${{ matrix.python-version }}"
       - uses: actions/setup-go@v5
+        if: matrix.batch == 'native'
         with:
           go-version: ">=1.17.0"
       - name: Set up Node.js
@@ -54,8 +79,10 @@ jobs:
           mkdir -p $HOME/.cache/go-build
           mkdir -p $HOME/go/bin
       - name: Install uv
-        shell: bash
-        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+        # Use the official action (not `curl | sh`): it adds uv to GITHUB_PATH (the real OS PATH),
+        # so the Python language server's subprocess `shutil.which("uv"/"uvx")` finds it on Windows.
+        # The bash installer only puts uv on the Git-Bash MSYS PATH, invisible to Windows processes.
+        uses: astral-sh/setup-uv@v5
       - name: Cache uv virtualenv
         id: cache-uv
         uses: actions/cache@v3
@@ -75,12 +102,15 @@ jobs:
         shell: bash
         run: uv pip list
       - name: Check formatting
+        # Lint is batch-independent; run it only once per OS instead of in every batch.
+        if: matrix.batch == 'catch-all'
         shell: bash
         run: uv run poe lint
       # Add Go bin directory to PATH for this workflow
       # GITHUB_PATH is a special file that GitHub Actions uses to modify PATH
       # Writing to this file adds the directory to the PATH for subsequent steps
       - name: Cache Go binaries
+        if: matrix.batch == 'native'
         id: cache-go-binaries
         uses: actions/cache@v3
         with:
@@ -89,7 +119,7 @@ jobs:
             ~/.cache/go-build
           key: go-binaries-${{ runner.os }}-gopls-latest
       - name: Install gopls
-        if: steps.cache-go-binaries.outputs.cache-hit != 'true'
+        if: matrix.batch == 'native' && steps.cache-go-binaries.outputs.cache-hit != 'true'
         shell: bash
         run: go install golang.org/x/tools/gopls@latest
 #      Erlang currently not tested in CI, random hangings on macos, always hangs on ubuntu
@@ -107,10 +137,12 @@ jobs:
 #          # Ensure erlang_ls is in PATH
 #          echo "$HOME/.local/bin" >> $GITHUB_PATH
       - name: Install clojure tools
+        if: matrix.batch == 'jvm'
         uses: DeLaGuardo/setup-clojure@13.4
         with:
           cli: latest
       - name: Install ccls (C/C++ Language Server)
+        if: matrix.batch == 'native'
         shell: bash
         run: |
           if [[ "${{ runner.os }}" == "Linux" ]]; then
@@ -129,18 +161,22 @@ jobs:
             exit 1
           fi
       - name: Setup Java (for JVM based languages)
+        if: matrix.batch == 'jvm'
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '21'
       - name: Setup .NET SDK (for F# and C# languages)
+        if: matrix.batch == 'jvm'
         uses: actions/setup-dotnet@v4
         with:
           dotnet-version: '10.0.x'
       - name: List .NET runtimes
+        if: matrix.batch == 'jvm'
         shell: bash
         run: dotnet --list-runtimes
       - name: Install Terraform
+        if: matrix.batch == 'other-langs'
         uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: "1.5.0"
@@ -151,25 +187,46 @@ jobs:
       # Installation of swift with the action screws with installation of ruby on macOS for some reason
       # We can try again when version 3 of the action is released, where they will also use swiftly
       # Until then, we use custom code to install swift. Sourcekit-lsp is installed automatically with swift
+      - name: Cache Swift toolchain (swiftly)
+        if: runner.os == 'macOS' && matrix.batch == 'swift'
+        id: cache-swiftly
+        uses: actions/cache@v3
+        with:
+          # macOS keeps the toolchain (incl. sourcekit-lsp) under ~/Library/Developer/Toolchains,
+          # NOT under ~/.swiftly -- both must be cached or the toolchain is lost on restore.
+          path: |
+            ~/.swiftly
+            ~/Library/Developer/Toolchains
+          key: swiftly-${{ runner.os }}-6.1.2-v2
       - name: Install Swift with swiftly (macOS)
-        if: runner.os == 'macOS'
+        if: runner.os == 'macOS' && matrix.batch == 'swift'
         run: |
-          echo "=== Installing swiftly on macOS ==="
-          curl -O https://download.swift.org/swiftly/darwin/swiftly.pkg && \
-          installer -pkg swiftly.pkg -target CurrentUserHomeDirectory && \
-          ~/.swiftly/bin/swiftly init --quiet-shell-followup && \
-          . "${SWIFTLY_HOME_DIR:-$HOME/.swiftly}/env.sh" && \
+          set -e
+          # Install swiftly itself only if its binary isn't already present (e.g. restored from cache).
+          if [ ! -x "$HOME/.swiftly/bin/swiftly" ]; then
+            echo "=== Installing swiftly ==="
+            curl -O https://download.swift.org/swiftly/darwin/swiftly.pkg
+            installer -pkg swiftly.pkg -target CurrentUserHomeDirectory
+            "$HOME/.swiftly/bin/swiftly" init --quiet-shell-followup
+          fi
+          . "${SWIFTLY_HOME_DIR:-$HOME/.swiftly}/env.sh"
           hash -r
-          swiftly install --use 6.1.2
-          swiftly use 6.1.2
-          echo "~/.swiftly/bin" >> $GITHUB_PATH
-          echo "Swiftly installed successfully"
-          # Verify sourcekit-lsp is working before proceeding
-          echo "=== Verifying sourcekit-lsp installation ==="
-          which sourcekit-lsp || echo "Warning: sourcekit-lsp not found in PATH"
-          sourcekit-lsp --help || echo "Warning: sourcekit-lsp not responding"
+          # Always run install: it downloads on a cache miss and is a fast no-op when the toolchain is
+          # genuinely present from cache. No `--use` -- that triggers the interactive .swift-version prompt.
+          swiftly install 6.1.2
+          # Locate sourcekit-lsp directly (macOS toolchains live under ~/Library/Developer/Toolchains)
+          # and put its directory on PATH -- avoids `swiftly use`, which prompts/hangs in CI.
+          SK="$(find "$HOME/Library/Developer/Toolchains" "$HOME/.swiftly" -name sourcekit-lsp 2>/dev/null | head -1)"
+          if [ -z "$SK" ]; then
+            echo "ERROR: sourcekit-lsp not found after Swift install. Searched:"
+            ls -la "$HOME/Library/Developer/Toolchains" 2>/dev/null || echo "  (no ~/Library/Developer/Toolchains)"
+            ls -la "$HOME/.swiftly" 2>/dev/null || echo "  (no ~/.swiftly)"
+            exit 1
+          fi
+          echo "sourcekit-lsp: $SK"
+          dirname "$SK" >> "$GITHUB_PATH"
       - name: Install Swift with swiftly (Ubuntu)
-        if: runner.os == 'Linux'
+        if: runner.os == 'Linux' && matrix.batch == 'swift'
         run: |
           echo "=== Installing swiftly on Ubuntu ==="
           # Install dependencies BEFORE Swift to avoid exit code 1
@@ -190,38 +247,48 @@ jobs:
           which sourcekit-lsp || echo "Warning: sourcekit-lsp not found in PATH"
           sourcekit-lsp --help || echo "Warning: sourcekit-lsp not responding"
       - name: Install Ruby
+        if: matrix.batch == 'scripting'
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: '3.4'
       - name: Install Ruby language server
+        if: matrix.batch == 'scripting'
         shell: bash
         run: gem install ruby-lsp
       - name: Install Zig
+        if: matrix.batch == 'native'
         uses: mlugg/setup-zig@v2
         with:
           version: 0.14.1
+      - name: Cache ZLS (Zig Language Server)
+        if: matrix.batch == 'native'
+        id: cache-zls
+        uses: actions/cache@v3
+        with:
+          path: ~/.zls
+          key: zls-${{ runner.os }}-0.14.0
       - name: Install ZLS (Zig Language Server)
+        if: matrix.batch == 'native'
         shell: bash
         run: |
-          if [[ "${{ runner.os }}" == "Linux" ]]; then
-            wget https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-linux.tar.xz
-            tar -xf zls-x86_64-linux.tar.xz
-            sudo mv zls /usr/local/bin/
-            rm zls-x86_64-linux.tar.xz
-          elif [[ "${{ runner.os }}" == "macOS" ]]; then
-            wget https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-macos.tar.xz
-            tar -xf zls-x86_64-macos.tar.xz
-            sudo mv zls /usr/local/bin/
-            rm zls-x86_64-macos.tar.xz
-          elif [[ "${{ runner.os }}" == "Windows" ]]; then
-            curl -L -o zls.zip https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-windows.zip
-            unzip -o zls.zip
-            mkdir -p "$HOME/bin"
-            mv zls.exe "$HOME/bin/"
-            echo "$HOME/bin" >> $GITHUB_PATH
-            rm zls.zip
+          ZLS_DIR="$HOME/.zls"
+          if [[ "${{ steps.cache-zls.outputs.cache-hit }}" != "true" ]]; then
+            mkdir -p "$ZLS_DIR"
+            if [[ "${{ runner.os }}" == "Linux" ]]; then
+              wget -O /tmp/zls.tar.xz https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-linux.tar.xz
+              tar -xf /tmp/zls.tar.xz -C "$ZLS_DIR" && rm /tmp/zls.tar.xz
+            elif [[ "${{ runner.os }}" == "macOS" ]]; then
+              wget -O /tmp/zls.tar.xz https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-macos.tar.xz
+              tar -xf /tmp/zls.tar.xz -C "$ZLS_DIR" && rm /tmp/zls.tar.xz
+            elif [[ "${{ runner.os }}" == "Windows" ]]; then
+              curl -L -o zls.zip https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-windows.zip
+              unzip -o zls.zip -d "$ZLS_DIR" && rm zls.zip
+            fi
+            chmod +x "$ZLS_DIR/zls" 2>/dev/null || true
           fi
+          echo "$ZLS_DIR" >> "$GITHUB_PATH"
       - name: Install verible-verilog-ls (SystemVerilog Language Server)
+        if: matrix.batch == 'other-langs'
         shell: bash
         run: |
           VERIBLE_VERSION="v0.0-4051-g9fdb4057"
@@ -252,6 +319,7 @@ jobs:
             echo "WARNING: verible-verilog-ls not found in PATH"
           fi
       - name: Install Lua Language Server
+        if: matrix.batch == 'scripting'
         shell: bash
         run: |
           LUA_LS_VERSION="3.15.0"
@@ -297,18 +365,22 @@ jobs:
             rm lua-ls.zip
           fi
       - name: Install ansible-core and ansible-lint (for Ansible language server tests)
+        if: matrix.batch == 'other-langs'
         shell: bash
         run: uv run pip install ansible-core ansible-lint
       - name: Install Haxe
+        if: matrix.batch == 'scripting'
         # The Haxe LS binary (server.js) is auto-downloaded and runs on Node.js,
         # but it delegates to the Haxe compiler for code analysis at runtime.
         uses: krdlab/setup-haxe@v2
         with:
           haxe-version: 4.3.7
       - name: Install Elm
+        if: matrix.batch == 'web'
         shell: bash
         run: npm install -g elm@0.19.1-6
       - name: Install Regal (Rego Language Server)
+        if: matrix.batch == 'other-langs'
         shell: bash
         run: |
           REGAL_VERSION="0.39.0"
@@ -335,35 +407,39 @@ jobs:
             mv regal.exe "$HOME/bin/"
             echo "$HOME/bin" >> $GITHUB_PATH
           fi
+      - name: Cache Free Pascal Compiler
+        if: matrix.batch == 'native'
+        id: cache-fpc
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/fpc
+            ~/fpcsrc
+          key: fpc-${{ runner.os }}-3.2.2
       - name: Install Free Pascal Compiler
+        if: matrix.batch == 'native'
         shell: bash
         run: |
+          # Only the downloads are gated on the cache; the PP/FPCDIR/PATH detection always runs
+          # (it works the same on freshly-downloaded or cache-restored ~/fpc + ~/fpcsrc).
+          FPC_CACHE_HIT="${{ steps.cache-fpc.outputs.cache-hit }}"
+          FPC_VERSION="3.2.2"
           if [[ "${{ runner.os }}" == "Linux" ]]; then
             sudo apt-get update
             sudo apt-get install -y fpc fpc-source
-            # Set environment variables for pasls
             echo "PP=/usr/bin/fpc" >> $GITHUB_ENV
-            # Find FPC source directory (version may vary) - remove trailing slash
             FPCDIR=$(ls -d /usr/share/fpcsrc/*/ 2>/dev/null | head -1 | sed 's:/$::')
-            if [[ -z "$FPCDIR" ]]; then
-              FPCDIR="/usr/share/fpcsrc"
-            fi
+            [[ -z "$FPCDIR" ]] && FPCDIR="/usr/share/fpcsrc"
             echo "FPCDIR=$FPCDIR" >> $GITHUB_ENV
-            echo "=== FPC source directory structure ==="
-            ls -la "$FPCDIR" || echo "FPCDIR not found"
-            ls -la "$FPCDIR/rtl" 2>/dev/null || echo "rtl subdirectory not found"
           elif [[ "${{ runner.os }}" == "macOS" ]]; then
-            brew install fpc
-            # Download FPC source from SourceForge (fpc-src-laz cask is incompatible with ARM64)
-            FPC_VERSION="3.2.2"
-            curl -L -o fpc-source.tar.gz "https://sourceforge.net/projects/freepascal/files/Source/${FPC_VERSION}/fpc-${FPC_VERSION}.source.tar.gz/download"
-            mkdir -p "$HOME/fpcsrc"
-            tar -xzf fpc-source.tar.gz -C "$HOME/fpcsrc"
-            rm fpc-source.tar.gz
-            # Check extracted directory structure (might be nested)
-            echo "=== Extracted FPC source structure ==="
-            ls -la "$HOME/fpcsrc"
-            # Find the actual FPC source root (contains rtl, packages, etc.)
+            command -v fpc >/dev/null || brew install fpc
+            if [[ "$FPC_CACHE_HIT" != "true" ]]; then
+              # FPC source from SourceForge (fpc-src-laz cask is incompatible with ARM64)
+              curl -L -o fpc-source.tar.gz "https://sourceforge.net/projects/freepascal/files/Source/${FPC_VERSION}/fpc-${FPC_VERSION}.source.tar.gz/download"
+              mkdir -p "$HOME/fpcsrc"
+              tar -xzf fpc-source.tar.gz -C "$HOME/fpcsrc"
+              rm fpc-source.tar.gz
+            fi
             if [[ -d "$HOME/fpcsrc/fpc-${FPC_VERSION}/rtl" ]]; then
               FPCDIR="$HOME/fpcsrc/fpc-${FPC_VERSION}"
             elif [[ -d "$HOME/fpcsrc/fpc-${FPC_VERSION}/fpc-${FPC_VERSION}/rtl" ]]; then
@@ -373,34 +449,26 @@ jobs:
             fi
             echo "PP=$(which fpc)" >> $GITHUB_ENV
             echo "FPCDIR=$FPCDIR" >> $GITHUB_ENV
-            echo "=== FPC source directory ==="
-            ls -la "$FPCDIR" || echo "FPCDIR not found"
           elif [[ "${{ runner.os }}" == "Windows" ]]; then
-            FPC_VERSION="3.2.2"
-            # Download freepascal-ootb (includes FPC compiler)
-            curl -L -o fpc-ootb.zip https://github.com/fredvs/freepascal-ootb/releases/download/${FPC_VERSION}/fpc-ootb-322-x86_64-win64.zip
-            mkdir -p "$HOME/fpc"
-            unzip -q fpc-ootb.zip -d "$HOME/fpc"
-            rm fpc-ootb.zip
-            # Download FPC source from SourceForge (fpc-ootb only has compiled units, not source)
-            curl -L -o fpc-source.zip "https://sourceforge.net/projects/freepascal/files/Source/${FPC_VERSION}/fpc-${FPC_VERSION}.source.zip/download"
-            mkdir -p "$HOME/fpcsrc"
-            unzip -q fpc-source.zip -d "$HOME/fpcsrc"
-            rm fpc-source.zip
-            # Find fpc executable (fpc-ootb uses fpc-ootb.exe as the compiler)
-            echo "=== FPC directory structure ==="
-            find "$HOME/fpc" -name "*.exe" -type f 2>/dev/null | head -10
+            if [[ "$FPC_CACHE_HIT" != "true" ]]; then
+              # freepascal-ootb (compiler) + source (ootb ships compiled units only, not source)
+              curl -L -o fpc-ootb.zip https://github.com/fredvs/freepascal-ootb/releases/download/${FPC_VERSION}/fpc-ootb-322-x86_64-win64.zip
+              mkdir -p "$HOME/fpc"
+              unzip -q fpc-ootb.zip -d "$HOME/fpc"
+              rm fpc-ootb.zip
+              curl -L -o fpc-source.zip "https://sourceforge.net/projects/freepascal/files/Source/${FPC_VERSION}/fpc-${FPC_VERSION}.source.zip/download"
+              mkdir -p "$HOME/fpcsrc"
+              unzip -q fpc-source.zip -d "$HOME/fpcsrc"
+              rm fpc-source.zip
+            fi
             FPC_EXE=$(find "$HOME/fpc" -name "fpc-ootb-64.exe" -type f 2>/dev/null | head -1)
             echo "Found FPC executable: $FPC_EXE"
-            echo "Found FPC source dir: $HOME/fpcsrc/fpc-${FPC_VERSION}"
-            # Set environment variables for pasls
             echo "PP=$FPC_EXE" >> $GITHUB_ENV
             echo "FPCDIR=$HOME/fpcsrc/fpc-${FPC_VERSION}" >> $GITHUB_ENV
-            # Add FPC bin directory to PATH
-            FPC_BIN_DIR=$(dirname "$FPC_EXE")
-            echo "$FPC_BIN_DIR" >> $GITHUB_PATH
+            echo "$(dirname "$FPC_EXE")" >> $GITHUB_PATH
           fi
       - name: Verify FPC installation
+        if: matrix.batch == 'native'
         shell: bash
         run: |
           echo "=== Environment variables ==="
@@ -441,11 +509,12 @@ jobs:
             echo "ERROR: FPCDIR does not exist: $FPCDIR"
             exit 1
           fi
-      - name: Build Lean 4 test project
-        uses: leanprover/lean-action@v1
-        with:
-          lake-package-directory: test/resources/repos/lean4/test_repo
+      # Lean 4 is intentionally NOT installed/built here: the toolchain is ~1 GB (it *is* the language
+      # server) and lean4 is already covered by the Linux maximal container (test-parallel.yml's niche
+      # batch). With no `lean` on PATH, the central language_tests_enabled(Language.LEAN4) guard simply
+      # skips lean4 on the Windows/macOS runners -- no per-OS toolchain download.
       - name: Install npm deps for Angular test repo
+        if: matrix.batch == 'web'
         shell: bash
         run: |
           if [ -d test/resources/repos/angular/test_repo ]; then
@@ -457,17 +526,36 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.serena/language_servers/static
-          key: language-servers-${{ runner.os }}-v1
+          key: language-servers-${{ runner.os }}-${{ matrix.batch }}-v1
           restore-keys: |
-            language-servers-${{ runner.os }}-
+            language-servers-${{ runner.os }}-${{ matrix.batch }}-
       - name: Report free disk space
         if: runner.os == 'Linux'
         run: |
           echo "Free disk space before tests:"
           df -h
+      - name: Compute marker expression for this batch
+        id: markers
+        shell: bash
+        run: |
+          case "${{ matrix.batch }}" in
+            jvm)         expr="$MARKERS_JVM" ;;
+            native)      expr="$MARKERS_NATIVE" ;;
+            swift)       expr="$MARKERS_SWIFT" ;;
+            web)         expr="$MARKERS_WEB" ;;
+            python)      expr="$MARKERS_PYTHON" ;;
+            scripting)   expr="$MARKERS_SCRIPTING" ;;
+            other-langs) expr="$MARKERS_OTHER_LANGS" ;;
+            catch-all)   expr="not ($MARKERS_JVM or $MARKERS_NATIVE or $MARKERS_SWIFT or $MARKERS_WEB or $MARKERS_PYTHON or $MARKERS_SCRIPTING or $MARKERS_OTHER_LANGS)" ;;
+            *)           echo "Unknown batch: ${{ matrix.batch }}" >&2; exit 1 ;;
+          esac
+          echo "Marker expression: $expr"
+          echo "expr=$expr" >> "$GITHUB_OUTPUT"
       - name: Test with pytest
         shell: bash
-        run: uv run poe test -q --tb=short
+        run: uv run poe test -m "${{ steps.markers.outputs.expr }}" -q --tb=short
       - name: Type-checking with mypy
+        # Type-checking is batch-independent; run it only once per OS instead of in every batch.
+        if: matrix.batch == 'catch-all'
         shell: bash
         run: uv run poe type-check