Scala: wait for Metals' reported work, not a fixed 5 seconds (#1859)

Cross-file queries waited 5s after the first file was opened. On a cold Metals
that is long before it has imported the build, indexed, and — crucially —
compiled: references come from SemanticDB, which the build server writes as it
compiles, so a query made in between returns some of the references rather than
none, with nothing to mark it partial. Against a ~700-file Scala 3 build the
first find-references returned 124 of 242.

Metals already reports all of this as work-done progress; Serena did not declare
`window.workDoneProgress`, so none of it arrived. Declare it, track the tokens,
and wait for them before the first cross-file query — the same event-based shape
as TypeScript's indexing wait (#1639), whose hooks this uses.

Readiness is "nothing outstanding for a moment", not "nothing outstanding":
Metals hands off between its phases rather than overlapping them, so the token
set empties for around a second between the import ending and indexing starting,
and a plain drain returns in that gap. Measured on the same build, the first
find-references now returns all 242, taking 32s rather than 22s.

src/solidlsp/language_servers/scala_language_server.py

@@ -6,6 +6,8 @@ import logging
 import os
 import shutil
 import subprocess
+import threading
+import time
 from enum import Enum
 
 from overrides import override
@@ -31,6 +33,9 @@ DEFAULT_ON_STALE_LOCK = "auto-clean"
 DEFAULT_LOG_MULTI_INSTANCE_NOTICE = True
 DEFAULT_AUTO_IMPORT_BUILD = True
 DEFAULT_PROJECT_ROOT_SCAN_DEPTH = 3
+DEFAULT_INDEXING_TIMEOUT = 180.0
+DEFAULT_INDEXING_START_GRACE = 15.0
+DEFAULT_INDEXING_QUIET_PERIOD = 3.0
 
 # The `window/showMessageRequest` actions Serena answers affirmatively: the ones standing between
 # an un-imported workspace and a build server. Everything else is dismissed, since a prompt we do
@@ -81,6 +86,139 @@ BUILD_ROOT_MARKER_JSON_DIRS = (".bloop", ".bsp")
 BUILD_ROOT_SCAN_SKIP_DIRS = frozenset({"node_modules", "out", "project", "src", "target", "venv"})
 
 
+class IndexingOutcome(Enum):
+    """How a wait for Metals' outstanding work ended."""
+
+    NO_WORK = "no-work"
+    """Metals reported nothing at all within the start grace — it may not report progress."""
+
+    IDLE = "idle"
+    """Every task Metals reported has finished."""
+
+    TIMEOUT = "timeout"
+    """Work was still outstanding when the timeout expired."""
+
+
+class MetalsProgressTracker:
+    """
+    Tracks the work-done progress Metals reports, so that a cross-file query can wait for its
+    import, indexing and compilation rather than for a fixed period. References in particular
+    are served from SemanticDB, which only exists once the build server has compiled the
+    sources, and that finishes well after indexing does.
+
+    Metals keeps reporting progress for the rest of the session — it loads a presentation
+    compiler for essentially every file it is asked about — so the tracker answers "is anything
+    outstanding right now", never "has the server finished for good".
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._active: dict[str, str] = {}
+        self._seen_work = False
+        self._idle = threading.Event()
+        self._idle.set()
+
+    def expect_work(self) -> None:
+        """
+        Mark the tracker busy ahead of an action that is expected to make Metals do something,
+        so that a wait started before the first token arrives blocks rather than returning at once.
+        """
+        self._idle.clear()
+
+    def on_create(self, params: dict) -> dict:
+        """
+        Handle `window/workDoneProgress/create`, which precedes the token's first notification.
+
+        Tracking from creation rather than from `begin` means work announced just before a query
+        is waited for even if it has not started yet.
+
+        :param params: the request's `WorkDoneProgressCreateParams`
+        :return: the empty result the request expects
+        """
+        with self._lock:
+            self._active.setdefault(str(params.get("token", "")), "")
+            self._seen_work = True
+            self._idle.clear()
+        return {}
+
+    def on_progress(self, params: dict) -> None:
+        """
+        Handle a `$/progress` notification, tracking the token's title for diagnostics.
+
+        :param params: the notification's `ProgressParams`
+        """
+        token = str(params.get("token", ""))
+        value = params.get("value") or {}
+        kind = value.get("kind")
+        if kind == "begin":
+            title = str(value.get("title", ""))
+            with self._lock:
+                self._active[token] = title
+                self._seen_work = True
+                self._idle.clear()
+            log.info(f"Metals progress [{token}] started: {title}")
+        elif kind == "end":
+            with self._lock:
+                title = self._active.pop(token, "")
+                if not self._active:
+                    self._idle.set()
+            log.info(f"Metals progress [{token}] ended: {title}")
+
+    def wait_until_idle(self, timeout: float, start_grace: float, quiet_period: float) -> IndexingOutcome:
+        """
+        Wait for everything Metals is doing to finish.
+
+        Metals hands off between its phases rather than overlapping them — the import ends, then
+        indexing begins, then a compilation per module — so its set of tokens empties for a
+        second or so in between. Readiness is therefore "nothing outstanding for `quiet_period`",
+        not "nothing outstanding", which would return in the first such gap.
+
+        :param timeout: how long to wait, in seconds, once work is known to be outstanding
+        :param start_grace: how long to wait, in seconds, for work to appear at all
+        :param quiet_period: how long, in seconds, Metals must report nothing to count as finished
+        :return: how the wait ended
+        """
+        grace_deadline = time.monotonic() + start_grace
+        while time.monotonic() < grace_deadline:
+            with self._lock:
+                if self._active or self._seen_work:
+                    break
+            time.sleep(0.05)
+
+        with self._lock:
+            if not self._active and not self._seen_work:
+                self._idle.set()
+                return IndexingOutcome.NO_WORK
+
+        deadline = time.monotonic() + timeout
+        while True:
+            remaining = deadline - time.monotonic()
+            if remaining <= 0 or not self._idle.wait(timeout=remaining):
+                return IndexingOutcome.TIMEOUT
+            if self._stays_idle(quiet_period, deadline):
+                return IndexingOutcome.IDLE
+
+    def _stays_idle(self, quiet_period: float, deadline: float) -> bool:
+        """
+        :param quiet_period: how long, in seconds, nothing new must be reported
+        :param deadline: the monotonic time at which to give up regardless
+        :return: whether the tracker stayed idle for the whole quiet period
+        """
+        quiet_end = time.monotonic() + quiet_period
+        while time.monotonic() < min(quiet_end, deadline):
+            if not self._idle.is_set():
+                return False
+            time.sleep(0.05)
+        return self._idle.is_set()
+
+    def describe(self) -> str:
+        """:return: compact diagnostic state of Metals' outstanding work."""
+        with self._lock:
+            titles = sorted(title for title in self._active.values() if title)
+            idle = self._idle.is_set()
+        return f"idle={idle}, active={', '.join(titles) or '<none>'}"
+
+
 class StaleLockMode(Enum):
     """Mode for handling stale Metals H2 database locks."""
 
@@ -238,6 +376,18 @@ def _parse_project_root_scan_depth(value: object) -> int:
     return value
 
 
+def _parse_positive_float(value: object, name: str, default: float) -> float:
+    """
+    Validate a positive-number setting, falling back to the default if it is unusable.
+    """
+    if value is None:
+        return default
+    if isinstance(value, bool) or not isinstance(value, int | float) or value <= 0:
+        log.warning(f"Invalid {name} value {value!r}, expected a positive number; using {default}")
+        return default
+    return float(value)
+
+
 def _get_scala_settings(solidlsp_settings: SolidLSPSettings) -> dict[str, object]:
     """
     Extract Scala-specific settings with defaults applied.
@@ -250,6 +400,9 @@ def _get_scala_settings(solidlsp_settings: SolidLSPSettings) -> dict[str, object
         - auto_import_build: bool
         - project_roots: list[str] | None
         - project_root_scan_depth: int
+        - indexing_timeout: float
+        - indexing_start_grace: float
+        - indexing_quiet_period: float
     """
     from solidlsp.ls_config import LanguageServerId
 
@@ -261,6 +414,9 @@ def _get_scala_settings(solidlsp_settings: SolidLSPSettings) -> dict[str, object
         "auto_import_build": DEFAULT_AUTO_IMPORT_BUILD,
         "project_roots": None,
         "project_root_scan_depth": DEFAULT_PROJECT_ROOT_SCAN_DEPTH,
+        "indexing_timeout": DEFAULT_INDEXING_TIMEOUT,
+        "indexing_start_grace": DEFAULT_INDEXING_START_GRACE,
+        "indexing_quiet_period": DEFAULT_INDEXING_QUIET_PERIOD,
     }
 
     if not solidlsp_settings.ls_specific_settings:
@@ -284,6 +440,13 @@ def _get_scala_settings(solidlsp_settings: SolidLSPSettings) -> dict[str, object
         "auto_import_build": scala_settings.get("auto_import_build", DEFAULT_AUTO_IMPORT_BUILD),
         "project_roots": _parse_project_roots(scala_settings.get("project_roots")),
         "project_root_scan_depth": _parse_project_root_scan_depth(scala_settings.get("project_root_scan_depth")),
+        "indexing_timeout": _parse_positive_float(scala_settings.get("indexing_timeout"), "indexing_timeout", DEFAULT_INDEXING_TIMEOUT),
+        "indexing_start_grace": _parse_positive_float(
+            scala_settings.get("indexing_start_grace"), "indexing_start_grace", DEFAULT_INDEXING_START_GRACE
+        ),
+        "indexing_quiet_period": _parse_positive_float(
+            scala_settings.get("indexing_quiet_period"), "indexing_quiet_period", DEFAULT_INDEXING_QUIET_PERIOD
+        ),
     }
 
 
@@ -312,6 +475,20 @@ class ScalaLanguageServer(SolidLanguageServer):
             project_roots: ['backend', 'tooling/plugin']
             # How many levels below the repository root auto-detection searches
             project_root_scan_depth: 3
+            # How long to wait, in seconds, for Metals to finish indexing and compiling
+            # before the first cross-file query (see "Indexing")
+            indexing_timeout: 180
+            # How long to wait for that work to *begin* before concluding there is none
+            indexing_start_grace: 15
+            # How long Metals must report nothing for its work to count as finished
+            indexing_quiet_period: 3
+
+    Indexing:
+        Metals reports its import, indexing and compilation as LSP work-done progress, and
+        references are only complete once the build server has compiled the sources that
+        produce SemanticDB. The first cross-file query of a session therefore waits for that
+        work rather than for a fixed period; `indexing_timeout` bounds the wait, after which
+        the query proceeds against whatever Metals has so far.
 
     Build import:
         Metals asks, via `window/showMessageRequest`, whether to import a workspace it has not
@@ -342,7 +519,13 @@ class ScalaLanguageServer(SolidLanguageServer):
         for build_root in self._build_roots:
             self._check_metals_db_status(build_root, solidlsp_settings)
 
-        self._auto_import_build: bool = _get_scala_settings(solidlsp_settings)["auto_import_build"]  # type: ignore[assignment]
+        settings = _get_scala_settings(solidlsp_settings)
+        self._auto_import_build: bool = settings["auto_import_build"]  # type: ignore[assignment]
+        self._indexing_timeout: float = settings["indexing_timeout"]  # type: ignore[assignment]
+        self._indexing_start_grace: float = settings["indexing_start_grace"]  # type: ignore[assignment]
+        self._indexing_quiet_period: float = settings["indexing_quiet_period"]  # type: ignore[assignment]
+
+        self._progress = MetalsProgressTracker()
 
         scala_lsp_executable_path = self._setup_runtime_dependencies(config, solidlsp_settings)
         super().__init__(
@@ -552,18 +735,54 @@ class ScalaLanguageServer(SolidLanguageServer):
                 "copyWorksheetOutputProvider": False,
                 "doctorVisibilityProvider": False,
             },
-            "capabilities": {"textDocument": {"documentSymbol": {"hierarchicalDocumentSymbolSupport": True}}},
+            "capabilities": {
+                "textDocument": {"documentSymbol": {"hierarchicalDocumentSymbolSupport": True}},
+                # without this Metals never reports what it is doing, so there is nothing to
+                # wait on before the first cross-file query
+                "window": {"workDoneProgress": True},
+            },
         }
         return initialize_params
 
     def _answer_show_message_request(self, params: dict) -> dict | None:
         return choose_show_message_request_action(params, auto_import_build=self._auto_import_build)
 
+    @override
+    def _pre_open_for_cross_file_references(self) -> None:
+        if not self._has_waited_for_cross_file_references:
+            self._progress.expect_work()
+
+    @override
+    def _wait_for_cross_file_references_if_needed(self) -> None:
+        if self._has_waited_for_cross_file_references:
+            return
+
+        # Opening a file is what makes Metals connect to its build server, so the work only
+        # begins after the didOpen that precedes this call.
+        outcome = self._progress.wait_until_idle(
+            timeout=self._indexing_timeout,
+            start_grace=self._indexing_start_grace,
+            quiet_period=self._indexing_quiet_period,
+        )
+        if outcome == IndexingOutcome.NO_WORK:
+            log.info(f"Metals reported no work within {self._indexing_start_grace:.0f}s; proceeding")
+        elif outcome == IndexingOutcome.IDLE:
+            log.info("Metals indexing complete")
+        else:
+            log.warning(
+                "Metals was still working after %.0fs; proceeding, so cross-file results may be incomplete (%s)",
+                self._indexing_timeout,
+                self._progress.describe(),
+            )
+        self._has_waited_for_cross_file_references = True
+
     def _start_server(self) -> None:
         """
         Starts the Scala Language Server
         """
         self.server.on_request("window/showMessageRequest", self._answer_show_message_request)
+        self.server.on_request("window/workDoneProgress/create", self._progress.on_create)
+        self.server.on_notification("$/progress", self._progress.on_progress)
 
         log.info("Starting Scala server process")
         self.server.start()
@@ -573,7 +792,3 @@ class ScalaLanguageServer(SolidLanguageServer):
         initialize_params = self._create_initialize_params()
         self.server.send.initialize(initialize_params)
         self.server.notify.initialized({})
-
-    @override
-    def _get_wait_time_for_cross_file_referencing(self) -> float:
-        return 5