From 7bef5a6ee35214bd1bee74429827baae83ae10f2 Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Sun, 7 Dec 2025 20:52:26 +0700 Subject: [PATCH 1/5] Changed dashboard listen address to 127.0.0.1 Also minor refactoring in SerenaConfig that allows not repeating the defaults (as field defaults and in `from_config_file`) --- src/serena/agent.py | 8 +++++-- src/serena/config/serena_config.py | 35 ++++++++++++++++++------------ src/serena/dashboard.py | 6 ++--- src/serena/util/general.py | 22 ++++++++++++++++++- 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index c02e1051..2d728610 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -271,8 +271,12 @@ class SerenaAgent: if self.serena_config.web_dashboard: self._dashboard_thread, port = SerenaDashboardAPI( get_memory_log_handler(), tool_names, agent=self, tool_usage_stats=self._tool_usage_stats - ).run_in_thread() - dashboard_url = f"http://127.0.0.1:{port}/dashboard/index.html" + ).run_in_thread(host=self.serena_config.web_dashboard_host) + host_for_url = self.serena_config.web_dashboard_host + # this usually means we are in docker, so we use localhost for URL (the user has to port-forward then) + if host_for_url == "0.0.0.0": + host_for_url = "127.0.0.1" + dashboard_url = f"http://{self.serena_config.web_dashboard_host}:{port}/dashboard/index.html" log.info("Serena web dashboard started at %s", dashboard_url) if self.serena_config.web_dashboard_open_on_launch: # open the dashboard URL in the default web browser (using a separate process to control diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index c5345d48..cfe3f0b1 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -28,7 +28,7 @@ from serena.constants import ( SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME, ) -from serena.util.general import load_yaml, save_yaml +from serena.util.general import get_dataclass_default, load_yaml, save_yaml from serena.util.inspection import determine_programming_language_composition from solidlsp.ls_config import Language @@ -342,6 +342,8 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): trace_lsp_communication: bool = False web_dashboard: bool = True web_dashboard_open_on_launch: bool = True + web_dashboard_host: str = "127.0.0.1" + """Will be automatically set to '0.0.0.0' when running in Docker. Not loaded from config file.""" tool_timeout: float = DEFAULT_TOOL_TIMEOUT loaded_commented_yaml: CommentedMap | None = None config_file_path: str | None = None @@ -373,6 +375,10 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): CONFIG_FILE = "serena_config.yml" CONFIG_FILE_DOCKER = "serena_config.docker.yml" # Docker-specific config file; auto-generated if missing, mounted via docker-compose for user customization + def __post_init__(self) -> None: + if is_running_in_docker(): + self.web_dashboard_host = "0.0.0.0" + def _tostring_includes(self) -> list[str]: return ["config_file_path"] @@ -456,23 +462,24 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): instance.projects.append(project) # set other configuration parameters + def get_value_or_default(field_name: str) -> Any: + return loaded_commented_yaml.get(field_name, get_dataclass_default(SerenaConfig, field_name)) + if is_running_in_docker(): instance.gui_log_window_enabled = False # not supported in Docker else: - instance.gui_log_window_enabled = loaded_commented_yaml.get("gui_log_window", False) + instance.gui_log_window_enabled = get_value_or_default("gui_log_window_enabled") instance.log_level = loaded_commented_yaml.get("log_level", loaded_commented_yaml.get("gui_log_level", logging.INFO)) - instance.web_dashboard = loaded_commented_yaml.get("web_dashboard", True) - instance.web_dashboard_open_on_launch = loaded_commented_yaml.get("web_dashboard_open_on_launch", True) - instance.tool_timeout = loaded_commented_yaml.get("tool_timeout", DEFAULT_TOOL_TIMEOUT) - instance.trace_lsp_communication = loaded_commented_yaml.get("trace_lsp_communication", False) - instance.excluded_tools = loaded_commented_yaml.get("excluded_tools", []) - instance.included_optional_tools = loaded_commented_yaml.get("included_optional_tools", []) - instance.jetbrains = loaded_commented_yaml.get("jetbrains", False) - instance.token_count_estimator = loaded_commented_yaml.get( - "token_count_estimator", RegisteredTokenCountEstimator.TIKTOKEN_GPT4O.name - ) - instance.default_max_tool_answer_chars = loaded_commented_yaml.get("default_max_tool_answer_chars", 150_000) - instance.ls_specific_settings = loaded_commented_yaml.get("ls_specific_settings", {}) + instance.web_dashboard = get_value_or_default("web_dashboard") + instance.web_dashboard_open_on_launch = get_value_or_default("web_dashboard_open_on_launch") + instance.tool_timeout = get_value_or_default("tool_timeout") + instance.trace_lsp_communication = get_value_or_default("trace_lsp_communication") + instance.excluded_tools = get_value_or_default("excluded_tools") + instance.included_optional_tools = get_value_or_default("included_optional_tools") + instance.jetbrains = get_value_or_default("jetbrains") + instance.token_count_estimator = get_value_or_default("token_count_estimator") + instance.default_max_tool_answer_chars = get_value_or_default("default_max_tool_answer_chars") + instance.ls_specific_settings = get_value_or_default("ls_specific_settings") # re-save the configuration file if any migrations were performed if num_project_migrations > 0: diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index 6005bcad..5b414e2d 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -560,7 +560,7 @@ class SerenaDashboardAPI: raise RuntimeError(f"No free ports found starting from {start_port}") - def run(self, host: str = "0.0.0.0", port: int = 0x5EDA) -> int: + def run(self, host: str, port: int) -> int: """ Runs the dashboard on the given host and port and returns the port number. """ @@ -572,8 +572,8 @@ class SerenaDashboardAPI: self._app.run(host=host, port=port, debug=False, use_reloader=False, threaded=True) return port - def run_in_thread(self) -> tuple[threading.Thread, int]: + def run_in_thread(self, host: str) -> tuple[threading.Thread, int]: port = self._find_first_free_port(0x5EDA) - thread = threading.Thread(target=lambda: self.run(port=port), daemon=True) + thread = threading.Thread(target=lambda: self.run(host=host, port=port), daemon=True) thread.start() return thread, port diff --git a/src/serena/util/general.py b/src/serena/util/general.py index 4a1397e2..f380c699 100644 --- a/src/serena/util/general.py +++ b/src/serena/util/general.py @@ -1,5 +1,6 @@ import os -from typing import Literal, overload +from dataclasses import MISSING, Field +from typing import Any, Literal, cast, overload from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap @@ -32,3 +33,22 @@ def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = Fa os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding=SERENA_FILE_ENCODING) as f: yaml.dump(data, f) + + +def get_dataclass_default(cls: type, field_name: str) -> Any: + """ + Gets the default value of a dataclass field. + + :param cls: The dataclass type. + :param field_name: The name of the field. + :return: The default value of the field (either from default or default_factory). + """ + field = cast(Field, cls.__dataclass_fields__[field_name]) # type: ignore[attr-defined] + + if field.default is not MISSING: + return field.default + + if field.default_factory is not MISSING: # default_factory is a function + return field.default_factory() + + raise AttributeError(f"{field_name} has no default") From 8bd13e44b3e67ee472b3df7e0a83257badd4a19c Mon Sep 17 00:00:00 2001 From: Michael Panchenko Date: Wed, 10 Dec 2025 16:00:04 +0700 Subject: [PATCH 2/5] Made dashboard_listen_address configurable (except when running in docker) --- src/serena/agent.py | 15 ++++++++------- src/serena/config/serena_config.py | 8 +++++--- src/serena/resources/serena_config.template.yml | 7 +++++++ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/serena/agent.py b/src/serena/agent.py index 2d728610..8b198c5a 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -18,7 +18,7 @@ from interprompt.jinja_template import JinjaTemplate from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition +from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, is_running_in_docker from serena.dashboard import SerenaDashboardAPI from serena.ls_manager import LanguageServerManager from serena.project import Project @@ -271,12 +271,13 @@ class SerenaAgent: if self.serena_config.web_dashboard: self._dashboard_thread, port = SerenaDashboardAPI( get_memory_log_handler(), tool_names, agent=self, tool_usage_stats=self._tool_usage_stats - ).run_in_thread(host=self.serena_config.web_dashboard_host) - host_for_url = self.serena_config.web_dashboard_host - # this usually means we are in docker, so we use localhost for URL (the user has to port-forward then) - if host_for_url == "0.0.0.0": - host_for_url = "127.0.0.1" - dashboard_url = f"http://{self.serena_config.web_dashboard_host}:{port}/dashboard/index.html" + ).run_in_thread(host=self.serena_config.web_dashboard_listen_address) + dashboard_listen_address = self.serena_config.web_dashboard_listen_address + if is_running_in_docker(): + # we use localhost for URL instead of the 0.0.0.0 that is configured for docker + # (the user has to port-forward then) + dashboard_listen_address = "127.0.0.1" + dashboard_url = f"http://{dashboard_listen_address}:{port}/dashboard/index.html" log.info("Serena web dashboard started at %s", dashboard_url) if self.serena_config.web_dashboard_open_on_launch: # open the dashboard URL in the default web browser (using a separate process to control diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index cfe3f0b1..6008f3d7 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -342,8 +342,8 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): trace_lsp_communication: bool = False web_dashboard: bool = True web_dashboard_open_on_launch: bool = True - web_dashboard_host: str = "127.0.0.1" - """Will be automatically set to '0.0.0.0' when running in Docker. Not loaded from config file.""" + web_dashboard_listen_address: str = "127.0.0.1" + """When running in Docker, this will be automatically set to '0.0.0.0' ignoring any configuration by the user.""" tool_timeout: float = DEFAULT_TOOL_TIMEOUT loaded_commented_yaml: CommentedMap | None = None config_file_path: str | None = None @@ -377,7 +377,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): def __post_init__(self) -> None: if is_running_in_docker(): - self.web_dashboard_host = "0.0.0.0" + self.web_dashboard_listen_address = "0.0.0.0" def _tostring_includes(self) -> list[str]: return ["config_file_path"] @@ -469,6 +469,8 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): instance.gui_log_window_enabled = False # not supported in Docker else: instance.gui_log_window_enabled = get_value_or_default("gui_log_window_enabled") + # in docker the post_init takes care of setting the address to 0.0.0.0, we ignore the user's setting + instance.web_dashboard_listen_address = get_value_or_default("web_dashboard_listen_address") instance.log_level = loaded_commented_yaml.get("log_level", loaded_commented_yaml.get("gui_log_level", logging.INFO)) instance.web_dashboard = get_value_or_default("web_dashboard") instance.web_dashboard_open_on_launch = get_value_or_default("web_dashboard_open_on_launch") diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index 6a6ae491..75aa5cff 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -20,12 +20,19 @@ web_dashboard: True # shows Serena's current session logs - as an alternative to the GUI log window which # is supported on all platforms. +web_dashboard_listen_address: 127.0.0.1 +# The address where the web dashboard will listen on (by default, localhost) +# Important: when running serena in docker, this configuration entry will be ignored and the +# address will be set to 0.0.0.0. (as it's running in a sandbox, this does not pose a safety issue) +# You can then port-forward it to a host of your choice, e.g., localhost + web_dashboard_open_on_launch: True # whether to open a browser window with the web dashboard when Serena starts (provided that web_dashboard # is enabled). If set to False, you can still open the dashboard manually by navigating to # http://localhost:24282/dashboard/ in your web browser (24282 = 0x5EDA, SErena DAshboard). # If you have multiple instances running, a higher port will be used; try port 24283, 24284, etc. + log_level: 20 # the minimum log level for the GUI log window and the dashboard (10 = debug, 20 = info, 30 = warning, 40 = error) From d452c2e413e3832f7bd9fd57435df3cdd4a50656 Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Wed, 10 Dec 2025 12:09:38 +0100 Subject: [PATCH 3/5] Do away with Docker-specific case distinctions; instead, configure Docker properly --- DOCKER.md | 66 +++++++++---------- Dockerfile | 28 +++----- compose.yaml | 23 +------ docs/02-usage/020_running.md | 26 ++++---- src/serena/agent.py | 12 ++-- src/serena/config/serena_config.py | 48 ++++---------- src/serena/dashboard.py | 1 + .../resources/serena_config.template.yml | 6 +- 8 files changed, 75 insertions(+), 135 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index 03432451..bb77040c 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -1,6 +1,6 @@ # Docker Setup for Serena (Experimental) -⚠️ **EXPERIMENTAL FEATURE**: The Docker setup for Serena is currently experimental and has several limitations. Please read this entire document before using Docker with Serena. +⚠️ **EXPERIMENTAL FEATURE**: The Docker setup for Serena is still experimental and has some limitations. Please read this entire document before using Docker with Serena. ## Overview @@ -12,27 +12,42 @@ Docker support allows you to run Serena in an isolated container environment, wh - **Consistent dependencies**: No need to manage language servers and dependencies on your host system - **Cross-platform support**: Works consistently across Windows, macOS, and Linux -## Important Limitations and Caveats +## Important Usage Pointers -### 1. Configuration File Conflicts +### Configuration -⚠️ **Critical**: Docker uses a separate configuration file (`serena_config.docker.yml`) to avoid path conflicts. When running in Docker: -- Container paths will be stored in the configuration (e.g., `/workspaces/serena/...`) -- These paths are incompatible with non-Docker usage -- After using Docker, you cannot directly switch back to non-Docker usage without manual configuration adjustment +Serena's configuration and log files are stored in the container in `/workspaces/serena/config/`. +Any local configuration you may have for Serena will not apply; the container uses its own separate configuration. -### 2. Project Activation Limitations +You can mount a local configuration/data directory to persist settings across container restarts +(which will also contain session log files). +Simply mount your local directory to `/workspaces/serena/config` in the container. +Initially, be sure to add a `serena_config.yml` file to the mounted directory which applies the following +special settings for Docker usage: +``` +# Disable the GUI log window since it's not supported in Docker +gui_log_window: False +# Listen on all interfaces for the web dashboard to be accessible from outside the container +web_dashboard_listen_address: 0.0.0.0 +# Disable opening the web dashboard on launch (not possible within the container) +web_dashboard_open_on_launch: False +``` +Set other configuration options as needed. + +### Project Activation Limitations - **Only mounted directories work**: Projects must be mounted as volumes to be accessible - Projects outside the mounted directories cannot be activated or accessed -- Default setup only mounts the current directory +- Since projects are not remembered across container restarts (unless you mount a local configuration as described above), + activate them using the full path (e.g. `/workspaces/projects/my-project`) when using dynamic project activation -### 3. GUI Window Disabled +### Language Support Limitations -- The GUI log window option is automatically disabled in Docker environments -- Use the web dashboard instead (see below) +The default Docker image does not include dependencies for languages that +require explicit system-level installations. +Only languages that install their requirements on the fly will work out of the box. -### 4. Dashboard Port Configuration +### Dashboard Port Configuration The web dashboard runs on port 24282 (0x5EDA) by default. You can configure this using environment variables: @@ -46,7 +61,7 @@ SERENA_DASHBOARD_PORT=8080 docker-compose up serena ⚠️ **Note**: If the local port is occupied, you'll need to specify a different port using the environment variable. -### 5. Line Ending Issues on Windows +### Line Ending Issues on Windows ⚠️ **Windows Users**: Be aware of potential line ending inconsistencies: - Files edited within the Docker container may use Unix line endings (LF) @@ -70,7 +85,7 @@ SERENA_DASHBOARD_PORT=8080 docker-compose up serena Note: Edit the `compose.yaml` file to customize volume mounts for your projects. -### Using Docker directly +### Building the Docker Image Manually ```bash # Build the image @@ -157,24 +172,3 @@ Ensure projects are properly mounted: - Check volume mounts in `docker-compose.yaml` - Use absolute paths for external projects - Verify permissions on mounted directories - -## Migration Path - -To switch between Docker and non-Docker usage: - -1. **Docker to Non-Docker**: - - Manually edit project paths in `serena_config.yml` - - Change container paths to host paths - - Or use separate config files for each environment - -2. **Non-Docker to Docker**: - - Projects will be re-registered with container paths - - Original config remains unchanged - -## Future Improvements - -We're working on: -- Automatic config migration between environments -- Better project path handling -- Dynamic port allocation -- Windows line-ending handling. diff --git a/Dockerfile b/Dockerfile index d84ed7d0..1dccaf40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,28 +53,21 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ # Set the working directory WORKDIR /workspaces/serena -# Development target -FROM base AS development # Copy all files for development COPY . /workspaces/serena/ -# Create virtual environment and install dependencies with dev extras -RUN uv venv -RUN . .venv/bin/activate -RUN uv pip install --all-extras -r pyproject.toml -e . -ENV PATH="/workspaces/serena/.venv/bin:${PATH}" +# Install sed +RUN apt-get update && apt-get install -y sed -# Entrypoint to ensure environment is activated -ENTRYPOINT ["/bin/bash", "-c", "source .venv/bin/activate && $0 $@"] +# Create Serena configuration +ENV SERENA_HOME=/workspaces/serena/config +RUN mkdir -p $SERENA_HOME +RUN cp src/serena/resources/serena_config.template.yml $SERENA_HOME/serena_config.yml +RUN sed -i 's/^gui_log_window: .*/gui_log_window: False/' $SERENA_HOME/serena_config.yml +RUN sed -i 's/^web_dashboard_listen_address: .*/web_dashboard_listen_address: 0.0.0.0/' $SERENA_HOME/serena_config.yml +RUN sed -i 's/^web_dashboard_open_on_launch: .*/web_dashboard_open_on_launch: False/' $SERENA_HOME/serena_config.yml -# Production target -FROM base AS production -# Copy only necessary files for production -COPY pyproject.toml /workspaces/serena/ -COPY README.md /workspaces/serena/ -COPY src/ /workspaces/serena/src/ - -# Create virtual environment and install dependencies (production only) +# Create virtual environment and install dependencies RUN uv venv RUN . .venv/bin/activate RUN uv pip install -r pyproject.toml -e . @@ -82,4 +75,3 @@ ENV PATH="/workspaces/serena/.venv/bin:${PATH}" # Entrypoint to ensure environment is activated ENTRYPOINT ["/bin/bash", "-c", "source .venv/bin/activate && $0 $@"] - diff --git a/compose.yaml b/compose.yaml index 48923e27..d7c8becd 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,7 +1,7 @@ services: serena: image: serena:latest - # To work with projects, you must mount them as volumes: + # To work with projects, you must mount them into /workspace/ in the container: # volumes: # - ./my-project:/workspace/my-project # - /path/to/another/project:/workspace/another-project @@ -16,23 +16,6 @@ services: - SERENA_DOCKER=1 command: - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0" - # Add the context for the IDE assistant - # - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide-assistant" + # Alternatively add further arguments, e.g. a context + # - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide" - serena-dev: - image: serena:dev - build: - context: ./ - dockerfile: Dockerfile - target: development - tty: true - stdin_open: true - environment: - - SERENA_DOCKER=1 - volumes: - - .:/workspaces/serena - ports: - - "${SERENA_PORT:-9121}:9121" # MCP server port - - "${SERENA_DASHBOARD_PORT:-24282}:24282" # Dashboard port - command: - - "uv run --directory . serena-mcp-server" diff --git a/docs/02-usage/020_running.md b/docs/02-usage/020_running.md index ede78ec2..231dd70c 100644 --- a/docs/02-usage/020_running.md +++ b/docs/02-usage/020_running.md @@ -47,28 +47,30 @@ As a consequence, you will need to specify paths when using CLI commands that wo ::: (docker)= -### Using Docker (Experimental) +### Using Docker -:::{warning} -Docker support is currently experimental with several limitations. -::: +The Docker approach offers several advantages: -You can run the Serena MCP server directly via docker as follows, +* better security isolation for shell command execution +* no need to install language servers and dependencies locally +* consistent environment across different systems + +You can run the Serena MCP server directly via Docker as follows, assuming that the projects you want to work on are all located in `/path/to/your/projects`: ```shell docker run --rm -i --network host -v /path/to/your/projects:/workspaces/projects ghcr.io/oraios/serena:latest serena ``` -Replace `/path/to/your/projects` with the absolute path to your projects directory. The Docker approach provides: +This command mounts your projects into the container under `/workspaces/projects`, so when working with projects, +you need to refer to them using the respective path (e.g. `/workspaces/projects/my-project`). -* Better security isolation for shell command execution -* No need to install language servers and dependencies locally -* Consistent environment across different systems +Alternatively, you may use Docker compose with the `compose.yml` file provided in the repository. +See our [advanced Docker usage](https://github.com/oraios/serena/blob/main/DOCKER.md) documentation for more detailed instructions, configuration options, and limitations. -Alternatively, use docker compose with the `compose.yml` file provided in the repository. - -See our [Docker Setup](https://github.com/oraios/serena/blob/main/DOCKER.md) documentation for more detailed setup instructions, configuration options, and known limitations. +:::{note} +Docker usage is subject to limitations; see the [advanced Docker usage](https://github.com/oraios/serena/blob/main/DOCKER.md) documentation for details. +::: ### Using Nix diff --git a/src/serena/agent.py b/src/serena/agent.py index 8b198c5a..394c14bb 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -18,7 +18,7 @@ from interprompt.jinja_template import JinjaTemplate from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, is_running_in_docker +from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition from serena.dashboard import SerenaDashboardAPI from serena.ls_manager import LanguageServerManager from serena.project import Project @@ -272,12 +272,10 @@ class SerenaAgent: self._dashboard_thread, port = SerenaDashboardAPI( get_memory_log_handler(), tool_names, agent=self, tool_usage_stats=self._tool_usage_stats ).run_in_thread(host=self.serena_config.web_dashboard_listen_address) - dashboard_listen_address = self.serena_config.web_dashboard_listen_address - if is_running_in_docker(): - # we use localhost for URL instead of the 0.0.0.0 that is configured for docker - # (the user has to port-forward then) - dashboard_listen_address = "127.0.0.1" - dashboard_url = f"http://{dashboard_listen_address}:{port}/dashboard/index.html" + dashboard_host = self.serena_config.web_dashboard_listen_address + if dashboard_host == "0.0.0.0": + dashboard_host = "localhost" + dashboard_url = f"http://{dashboard_host}:{port}/dashboard/index.html" log.info("Serena web dashboard started at %s", dashboard_url) if self.serena_config.web_dashboard_open_on_launch: # open the dashboard URL in the default web browser (using a separate process to control diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 83d6bcb4..75f916b8 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -106,19 +106,6 @@ def get_serena_managed_in_project_dir(project_root: str | Path) -> str: return os.path.join(project_root, SERENA_MANAGED_DIR_NAME) -def is_running_in_docker() -> bool: - """Check if we're running inside a Docker container.""" - # Check for Docker-specific files - if os.path.exists("/.dockerenv"): - return True - # Check cgroup for docker references - try: - with open("/proc/self/cgroup") as f: - return "docker" in f.read() - except FileNotFoundError: - return False - - @dataclass(kw_only=True) class ProjectConfig(ToolInclusionDefinition, ToStringMixin): project_name: str @@ -368,7 +355,6 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): web_dashboard: bool = True web_dashboard_open_on_launch: bool = True web_dashboard_listen_address: str = "127.0.0.1" - """When running in Docker, this will be automatically set to '0.0.0.0' ignoring any configuration by the user.""" tool_timeout: float = DEFAULT_TOOL_TIMEOUT loaded_commented_yaml: CommentedMap | None = None config_file_path: str | None = None @@ -398,11 +384,6 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): """Advanced configuration option allowing to configure language server implementation specific options, see SolidLSPSettings for more info.""" CONFIG_FILE = "serena_config.yml" - CONFIG_FILE_DOCKER = "serena_config.docker.yml" # Docker-specific config file; auto-generated if missing, mounted via docker-compose for user customization - - def __post_init__(self) -> None: - if is_running_in_docker(): - self.web_dashboard_listen_address = "0.0.0.0" def _tostring_includes(self) -> list[str]: return ["config_file_path"] @@ -423,20 +404,17 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): """ :return: the location where the Serena configuration file is stored/should be stored """ - if is_running_in_docker(): - return os.path.join(REPO_ROOT, cls.CONFIG_FILE_DOCKER) - else: - config_path = os.path.join(SerenaPaths().serena_user_home_dir, cls.CONFIG_FILE) + config_path = os.path.join(SerenaPaths().serena_user_home_dir, cls.CONFIG_FILE) - # if the config file does not exist, check if we can migrate it from the old location - if not os.path.exists(config_path): - old_config_path = os.path.join(REPO_ROOT, cls.CONFIG_FILE) - if os.path.exists(old_config_path): - log.info(f"Moving Serena configuration file from {old_config_path} to {config_path}") - os.makedirs(os.path.dirname(config_path), exist_ok=True) - shutil.move(old_config_path, config_path) + # if the config file does not exist, check if we can migrate it from the old location + if not os.path.exists(config_path): + old_config_path = os.path.join(REPO_ROOT, cls.CONFIG_FILE) + if os.path.exists(old_config_path): + log.info(f"Moving Serena configuration file from {old_config_path} to {config_path}") + os.makedirs(os.path.dirname(config_path), exist_ok=True) + shutil.move(old_config_path, config_path) - return config_path + return config_path @classmethod def from_config_file(cls, generate_if_missing: bool = True) -> "SerenaConfig": @@ -490,12 +468,8 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): def get_value_or_default(field_name: str) -> Any: return loaded_commented_yaml.get(field_name, get_dataclass_default(SerenaConfig, field_name)) - if is_running_in_docker(): - instance.gui_log_window_enabled = False # not supported in Docker - else: - instance.gui_log_window_enabled = get_value_or_default("gui_log_window_enabled") - # in docker the post_init takes care of setting the address to 0.0.0.0, we ignore the user's setting - instance.web_dashboard_listen_address = get_value_or_default("web_dashboard_listen_address") + instance.gui_log_window_enabled = get_value_or_default("gui_log_window_enabled") + instance.web_dashboard_listen_address = get_value_or_default("web_dashboard_listen_address") instance.log_level = loaded_commented_yaml.get("log_level", loaded_commented_yaml.get("gui_log_level", logging.INFO)) instance.web_dashboard = get_value_or_default("web_dashboard") instance.web_dashboard_open_on_launch = get_value_or_default("web_dashboard_open_on_launch") diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index 5b414e2d..9002c3bf 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -574,6 +574,7 @@ class SerenaDashboardAPI: def run_in_thread(self, host: str) -> tuple[threading.Thread, int]: port = self._find_first_free_port(0x5EDA) + log.info("Starting dashboard (listen_address=%s, port=%d)", host, port) thread = threading.Thread(target=lambda: self.run(host=host, port=port), daemon=True) thread.start() return thread, port diff --git a/src/serena/resources/serena_config.template.yml b/src/serena/resources/serena_config.template.yml index 75aa5cff..974fe7e5 100644 --- a/src/serena/resources/serena_config.template.yml +++ b/src/serena/resources/serena_config.template.yml @@ -21,10 +21,7 @@ web_dashboard: True # is supported on all platforms. web_dashboard_listen_address: 127.0.0.1 -# The address where the web dashboard will listen on (by default, localhost) -# Important: when running serena in docker, this configuration entry will be ignored and the -# address will be set to 0.0.0.0. (as it's running in a sandbox, this does not pose a safety issue) -# You can then port-forward it to a host of your choice, e.g., localhost +# The address where the web dashboard will listen on web_dashboard_open_on_launch: True # whether to open a browser window with the web dashboard when Serena starts (provided that web_dashboard @@ -32,7 +29,6 @@ web_dashboard_open_on_launch: True # http://localhost:24282/dashboard/ in your web browser (24282 = 0x5EDA, SErena DAshboard). # If you have multiple instances running, a higher port will be used; try port 24283, 24284, etc. - log_level: 20 # the minimum log level for the GUI log window and the dashboard (10 = debug, 20 = info, 30 = warning, 40 = error) From 7ad44edd18fe9e28d0c77c3e24893f0248a792ad Mon Sep 17 00:00:00 2001 From: Dominik Jain Date: Thu, 11 Dec 2025 00:06:46 +0100 Subject: [PATCH 4/5] Do not use listen_address 0.0.0.0 for Docker (has no desirable effect) --- DOCKER.md | 2 -- Dockerfile | 1 - 2 files changed, 3 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index bb77040c..08348182 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -27,8 +27,6 @@ special settings for Docker usage: ``` # Disable the GUI log window since it's not supported in Docker gui_log_window: False -# Listen on all interfaces for the web dashboard to be accessible from outside the container -web_dashboard_listen_address: 0.0.0.0 # Disable opening the web dashboard on launch (not possible within the container) web_dashboard_open_on_launch: False ``` diff --git a/Dockerfile b/Dockerfile index 1dccaf40..e58d821b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,7 +64,6 @@ ENV SERENA_HOME=/workspaces/serena/config RUN mkdir -p $SERENA_HOME RUN cp src/serena/resources/serena_config.template.yml $SERENA_HOME/serena_config.yml RUN sed -i 's/^gui_log_window: .*/gui_log_window: False/' $SERENA_HOME/serena_config.yml -RUN sed -i 's/^web_dashboard_listen_address: .*/web_dashboard_listen_address: 0.0.0.0/' $SERENA_HOME/serena_config.yml RUN sed -i 's/^web_dashboard_open_on_launch: .*/web_dashboard_open_on_launch: False/' $SERENA_HOME/serena_config.yml # Create virtual environment and install dependencies From 2ec96089f6194d0b15be691b022b0fd39cbcd6fc Mon Sep 17 00:00:00 2001 From: Lenz Gschwendtner Date: Thu, 11 Dec 2025 08:57:13 +0000 Subject: [PATCH 5/5] updated: elixir language server, replaced with expert (#783) --- .github/workflows/pytest.yml | 4 +- docs/01-about/020_programming-languages.md | 2 +- pyproject.toml | 1 + .../language_servers/elixir_tools/README.md | 41 ++- .../elixir_tools/elixir_tools.py | 255 ++++++++---------- src/solidlsp/ls.py | 3 + test/solidlsp/elixir/__init__.py | 19 +- test/solidlsp/elixir/test_elixir_basic.py | 4 +- .../elixir/test_elixir_ignored_dirs.py | 49 +++- .../elixir/test_elixir_integration.py | 4 +- .../elixir/test_elixir_symbol_retrieval.py | 28 +- 11 files changed, 210 insertions(+), 200 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 67f38ffe..622b6c4b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -90,8 +90,8 @@ jobs: if: runner.os != 'Windows' uses: erlef/setup-beam@v1 with: - elixir-version: "1.18.4" - otp-version: "26.1" + elixir-version: "1.19.3" + otp-version: "28" # Erlang currently not tested in CI, random hangings on macos, always hangs on ubuntu # In local tests, erlang seems to work though # - name: Install Erlang Language Server diff --git a/docs/01-about/020_programming-languages.md b/docs/01-about/020_programming-languages.md index 692f3b2a..3637ce04 100644 --- a/docs/01-about/020_programming-languages.md +++ b/docs/01-about/020_programming-languages.md @@ -20,7 +20,7 @@ With Serena, we provide direct, out-of-the-box support for: * **Clojure** * **Dart** * **Elixir** - (requires installation of NextLS and Elixir; Windows not supported) + (requires Elixir installation; Expert language server is downloaded automatically) * **Elm** (requires Elm compiler) * **Erlang** diff --git a/pyproject.toml b/pyproject.toml index 9fce285f..e6353018 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -301,6 +301,7 @@ markers = [ "fortran: language server running for Fortran", "haskell: Haskell language server tests", "yaml: language server running for YAML", + "slow: tests that require additional Expert instances and have long startup times (~60-90s each)", ] [tool.codespell] diff --git a/src/solidlsp/language_servers/elixir_tools/README.md b/src/solidlsp/language_servers/elixir_tools/README.md index b3c56d17..56f2e03c 100644 --- a/src/solidlsp/language_servers/elixir_tools/README.md +++ b/src/solidlsp/language_servers/elixir_tools/README.md @@ -1,18 +1,6 @@ # Elixir Language Server Integration -This directory contains the integration for Elixir language support using [Next LS](https://github.com/elixir-tools/next-ls) from the elixir-tools project. - -> **⚠️ Windows Not Supported**: Next LS does not provide Windows binaries, so Elixir language server integration is only available on Linux and macOS. - -## Known Issues - -### Next LS v0.23.3 Timeout Enumeration Bug -There is a known intermittent bug in Next LS v0.23.3 where `textDocument/definition` requests can fail with: -``` -Protocol.UndefinedError: protocol Enumerable not implemented for :timeout of type Atom -``` - -This bug is tracked in [Next LS Issue #543](https://github.com/elixir-tools/next-ls/issues/543) and primarily occurs in CI environments. The affected test (`test_request_defining_symbol_none`) is marked as expected to fail until this upstream bug is resolved. +This directory contains the integration for Elixir language support using [Expert](https://github.com/elixir-lang/expert), the official Elixir language server. ## Prerequisites @@ -22,9 +10,10 @@ Before using the Elixir language server integration, you need to have: - Install from: https://elixir-lang.org/install.html - Verify with: `elixir --version` -2. **Next LS** installed and available in your PATH - - Install from: https://github.com/elixir-tools/next-ls#installation - - Verify with: `nextls --version` +2. **Expert** (optional - will be downloaded automatically if not found) + - Expert binaries are automatically downloaded from GitHub releases + - Manual installation: https://github.com/elixir-lang/expert#installation + - If installed manually, ensure `expert` is in your PATH ## Features @@ -42,13 +31,20 @@ The Elixir integration provides: ## Configuration -The integration uses the default Next LS configuration with: +The integration uses the default Expert configuration with: - **MIX_ENV**: `dev` - **MIX_TARGET**: `host` - **Experimental completions**: Disabled by default - **Credo extension**: Enabled by default +### Version Management (asdf) + +Expert automatically respects project-specific Elixir versions when using asdf: +- If a `.tool-versions` file exists in the project root, Expert will use the specified Elixir version +- Expert is launched from the project directory, allowing it to pick up project configuration +- No additional configuration needed - just ensure asdf is installed and the project has a `.tool-versions` file + ## Usage The Elixir language server is automatically selected when working with Elixir projects. It will be used for: @@ -63,14 +59,14 @@ The Elixir language server is automatically selected when working with Elixir pr ### Important: Project Compilation -Next LS requires your Elixir project to be **compiled** for optimal performance, especially for: +Expert requires your Elixir project to be **compiled** for optimal performance, especially for: - Cross-file reference resolution - Complete symbol information - Accurate go-to-definition **For production use**: Ensure your project is compiled with `mix compile` before using the language server. -**For testing**: The test suite automatically compiles the test repositories before running tests to ensure optimal Next LS performance. +**For testing**: The test suite automatically compiles the test repositories before running tests to ensure optimal Expert performance. ## Testing @@ -83,8 +79,9 @@ pytest test/solidlsp/elixir/ -m elixir ## Implementation Details - **Main class**: `ElixirTools` in `elixir_tools.py` -- **Initialization parameters**: Defined in `initialize_params.json` - **Language identifier**: `"elixir"` -- **Command**: `nextls --stdio` +- **Command**: `expert --stdio` +- **Supported platforms**: Linux (x64, arm64), macOS (x64, arm64), Windows (x64, arm64) +- **Binary distribution**: Downloaded from [GitHub releases](https://github.com/elixir-lang/expert/releases) -The implementation follows the same patterns as other language servers in this project, inheriting from `SolidLanguageServer` and providing Elixir-specific configuration and behavior. \ No newline at end of file +The implementation follows the same patterns as other language servers in this project, inheriting from `SolidLanguageServer` and providing Elixir-specific configuration and behavior. diff --git a/src/solidlsp/language_servers/elixir_tools/elixir_tools.py b/src/solidlsp/language_servers/elixir_tools/elixir_tools.py index 926ee792..de13725e 100644 --- a/src/solidlsp/language_servers/elixir_tools/elixir_tools.py +++ b/src/solidlsp/language_servers/elixir_tools/elixir_tools.py @@ -11,7 +11,6 @@ from overrides import override from solidlsp.ls import SolidLanguageServer from solidlsp.ls_config import LanguageServerConfig from solidlsp.ls_utils import FileUtils, PlatformId, PlatformUtils -from solidlsp.lsp_protocol_handler import lsp_types from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo from solidlsp.settings import SolidLSPSettings @@ -23,12 +22,12 @@ log = logging.getLogger(__name__) class ElixirTools(SolidLanguageServer): """ - Provides Elixir specific instantiation of the LanguageServer class using Next LS from elixir-tools. + Provides Elixir specific instantiation of the LanguageServer class using Expert, the official Elixir language server. """ @override def _get_wait_time_for_cross_file_referencing(self) -> float: - return 10.0 # Elixir projects need a lot of time to compile and index before cross-file references work + return 10.0 # Elixir projects need time to compile and index before cross-file references work @override def is_ignored_dirname(self, dirname: str) -> bool: @@ -38,7 +37,8 @@ class ElixirTools(SolidLanguageServer): # - node_modules: if the project has JavaScript components # - .elixir_ls: ElixirLS artifacts (in case both are present) # - cover: coverage reports - return super().is_ignored_dirname(dirname) or dirname in ["_build", "deps", "node_modules", ".elixir_ls", "cover"] + # - .expert: Expert artifacts + return super().is_ignored_dirname(dirname) or dirname in ["_build", "deps", "node_modules", ".elixir_ls", ".expert", "cover"] @override def is_ignored_path(self, relative_path: str, ignore_unsupported_files: bool = True) -> bool: @@ -49,42 +49,6 @@ class ElixirTools(SolidLanguageServer): return super().is_ignored_path(relative_path, ignore_unsupported_files) - @staticmethod - def _is_next_ls_internal_file(abs_path: str) -> bool: - """Check if an absolute path is a Next LS internal file that should be ignored.""" - return any( - pattern in abs_path - for pattern in [ - ".burrito", # Next LS runtime directory - "next_ls_erts-", # Next LS Erlang runtime - "_next_ls_private_", # Next LS private files - "/priv/monkey/", # Next LS monkey patching directory - ] - ) - - @override - def _send_references_request(self, relative_file_path: str, line: int, column: int) -> list[lsp_types.Location] | None: - """Override to filter out Next LS internal files from references.""" - from solidlsp.ls_utils import PathUtils - - # Get the raw response from the parent implementation - raw_response = super()._send_references_request(relative_file_path, line, column) - - if raw_response is None: - return None - - # Filter out Next LS internal files - filtered_response = [] - for item in raw_response: - if isinstance(item, dict) and "uri" in item: - abs_path = PathUtils.uri_to_path(item["uri"]) - if self._is_next_ls_internal_file(abs_path): - log.debug(f"Filtering out Next LS internal file: {abs_path}") - continue - filtered_response.append(item) - - return filtered_response - @classmethod def _get_elixir_version(cls) -> str | None: """Get the installed Elixir version or None if not found.""" @@ -99,8 +63,8 @@ class ElixirTools(SolidLanguageServer): @classmethod def _setup_runtime_dependencies(cls, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings) -> str: """ - Setup runtime dependencies for Next LS. - Downloads the Next LS binary for the current platform and returns the path to the executable. + Setup runtime dependencies for Expert. + Downloads the Expert binary for the current platform and returns the path to the executable. """ # Check if Elixir is available first elixir_version = cls._get_elixir_version() @@ -111,113 +75,137 @@ class ElixirTools(SolidLanguageServer): log.info(f"Found Elixir: {elixir_version}") - platform_id = PlatformUtils.get_platform_id() + # First, check if expert is already in PATH (user may have installed it manually) + import shutil - # Check for Windows and provide a helpful error message - if platform_id.value.startswith("win"): - raise RuntimeError( - "Windows is not supported by Next LS. The Next LS project does not provide Windows binaries. " - "Consider using Windows Subsystem for Linux (WSL) or a virtual machine with Linux/macOS." - ) + expert_in_path = shutil.which("expert") + if expert_in_path: + log.info(f"Found Expert in PATH: {expert_in_path}") + return expert_in_path + + platform_id = PlatformUtils.get_platform_id() valid_platforms = [ PlatformId.LINUX_x64, PlatformId.LINUX_arm64, PlatformId.OSX_x64, PlatformId.OSX_arm64, + PlatformId.WIN_x64, + PlatformId.WIN_arm64, ] - assert platform_id in valid_platforms, f"Platform {platform_id} is not supported for Next LS at the moment" + assert platform_id in valid_platforms, f"Platform {platform_id} is not supported for Expert at the moment" - next_ls_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "next-ls") + expert_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "expert") - NEXTLS_VERSION = "v0.23.4" + EXPERT_VERSION = "nightly" # Define runtime dependencies inline runtime_deps = { PlatformId.LINUX_x64: RuntimeDependency( - id="next_ls_linux_amd64", + id="expert_linux_amd64", platform_id="linux-x64", - url=f"https://github.com/elixir-tools/next-ls/releases/download/{NEXTLS_VERSION}/next_ls_linux_amd64", + url=f"https://github.com/elixir-lang/expert/releases/download/{EXPERT_VERSION}/expert_linux_amd64", archive_type="binary", - binary_name="next_ls_linux_amd64", - extract_path="next_ls", + binary_name="expert_linux_amd64", + extract_path="expert", ), PlatformId.LINUX_arm64: RuntimeDependency( - id="next_ls_linux_arm64", + id="expert_linux_arm64", platform_id="linux-arm64", - url=f"https://github.com/elixir-tools/next-ls/releases/download/{NEXTLS_VERSION}/next_ls_linux_arm64", + url=f"https://github.com/elixir-lang/expert/releases/download/{EXPERT_VERSION}/expert_linux_arm64", archive_type="binary", - binary_name="next_ls_linux_arm64", - extract_path="next_ls", + binary_name="expert_linux_arm64", + extract_path="expert", ), PlatformId.OSX_x64: RuntimeDependency( - id="next_ls_darwin_amd64", + id="expert_darwin_amd64", platform_id="osx-x64", - url=f"https://github.com/elixir-tools/next-ls/releases/download/{NEXTLS_VERSION}/next_ls_darwin_amd64", + url=f"https://github.com/elixir-lang/expert/releases/download/{EXPERT_VERSION}/expert_darwin_amd64", archive_type="binary", - binary_name="next_ls_darwin_amd64", - extract_path="next_ls", + binary_name="expert_darwin_amd64", + extract_path="expert", ), PlatformId.OSX_arm64: RuntimeDependency( - id="next_ls_darwin_arm64", + id="expert_darwin_arm64", platform_id="osx-arm64", - url=f"https://github.com/elixir-tools/next-ls/releases/download/{NEXTLS_VERSION}/next_ls_darwin_arm64", + url=f"https://github.com/elixir-lang/expert/releases/download/{EXPERT_VERSION}/expert_darwin_arm64", archive_type="binary", - binary_name="next_ls_darwin_arm64", - extract_path="next_ls", + binary_name="expert_darwin_arm64", + extract_path="expert", + ), + PlatformId.WIN_x64: RuntimeDependency( + id="expert_windows_amd64", + platform_id="win-x64", + url=f"https://github.com/elixir-lang/expert/releases/download/{EXPERT_VERSION}/expert_windows_amd64.exe", + archive_type="binary", + binary_name="expert_windows_amd64.exe", + extract_path="expert.exe", + ), + PlatformId.WIN_arm64: RuntimeDependency( + id="expert_windows_arm64", + platform_id="win-arm64", + url=f"https://github.com/elixir-lang/expert/releases/download/{EXPERT_VERSION}/expert_windows_arm64.exe", + archive_type="binary", + binary_name="expert_windows_arm64.exe", + extract_path="expert.exe", ), } dependency = runtime_deps[platform_id] - executable_path = os.path.join(next_ls_dir, "nextls") + # On Windows, use .exe extension + executable_name = "expert.exe" if platform_id.value.startswith("win") else "expert" + executable_path = os.path.join(expert_dir, executable_name) assert dependency.binary_name is not None - binary_path = os.path.join(next_ls_dir, dependency.binary_name) + binary_path = os.path.join(expert_dir, dependency.binary_name) if not os.path.exists(executable_path): - log.info(f"Downloading Next LS binary from {dependency.url}") + log.info(f"Downloading Expert binary from {dependency.url}") assert dependency.url is not None FileUtils.download_file(dependency.url, binary_path) # Make the binary executable on Unix-like systems - os.chmod(binary_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) + if not platform_id.value.startswith("win"): + os.chmod(binary_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) - # Create a symlink with the expected name - if binary_path != executable_path: + # Create a symlink with the expected name on Unix-like systems + if binary_path != executable_path and not platform_id.value.startswith("win"): if os.path.exists(executable_path): os.remove(executable_path) os.symlink(os.path.basename(binary_path), executable_path) - assert os.path.exists(executable_path), f"Next LS executable not found at {executable_path}" + assert os.path.exists(executable_path), f"Expert executable not found at {executable_path}" - log.info(f"Next LS binary ready at: {executable_path}") + log.info(f"Expert binary ready at: {executable_path}") return executable_path def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings): - nextls_executable_path = self._setup_runtime_dependencies(config, solidlsp_settings) + expert_executable_path = self._setup_runtime_dependencies(config, solidlsp_settings) super().__init__( config, repository_root_path, - ProcessLaunchInfo(cmd=f'"{nextls_executable_path}" --stdio', cwd=repository_root_path), + ProcessLaunchInfo(cmd=f"{expert_executable_path} --stdio", cwd=repository_root_path), "elixir", solidlsp_settings, ) self.server_ready = threading.Event() self.request_id = 0 - # Set generous timeout for Next LS which can be slow to initialize and respond - self.set_request_timeout(180.0) # 60 seconds for all environments + # Set generous timeout for Expert which can be slow to initialize and respond + self.set_request_timeout(180.0) @staticmethod def _get_initialize_params(repository_absolute_path: str) -> InitializeParams: """ - Returns the initialize params for the Next LS Language Server. + Returns the initialize params for the Expert Language Server. """ - root_uri = pathlib.Path(repository_absolute_path).as_uri() + # Ensure the path is absolute + abs_path = os.path.abspath(repository_absolute_path) + root_uri = pathlib.Path(abs_path).as_uri() initialize_params = { "processId": os.getpid(), "locale": "en", - "rootPath": repository_absolute_path, + "rootPath": abs_path, "rootUri": root_uri, "initializationOptions": { "mix_env": "dev", @@ -275,89 +263,80 @@ class ElixirTools(SolidLanguageServer): return cast(InitializeParams, initialize_params) def _start_server(self) -> None: - """Start Next LS server process""" + """Start Expert server process""" def register_capability_handler(params: Any) -> None: + log.debug(f"LSP: client/registerCapability: {params}") return def window_log_message(msg: Any) -> None: - """Handle window/logMessage notifications from Next LS""" + """Handle window/logMessage notifications from Expert""" + message_type = msg.get("type", 4) # 1=Error, 2=Warning, 3=Info, 4=Log message_text = msg.get("message", "") - log.info(f"LSP: window/logMessage: {message_text}") - # Check for the specific Next LS readiness signal - # Based on Next LS source: "Runtime for folder #{name} is ready..." - if "Runtime for folder" in message_text and "is ready..." in message_text: - log.info("Next LS runtime is ready based on official log message") - self.server_ready.set() - - def do_nothing(params: Any) -> None: - return + # Log at appropriate level based on message type + if message_type == 1: + log.error(f"Expert: {message_text}") + elif message_type == 2: + log.warning(f"Expert: {message_text}") + else: + log.debug(f"Expert: {message_text}") def check_server_ready(params: Any) -> None: """ - Handle $/progress notifications from Next LS. - Keep as fallback for error detection, but primary readiness detection - is now done via window/logMessage handler. + Handle $/progress notifications from Expert. + Expert sends progress updates during compilation and indexing. + The server is considered ready when project build completes. """ value = params.get("value", {}) + kind = value.get("kind", "") + title = value.get("title", "") - # Check for initialization completion progress (fallback signal) - if value.get("kind") == "end": - message = value.get("message", "") - if "has initialized!" in message: - log.info("Next LS initialization progress completed") - # Note: We don't set server_ready here - we wait for the log message + if kind == "begin": + # Track when building the project starts (not "Building engine") + if title.startswith("Building ") and not title.startswith("Building engine"): + self._building_project = True + elif kind == "end": + # Project build completion is the main readiness signal + if getattr(self, "_building_project", False): + log.debug("Expert project build completed - server is ready") + self._building_project = False + self.server_ready.set() - def work_done_progress(params: Any) -> None: - """ - Handle $/workDoneProgress notifications from Next LS. - Keep for completeness but primary readiness detection is via window/logMessage. - """ - value = params.get("value", {}) - if value.get("kind") == "end": - log.info("Next LS work done progress completed") - # Note: We don't set server_ready here - we wait for the log message + def work_done_progress_create(params: Any) -> None: + """Handle window/workDoneProgress/create requests from Expert.""" + return + + def publish_diagnostics(params: Any) -> None: + """Handle textDocument/publishDiagnostics notifications.""" + return self.server.on_request("client/registerCapability", register_capability_handler) self.server.on_notification("window/logMessage", window_log_message) self.server.on_notification("$/progress", check_server_ready) - self.server.on_request("window/workDoneProgress/create", do_nothing) - self.server.on_notification("$/workDoneProgress", work_done_progress) - self.server.on_notification("textDocument/publishDiagnostics", do_nothing) + self.server.on_request("window/workDoneProgress/create", work_done_progress_create) + self.server.on_notification("textDocument/publishDiagnostics", publish_diagnostics) - log.info("Starting Next LS server process") + log.debug("Starting Expert server process") self.server.start() initialize_params = self._get_initialize_params(self.repository_root_path) - log.info("Sending initialize request from LSP client to LSP server and awaiting response") + log.debug("Sending initialize request to Expert") init_response = self.server.send.initialize(initialize_params) - # Verify server capabilities - be more lenient with Next LS - log.info(f"Next LS capabilities: {list(init_response['capabilities'].keys())}") - - # Next LS may not provide all capabilities immediately, so we check for basic ones + # Verify basic server capabilities assert "textDocumentSync" in init_response["capabilities"], f"Missing textDocumentSync in {init_response['capabilities']}" - # Some capabilities might be optional or provided later. This is expected, so we log as info - if "completionProvider" not in init_response["capabilities"]: - log.info("completionProvider not available in initial capabilities") - - if "definitionProvider" not in init_response["capabilities"]: - log.info("definitionProvider not available in initial capabilities") - self.server.notify.initialized({}) self.completions_available.set() - # Wait for Next LS to send the specific "Runtime for folder X is ready..." log message - # This is the authoritative signal that Next LS is truly ready for requests - ready_timeout = 180.0 - log.info(f"Waiting up to {ready_timeout} seconds for Next LS runtime readiness...") - + # Expert needs time to compile the project and build indexes on first run. + # This can take 2-3+ minutes for mid-sized codebases. + # After the first run, subsequent startups are much faster. + ready_timeout = 300.0 # 5 minutes + log.debug(f"Waiting up to {ready_timeout}s for Expert to compile and index...") if self.server_ready.wait(timeout=ready_timeout): - log.info("Next LS is ready and available for requests") - + log.debug("Expert is ready for requests") else: - error_msg = f"Next LS failed to initialize within {ready_timeout} seconds. This may indicate a problem with the Elixir installation, project compilation, or Next LS itself." - log.error(error_msg) - raise RuntimeError(error_msg) + log.warning(f"Expert did not signal readiness within {ready_timeout}s. Proceeding with requests anyway.") + self.server_ready.set() # Mark as ready anyway to allow requests diff --git a/src/solidlsp/ls.py b/src/solidlsp/ls.py index f9ce89b8..be66e36c 100644 --- a/src/solidlsp/ls.py +++ b/src/solidlsp/ls.py @@ -227,6 +227,9 @@ class SolidLanguageServer(ABC): if solidlsp_settings is None: solidlsp_settings = SolidLSPSettings() + # Ensure repository_root_path is absolute to avoid issues with file URIs + repository_root_path = os.path.abspath(repository_root_path) + ls_class = config.code_language.get_ls_class() # For now, we assume that all language server implementations have the same signature of the constructor # (which, unfortunately, differs from the signature of the base class). diff --git a/test/solidlsp/elixir/__init__.py b/test/solidlsp/elixir/__init__.py index 4d7e8860..f8532a0b 100644 --- a/test/solidlsp/elixir/__init__.py +++ b/test/solidlsp/elixir/__init__.py @@ -1,12 +1,5 @@ -import platform - - -def _test_nextls_available() -> str: - """Test if Next LS is available and return error reason if not.""" - # Check if we're on Windows (Next LS doesn't support Windows) - if platform.system() == "Windows": - return "Next LS does not support Windows" - +def _test_expert_available() -> str: + """Test if Expert is available and return error reason if not.""" # Try to import and check Elixir availability try: from solidlsp.language_servers.elixir_tools.elixir_tools import ElixirTools @@ -16,13 +9,13 @@ def _test_nextls_available() -> str: if not elixir_version: return "Elixir is not installed or not in PATH" - return "" # No error, Next LS should be available + return "" # No error, Expert should be available except ImportError as e: return f"Failed to import ElixirTools: {e}" except Exception as e: - return f"Error checking Next LS availability: {e}" + return f"Error checking Expert availability: {e}" -NEXTLS_UNAVAILABLE_REASON = _test_nextls_available() -NEXTLS_UNAVAILABLE = bool(NEXTLS_UNAVAILABLE_REASON) +EXPERT_UNAVAILABLE_REASON = _test_expert_available() +EXPERT_UNAVAILABLE = bool(EXPERT_UNAVAILABLE_REASON) diff --git a/test/solidlsp/elixir/test_elixir_basic.py b/test/solidlsp/elixir/test_elixir_basic.py index 44bf7378..fe18cd34 100644 --- a/test/solidlsp/elixir/test_elixir_basic.py +++ b/test/solidlsp/elixir/test_elixir_basic.py @@ -12,10 +12,10 @@ import pytest from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language -from . import NEXTLS_UNAVAILABLE, NEXTLS_UNAVAILABLE_REASON +from . import EXPERT_UNAVAILABLE, EXPERT_UNAVAILABLE_REASON # These marks will be applied to all tests in this module -pytestmark = [pytest.mark.elixir, pytest.mark.skipif(NEXTLS_UNAVAILABLE, reason=f"Next LS not available: {NEXTLS_UNAVAILABLE_REASON}")] +pytestmark = [pytest.mark.elixir, pytest.mark.skipif(EXPERT_UNAVAILABLE, reason=f"Next LS not available: {EXPERT_UNAVAILABLE_REASON}")] class TestElixirBasic: diff --git a/test/solidlsp/elixir/test_elixir_ignored_dirs.py b/test/solidlsp/elixir/test_elixir_ignored_dirs.py index 71bf3a3f..d74c58d2 100644 --- a/test/solidlsp/elixir/test_elixir_ignored_dirs.py +++ b/test/solidlsp/elixir/test_elixir_ignored_dirs.py @@ -1,3 +1,4 @@ +import os from collections.abc import Generator from pathlib import Path @@ -7,23 +8,38 @@ from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from test.conftest import start_ls_context -from . import NEXTLS_UNAVAILABLE, NEXTLS_UNAVAILABLE_REASON +from . import EXPERT_UNAVAILABLE, EXPERT_UNAVAILABLE_REASON # These marks will be applied to all tests in this module -pytestmark = [pytest.mark.elixir, pytest.mark.skipif(NEXTLS_UNAVAILABLE, reason=f"Next LS not available: {NEXTLS_UNAVAILABLE_REASON}")] +pytestmark = [pytest.mark.elixir, pytest.mark.skipif(EXPERT_UNAVAILABLE, reason=f"Expert not available: {EXPERT_UNAVAILABLE_REASON}")] + +# Skip slow tests in CI - they require multiple Expert instances which is too slow +IN_CI = bool(os.environ.get("CI") or os.environ.get("GITHUB_ACTIONS")) +SKIP_SLOW_IN_CI = pytest.mark.skipif( + IN_CI, + reason="Slow tests skipped in CI - require multiple Expert instances (~60-90s each)", +) -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def ls_with_ignored_dirs() -> Generator[SolidLanguageServer, None, None]: - """Fixture to set up an LS for the elixir test repo with the 'scripts' directory ignored.""" + """Fixture to set up an LS for the elixir test repo with the 'scripts' directory ignored. + + Uses session scope to avoid restarting Expert for each test. + """ ignored_paths = ["scripts", "ignored_dir"] with start_ls_context(language=Language.ELIXIR, ignored_paths=ignored_paths) as ls: yield ls -@pytest.mark.parametrize("ls_with_ignored_dirs", [Language.ELIXIR], indirect=True) +@pytest.mark.slow +@SKIP_SLOW_IN_CI def test_symbol_tree_ignores_dir(ls_with_ignored_dirs: SolidLanguageServer): - """Tests that request_full_symbol_tree ignores the configured directory.""" + """Tests that request_full_symbol_tree ignores the configured directory. + + Note: This test uses a separate Expert instance with custom ignored paths, + which adds ~60-90s startup time. + """ root = ls_with_ignored_dirs.request_full_symbol_tree()[0] root_children = root["children"] children_names = {child["name"] for child in root_children} @@ -35,9 +51,14 @@ def test_symbol_tree_ignores_dir(ls_with_ignored_dirs: SolidLanguageServer): assert "ignored_dir" not in children_names, f"ignored_dir should not be in {children_names}" -@pytest.mark.parametrize("ls_with_ignored_dirs", [Language.ELIXIR], indirect=True) +@pytest.mark.slow +@SKIP_SLOW_IN_CI def test_find_references_ignores_dir(ls_with_ignored_dirs: SolidLanguageServer): - """Tests that find_references ignores the configured directory.""" + """Tests that find_references ignores the configured directory. + + Note: This test uses a separate Expert instance with custom ignored paths, + which adds ~60-90s startup time. + """ # Location of User struct, which is referenced in scripts and ignored_dir definition_file = "lib/models.ex" @@ -60,9 +81,15 @@ def test_find_references_ignores_dir(ls_with_ignored_dirs: SolidLanguageServer): assert not any("ignored_dir" in ref["relativePath"] for ref in references), "ignored_dir should be ignored" +@pytest.mark.slow +@SKIP_SLOW_IN_CI @pytest.mark.parametrize("repo_path", [Language.ELIXIR], indirect=True) def test_refs_and_symbols_with_glob_patterns(repo_path: Path) -> None: - """Tests that refs and symbols with glob patterns are ignored.""" + """Tests that refs and symbols with glob patterns are ignored. + + Note: This test uses a separate Expert instance with custom ignored paths, + which adds ~60-90s startup time. + """ ignored_paths = ["*cripts", "ignored_*"] # codespell:ignore cripts with start_ls_context(language=Language.ELIXIR, repo_path=str(repo_path), ignored_paths=ignored_paths) as ls: @@ -114,6 +141,10 @@ def test_default_ignored_directories(language_server: SolidLanguageServer): assert not language_server.is_ignored_dirname("priv"), "priv should not be ignored" +@pytest.mark.xfail( + reason="Expert 0.1.0 bug: document_symbols may return nil for some files (flaky)", + raises=Exception, +) @pytest.mark.parametrize("language_server", [Language.ELIXIR], indirect=True) def test_symbol_tree_excludes_build_dirs(language_server: SolidLanguageServer): """Test that symbol tree excludes build and dependency directories.""" diff --git a/test/solidlsp/elixir/test_elixir_integration.py b/test/solidlsp/elixir/test_elixir_integration.py index 553d2569..eff618b2 100644 --- a/test/solidlsp/elixir/test_elixir_integration.py +++ b/test/solidlsp/elixir/test_elixir_integration.py @@ -14,10 +14,10 @@ from serena.project import Project from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language -from . import NEXTLS_UNAVAILABLE, NEXTLS_UNAVAILABLE_REASON +from . import EXPERT_UNAVAILABLE, EXPERT_UNAVAILABLE_REASON # These marks will be applied to all tests in this module -pytestmark = [pytest.mark.elixir, pytest.mark.skipif(NEXTLS_UNAVAILABLE, reason=f"Next LS not available: {NEXTLS_UNAVAILABLE_REASON}")] +pytestmark = [pytest.mark.elixir, pytest.mark.skipif(EXPERT_UNAVAILABLE, reason=f"Next LS not available: {EXPERT_UNAVAILABLE_REASON}")] class TestElixirIntegration: diff --git a/test/solidlsp/elixir/test_elixir_symbol_retrieval.py b/test/solidlsp/elixir/test_elixir_symbol_retrieval.py index b8c058cc..7fe40a2d 100644 --- a/test/solidlsp/elixir/test_elixir_symbol_retrieval.py +++ b/test/solidlsp/elixir/test_elixir_symbol_retrieval.py @@ -15,15 +15,18 @@ from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language from solidlsp.ls_types import SymbolKind -from . import NEXTLS_UNAVAILABLE, NEXTLS_UNAVAILABLE_REASON +from . import EXPERT_UNAVAILABLE, EXPERT_UNAVAILABLE_REASON # These marks will be applied to all tests in this module -pytestmark = [pytest.mark.elixir, pytest.mark.skipif(NEXTLS_UNAVAILABLE, reason=f"Next LS not available: {NEXTLS_UNAVAILABLE_REASON}")] +pytestmark = [pytest.mark.elixir, pytest.mark.skipif(EXPERT_UNAVAILABLE, reason=f"Next LS not available: {EXPERT_UNAVAILABLE_REASON}")] class TestElixirLanguageServerSymbols: """Test the Elixir language server's symbol-related functionality.""" + @pytest.mark.xfail( + reason="Expert 0.1.0 bug: document_symbols returns nil for some files (FunctionClauseError in XPExpert.EngineApi.document_symbols/2)" + ) @pytest.mark.parametrize("language_server", [Language.ELIXIR], indirect=True) def test_request_containing_symbol_function(self, language_server: SolidLanguageServer) -> None: """Test request_containing_symbol for a function.""" @@ -159,6 +162,9 @@ class TestElixirLanguageServerSymbols: pass # Tests for request_defining_symbol + @pytest.mark.xfail( + reason="Expert 0.1.0 bug: definition request crashes (FunctionClauseError in XPExpert.Protocol.Conversions.to_elixir/2)" + ) @pytest.mark.parametrize("language_server", [Language.ELIXIR], indirect=True) def test_request_defining_symbol_function_call(self, language_server: SolidLanguageServer) -> None: """Test request_defining_symbol for a function call.""" @@ -183,6 +189,9 @@ class TestElixirLanguageServerSymbols: if "location" in defining_symbol and "uri" in defining_symbol["location"]: assert "models.ex" in defining_symbol["location"]["uri"] + @pytest.mark.xfail( + reason="Expert 0.1.0 bug: definition request crashes (FunctionClauseError in XPExpert.Protocol.Conversions.to_elixir/2)" + ) @pytest.mark.parametrize("language_server", [Language.ELIXIR], indirect=True) def test_request_defining_symbol_struct_usage(self, language_server: SolidLanguageServer) -> None: """Test request_defining_symbol for a struct usage.""" @@ -205,10 +214,7 @@ class TestElixirLanguageServerSymbols: assert "User" in defining_symbol.get("name", "") @pytest.mark.xfail( - reason="Known intermittent bug in Next LS v0.23.3: Protocol.UndefinedError for :timeout atom. " - "Occurs in CI environments but may pass locally. " - "See https://github.com/elixir-tools/next-ls/issues/543", - strict=False, + reason="Expert 0.1.0 bug: definition request crashes (FunctionClauseError in XPExpert.Protocol.Conversions.to_elixir/2)" ) @pytest.mark.parametrize("language_server", [Language.ELIXIR], indirect=True) def test_request_defining_symbol_none(self, language_server: SolidLanguageServer) -> None: @@ -271,7 +277,7 @@ class TestElixirLanguageServerSymbols: break if lib_dir: - # Next LS returns module names instead of file names (e.g., 'services' instead of 'services.ex') + # Expert returns module names instead of file names (e.g., 'services' instead of 'services.ex') file_names = [child["name"] for child in lib_dir.get("children", [])] expected_modules = ["models", "services", "examples", "utils", "test_repo"] found_modules = [name for name in expected_modules if name in file_names] @@ -284,7 +290,7 @@ class TestElixirLanguageServerSymbols: # Should get an overview of the lib directory assert lib_overview is not None - # Next LS returns keys like 'lib/services.ex' instead of just 'lib' + # Expert returns keys like 'lib/services.ex' instead of just 'lib' overview_keys = list(lib_overview.keys()) if hasattr(lib_overview, "keys") else [] lib_files = [key for key in overview_keys if key.startswith("lib/")] assert len(lib_files) > 0, f"Expected to find lib/ files in overview keys: {overview_keys}" @@ -298,10 +304,10 @@ class TestElixirLanguageServerSymbols: # @pytest.mark.parametrize("language_server", [Language.ELIXIR], indirect=True) # def test_request_document_overview(self, language_server: SolidLanguageServer) -> None: # """Test request_document_overview functionality.""" - # # COMMENTED OUT: Next LS document overview doesn't contain expected terms - # # Next LS return value: [('TestRepo.Models', 2, 0, 0)] - only module info, no detailed content + # # COMMENTED OUT: Expert document overview doesn't contain expected terms + # # Expert return value: [('TestRepo.Models', 2, 0, 0)] - only module info, no detailed content # # Expected terms like 'user', 'item', 'order', 'struct', 'defmodule' are not present - # # This appears to be a limitation of Next LS document overview functionality + # # This appears to be a limitation of Expert document overview functionality # # # file_path = os.path.join("lib", "models.ex") # doc_overview = language_server.request_document_overview(file_path)