tiennm99/serena
1
0 Fork 0
mirror of https://github.com/tiennm99/serena.git synced 2026-10-14 10:19:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fsharp

Browse Source 
Conflicts:
	CHANGELOG.md
	README.md
	pyproject.toml
	src/solidlsp/ls_config.py
This commit is contained in:
Dominik Jain authored and Dominik Jain committed 2025-12-07 19:43:09 +01:00
2 parents 4cc0cabcb7 e7f1260b3a
commit 1938ed812c
214 files changed
+14246 -4666
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/docs.yaml
+63
View File
@@ -0,0 +1,63 @@
name: Docs Build
on:
push:
branches: [ main ]
workflow_dispatch:
permissions:
contents: read
pages: write
id-token: write
concurrency:
group: "pages"
cancel-in-progress: true
jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.11
- name: Install uv
run: pip install uv
- name: Cache dependencies
uses: actions/cache@v4
with:
path: ~/.cache/uv
key: uv-${{ hashFiles('pyproject.toml') }}
- name: Install dependencies
run: |
uv venv
uv pip install -e ".[dev]"
- name: Build docs
run: uv run poe doc-build
continue-on-error: false
- name: Upload Pages artifact
uses: actions/upload-pages-artifact@v3
with:
path: docs/_build
deploy:
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
runs-on: ubuntu-latest
needs: build
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4
.github/workflows/lint_and_docs.yaml
-47
View File
@@ -1,47 +0,0 @@
name: Linting, Types and Docs Check
on:
workflow_dispatch: # Manual trigger only - workflow disabled from auto-running
jobs:
check:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
id-token: write
pages: write
actions: write
contents: read
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
# use uv package manager
- name: Install uv
run: pip install uv
- uses: actions/cache@v3
name: Cache dependencies
with:
path: ~/.cache/uv
key: uv-${{ hashFiles('pyproject.toml') }}
- name: Install dependencies
run: |
uv venv
uv pip install -e ".[dev]"
- name: Lint
run: uv run poe lint
- name: Types
run: uv run poe type-check
- name: Docs
run: uv run poe doc-build
- name: Upload artifact
uses: actions/upload-pages-artifact@v2
with:
path: "docs/_build"
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v2
.github/workflows/pytest.yml
+81 -20
View File
@@ -21,6 +21,18 @@ jobs:
python-version: ["3.11"]
steps:
- uses: actions/checkout@v3
- name: Free disk space
if: runner.os == 'Linux'
run: |
df -h
sudo rm -rf /usr/local/lib/android
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache
sudo apt-get clean
sudo apt-get autoremove -y
docker system prune -af || true
df -h
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
@@ -38,6 +50,27 @@ jobs:
mkdir -p $HOME/.serena/language_servers/static
mkdir -p $HOME/.cache/go-build
mkdir -p $HOME/go/bin
- name: Install uv
shell: bash
run: curl -LsSf https://astral.sh/uv/install.sh | sh
- name: Cache uv virtualenv
id: cache-uv
uses: actions/cache@v3
with:
path: .venv
key: uv-venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('uv.lock') }}
- name: Create virtual environment
shell: bash
run: |
if [ ! -d ".venv" ]; then
uv venv
fi
- name: Install dependencies
shell: bash
run: uv pip install -e ".[dev]"
- name: Check formatting
shell: bash
run: uv run poe lint
# Add Go bin directory to PATH for this workflow
# GITHUB_PATH is a special file that GitHub Actions uses to modify PATH
# Writing to this file adds the directory to the PATH for subsequent steps
@@ -151,6 +184,47 @@ jobs:
shell: bash
run: |
Rscript -e "install.packages('languageserver', repos='https://cloud.r-project.org')"
- name: Set up Julia
uses: julia-actions/setup-julia@v2
with:
version: '1.10'
- name: Install Julia LanguageServer
shell: bash
run: julia -e 'using Pkg; Pkg.add("LanguageServer")'
- name: Setup Haskell toolchain
if: runner.os != 'Windows'
uses: haskell/ghcup-setup@v1
with:
ghc: '9.12.2'
cabal: '3.10.3.0'
hls: '2.11.0.0'
- name: Verify Haskell tools
if: runner.os != 'Windows'
run: |
echo "Verifying installed Haskell tools:"
which ghc && ghc --version
which cabal && cabal --version
# HLS verification - non-blocking in case of version incompatibility
if command -v haskell-language-server-wrapper &>/dev/null; then
echo "Found haskell-language-server-wrapper"
haskell-language-server-wrapper --version || echo "WARNING: HLS wrapper found but version check failed"
elif command -v haskell-language-server &>/dev/null; then
echo "Found haskell-language-server"
haskell-language-server --version || echo "WARNING: HLS found but version check failed"
else
echo "WARNING: HLS not found (may be incompatible with GHC 9.12.2)"
echo "This is not a critical error - tests will use HLS if available at runtime"
fi
shell: bash
- name: Pre-build Haskell test project for HLS
if: runner.os != 'Windows'
run: |
cd test/resources/repos/haskell/test_repo
cabal update
cabal build --only-dependencies
cabal build
echo "Haskell test project built successfully"
shell: bash
- name: Install Zig
uses: goto-bus-stop/setup-zig@v2
with:
@@ -293,15 +367,6 @@ jobs:
mv regal.exe "$HOME/bin/"
echo "$HOME/bin" >> $GITHUB_PATH
fi
- name: Install uv
shell: bash
run: curl -LsSf https://astral.sh/uv/install.sh | sh
- name: Cache uv virtualenv
id: cache-uv
uses: actions/cache@v3
with:
path: .venv
key: uv-venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('uv.lock') }}
- name: Cache language servers
id: cache-language-servers
uses: actions/cache@v3
@@ -310,18 +375,14 @@ jobs:
key: language-servers-${{ runner.os }}-v1
restore-keys: |
language-servers-${{ runner.os }}-
- name: Create virtual environment
shell: bash
- name: Report free disk space
if: runner.os == 'Linux'
run: |
if [ ! -d ".venv" ]; then
uv venv
fi
- name: Install dependencies
shell: bash
run: uv pip install -e ".[dev]"
- name: Check formatting
shell: bash
run: uv run poe lint
echo "Free disk space before tests:"
df -h
- name: Test with pytest
shell: bash
run: uv run poe test
- name: Type-checking with mypy
shell: bash
run: uv run poe type-check
.gitignore
+23 -9
View File
@@ -200,11 +200,7 @@ pylint.html
/*.yml
!*.template.yml
/agent-ui
# dynamic LS installations
/src/multilspy/language_servers/*/static
/src/solidlsp/language_servers/*/static
/src/solidlsp/language_servers/static
/test/**/.serena
# clojure-lsp temporary files
.calva/
@@ -212,12 +208,11 @@ pylint.html
.cpcache/
.lsp/
# Cache is in .multilspy
**/.multilspy/**
# temporary and backup files
*.bak
*.tmp
tmp/
.serena/
.vscode/
# Claude settings
@@ -231,6 +226,9 @@ tmp/
# Exception: Don't ignore Nix test repository lib directory (contains source code)
!/test/resources/repos/nix/test_repo/lib
# Exception: Don't ignore Julia test repository lib directory (contains source code)
!/test/resources/repos/julia/test_repo/lib
# Swift
/test/resources/repos/swift/test_repo/.build
/test/resources/repos/swift/test_repo/.swiftpm
@@ -238,3 +236,19 @@ tmp/
# Elm
/test/resources/repos/elm/test_repo/.elm/
/test/resources/repos/elm/test_repo/elm-stuff/
# Scala
.metals/
.bsp/
.scala-build/
.bloop/
bootstrap
test/resources/repos/scala/.bloop/
# Haskell
.stack-work/
*.cabal
stack.yaml.lock
dist-newstyle/
cabal.project.local*
.ghc.environment.*
.serena/.gitignore
+1
View File
@@ -0,0 +1 @@
/cache
.serena/memories/adding_new_language_support_guide.md
+15 -19
View File
@@ -21,39 +21,35 @@ Have a look at `intelephense.py` for a reference implementation of a language se
dependencies, and on `pyright_server.py` that does not need any additional dependencies
because the language server can be installed directly as python package.
```python
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
class NewLanguageServer(SolidLanguageServer):
"""
Language server implementation for NewLanguage.
"""
def __init__(self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str):
def __init__(self, config: LanguageServerConfig, repository_root_path: str):
# Determine language server command
cmd = self._get_language_server_command()
super().__init__(
config,
logger,
repository_root_path,
super().__init__(config,
ProcessLaunchInfo(cmd=cmd, cwd=repository_root_path),
"new_language", # Language ID for LSP
)
"new_language",,
def _get_language_server_command(self) -> list[str]:
"""Get the command to start the language server."""
# Example: return ["new-language-server", "--stdio"]
pass
@override
def is_ignored_dirname(self, dirname: str) -> bool:
"""Define language-specific directories to ignore."""
return super().is_ignored_dirname(dirname) or dirname in ["build", "dist", "target"]
return super().is_ignored_dirname(dirname) or dirname in ["build",
"dist", "target"]
```
### 1.2 Language Server Discovery and Installation
@@ -62,7 +58,7 @@ For languages requiring automatic installation, implement download logic similar
```python
@classmethod
def _ensure_server_installed(cls, logger: LanguageServerLogger) -> str:
def _ensure_server_installed(cls) -> str:
"""Ensure language server is installed and return path."""
# Check system installation first
system_server = shutil.which("new-language-server")
@@ -70,10 +66,10 @@ def _ensure_server_installed(cls, logger: LanguageServerLogger) -> str:
return system_server
# Download and install if needed
server_path = cls._download_and_install_server(logger)
server_path = cls._download_and_install_server()
return server_path
def _download_and_install_server(cls, logger: LanguageServerLogger) -> str:
def _download_and_install_server(cls) -> str:
"""Download and install the language server."""
# Implementation specific to your language server
pass
@@ -129,12 +125,12 @@ In `src/solidlsp/ls.py`, add your language to the `create` method:
```python
@classmethod
def create(cls, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str) -> "SolidLanguageServer":
def create(cls, config: LanguageServerConfig, repository_root_path: str) -> "SolidLanguageServer":
match config.code_language:
# Existing cases...
case Language.NEW_LANGUAGE:
from solidlsp.language_servers.new_language_server import NewLanguageServer
return NewLanguageServer(config, logger, repository_root_path)
return NewLanguageServer(config, repository_root_path)
```
## Step 3: Test Repository
.serena/project.yml
+6 -7
View File
@@ -1,11 +1,5 @@
# language of the project (csharp, python, rust, java, typescript, javascript, go, cpp, or ruby)
# Special requirements:
# * csharp: Requires the presence of a .sln file in the project folder.
language: python
# whether to use the project's gitignore file to ignore files
# Added on 2025-04-07
ignore_all_files_in_gitignore: true
# list of additional paths to ignore
# same syntax as gitignore, so you can use * and **
# Was previously called `ignored_dirs`, please update your config if you are using that.
@@ -64,3 +58,8 @@ excluded_tools: []
initial_prompt: ""
project_name: "serena"
languages:
- python
- typescript
included_optional_tools: []
encoding: utf-8
CHANGELOG.md
+38 -8
View File
@@ -1,7 +1,43 @@
# latest
Status of the `main` branch. Changes prior to the next official version change will appear here.
* General:
* Add monorepo/multi-language support
* Project configuration files (`project.yml`) can now define multiple languages.
Auto-detection adds only the most prominent language by default.
* Additional languages can be conveniently added via the Dashboard while a project is already activated.
* Support overloaded symbols in `FindSymbolTool` and related tools
* Name paths of overloaded symbols now include an index (e.g., `myOverloadedFunction[2]`)
* Responses of the Java language server, which handled this in its own way, are now adapted accordingly,
solving several issues related to retrieval problems in Java projects
* Major extensions to the dashboard, which now serves as a central web interface for Serena
* View current configuration
* View the executions, with the possibility to cancel running/scheduled executions
* View tool usage statistics
* View and create memories and edit the serena configuration file
* New two-tier caching of language server document symbols and considerable performance improvements surrounding symbol retrieval/indexing
* Various fixes related to indexing, special paths and determination of ignored paths
* Decreased `TOOL_DEFAULT_MAX_ANSWER_LENGTH` to be in accordance with (below) typical max-tokens configurations
* Allow passing language server specific settings through `ls_specific_settings` field (in `serena_config.yml`)
* Add notion of a "single-project context" (flag `single_project`), allowing user-defined contexts to behave
like the built-in `ide-assistant` context (where the available tools are restricted to ones required by the active
project and project changes are disabled)
* Client support:
* New mode `oaicompat-agent` and extensions enhancing OpenAI tool compatibility, permitting Serena to work with llama.cpp
* Tools:
* Added `RenameSymbolTool` for renaming symbols across the codebase (if LS supports this operation).
* Replaced `ReplaceRegexTool` with `ReplaceContentTool`, which supports both plain text and regex-based replacements
(and which requires no escaping in the replacement text, making it more robust)
* Language support:
* **Add support for Fortran** via fortls language server (requires `pip install fortls`)
* **Add support for Julia** via LanguageServer.jl
* **Add support for Haskell** via Haskell Language Server (HLS) with automatic discovery via ghcup, stack, or system PATH; supports both Stack and Cabal projects
* **Add support for Scala** via Metals language server (requires some [manual setup](docs/03-special-guides/scala_setup_guide_for_serena.md))
* **Add support for F#** via FsAutoComplete/Ionide LSP server.
* **Add support for Elm** via @elm-tooling/elm-language-server (automatically downloads if not installed; requires Elm compiler)
* **Add support for Perl** via Perl::LanguageServer with LSP integration for .pl, .pm, and .t files
@@ -10,19 +46,13 @@ Status of the `main` branch. Changes prior to the next official version change w
* **Add support for Zig** via ZLS (cross-file references may not fully work on Windows)
* **Add support for Lua** via lua-language-server
* **Add support for Nix** requires nixd installation (Windows not supported)
* **Add experimental support for YAML** via yaml-language-server with LSP integration for .yaml and .yml files
* **Dart now officially supported**: Dart was always working, but now tests were added, and it is promoted to "officially supported"
* **Rust now uses already installed rustup**: The rust-analyzer is no longer bundled with Serena. Instead, it uses the rust-analyzer from your Rust toolchain managed by rustup. This ensures compatibility with your Rust version and eliminates outdated bundled binaries.
* **Kotlin now officially supported**: We now use the official Kotlin LS, tests run through and performance is good, even though the LS is in an early development stage.
* **Kotlin now officially supported**: We now use the official Kotlin LS, tests run through and performance is good, even though the LS is in an early development stage.
* **Add support for Erlang** experimental, may hang or be slow, uses the recently archived [erlang_ls](https://github.com/erlang-ls/erlang_ls)
* **Ruby dual language server support**: Added ruby-lsp as the modern primary Ruby language server. Solargraph remains available as an experimental legacy option. ruby-lsp supports both .rb and .erb files, while Solargraph supports .rb files only.
* Client support:
* New mode `oaicompat-agent` and extensions in the openai tool compatibility, **permitting Serena to work with llama.cpp**
* General:
* Various fixes related to indexing, special paths and determation of ignored paths
* Decreased `TOOL_DEFAULT_MAX_ANSWER_LENGTH` to be in accordance with (below) typical max-tokens configurations
* Allow passing language server specific settings through `ls_specific_settings` field (in `serena_config.yml`)
# 0.1.4
DOCKER.md
+1 -1
View File
@@ -98,7 +98,7 @@ services:
- /path/to/another/project:/workspace/another-project
# Add the context for the IDE assistant option:
command:
- "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide-assistant"
- "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context claude-code"
```
See the [Docker Merge Compose files documentation](https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/) for more details on using merge files.
README.md
+46 -765
View File
@@ -8,46 +8,39 @@
* :wrench: Serena provides essential **semantic code retrieval and editing tools** that are akin to an IDE's capabilities, extracting code entities at the symbol level and exploiting relational structure. When combined with an existing coding agent, these tools greatly enhance (token) efficiency.
* :free: Serena is **free & open-source**, enhancing the capabilities of LLMs you already have access to free of charge.
You can think of Serena as providing IDE-like tools to your LLM/coding agent. With it, the agent no longer needs to read entire
files, perform grep-like searches or string replacements to find and edit the right code. Instead, it can use code centered tools like `find_symbol`, `find_referencing_symbols` and `insert_after_symbol`.
You can think of Serena as providing IDE-like tools to your LLM/coding agent.
With it, the agent no longer needs to read entire files, perform grep-like searches or basic string replacements to find the right parts of the code and to edit code.
Instead, it can use code-centric tools like `find_symbol`, `find_referencing_symbols` and `insert_after_symbol`.
<p align="center">
<em>Serena is under active development! See the latest updates, upcoming features, and lessons learned to stay up to date.</em>
</p>
<p align="center">
<a href="CHANGELOG.md">
<img src="https://img.shields.io/badge/Updates-1e293b?style=flat&logo=rss&logoColor=white&labelColor=1e293b" alt="Changelog" />
</a>
<a href="roadmap.md">
<img src="https://img.shields.io/badge/Roadmap-14532d?style=flat&logo=target&logoColor=white&labelColor=14532d" alt="Roadmap" />
</a>
<a href="lessons_learned.md">
<img src="https://img.shields.io/badge/Lessons-Learned-7c4700?style=flat&logo=readthedocs&logoColor=white&labelColor=7c4700" alt="Lessons Learned" />
</a>
<a href="CHANGELOG.md"><img src="https://img.shields.io/badge/Updates-1e293b?style=flat&logo=rss&logoColor=white&labelColor=1e293b" alt="Changelog" /></a>
<a href="roadmap.md"><img src="https://img.shields.io/badge/Roadmap-14532d?style=flat&logo=target&logoColor=white&labelColor=14532d" alt="Roadmap" /></a>
<a href="lessons_learned.md"><img src="https://img.shields.io/badge/Lessons-Learned-7c4700?style=flat&logo=readthedocs&logoColor=white&labelColor=7c4700" alt="Lessons Learned" /></a>
</p>
### LLM Integration
## LLM Integration
Serena provides the necessary [tools](#list-of-tools) for coding workflows, but an LLM is required to do the actual work,
Serena provides the necessary [tools](https://oraios.github.io/serena/01-about/035_tools.html) for coding workflows, but an LLM is required to do the actual work,
orchestrating tool use.
For example, **supercharge the performance of Claude Code** with a [one-line shell command](#claude-code).
In general, Serena can be integrated with an LLM in several ways:
* by using the **model context protocol (MCP)**.
Serena provides an MCP server which integrates with
* Claude Code and Claude Desktop,
* Terminal-based clients like Codex, Gemini-CLI, Qwen3-Coder, rovodev, OpenHands CLI and others,
* terminal-based clients like Codex, Gemini-CLI, Qwen3-Coder, rovodev, OpenHands CLI and others,
* IDEs like VSCode, Cursor or IntelliJ,
* Extensions like Cline or Roo Code
* Local clients like [OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp), [Jan](https://jan.ai/docs/mcp-examples/browser/browserbase#enable-mcp), [Agno](https://docs.agno.com/introduction/playground) and others
* by using [mcpo to connect it to ChatGPT](docs/serena_on_chatgpt.md) or other clients that don't support MCP but do support tool calling via OpenAPI.
* by incorporating Serena's tools into an agent framework of your choice, as illustrated [here](docs/custom_agent.md).
* by using [mcpo to connect it to ChatGPT](docs/03-special-guides/serena_on_chatgpt.md) or other clients that don't support MCP but do support tool calling via OpenAPI.
* by incorporating Serena's tools into an agent framework of your choice, as illustrated [here](docs/03-special-guides/custom_agent.md).
Serena's tool implementation is decoupled from the framework-specific code and can thus easily be adapted to any agent framework.
### Serena in Action
## Serena in Action
#### Demonstration 1: Efficient Operation in Claude Code
@@ -74,44 +67,45 @@ complex projects! So not only is it free and open-source, it frequently achieves
than existing solutions that charge a premium.
Language servers provide support for a wide range of programming languages.
With Serena, we provide direct, out-of-the-box support for:
With Serena's LSP library, we provide **support for over 30 programming languages**, including
AL, Bash, C#, C/C++, Clojure, Dart, Elixir, Elm, Erlang, Fortran, Go, Haskell, Java, Javascript, Julia, Kotlin, Lua, Markdown, Nix, Perl, PHP, Python, R, Ruby, Rust, Scala, Swift, TypeScript, YAML and Zig.
* Python
* TypeScript/Javascript
* PHP (uses Intelephense LSP; set `INTELEPHENSE_LICENSE_KEY` environment variable for premium features)
* Go (requires installation of gopls)
* R (requires installation of the `languageserver` R package)
* Rust (requires [rustup](https://rustup.rs/) - uses rust-analyzer from your toolchain)
* C/C++ (you may experience issues with finding references, we are working on it)
* Zig (requires installation of ZLS - Zig Language Server)
* C#
* F# (requires .NET SDK 8.0+ and uses FsAutoComplete/Ionide LSP server)
* Ruby (by default, uses [ruby-lsp](https://github.com/Shopify/ruby-lsp), specify ruby_solargraph as your language to use the previous solargraph based implementation)
* Swift
* Kotlin (uses the pre-alpha [official kotlin LS](https://github.com/Kotlin/kotlin-lsp), some issues may appear)
* Java (_Note_: startup is slow, initial startup especially so. There may be issues with java on macos and linux, we are working on it.)
* Clojure
* Dart
* Bash
* Lua (automatically downloads lua-language-server if not installed)
* Nix (requires nixd installation)
* Elixir (requires installation of NextLS and Elixir; **Windows not supported**)
* Elm (automatically downloads elm-language-server if not installed; requires Elm compiler)
* Erlang (requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls), experimental, might be slow or hang)
* Perl (requires installation of Perl::LanguageServer)
* AL
* Markdown (automatically downloads marksman if not installed, experimental, must be explicitly specified via `--language markdown` when generating project config - primarily useful for documentation-heavy projects)
> [!IMPORTANT]
> Some languages require additional dependencies to be installed; see the [Language Support](https://oraios.github.io/serena/01-about/020_programming-languages.html) page for details.
Support for further languages can easily be added by providing a shallow adapter for a new language server implementation,
see Serena's [memory on that](.serena/memories/adding_new_language_support_guide.md).
## Quick Start
### Community Feedback
**Prerequisites**. Serena is managed by *uv*. If you don’t already have it, you need to [install uv](https://docs.astral.sh/uv/getting-started/installation/) before proceeding.
**Starting the MCP Server**. The easiest way to start the Serena MCP server is by running the latest version from GitHub using uvx.
Issue this command to see available options:
```bash
uvx --from git+https://github.com/oraios/serena serena start-mcp-server --help
```
**Configuring Your Client**. To connect Serena to your preferred MCP client, you typically need to [configure a launch command in your client](https://oraios.github.io/serena/02-usage/030_clients.html).
Follow the link for specific instructions on how to set up Serena for Claude Code, Codex, Claude Desktop, MCP-enabled IDEs and other clients (such as local and web-based GUIs).
> [!TIP]
> While getting started quickly is easy, Serena is a powerful toolkit with many configuration options.
> We highly recommend reading through the [user guide](https://oraios.github.io/serena/02-usage/000_intro.html) to get the most out of Serena.
>
> Specifically, we recommend to read about ...
> * [Serena's project-based workflow](https://oraios.github.io/serena/02-usage/040_workflow.html) and
> * [configuring Serena](https://oraios.github.io/serena/02-usage/050_configuration.html).
## User Guide
Please refer to the [user guide](https://oraios.github.io/serena/02-usage/000_intro.html) for detailed instructions on how to use Serena effectively.
## Community Feedback
Most users report that Serena has strong positive effects on the results of their coding agents, even when used within
very capable agents like Claude Code. Serena is often described to be a [game changer](https://www.reddit.com/r/ClaudeAI/comments/1lfsdll/try_out_serena_mcp_thank_me_later/), providing an enormous [productivity boost](https://www.reddit.com/r/ClaudeCode/comments/1mguoia/absolutely_insane_improvement_of_claude_code).
Serena excels at navigating and manipulating complex codebases, providing tools that support precise code retrieval and editing in the presence of large, strongly structured codebases.
However, when dealing with tasks that involve only very few/small files, you may not benefit from including Serena on top of your existing coding agent.
However, when dealing with tasks that involve only very few/small files, you may not benefit from including Serena on top of your existing coding agent.
In particular, when writing code from scratch, Serena will not provide much value initially, as the more complex structures that Serena handles more gracefully than simplistic, file-based approaches are yet to be created.
Several videos and blog posts have talked about Serena:
@@ -126,666 +120,6 @@ Several videos and blog posts have talked about Serena:
* [Serena with Claude Code (in Japanese)](https://blog.lai.so/serena/)
* [Turning Claude Code into a Development Powerhouse](https://robertmarshall.dev/blog/turning-claude-code-into-a-development-powerhouse/)
## Table of Contents
<!-- Created with markdown-toc -i README.md -->
<!-- Install it with npm install -g markdown-toc -->
<!-- toc -->
- [Quick Start](#quick-start)
* [Running the Serena MCP Server](#running-the-serena-mcp-server)
+ [Usage](#usage)
+ [Using uvx](#using-uvx)
+ [Local Installation](#local-installation)
+ [Using Docker (Experimental)](#using-docker-experimental)
+ [Using Nix](#using-nix)
+ [Streamable HTTP Mode](#streamable-http-mode)
+ [Command-Line Arguments](#command-line-arguments)
* [Configuration](#configuration)
* [Project Activation & Indexing](#project-activation--indexing)
* [Claude Code](#claude-code)
* [Codex](#codex)
* [Other Terminal-Based Clients](#other-terminal-based-clients)
* [Claude Desktop](#claude-desktop)
* [MCP Coding Clients (Cline, Roo-Code, Cursor, Windsurf, etc.)](#mcp-coding-clients-cline-roo-code-cursor-windsurf-etc)
* [Local GUIs and Frameworks](#local-guis-and-frameworks)
- [Detailed Usage and Recommendations](#detailed-usage-and-recommendations)
* [Tool Execution](#tool-execution)
+ [Shell Execution and Editing Tools](#shell-execution-and-editing-tools)
* [Modes and Contexts](#modes-and-contexts)
+ [Contexts](#contexts)
+ [Modes](#modes)
+ [Customization](#customization)
* [Onboarding and Memories](#onboarding-and-memories)
* [Prepare Your Project](#prepare-your-project)
+ [Structure Your Codebase](#structure-your-codebase)
+ [Start from a Clean State](#start-from-a-clean-state)
+ [Logging, Linting, and Automated Tests](#logging-linting-and-automated-tests)
* [Prompting Strategies](#prompting-strategies)
* [Running Out of Context](#running-out-of-context)
* [Serena's Logs: The Dashboard and GUI Tool](#serenas-logs-the-dashboard-and-gui-tool)
- [Comparison with Other Coding Agents](#comparison-with-other-coding-agents)
* [Subscription-Based Coding Agents](#subscription-based-coding-agents)
* [API-Based Coding Agents](#api-based-coding-agents)
* [Other MCP-Based Coding Agents](#other-mcp-based-coding-agents)
- [Acknowledgements](#acknowledgements)
* [Sponsors](#sponsors)
* [Community Contributions](#community-contributions)
* [Technologies](#technologies)
- [Customizing and Extending Serena](#customizing-and-extending-serena)
- [List of Tools](#list-of-tools)
<!-- tocstop -->
## Quick Start
Serena can be used in various ways, below you will find instructions for selected integrations.
* For coding with Claude, we recommend using Serena through [Claude Code](#claude-code) or [Claude Desktop](#claude-desktop). You can also use Serena in most other [terminal-based clients](#other-terminal-based-clients).
* If you want a GUI experience outside an IDE, you can use one of the many [local GUIs](#local-guis-and-frameworks) that support MCP servers.
You can also connect Serena to many web clients (including ChatGPT) using [mcpo](docs/serena_on_chatgpt.md).
* If you want to use Serena integrated in your IDE, see the section on [other MCP clients](#other-mcp-clients---cline-roo-code-cursor-windsurf-etc).
* You can use Serena as a library for building your own applications. We try to keep the public API stable, but you should still
expect breaking changes and pin Serena to a fixed version if you use it as a dependency.
Serena is managed by `uv`, so you will need to [install it](https://docs.astral.sh/uv/getting-started/installation/).
### Running the Serena MCP Server
You have several options for running the MCP server, which are explained in the subsections below.
#### Usage
The typical usage involves the client (Claude Code, Claude Desktop, etc.) running
the MCP server as a subprocess (using stdio communication),
so the client needs to be provided with the command to run the MCP server.
(Alternatively, you can run the MCP server in Streamable HTTP or SSE mode and tell your client
how to connect to it.)
Note that no matter how you run the MCP server, Serena will, by default, start a small web-based dashboard on localhost that will display logs and allow shutting down the
MCP server (since many clients fail to clean up processes correctly).
This and other settings can be adjusted in the [configuration](#configuration) and/or by providing [command-line arguments](#command-line-arguments).
#### Using uvx
`uvx` can be used to run the latest version of Serena directly from the repository, without an explicit local installation.
```shell
uvx --from git+https://github.com/oraios/serena serena start-mcp-server
```
Explore the CLI to see some of the customization options that serena provides (more info on them below).
#### Local Installation
1. Clone the repository and change into it.
```shell
git clone https://github.com/oraios/serena
cd serena
```
2. Optionally edit the configuration file in your home directory with
```shell
uv run serena config edit
```
If you just want the default config, you can skip this part, and a config file will be created when you first run Serena.
3. Run the server with `uv`:
```shell
uv run serena start-mcp-server
```
When running from outside the serena installation directory, be sure to pass it, i.e., use
```shell
uv run --directory /abs/path/to/serena serena start-mcp-server
```
#### Using Docker (Experimental)
⚠️ Docker support is currently experimental with several limitations. Please read the [Docker documentation](DOCKER.md) for important caveats before using it.
You can run the Serena MCP server directly via docker as follows,
assuming that the projects you want to work on are all located in `/path/to/your/projects`:
```shell
docker run --rm -i --network host -v /path/to/your/projects:/workspaces/projects ghcr.io/oraios/serena:latest serena start-mcp-server --transport stdio
```
Replace `/path/to/your/projects` with the absolute path to your projects directory. The Docker approach provides:
* Better security isolation for shell command execution
* No need to install language servers and dependencies locally
* Consistent environment across different systems
Alternatively, use docker compose with the `compose.yml` file provided in the repository.
See the [Docker documentation](DOCKER.md) for detailed setup instructions, configuration options, and known limitations.
#### Using Nix
If you are using Nix and [have enabled the `nix-command` and `flakes` features](https://nixos.wiki/wiki/flakes), you can run Serena using the following command:
```bash
nix run github:oraios/serena -- start-mcp-server --transport stdio
```
You can also install Serena by referencing this repo (`github:oraios/serena`) and using it in your Nix flake. The package is exported as `serena`.
#### Streamable HTTP Mode
ℹ️ Note that MCP servers which use stdio as a protocol are somewhat unusual as far as client/server architectures go, as the server
necessarily has to be started by the client in order for communication to take place via the server's standard input/output stream.
In other words, you do not need to start the server yourself. The client application (e.g. Claude Desktop) takes care of this and
therefore needs to be configured with a launch command.
When using instead the *Streamable HTTP* mode, you control the server lifecycle yourself,
i.e. you start the server and provide the client with the URL to connect to it.
Simply provide `start-mcp-server` with the `--transport streamable-http` option and optionally provide the port.
For example, to run the Serena MCP server in Streamable HTTP mode on port 9121 using a local installation,
you would run this command from the Serena directory,
```shell
uv run serena start-mcp-server --transport streamable-http --port 9121
```
and then configure your client to connect to `http://localhost:9121/mcp`.
ℹ️ Note that SSE transport is supported as well, but its use is discouraged.
Use Streamable HTTP instead.
#### Command-Line Arguments
The Serena MCP server supports a wide range of additional command-line options, including the option to run in Streamable HTTP or SSE mode
and to adapt Serena to various [contexts and modes of operation](#modes-and-contexts).
Run with parameter `--help` to get a list of available options.
### Configuration
Serena is very flexible in terms of configuration. While for most users, the default configurations will work,
you can fully adjust it to your needs by editing a few yaml files. You can disable tools, change Serena's instructions
(what we denote as the `system_prompt`), adjust the output of tools that just provide a prompt, and even adjust tool descriptions.
Serena is configured in four places:
1. The `serena_config.yml` for general settings that apply to all clients and projects.
It is located in your user directory under `.serena/serena_config.yml`.
If you do not explicitly create the file, it will be auto-generated when you first run Serena.
You can edit it directly or use
```shell
uvx --from git+https://github.com/oraios/serena serena config edit
```
(or use the `--directory` command version).
2. In the arguments passed to the `start-mcp-server` in your client's config (see below),
which will apply to all sessions started by the respective client. In particular, the [context](#contexts) parameter
should be set appropriately for Serena to be best adjusted to existing tools and capabilities of your client.
See for a detailed explanation. You can override all entries from the `serena_config.yml` through command line arguments.
3. In the `.serena/project.yml` file within your project. This will hold project-level configuration that is used whenever
that project is activated. This file will be autogenerated when you first use Serena on that project, but you can also
generate it explicitly with
```shell
uvx --from git+https://github.com/oraios/serena serena project generate-yml
```
(or use the `--directory` command version).
4. Through the context and modes. Explore the [modes and contexts](#modes-and-contexts) section for more details.
After the initial setup, continue with one of the sections below, depending on how you
want to use Serena.
### Project Activation & Indexing
If you are mostly working with the same project, you can configure to always activate it at startup
by passing `--project <path_or_name>` to the `start-mcp-server` command in your client's MCP config.
This is especially useful for clients which configure MCP servers on a per-project basis, like Claude Code.
Otherwise, the recommended way is to just ask the LLM to activate a project by providing it an absolute path to, or,
in case the project was activated in the past, by its name. The default project name is the directory name.
* "Activate the project /path/to/my_project"
* "Activate the project my_project"
All projects that have been activated will be automatically added to your `serena_config.yml`, and for each
project, the file `.serena/project.yml` will be generated. You can adjust the latter, e.g., by changing the name
(which you refer to during the activation) or other options. Make sure to not have two different projects with the
same name.
ℹ️ For larger projects, we recommend that you index your project to accelerate Serena's tools; otherwise the first
tool application may be very slow.
To do so, run this from the project directory (or pass the path to the project as an argument):
```shell
uvx --from git+https://github.com/oraios/serena serena project index
```
(or use the `--directory` command version).
### Claude Code
Serena is a great way to make Claude Code both cheaper and more powerful!
From your project directory, add serena with a command like this,
```shell
claude mcp add serena -- <serena-mcp-server> --context ide-assistant --project $(pwd)
```
where `<serena-mcp-server>` is your way of [running the Serena MCP server](#running-the-serena-mcp-server).
For example, when using `uvx`, you would run
```shell
claude mcp add serena -- uvx --from git+https://github.com/oraios/serena serena start-mcp-server --context ide-assistant --project $(pwd)
```
ℹ️ Serena comes with an instruction text, and Claude needs to read it to properly use Serena's tools.
As of version `v1.0.52`, claude code reads the instructions of the MCP server, so this **is handled automatically**.
If you are using an older version, or if Claude fails to read the instructions, you can ask it explicitly
to "read Serena's initial instructions" or run `/mcp__serena__initial_instructions` to load the instruction text.
If you want to make use of that, you will have to enable the corresponding tool explicitly by adding `initial_instructions` to the `included_optional_tools`
in your config.
Note that you may have to make Claude read the instructions when you start a new conversation and after any compacting operation to ensure Claude remains properly configured to use Serena's tools.
### Codex
Serena works with OpenAI's Codex CLI out of the box, but you have to use the `codex` context for it to work properly. (The technical reason is that Codex doesn't fully support the MCP specifications, so some massaging of tools is required.).
Unlike Claude Code, in Codex you add an MCP server globally and not per project. Add the following to
`~/.codex/config.toml` (create the file if it does not exist):
```toml
[mcp_servers.serena]
command = "uvx"
args = ["--from", "git+https://github.com/oraios/serena", "serena", "start-mcp-server", "--context", "codex"]
```
After codex has started, you need to activate the project, which you can do by saying:
"Activate the current dir as project using serena"
> If you don't activate the project, you will not be able to use Serena's tools!
That's it! Have a look at `~/.codex/log/codex-tui.log` to see if any errors occurred.
The Serena dashboard will run if you have not disabled it in the configuration, but due to Codex's sandboxing the webbrowser
may not open automatically. You can open it manually by going to `http://localhost:24282/dashboard/index.html` (or a higher port, if
that was already taken).
> Codex will often show the tools as `failed` even though they are successfully executed. This is not a problem, seems to be a bug in Codex. Despite the error message, everything works as expected.
### Other Terminal-Based Clients
There are many terminal-based coding assistants that support MCP servers, such as [Codex](https://github.com/openai/codex?tab=readme-ov-file#model-context-protocol-mcp),
[Gemini-CLI](https://github.com/google-gemini/gemini-cli), [Qwen3-Coder](https://github.com/QwenLM/Qwen3-Coder),
[rovodev](https://community.atlassian.com/forums/Rovo-for-Software-Teams-Beta/Introducing-Rovo-Dev-CLI-AI-Powered-Development-in-your-terminal/ba-p/3043623),
the [OpenHands CLI](https://docs.all-hands.dev/usage/how-to/cli-mode) and [opencode](https://github.com/sst/opencode).
They generally benefit from the symbolic tools provided by Serena. You might want to customize some aspects of Serena
by writing your own context, modes or prompts to adjust it to your workflow, to other MCP servers you are using, and to
the client's internal capabilities.
### Claude Desktop
For [Claude Desktop](https://claude.ai/download) (available for Windows and macOS), go to File / Settings / Developer / MCP Servers / Edit Config,
which will let you open the JSON file `claude_desktop_config.json`.
Add the `serena` MCP server configuration, using a [run command](#running-the-serena-mcp-server) depending on your setup.
* local installation:
```json
{
"mcpServers": {
"serena": {
"command": "/abs/path/to/uv",
"args": ["run", "--directory", "/abs/path/to/serena", "serena", "start-mcp-server"]
}
}
}
```
* uvx:
```json
{
"mcpServers": {
"serena": {
"command": "/abs/path/to/uvx",
"args": ["--from", "git+https://github.com/oraios/serena", "serena", "start-mcp-server"]
}
}
}
```
* docker:
```json
{
"mcpServers": {
"serena": {
"command": "docker",
"args": ["run", "--rm", "-i", "--network", "host", "-v", "/path/to/your/projects:/workspaces/projects", "ghcr.io/oraios/serena:latest", "serena", "start-mcp-server", "--transport", "stdio"]
}
}
}
```
If you are using paths containing backslashes for paths on Windows
(note that you can also just use forward slashes), be sure to escape them correctly (`\\`).
That's it! Save the config and then restart Claude Desktop. You are ready for activating your first project.
ℹ️ You can further customize the run command using additional arguments (see [above](#command-line-arguments)).
Note: on Windows and macOS there are official Claude Desktop applications by Anthropic, for Linux there is an [open-source
community version](https://github.com/aaddrick/claude-desktop-debian).
⚠️ Be sure to fully quit the Claude Desktop application, as closing Claude will just minimize it to the system tray – at least on Windows.
⚠️ Some clients may leave behind zombie processes. You will have to find and terminate them manually then.
With Serena, you can activate the [dashboard](#serenas-logs-the-dashboard-and-gui-tool) to prevent unnoted processes and also use the dashboard
for shutting down Serena.
After restarting, you should see Serena's tools in your chat interface (notice the small hammer icon).
For more information on MCP servers with Claude Desktop, see [the official quick start guide](https://modelcontextprotocol.io/quickstart/user).
### MCP Coding Clients (Cline, Roo-Code, Cursor, Windsurf, etc.)
Being an MCP Server, Serena can be included in any MCP Client. The same configuration as above,
perhaps with small client-specific modifications, should work. Most of the popular
existing coding assistants (IDE extensions or VSCode-like IDEs) support connections
to MCP Servers. It is **recommended to use the `ide-assistant` context** for these integrations by adding `"--context", "ide-assistant"` to the `args` in your MCP client's configuration. Including Serena generally boosts their performance
by providing them tools for symbolic operations.
In this case, the billing for the usage continues to be controlled by the client of your choice
(unlike with the Claude Desktop client). But you may still want to use Serena through such an approach,
e.g., for one of the following reasons:
1. You are already using a coding assistant (say Cline or Cursor) and just want to make it more powerful.
2. You are on Linux and don't want to use the [community-created Claude Desktop](https://github.com/aaddrick/claude-desktop-debian).
3. You want tighter integration of Serena into your IDE and don't mind paying for that.
### Local GUIs and Frameworks
Over the last months, several technologies have emerged that allow you to run a powerful local GUI
and connect it to an MCP server. They will work with Serena out of the box.
Some of the leading open source GUI technologies offering this are
[Jan](https://jan.ai/docs/mcp), [OpenHands](https://github.com/All-Hands-AI/OpenHands/),
[OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp) and [Agno](https://docs.agno.com/introduction/playground).
They allow combining Serena with almost any LLM (including locally running ones) and offer various other integrations.
## Detailed Usage and Recommendations
### Tool Execution
Serena combines tools for semantic code retrieval with editing capabilities and shell execution.
Serena's behavior can be further customized through [Modes and Contexts](#modes-and-contexts).
Find the complete list of tools [below](#full-list-of-tools).
The use of all tools is generally recommended, as this allows Serena to provide the most value:
Only by executing shell commands (in particular, tests) can Serena identify and correct mistakes
autonomously.
#### Shell Execution and Editing Tools
Many clients have their own shell execution tool, and by default Serena's shell tool will be disabled in them
(e.g., when using the `ide-assistant` or `codex` context). However, when using Serena through something like
Claude Desktop or ChatGPT, it is recommended to enable Serena's `execute_shell_command` tool to allow
agentic behavior.
It should be noted that the `execute_shell_command` tool allows for arbitrary code execution.
When using Serena as an MCP Server, clients will typically ask the user for permission
before executing a tool, so as long as the user inspects execution parameters beforehand,
this should not be a problem.
However, if you have concerns, you can choose to disable certain commands in your project's configuration file.
If you only want to use Serena purely for analyzing code and suggesting implementations
without modifying the codebase, you can enable read-only mode by setting `read_only: true` in your project configuration file.
This will automatically disable all editing tools and prevent any modifications to your codebase while still
allowing all analysis and exploration capabilities.
In general, be sure to back up your work and use a version control system in order to avoid
losing any work.
### Modes and Contexts
Serena's behavior and toolset can be adjusted using contexts and modes.
These allow for a high degree of customization to best suit your workflow and the environment Serena is operating in.
#### Contexts
A context defines the general environment in which Serena is operating.
It influences the initial system prompt and the set of available tools.
A context is set at startup when launching Serena (e.g., via CLI options for an MCP server or in the agent script) and cannot be changed during an active session.
Serena comes with pre-defined contexts:
* `desktop-app`: Tailored for use with desktop applications like Claude Desktop. This is the default.
* `agent`: Designed for scenarios where Serena acts as a more autonomous agent, for example, when used with Agno.
* `ide-assistant`: Optimized for integration into IDEs like VSCode, Cursor, or Cline, focusing on in-editor coding assistance.
Choose the context that best matches the type of integration you are using.
When launching Serena, specify the context using `--context <context-name>`.
Note that for cases where parameter lists are specified (e.g. Claude Desktop), you must add two parameters to the list.
If you are using a local server (such as Llama.cpp) which requires you to use OpenAI-compatible tool descriptions, use context `oaicompat-agent` instead of `agent`.
#### Modes
Modes further refine Serena's behavior for specific types of tasks or interaction styles. Multiple modes can be active simultaneously, allowing you to combine their effects. Modes influence the system prompt and can also alter the set of available tools by excluding certain ones.
Examples of built-in modes include:
* `planning`: Focuses Serena on planning and analysis tasks.
* `editing`: Optimizes Serena for direct code modification tasks.
* `interactive`: Suitable for a conversational, back-and-forth interaction style.
* `one-shot`: Configures Serena for tasks that should be completed in a single response, often used with `planning` for generating reports or initial plans.
* `no-onboarding`: Skips the initial onboarding process if it's not needed for a particular session.
* `onboarding`: (Usually triggered automatically) Focuses on the project onboarding process.
Modes can be set at startup (similar to contexts) but can also be _switched dynamically_ during a session. You can instruct the LLM to use the `switch_modes` tool to activate a different set of modes (e.g., "switch to planning and one-shot modes").
When launching Serena, specify modes using `--mode <mode-name>`; multiple modes can be specified, e.g. `--mode planning --mode no-onboarding`.
:warning: **Mode Compatibility**: While you can combine modes, some may be semantically incompatible (e.g., `interactive` and `one-shot`). Serena currently does not prevent incompatible combinations; it is up to the user to choose sensible mode configurations.
#### Customization
You can create your own contexts and modes to precisely tailor Serena to your needs in two ways:
* You can use Serena's CLI to manage modes and contexts. Check out
```shell
uvx --from git+https://github.com/oraios/serena serena mode --help
```
and
```shell
uvx --from git+https://github.com/oraios/serena serena context --help
```
_NOTE_: Custom contexts/modes are simply YAML files in `<home>/.serena`, they are automatically registered and available for use by their name (filename without the `.yml` extension). If you don't want to use Serena's CLI, you can create and manage them in any way you see fit.
* **Using external YAML files**: When starting Serena, you can also provide an absolute path to a custom `.yml` file for a context or mode.
This customization allows for deep integration and adaptation of Serena to specific project requirements or personal preferences.
### Onboarding and Memories
By default, Serena will perform an **onboarding process** when
it is started for the first time for a project.
The goal of the onboarding is for Serena to get familiar with the project
and to store memories, which it can then draw upon in future interactions.
If an LLM should fail to complete the onboarding and does not actually write the
respective memories to disk, you may need to ask it to do so explicitly.
The onboarding will usually read a lot of content from the project, thus filling
up the context. It can therefore be advisable to switch to another conversation
once the onboarding is complete.
After the onboarding, we recommend that you have a quick look at the memories and,
if necessary, edit them or add additional ones.
**Memories** are files stored in `.serena/memories/` in the project directory,
which the agent can choose to read in subsequent interactions.
Feel free to read and adjust them as needed; you can also add new ones manually.
Every file in the `.serena/memories/` directory is a memory file.
Whenever Serena starts working on a project, the list of memories is
provided, and the agent can decide to read them.
We found that memories can significantly improve the user experience with Serena.
### Prepare Your Project
#### Structure Your Codebase
Serena uses the code structure for finding, reading and editing code. This means that it will
work well with well-structured code but may perform poorly on fully unstructured one (like a "God class"
with enormous, non-modular functions).
Furthermore, for languages that are not statically typed, type annotations are highly beneficial.
#### Start from a Clean State
It is best to start a code generation task from a clean git state. Not only will
this make it easier for you to inspect the changes, but also the model itself will
have a chance of seeing what it has changed by calling `git diff` and thereby
correct itself or continue working in a followup conversation if needed.
:warning: **Important**: since Serena will write to files using the system-native line endings
and it might want to look at the git diff, it is important to
set `git config core.autocrlf` to `true` on Windows.
With `git config core.autocrlf` set to `false` on Windows, you may end up with huge diffs
only due to line endings. It is generally a good idea to globally enable this git setting on Windows:
```shell
git config --global core.autocrlf true
```
#### Logging, Linting, and Automated Tests
Serena can successfully complete tasks in an _agent loop_, where it iteratively
acquires information, performs actions, and reflects on the results.
However, Serena cannot use a debugger; it must rely on the results of program executions,
linting results, and test results to assess the correctness of its actions.
Therefore, software that is designed to meaningful interpretable outputs (e.g. log messages)
and that has a good test coverage is much easier to work with for Serena.
We generally recommend to start an editing task from a state where all linting checks and tests pass.
### Prompting Strategies
We found that it is often a good idea to spend some time conceptualizing and planning a task
before actually implementing it, especially for non-trivial task. This helps both in achieving
better results and in increasing the feeling of control and staying in the loop. You can
make a detailed plan in one session, where Serena may read a lot of your code to build up the context,
and then continue with the implementation in another (potentially after creating suitable memories).
### Running Out of Context
For long and complicated tasks, or tasks where Serena has read a lot of content, you
may come close to the limits of context tokens. In that case, it is often a good idea to continue
in a new conversation. Serena has a dedicated tool to create a summary of the current state
of the progress and all relevant info for continuing it. You can request to create this summary and
write it to a memory. Then, in a new conversation, you can just ask Serena to read the memory and
continue with the task. In our experience, this worked really well. On the up-side, since in a
single session there is no summarization involved, Serena does not usually get lost (unlike some
other agents that summarize under the hood), and it is also instructed to occasionally check whether
it's on the right track.
Moreover, Serena is instructed to be frugal with context
(e.g., to not read bodies of code symbols unnecessarily),
but we found that Claude is not always very good in being frugal (Gemini seemed better at it).
You can explicitly instruct it to not read the bodies if you know that it's not needed.
### Serena's Logs: The Dashboard and GUI Tool
Serena provides two convenient ways of accessing the logs of the current session:
* via the **web-based dashboard** (enabled by default)
This is supported on all platforms.
By default, it will be accessible at `http://localhost:24282/dashboard/index.html`,
but a higher port may be used if the default port is unavailable/multiple instances are running.
* via the **GUI tool** (disabled by default)
This is mainly supported on Windows, but it may also work on Linux; macOS is unsupported.
Both can be enabled, configured or disabled in Serena's configuration file (`serena_config.yml`, see above).
If enabled, they will automatically be opened as soon as the Serena agent/MCP server is started.
The web dashboard will display usage statistics of Serena's tools if you set `record_tool_usage_stats: True` in your config.
In addition to viewing logs, both tools allow to shut down the Serena agent.
This function is provided, because clients like Claude Desktop may fail to terminate the MCP server subprocess
when they themselves are closed.
## Comparison with Other Coding Agents
To our knowledge, Serena is the first fully-featured coding agent where the
entire functionality
is available through an MCP server, thus not requiring API keys or
subscriptions.
### Subscription-Based Coding Agents
Many prominent subscription-based coding agents are parts of IDEs like
Windsurf, Cursor and VSCode.
Serena's functionality is similar to Cursor's Agent, Windsurf's Cascade or
VSCode's agent mode.
Serena has the advantage of not requiring a subscription.
A potential disadvantage is that it
is not directly integrated into an IDE, so the inspection of newly written code
is not as seamless.
More technical differences are:
* Serena is not bound to a specific IDE or CLI.
Serena's MCP server can be used with any MCP client (including some IDEs),
and the Agno-based agent provides additional ways of applying its functionality.
* Serena is not bound to a specific large language model or API.
* Serena navigates and edits code using a language server, so it has a symbolic
understanding of the code.
IDE-based tools often use a RAG-based or purely text-based approach, which is often
less powerful, especially for large codebases.
* Serena is open-source and has a small codebase, so it can be easily extended
and modified.
### API-Based Coding Agents
An alternative to subscription-based agents are API-based agents like Claude
Code, Cline, Aider, Roo Code and others, where the usage costs map directly
to the API costs of the underlying LLM.
Some of them (like Cline) can even be included in IDEs as an extension.
They are often very powerful and their main downside are the (potentially very
high) API costs.
Serena itself can be used as an API-based agent (see the section on Agno above).
We have not yet written a CLI tool or a
dedicated IDE extension for Serena (and there is probably no need for the latter, as
Serena can already be used with any IDE that supports MCP servers).
If there is demand for a Serena as a CLI tool like Claude Code, we will
consider writing one.
The main difference between Serena and other API-based agents is that Serena can
also be used as an MCP server, thus not requiring
an API key and bypassing the API costs. This is a unique feature of Serena.
### Other MCP-Based Coding Agents
There are other MCP servers designed for coding, like [DesktopCommander](https://github.com/wonderwhy-er/DesktopCommanderMCP) and
[codemcp](https://github.com/ezyang/codemcp).
However, to the best of our knowledge, none of them provide semantic code
retrieval and editing tools; they rely purely on text-based analysis.
It is the integration of language servers and the MCP that makes Serena unique
and so powerful for challenging coding tasks, especially in the context of
larger codebases.
## Acknowledgements
### Sponsors
@@ -815,16 +149,12 @@ what it is today.
We built Serena on top of multiple existing open-source technologies, the most important ones being:
1. [multilspy](https://github.com/microsoft/multilspy).
A library which wraps language server implementations and adapts them for interaction via Python
and which provided the basis for our library Solid-LSP (src/solidlsp).
A library which wraps language server implementations and adapts them for interaction via Python.
It provided the basis for our library Solid-LSP (`src/solidlsp`).
Solid-LSP provides pure synchronous LSP calls and extends the original library with the symbolic logic
that Serena required.
2. [Python MCP SDK](https://github.com/modelcontextprotocol/python-sdk)
3. [Agno](https://github.com/agno-agi/agno) and
the associated [agent-ui](https://github.com/agno-agi/agent-ui),
which we use to allow Serena to work with any model, beyond the ones
supporting the MCP.
4. All the language servers that we use through Solid-LSP.
3. All the language servers that we use through Solid-LSP.
Without these projects, Serena would not have been possible (or would have been significantly more difficult to build).
@@ -840,52 +170,3 @@ It is also relatively straightforward to add [support for a new programming lang
We look forward to seeing what the community will come up with!
For details on contributing, see [contributing guidelines](/CONTRIBUTING.md).
## List of Tools
Here is the list of Serena's default tools with a short description (output of `uv run serena tools list`):
* `activate_project`: Activates a project by name.
* `check_onboarding_performed`: Checks whether project onboarding was already performed.
* `create_text_file`: Creates/overwrites a file in the project directory.
* `delete_memory`: Deletes a memory from Serena's project-specific memory store.
* `execute_shell_command`: Executes a shell command.
* `find_file`: Finds files in the given relative paths
* `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type).
* `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
* `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
* `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
* `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
* `list_dir`: Lists files and directories in the given directory (optionally with recursion).
* `list_memories`: Lists memories in Serena's project-specific memory store.
* `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
* `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context).
* `read_file`: Reads a file within the project directory.
* `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
* `replace_regex`: Replaces content in a file by using regular expressions.
* `replace_symbol_body`: Replaces the full definition of a symbol.
* `search_for_pattern`: Performs a search for a pattern in the project.
* `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
* `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
* `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
* `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
There are several tools that are disabled by default, and have to be enabled explicitly, e.g., through the context or modes.
Note that several of our default contexts do enable some of these tools. For example, the `desktop-app` context enables the `execute_shell_command` tool.
The full list of optional tools is (output of `uv run serena tools list --only-optional`):
* `delete_lines`: Deletes a range of lines within a file.
* `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes.
* `initial_instructions`: Gets the initial instructions for the current project.
Should only be used in settings where the system prompt cannot be set,
e.g. in clients you have no control over, like Claude Desktop.
* `insert_at_line`: Inserts content at a given line in a file.
* `jet_brains_find_referencing_symbols`: Finds symbols that reference the given symbol
* `jet_brains_find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
* `jet_brains_get_symbols_overview`: Retrieves an overview of the top-level symbols within a specified file
* `remove_project`: Removes a project from the Serena configuration.
* `replace_lines`: Replaces a range of lines within a file with new content.
* `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
* `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
* `switch_modes`: Activates modes by providing a list of their names
docs/.gitignore
+4
View File
@@ -0,0 +1,4 @@
/_toc.yml
/jupyter_execute
/conf.py
/_build
docs/01-about/000_intro.md
+14
View File
@@ -0,0 +1,14 @@
# About Serena
* Serena is a powerful **coding agent toolkit** capable of turning an LLM into a fully-featured agent that works
**directly on your codebase**.
Unlike most other tools, it is not tied to an LLM, framework or an interface, making it easy to use it in a variety of ways.
* Serena provides essential **semantic code retrieval and editing tools** that are akin to an IDE's capabilities,
extracting code entities at the symbol level and exploiting relational structure.
When combined with an existing coding agent, these tools greatly enhance (token) efficiency.
* Serena is **free & open-source**, enhancing the capabilities of LLMs you already have access to free of charge.
Therefore, you can think of Serena as providing IDE-like tools to your LLM/coding agent.
With it, the agent no longer needs to read entire files, perform grep-like searches or string replacements to find and
edit the right code.
Instead, it can use code-centred tools like `find_symbol`, `find_referencing_symbols` and `insert_after_symbol`.
docs/01-about/010_llm-integration.md
+17
View File
@@ -0,0 +1,17 @@
# LLM Integration
Serena provides the necessary [tools](035_tools) for coding workflows, but an LLM is required to do the actual work,
orchestrating tool use.
In general, Serena can be integrated with an LLM in several ways:
* by using the **model context protocol (MCP)**.
Serena provides an MCP server which integrates with
* Claude Code and Claude Desktop,
* Terminal-based clients like Codex, Gemini-CLI, Qwen3-Coder, rovodev, OpenHands CLI and others,
* IDEs like VSCode, Cursor or IntelliJ,
* Extensions like Cline or Roo Code
* Local clients like [OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp), [Jan](https://jan.ai/docs/mcp-examples/browser/browserbase#enable-mcp), [Agno](https://docs.agno.com/introduction/playground) and others
* by using [mcpo to connect it to ChatGPT](../03-special-guides/serena_on_chatgpt.md) or other clients that don't support MCP but do support tool calling via OpenAPI.
* by incorporating Serena's tools into an agent framework of your choice, as illustrated [here](../03-special-guides/custom_agent).
Serena's tool implementation is decoupled from the framework-specific code and can thus easily be adapted to any agent framework.
docs/01-about/020_programming-languages.md
+65
View File
@@ -0,0 +1,65 @@
# Language Support
Serena's semantic code analysis capabilities build on **language servers** using the widely implemented
language server protocol (LSP). The LSP provides a set of versatile code querying
and editing functionalities based on symbolic understanding of the code.
Equipped with these capabilities, Serena discovers and edits code just like a seasoned developer
making use of an IDE's capabilities would.
Serena can efficiently find the right context and do the right thing even in very large and
complex projects! So not only is it free and open-source, it frequently achieves better results
than existing solutions that charge a premium.
Language servers provide support for a wide range of programming languages.
With Serena, we provide direct, out-of-the-box support for:
* **AL**
* **Bash**
* **C#**
* **C/C++**
(you may experience issues with finding references, we are working on it)
* **Clojure**
* **Dart**
* **Elixir**
(requires installation of NextLS and Elixir; Windows not supported)
* **Elm**
(requires Elm compiler)
* **Erlang**
(requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls); experimental, might be slow or hang)
* **F#**
(requires .NET SDK 8.0+ and uses FsAutoComplete/Ionide LSP server)
* **Fortran**
(requires installation of fortls: `pip install fortls`)
* **Go**
(requires installation of `gopls`)
* **Haskell**
(automatically locates HLS via ghcup, stack, or system PATH; supports Stack and Cabal projects)
* **Java**
* **JavaScript**
* **Julia**
* **Kotlin**
(uses the pre-alpha [official kotlin LS](https://github.com/Kotlin/kotlin-lsp), some issues may appear)
* **Lua**
* **Markdown**
(must be explicitly specified via `--language markdown` when generating project config, primarily useful for documentation-heavy projects)
* **Nix**
(requires nixd installation)
* **Perl**
(requires installation of Perl::LanguageServer)
* **PHP**
(uses Intelephense LSP; set `INTELEPHENSE_LICENSE_KEY` environment variable for premium features)
* **Python**
* **R**
(requires installation of the `languageserver` R package)
* **Ruby**
(by default, uses [ruby-lsp](https://github.com/Shopify/ruby-lsp), specify ruby_solargraph as your language to use the previous solargraph based implementation)
* **Rust**
(requires [rustup](https://rustup.rs/) - uses rust-analyzer from your toolchain)
* **Scala**
(requires some [manual setup](../03-special-guides/scala_setup_guide_for_serena); uses Metals LSP)
* **Swift**
* **TypeScript**
* **Zig**
(requires installation of ZLS - Zig Language Server)
Support for further languages can easily be added by providing a shallow adapter for a new language server implementation,
see Serena's [memory on that](https://github.com/oraios/serena/blob/main/.serena/memories/adding_new_language_support_guide.md).
docs/01-about/030_serena-in-action.md
+24
View File
@@ -0,0 +1,24 @@
# Serena in Action
## Demonstration 1: Efficient Operation in Claude Code
A demonstration of Serena efficiently retrieving and editing code within Claude Code, thereby saving tokens and time. Efficient operations are not only useful for saving costs, but also for generally improving the generated code's quality. This effect may be less pronounced in very small projects, but often becomes of crucial importance in larger ones.
<video src="https://github.com/user-attachments/assets/ab78ebe0-f77d-43cc-879a-cc399efefd87"
controls
preload="metadata"
style="max-width: 100%; height: auto;">
Your browser does not support the video tag.
</video>
## Demonstration 2: Serena in Claude Desktop
A demonstration of Serena implementing a small feature for itself (a better log GUI) with Claude Desktop.
Note how Serena's tools enable Claude to find and edit the right symbols.
<video src="https://github.com/user-attachments/assets/6eaa9aa1-610d-4723-a2d6-bf1e487ba753"
controls
preload="metadata"
style="max-width: 100%; height: auto;">
Your browser does not support the video tag.
</video>
docs/01-about/035_tools.md
+47
View File
@@ -0,0 +1,47 @@
# List of Tools
Find the full list of Serena's tools below (output of `<serena> tools list --all`).
Note that in most configurations, only a subset of these tools will be enabled simultaneously (see the section on [configuration](../02-usage/050_configuration) for details).
* `activate_project`: Activates a project based on the project name or path.
* `check_onboarding_performed`: Checks whether project onboarding was already performed.
* `create_text_file`: Creates/overwrites a file in the project directory.
* `delete_lines`: Deletes a range of lines within a file.
* `delete_memory`: Deletes a memory from Serena's project-specific memory store.
* `execute_shell_command`: Executes a shell command.
* `find_file`: Finds files in the given relative paths
* `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type).
* `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filter
ed by type).
* `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools,
contexts, and modes.
* `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
* `initial_instructions`: Provides instructions on how to use the Serena toolbox.
* `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
* `insert_at_line`: Inserts content at a given line in a file.
* `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
* `jet_brains_find_referencing_symbols`: Finds symbols that reference the given symbol
* `jet_brains_find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (option
ally filtered by type).
* `jet_brains_get_symbols_overview`: Retrieves an overview of the top-level symbols within a specified file
* `list_dir`: Lists files and directories in the given directory (optionally with recursion).
* `list_memories`: Lists memories in Serena's project-specific memory store.
* `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
* `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with th
e necessary context).
* `read_file`: Reads a file within the project directory.
* `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
* `remove_project`: Removes a project from the Serena configuration.
* `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities.
* `replace_lines`: Replaces a range of lines within a file with new content.
* `replace_content`: Replaces content in a file (optionally using regular expressions).
* `replace_symbol_body`: Replaces the full definition of a symbol.
* `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
* `search_for_pattern`: Performs a search for a pattern in the project.
* `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
* `switch_modes`: Activates modes by providing a list of their names
* `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
* `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
* `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
* `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
docs/01-about/040_comparison-to-other-agents.md
+51
View File
@@ -0,0 +1,51 @@
# Comparison with Other Coding Agents
To our knowledge, Serena is the first fully-featured coding agent where the
entire functionality is made available through an MCP server,
thus not requiring additional API keys or subscriptions if access to an LLM
is already available through an MCP-compatible client.
## Subscription-Based Coding Agents
Many prominent subscription-based coding agents are parts of IDEs like
Windsurf, Cursor and VSCode.
Serena's functionality is similar to Cursor's Agent, Windsurf's Cascade or
VSCode's agent mode.
Serena has the advantage of not requiring a subscription.
More technical differences are:
* Serena navigates and edits code using a language server, so it has a symbolic
understanding of the code.
IDE-based tools often use a text search-based or purely text file-based approach, which is often
less powerful, especially for large codebases.
* Serena is not bound to a specific interface (IDE or CLI).
Serena's MCP server can be used with any MCP client (including some IDEs).
* Serena is not bound to a specific large language model or API.
* Serena is open-source and has a small codebase, so it can be easily extended
and modified.
## API-Based Coding Agents
An alternative to subscription-based agents are API-based agents like Claude
Code, Cline, Aider, Roo Code and others, where the usage costs map directly
to the API costs of the underlying LLM.
Some of them (like Cline) can even be included in IDEs as an extension.
They are often very powerful and their main downside are the (potentially very
high) API costs.
Serena itself can be used as an API-based agent (see the [section on Agno](../03-special-guides/custom_agent.md)).
The main difference between Serena and other API-based agents is that Serena can
also be used as an MCP server, thus not requiring
an API key and bypassing the API costs.
## Other MCP-Based Coding Agents
There are other MCP servers designed for coding, like [DesktopCommander](https://github.com/wonderwhy-er/DesktopCommanderMCP) and
[codemcp](https://github.com/ezyang/codemcp).
However, to the best of our knowledge, none of them provide semantic code
retrieval and editing tools; they rely purely on text-based analysis.
It is the integration of language servers and the MCP that makes Serena unique
and so powerful for challenging coding tasks, especially in the context of
larger codebases.
docs/01-about/050_acknowledgements.md
+34
View File
@@ -0,0 +1,34 @@
# Acknowledgements
## Sponsors
We are very grateful to our [sponsors](https://github.com/sponsors/oraios), who help us drive Serena's development.
The core team (the founders of [Oraios AI](https://oraios-ai.de/)) put in a lot of work in order to turn Serena into a useful open source project.
So far, there is no business model behind this project, and sponsors are our only source of income from it.
Sponsors help us dedicate more time to the project, managing contributions, and working on larger features (like better tooling based on more advanced
LSP features, VSCode integration, debugging via the DAP, and several others).
If you find this project useful to your work, or would like to accelerate the development of Serena, consider becoming a sponsor.
We are proud to announce that the Visual Studio Code team, together with Microsoft’s Open Source Programs Office and GitHub Open Source
have decided to sponsor Serena with a one-time contribution!
## Community Contributions
A significant part of Serena, especially support for various languages, was contributed by the open source community.
We are very grateful for the many contributors who made this possible and who played an important role in making Serena
what it is today.
## Technologies
We built Serena on top of multiple existing open-source technologies, the most important ones being:
1. [multilspy](https://github.com/microsoft/multilspy).
A library which wraps language server implementations and adapts them for interaction via Python
and which provided the basis for our library Solid-LSP (src/solidlsp).
Solid-LSP provides pure synchronous LSP calls and extends the original library with the symbolic logic
that Serena required.
2. [Python MCP SDK](https://github.com/modelcontextprotocol/python-sdk)
3. All the language servers that we use through Solid-LSP.
Without these projects, Serena would not have been possible (or would have been significantly more difficult to build).
docs/02-usage/000_intro.md
+6
View File
@@ -0,0 +1,6 @@
# Usage
Serena can be used in various ways and supports coding workflows through a project-based approach.
Its configuration is flexible and allows tailoring it to your specific needs.
In this section, you will find general usage instructions as well as concrete instructions for selected integrations.
docs/02-usage/010_prerequisites.md
+12
View File
@@ -0,0 +1,12 @@
# Prerequisites
## Package Manager: uv
Serena is managed by `uv`.
If you do not have it yet, install it following the instructions [here](https://docs.astral.sh/uv/getting-started/installation/).
## Language-Specific Requirements
Depending on the programming language you intend to use with Serena, you may need to install additional tools or SDKs if you
intend to use the language server backend of Serena.
See the [language support documentation](../01-about/020_programming-languages) for details.
docs/02-usage/020_running.md
+228
View File
@@ -0,0 +1,228 @@
# Running Serena
Serena is a command-line tool with a variety of sub-commands.
This section describes
* varies ways of running Serena
* how to run and configure the most important command, i.e. starting the MCP server
* other useful commands.
## Ways of Running Serena
In the following, we will refer to the command used to run Serena as `<serena>`,
which you should replace with the appropriate command based on your chosen method.
### Using uvx
`uvx` is part of `uv`. It can be used to run the latest version of Serena directly from the repository, without an explicit local installation.
uvx --from git+https://github.com/oraios/serena serena
Explore the CLI to see some of the customization options that serena provides (more info on them below).
### Local Installation
1. Clone the repository and change into it.
```shell
git clone https://github.com/oraios/serena
cd serena
```
2. Run Serena via
```shell
uv run serena
```
when within the serena installation directory.
From other directories, run it with the `--directory` option, i.e.
```shell
uv run --directory /abs/path/to/serena serena
```
:::{note}
Adding the `--directory` option results in the working directory being set to the Serena directory.
As a consequence, you will need to specify paths when using CLI commands that would otherwise operate on the current directory.
:::
(docker)=
### Using Docker (Experimental)
:::{warning}
Docker support is currently experimental with several limitations.
:::
You can run the Serena MCP server directly via docker as follows,
assuming that the projects you want to work on are all located in `/path/to/your/projects`:
```shell
docker run --rm -i --network host -v /path/to/your/projects:/workspaces/projects ghcr.io/oraios/serena:latest serena
```
Replace `/path/to/your/projects` with the absolute path to your projects directory. The Docker approach provides:
* Better security isolation for shell command execution
* No need to install language servers and dependencies locally
* Consistent environment across different systems
Alternatively, use docker compose with the `compose.yml` file provided in the repository.
See our [Docker Setup](https://github.com/oraios/serena/blob/main/DOCKER.md) documentation for more detailed setup instructions, configuration options, and known limitations.
### Using Nix
If you are using Nix and [have enabled the `nix-command` and `flakes` features](https://nixos.wiki/wiki/flakes), you can run Serena using the following command:
```bash
nix run github:oraios/serena -- <command> [options]
```
You can also install Serena by referencing this repo (`github:oraios/serena`) and using it in your Nix flake. The package is exported as `serena`.
(start-mcp-server)=
## Running the MCP Server
Given your preferred method of running Serena, you can start the MCP server using the `start-mcp-server` command:
<serena> start-mcp-server [options]
Note that no matter how you run the MCP server, Serena will, by default, start a web-based dashboard on localhost that will allow you to inspect
the server's operations, logs, and configuration.
### Standard I/O Mode
The typical usage involves the client (e.g. Claude Code, Codex or Cursor) running
the MCP server as a subprocess and using the process' stdin/stdout streams to communicate with it.
In order to launch the server, the client thus needs to be provided with the command to run the MCP server.
:::{note}
MCP servers which use stdio as a protocol are somewhat unusual as far as client/server architectures go, as the server
necessarily has to be started by the client in order for communication to take place via the server's standard input/output streams.
In other words, you do not need to start the server yourself. The client application (e.g. Claude Desktop) takes care of this and
therefore needs to be configured with a launch command.
:::
Communication over stdio is the default for the Serena MCP server, so in the simplest
case, you can simply run the `start-mcp-server` command without any additional options.
<serena> start-mcp-server
For example, to run the server in stdio mode via `uvx`, you would run:
uvx --from git+https://github.com/oraios/serena serena start-mcp-server
See the section ["Configuring Your MCP Client"](030_clients) for specific information on how to configure your MCP client (e.g. Claude Code, Codex, Cursor, etc.)
to use such a launch command.
(streamable-http)=
### Streamable HTTP Mode
When using instead the *Streamable HTTP* mode, you control the server lifecycle yourself,
i.e. you start the server and provide the client with the URL to connect to it.
Simply provide `start-mcp-server` with the `--transport streamable-http` option and optionally provide the desired port
via the `--port` option.
<serena> start-mcp-server --transport streamable-http --port <port>
For example, to run the Serena MCP server in streamable HTTP mode on port 9121 using uvx,
you would run
uvx --from git+https://github.com/oraios/serena serena start-mcp-server --transport streamable-http --port 9121
and then configure your client to connect to `http://localhost:9121/mcp`.
Note that while the legacy SSE transport is also supported (via `--transport sse` with corresponding /sse endpoint), its use is discouraged.
(mcp-args)=
### MCP Server Command-Line Arguments
The Serena MCP server supports a wide range of additional command-line options.
Use the command
<serena> start-mcp-server --help
to get a list of all available options.
Some useful options include:
* `--project <path|name>`: specify the project to work on by name or path.
* `--context <context>`: specify the operation [context](contexts) in which Serena shall operate
* `--mode <mode>`: specify one or more [modes](modes) to enable (can be passed several times)
* `--enable-web-dashboard <true|false>`: enable or disable the web dashboard (enabled by default)
## Other Commands
Serena provides several other commands in addition to `start-mcp-server`,
most of which are related to project setup and configuration.
To get a list of available commands, run:
<serena> --help
To get help on a specific command, run:
<serena> <command> --help
In general, add `--help` to any command or sub-command to get information about its usage and available options.
Here are some examples of commands you might find useful:
```bash
# get help about a sub-command
<serena> tools list --help
# list all available tools
<serena> tools list --all
# get detailed description of a specific tool
<serena> tools description find_symbol
# creating a new Serena project in the current directory
<serena> project create
# creating and immediately indexing a project
<serena> project create --index
# indexing the project in the current directory (auto-creates if needed)
<serena> project index
# run a health check on the project in the current directory
<serena> project health-check
# check if a path is ignored by the project
<serena> project is_ignored_path path/to/check
# edit Serena's configuration file
<serena> config edit
# list available contexts
<serena> context list
# create a new context
<serena> context create my-custom-context
# edit a custom context
<serena> context edit my-custom-context
# list available modes
<serena> mode list
# create a new mode
<serena> mode create my-custom-mode
# edit a custom mode
<serena> mode edit my-custom-mode
# list available prompt definitions
<serena> prompts list
# create an override for internal prompts
<serena> prompts create-override prompt-name
# edit a prompt override
<serena> prompts edit-override prompt-name
```
Explore the full set of commands and options using the CLI itself!
docs/02-usage/030_clients.md
+169
View File
@@ -0,0 +1,169 @@
# Connecting Your MCP Client
In the following, we provide default configurations for popular MCP-enabled clients.
Depending on your needs, you might want to customize Serena's behaviour by
* [adding command-line arguments](mcp-args)
* [adjusting configuration](050_configuration).
## Claude Code
Serena is a great way to make Claude Code both cheaper and more powerful!
From your project directory, add serena with a command like this,
```shell
claude mcp add serena -- <serena> start-mcp-server --context claude-code --project "$(pwd)"
```
where `<serena>` is [your way of running Serena](020_running).
For example, when using `uvx`, the above command becomes
```shell
claude mcp add serena -- uvx --from git+https://github.com/oraios/serena serena start-mcp-server --context claude-code --project "$(pwd)"
```
Note:
* We use the `ide-assistant` context to disable unnecessary tools (avoiding duplication
with Claude Code's built-in capabilities).
* We specify the current directory as the project directory with `--project "$(pwd)"`, such
that Serena is configured to work on the current project from the get-go, following
Claude Code's mode of operation.
Be sure to use at least `v1.0.52` of Claude Code (as earlier versions do not read MCP server system prompts upon startup).
## Codex
Serena works with OpenAI's Codex CLI out of the box, but you have to use the `codex` context for it to work properly. (The technical reason is that Codex doesn't fully support the MCP specifications, so some massaging of tools is required.).
Add a [run command](020_running) to `~/.codex/config.toml` to configure Serena for all Codex sessions;
create the file if it does not exist.
For example, when using `uvx`, add the following section:
```toml
[mcp_servers.serena]
command = "uvx"
args = ["--from", "git+https://github.com/oraios/serena", "serena", "start-mcp-server", "--context", "codex"]
```
After codex has started, you need to activate the project, which you can do by saying:
"Activate the current dir as project using serena"
> If you don't activate the project, you will not be able to use Serena's tools!
That's it! Have a look at `~/.codex/log/codex-tui.log` to see if any errors occurred.
Serena's dashboard will run if you have not disabled it in the configuration, but due to Codex's sandboxing, the web browser
may not open automatically. You can open it manually by going to `http://localhost:24282/dashboard/index.html` (or a higher port, if
that was already taken).
> Codex will often show the tools as `failed` even though they are successfully executed. This is not a problem, seems to be a bug in Codex. Despite the error message, everything works as expected.
## Claude Desktop
On Windows and macOS there are official [Claude Desktop applications by Anthropic](https://claude.ai/download), for Linux there is an [open-source
community version](https://github.com/aaddrick/claude-desktop-debian).
To configure MCP server settings, go to File / Settings / Developer / MCP Servers / Edit Config,
which will let you open the JSON file `claude_desktop_config.json`.
Add the `serena` MCP server configuration, using a [run command](020_running.md) depending on your setup.
* local installation:
```json
{
"mcpServers": {
"serena": {
"command": "/abs/path/to/uv",
"args": ["run", "--directory", "/abs/path/to/serena", "serena", "start-mcp-server"]
}
}
}
```
* uvx:
```json
{
"mcpServers": {
"serena": {
"command": "/abs/path/to/uvx",
"args": ["--from", "git+https://github.com/oraios/serena", "serena", "start-mcp-server"]
}
}
}
```
* docker:
```json
{
"mcpServers": {
"serena": {
"command": "docker",
"args": ["run", "--rm", "-i", "--network", "host", "-v", "/path/to/your/projects:/workspaces/projects", "ghcr.io/oraios/serena:latest", "serena", "start-mcp-server", "--transport", "stdio"]
}
}
}
```
If you are using paths containing backslashes for paths on Windows
(note that you can also just use forward slashes), be sure to escape them correctly (`\\`).
That's it! Save the config and then restart Claude Desktop. You are ready for activating your first project.
:::{attention}
Be sure to fully quit the Claude Desktop application via File / Exit, as regularly closing the application will just minimize it.
:::
After restarting, you should see Serena's tools in your chat interface (notice the small hammer icon).
For more information on MCP servers with Claude Desktop, see [the official quick start guide](https://modelcontextprotocol.io/quickstart/user).
## Other Clients
In general, Serena can be used with any MCP-enabled client.
To connect Serena to your favourite client, simply
1. determine how to add a custom MCP server to your client (refer to the client's documentation).
2. add a new MCP server entry by specifying either
* a [run command](start-mcp-server) that allows the client to start the MCP server in stdio mode as a subprocess, or
* the URL of the HTTP/SSE endpoint, having started the [Serena MCP server in HTTP/SSE mode](streamable-http) beforehand.
Below, we list some popular categories of clients that Serena can be used with.
### Terminal-Based Clients
There are many terminal-based coding assistants that support MCP servers, such as
* [Gemini-CLI](https://github.com/google-gemini/gemini-cli),
* [Qwen3-Coder](https://github.com/QwenLM/Qwen3-Coder),
* [rovodev](https://community.atlassian.com/forums/Rovo-for-Software-Teams-Beta/Introducing-Rovo-Dev-CLI-AI-Powered-Development-in-your-terminal/ba-p/3043623),
* [OpenHands CLI](https://docs.all-hands.dev/usage/how-to/cli-mode) and
* [opencode](https://github.com/sst/opencode).
They generally benefit from the symbolic tools provided by Serena. You might want to customize some aspects of Serena
by writing your own context, modes or prompts to adjust it to the client's respective internal capabilities (and your general workflow).
### MCP-Enabled IDEs and Coding Clients (Cline, Roo-Code, Cursor, Windsurf, etc.)
Most of the popular existing coding assistants (e.g. IDE extensions) and AI-enabled IDEs themselves support connections
to MCP Servers. Serena generally boosts performance by providing efficient tools for symbolic operations.
We generally **recommend to use the `ide-assistant` context** for these integrations by adding the arguments `--context ide-assistant`
in order to reduce tool duplication.
### Local GUIs and Agent Frameworks
Over the last months, several technologies have emerged that allow you to run a local GUI client
and connect it to an MCP server. The respective applications will typically work with Serena out of the box.
Some of the leading open source GUI applications are
* [Jan](https://jan.ai/docs/mcp),
* [OpenHands](https://github.com/All-Hands-AI/OpenHands/),
* [OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp) and
* [Agno](https://docs.agno.com/introduction/playground).
These applications allow to combine Serena with almost any LLM (including locally running ones)
and offer various other integrations.
docs/02-usage/040_workflow.md
+138
View File
@@ -0,0 +1,138 @@
# The Project Workflow
Serena uses a project-based workflow.
A **project** is simply a directory on your filesystem that contains code and other files
that you want Serena to work with.
Assuming that you have project you want to work with (which may initially be empty),
setting up a project with Serena typically involves the following steps:
1. **Project creation**: Configuring project settings for Serena (and indexing the project, if desired)
2. **Project activation**: Making Serena aware of the project you want to work with
3. **Onboarding**: Getting Serena familiar with the project (creating memories)
4. **Working on coding tasks**: Using Serena to help you with actual coding tasks in the project
(project-creation-indexing)=
## Project Creation & Indexing
You can create a project either
* implicitly, by just activating a directory as a project while already in a conversation; this will use default settings for your project (skip to the next section).
* explicitly, using the project creation command, or
### Explicit Project Creation
To explicitly create a project, use the following command while in the project directory:
<serena> project create [options]
For instance, when using `uvx`, run
uvx --from git+https://github.com/oraios/serena serena project create [options]
* For an empty project, you will need to specify the programming language
(e.g., `--language python`).
* For an existing project, the main programming language will be detected automatically,
but you can choose to explicitly specify multiple languages by passing the `--language` parameter
multiple times (e.g. `--language python --language typescript`).
* You can optionally specify a custom project name with `--name "My Project"`.
* You can immediately index the project after creation with `--index`.
After creation, you can adjust the project settings in the generated `.serena/project.yml` file.
(indexing)=
### Indexing
Especially for larger project, it is advisable to index the project after creation (in order to avoid
delays during MCP server startup or the first tool application):
While in the project directory, run this command:
<serena> project index
Indexing has to be called only once. During regular usage, Serena will automatically update the index whenever files change.
## Project Activation
Project activation makes Serena aware of the project you want to work with.
You can either choose to do this
* while in a conversation, by telling the model to activate a project, e.g.,
* "Activate the project /path/to/my_project" (for first-time activation with auto-creation)
* "Activate the project my_project"
Note that this option requires the `activate_project` tool to be active,
which it isn't in the (default version) of context `ide-assistant` if a project is provided at startup.
(The tool is deactivated because we assume that in the ide-assistant context the user will only work on the open project and have
no need to switch it.)
* when the MCP server starts, by passing the project path or name as a command-line argument
(e.g. when working on a fixed project in `ide-assistant` mode): `--project <path|name>`
## Onboarding & Memories
By default, Serena will perform an **onboarding process** when
it is started for the first time for a project.
The goal of the onboarding is for Serena to get familiar with the project
and to store memories, which it can then draw upon in future interactions.
If an LLM should fail to complete the onboarding and does not actually write the
respective memories to disk, you may need to ask it to do so explicitly.
The onboarding will usually read a lot of content from the project, thus filling
up the context. It can therefore be advisable to switch to another conversation
once the onboarding is complete.
After the onboarding, we recommend that you have a quick look at the memories and,
if necessary, edit them or add additional ones.
**Memories** are files stored in `.serena/memories/` in the project directory,
which the agent can choose to read in subsequent interactions.
Feel free to read and adjust them as needed; you can also add new ones manually.
Every file in the `.serena/memories/` directory is a memory file.
Whenever Serena starts working on a project, the list of memories is
provided, and the agent can decide to read them.
We found that memories can significantly improve the user experience with Serena.
## Preparing Your Project
When using Serena to work on your project, it can be helpful to follow a few best practices.
### Structure Your Codebase
Serena uses the code structure for finding, reading and editing code. This means that it will
work well with well-structured code but may perform poorly on fully unstructured one (like a "God class"
with enormous, non-modular functions).
Furthermore, for languages that are not statically typed, the use of type annotations (if supported)
are highly beneficial.
### Start from a Clean State
It is best to start a code generation task from a clean git state. Not only will
this make it easier for you to inspect the changes, but also the model itself will
have a chance of seeing what it has changed by calling `git diff` and thereby
correct itself or continue working in a followup conversation if needed.
### Use Platform-Native Line Endings
**Important**: since Serena will write to files using the system-native line endings
and it might want to look at the git diff, it is important to
set `git config core.autocrlf` to `true` on Windows.
With `git config core.autocrlf` set to `false` on Windows, you may end up with huge diffs
due to line endings only.
It is generally a good idea to globally enable this git setting on Windows:
```shell
git config --global core.autocrlf true
```
### Logging, Linting, and Automated Tests
Serena can successfully complete tasks in an _agent loop_, where it iteratively
acquires information, performs actions, and reflects on the results.
However, Serena cannot use a debugger; it must rely on the results of program executions,
linting results, and test results to assess the correctness of its actions.
Therefore, software that is designed to meaningful interpretable outputs (e.g. log messages)
and that has a good test coverage is much easier to work with for Serena.
We generally recommend to start an editing task from a state where all linting checks and tests pass.
docs/02-usage/050_configuration.md
+135
View File
@@ -0,0 +1,135 @@
# Configuration
Serena is very flexible in terms of configuration. While for most users, the default configurations will work,
you can fully adjust it to your needs by editing a few yaml files. You can disable tools, change Serena's instructions
(what we denote as the `system_prompt`), adjust the output of tools that just provide a prompt, and even adjust tool descriptions.
Serena is configured in four places:
1. The `serena_config.yml` for general settings that apply to all clients and projects.
It is located in your user directory under `.serena/serena_config.yml`.
If you do not explicitly create the file, it will be auto-generated when you first run Serena.
You can edit it directly or use
```shell
uvx --from git+https://github.com/oraios/serena serena config edit
```
(or use the `--directory` command version).
2. In the arguments passed to the `start-mcp-server` in your client's config (see below),
which will apply to all sessions started by the respective client. In particular, the [context](contexts) parameter
should be set appropriately for Serena to be best adjusted to existing tools and capabilities of your client.
See for a detailed explanation. You can override all entries from the `serena_config.yml` through command line arguments.
3. In the `.serena/project.yml` file within your project. This will hold project-level configuration that is used whenever
that project is activated. This file will be autogenerated when you first use Serena on that project, but you can also
create it explicitly with `serena project create [options]` (see the [](project-creation-indexing)
for details on available options).
4. Through the context and modes (see below).
After the initial setup, continue with one of the sections below, depending on how you
want to use Serena.
## Modes and Contexts
Serena's behavior and toolset can be adjusted using contexts and modes.
These allow for a high degree of customization to best suit your workflow and the environment Serena is operating in.
(contexts)=
### Contexts
A **context** defines the general environment in which Serena is operating.
It influences the initial system prompt and the set of available tools.
A context is set at startup when launching Serena (e.g., via CLI options for an MCP server or in the agent script) and cannot be changed during an active session.
Serena comes with pre-defined contexts:
* `desktop-app`: Tailored for use with desktop applications like Claude Desktop. This is the default.
The full set of Serena's tools is provided, as the application is assumed to have no prior coding-specific capabilities.
* `claude-code`: Optimized for use with Claude Code, it disables tools that would duplicate Claude Code's built-in capabilities.
* `codex`: Optimized for use with OpenAI Codex.
* `ide`: Generic context for IDE assistants/coding agents, e.g. VSCode, Cursor, or Cline, focusing on augmenting existing capabilities.
Basic file operations and shell execution are assumed to be handled by the assistant's own capabilities.
* `agent`: Designed for scenarios where Serena acts as a more autonomous agent, for example, when used with Agno.
* Choose the context that best matches the type of integration you are using.
Find the concrete definitions of these modes [here](https://github.com/oraios/serena/tree/main/src/serena/resources/config/contexts).
When launching Serena, specify the context using `--context <context-name>`.
Note that for cases where parameter lists are specified (e.g. Claude Desktop), you must add two parameters to the list.
If you are using a local server (such as Llama.cpp) which requires you to use OpenAI-compatible tool descriptions, use context `oaicompat-agent` instead of `agent`.
You can manage contexts using the `context` command,
<serena> context --help
<serena> context list
<serena> context create <context-name>
<serena> context edit <context-name>
<serena> context delete <context-name>
where `<serena>` is [your way of running Serena](020_running).
(modes)=
### Modes
Modes further refine Serena's behavior for specific types of tasks or interaction styles. Multiple modes can be active simultaneously, allowing you to combine their effects. Modes influence the system prompt and can also alter the set of available tools by excluding certain ones.
Examples of built-in modes include:
* `planning`: Focuses Serena on planning and analysis tasks.
* `editing`: Optimizes Serena for direct code modification tasks.
* `interactive`: Suitable for a conversational, back-and-forth interaction style.
* `one-shot`: Configures Serena for tasks that should be completed in a single response, often used with `planning` for generating reports or initial plans.
* `no-onboarding`: Skips the initial onboarding process if it's not needed for a particular session but retains the memory tools (assuming initial memories were created externally).
* `onboarding`: Focuses on the project onboarding process.
* `no-memories`: Disables all memory tools (and tools building on memories such as onboarding tools)
Find the concrete definitions of these modes [here](https://github.com/oraios/serena/tree/main/src/serena/resources/config/modes).
Modes can be set at startup (similar to contexts) but can also be _switched dynamically_ during a session.
You can instruct the LLM to use the `switch_modes` tool to activate a different set of modes (e.g., "Switch to planning and one-shot modes").
When launching Serena, specify modes using `--mode <mode-name>`; multiple modes can be specified, e.g. `--mode planning --mode no-onboarding`.
:::{attention}
**Mode Compatibility**: While you can combine modes, some may be semantically incompatible (e.g., `interactive` and `one-shot`).
Serena currently does not prevent incompatible combinations; it is up to the user to choose sensible mode configurations.
:::
You can manage modes using the `mode` command,
<serena> mode --help
<serena> mode list
<serena> mode create <mode-name>
<serena> mode edit <mode-name>
<serena> mode delete <mode-name>
where `<serena>` is [your way of running Serena](020_running).
## Advanced Configuration
For advanced users, Serena's configuration can be further customized.
### Serena Data Directory
The Serena user data directory (where configuration, language server files, logs, etc. are stored) defaults to `~/.serena`.
You can change this location by setting the `SERENA_HOME` environment variable to your desired path.
### Custom Prompts
All of Serena's prompts can be fully customized.
We define prompt as jinja templates in yaml files, and you can inspect our default prompts [here](https://github.com/oraios/serena/tree/main/src/serena/resources/config/prompt_templates).
To override a prompt, simply add a .yml file to the `prompt_templates` folder in your Serena data directory
which defines the prompt with the same name as the default prompt you want to override.
For example, to override the `system_prompt`, you could create a file `~/.serena/prompt_templates/system_prompt.yml` (assuming default Serena data folder location)
with content like:
```yaml
prompts:
system_prompt: |
Whatever you want ...
```
It is advisable to use the default prompt as a starting point and modify it to suit your needs.
docs/02-usage/060_dashboard.md
+22
View File
@@ -0,0 +1,22 @@
# The Dashboard and GUI Tool
Serena comes with built-in tools for monitoring and managing the current session:
* the **web-based dashboard** (enabled by default)
The dashboard provides detailed information on your Serena session, the current configuration and provides access to logs.
Some settings (e.g. the current set of active programming languages) can also be directly modified through the dashboard.
The dashboard is supported on all platforms.
By default, it will be accessible at `http://localhost:24282/dashboard/index.html`,
but a higher port may be used if the default port is unavailable/multiple instances are running.
* the **GUI tool** (disabled by default)
The GUI tool is a native application window which displays logs.
This is mainly supported on Windows, but it may also work on Linux; macOS is unsupported.
Both can be configured in Serena's [configuration](050_configuration) file (`serena_config.yml`).
If enabled, they will automatically be opened as soon as the Serena agent/MCP server is started.
docs/02-usage/070_security.md
+12
View File
@@ -0,0 +1,12 @@
# Security Considerations
As fundamental abilities for a coding agent, Serena contains tools for executing shell commands and modifying files.
Therefore, if the respective tool calls are not monitored or restricted (and execution takes place in a sensitive environment),
there is a risk of unintended consequences.
Therefore, to reduce the risk of unintended consequences when using Serena, it is recommended to
* back up your work regularly (e.g. use a version control system like Git),
* monitor tool executions carefully (e.g. via your MCP client, provided that it supports it),
* consider enabling read-only mode for your project (set `read_only: True` in project.yml) if you only want to analyze code without modifying it,
* restrict the set of allowed tools via the [configuration](050_configuration),
* use a sandboxed environment for running Serena (e.g. by [using Docker](docker)).
docs/02-usage/999_additional-usage.md
+32
View File
@@ -0,0 +1,32 @@
# Additional Usage Pointers
## Prompting Strategies
We found that it is often a good idea to spend some time conceptualizing and planning a task
before actually implementing it, especially for non-trivial task. This helps both in achieving
better results and in increasing the feeling of control and staying in the loop. You can
make a detailed plan in one session, where Serena may read a lot of your code to build up the context,
and then continue with the implementation in another (potentially after creating suitable memories).
## Running Out of Context
For long and complicated tasks, or tasks where Serena has read a lot of content, you
may come close to the limits of context tokens. In that case, it is often a good idea to continue
in a new conversation. Serena has a dedicated tool to create a summary of the current state
of the progress and all relevant info for continuing it. You can request to create this summary and
write it to a memory. Then, in a new conversation, you can just ask Serena to read the memory and
continue with the task. In our experience, this worked really well. On the up-side, since in a
single session there is no summarization involved, Serena does not usually get lost (unlike some
other agents that summarize under the hood), and it is also instructed to occasionally check whether
it's on the right track.
Serena instructs the LLM to be economical in general, so the problem of running out of context
should not occur too often, unless the task is very large or complicated.
## Serena and Git Worktrees
[git-worktree](https://git-scm.com/docs/git-worktree) can be an excellent way to parallelize your work. More on this in [Anthropic: Run parallel Claude Code sessions with Git worktrees](https://docs.claude.com/en/docs/claude-code/common-workflows#run-parallel-claude-code-sessions-with-git-worktrees).
When it comes to serena AND git-worktree AND larger projects (that take longer to index),
the recommended way is to COPY your `$ORIG_PROJECT/.serena/cache` to `$GIT_WORKTREE/.serena/cache`.
Perform [pre-indexing of your project](indexing) to avoid having to re-index per each worktree you create.
docs/03-special-guides/000_intro.md
+3
View File
@@ -0,0 +1,3 @@
# Special Guides
This section contains special guides for certain topics that require more in-depth explanations.
docs/custom_agent.md → docs/03-special-guides/custom_agent.md
+1 -1
View File
@@ -63,5 +63,5 @@ configuration file (`.yml`).
It should be straightforward to incorporate Serena into any
agent framework (like [pydantic-ai](https://ai.pydantic.dev/), [langgraph](https://langchain-ai.github.io/langgraph/tutorials/introduction/) or others).
Typically, you need only to write an adapter for Serena's tools to the tool representation in the framework of your choice,
as was done by us for Agno with [SerenaAgnoToolkit](/src/serena/agno.py).
as was done by us for Agno with `SerenaAgnoToolkit` (see `/src/serena/agno.py`).
docs/03-special-guides/scala_setup_guide_for_serena.md
+67
View File
@@ -0,0 +1,67 @@
# Scala Setup Guide for Serena
This guide explains how to prepare a Scala project so that Serena can provide reliable code intelligence via Metals (Scala LSP) and how to run Scala tests manually.
Serena automatically bootstraps the Metals language server using Coursier when needed. Your project, however, must be importable by a build server (BSP) — typically via Bloop or sbt’s built‑in BSP — so that Metals can compile and index your code.
---
## Prerequisites
Install the following on your system and ensure they are available on `PATH`:
- Java Development Kit (JDK). A modern LTS (e.g., 17 or 21) is recommended.
- `sbt`
- Coursier command (`cs`) or the legacy `coursier` launcher
- Serena uses `cs` if available; if only `coursier` exists, it will attempt to install `cs`. If neither is present, install Coursier first.
---
## Quick Start (Recommended: VS Code + Metals auto‑import)
1. Open your Scala project in VS Code.
2. When prompted by Metals, accept “Import build”. Wait until the import and initial compile/indexing finish.
3. Run the “Connect to build server” command (id: `build.connect`).
4. Once the import completes, start Serena in your project root and use it.
This flow ensures the `.bloop/` and (if applicable) `.metals/` directories are created and your build is known to the build server that Metals uses.
---
## Manual Setup (No VS Code)
Follow these steps if you prefer a manual setup or you are not using VS Code:
These instructions cover the setup for projects that use sbt as the build tool, with Bloop as the BSP server.
1. Add Bloop to `project/plugins.sbt` in your Scala project:
```scala
// project/plugins.sbt
addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "<version>")
```
Replace `<version>` with an appropriate current version from the Metals documentation.
2. Export Bloop configuration with sources:
```bash
sbt -Dbloop.export-jar-classifiers=sources bloopInstall
```
This creates a `.bloop/` directory containing your project’s build metadata for the BSP server.
3. Compile from sbt to verify the build:
```bash
sbt compile
```
4. Start Serena in your project root. Serena will bootstrap Metals (if not already present) and connect to the build server using the configuration exported above.
---
## Using Serena with Scala
- Serena automatically detects Scala files (`*.scala`, `*.sbt`) and will start a Metals process per project when needed.
- On first run, you may see messages like “Bootstrapping metals…” in the Serena logs — this is expected.
- Optimal results require that your project compiles successfully via the build server (BSP). If compilation fails, fix build errors in `sbt` first.
Notes:
- Ensure you completed the manual or auto‑import steps so that the build is compiled and indexed; otherwise, code navigation and references may be incomplete until the first successful compile.
## Reference
- Metals + sbt: [https://scalameta.org/metals/docs/build-tools/sbt](https://scalameta.org/metals/docs/build-tools/sbt)
docs/serena_on_chatgpt.md → docs/03-special-guides/serena_on_chatgpt.md
View File
File renamed without changes.
docs/_config.yml
+154
View File
@@ -0,0 +1,154 @@
# Book settings
# Learn more at https://jupyterbook.org/customize/config.html
#######################################################################################
# A default configuration that will be loaded for all jupyter books
# Users are expected to override these values in their own `_config.yml` file.
# This is also the "master list" of all allowed keys and values.
#######################################################################################
# Book settings
title : Serena Documentation # The title of the book. Will be placed in the left navbar.
author : Oraios AI & Oraios Software # The author of the book
copyright : "2025 by Serena contributors" # Copyright year to be placed in the footer
# Patterns to skip when building the book. Can be glob-style (e.g. "*skip.ipynb")
exclude_patterns : ['**.ipynb_checkpoints', '.DS_Store', 'Thumbs.db', '_build', 'jupyter_execute', '.jupyter_cache', '.pytest_cache', 'docs/autogen_rst.py', 'docs/create_toc.py']
# Auto-exclude files not in the toc
only_build_toc_files : true
#######################################################################################
# Execution settings
execute:
# NOTE: Notebooks are not executed, because test_notebooks.py executes them and stores them with outputs in the docs/ folder
# NOTE: If changed, repeat below in `nb_execution_mode`.
execute_notebooks : "off" # Whether to execute notebooks at build time. Must be one of ("auto", "force", "cache", "off")
cache : "" # A path to the jupyter cache that will be used to store execution artifacts. Defaults to `_build/.jupyter_cache/`
exclude_patterns : [] # A list of patterns to *skip* in execution (e.g. a notebook that takes a really long time)
timeout : 1000 # The maximum time (in seconds) each notebook cell is allowed to run.
run_in_temp : false # If `True`, then a temporary directory will be created and used as the command working directory (cwd),
# otherwise the notebook's parent directory will be the cwd.
allow_errors : true # If `False`, when a code cell raises an error the execution is stopped, otherwise all cells are always run.
stderr_output : show # One of 'show', 'remove', 'remove-warn', 'warn', 'error', 'severe'
#######################################################################################
# Parse and render settings
parse:
myst_enable_extensions: # default extensions to enable in the myst parser. See https://myst-parser.readthedocs.io/en/latest/using/syntax-optional.html
- amsmath
- colon_fence
# - deflist
- dollarmath
# - html_admonition
# - html_image
- linkify
# - replacements
# - smartquotes
- substitution
- tasklist
- html_admonition
- html_image
myst_url_schemes: [ mailto, http, https ] # URI schemes that will be recognised as external URLs in Markdown links
myst_dmath_double_inline: true # Allow display math ($$) within an inline context
#######################################################################################
# HTML-specific settings
html:
#favicon : # A path to a favicon image
use_edit_page_button : false # Whether to add an "edit this page" button to pages. If `true`, repository information in repository: must be filled in
use_repository_button : false # Whether to add a link to your repository button
use_issues_button : false # Whether to add an "open an issue" button
use_multitoc_numbering : true # Continuous numbering across parts/chapters
use_darkmode_button : false
extra_footer : ""
home_page_in_navbar : true # Whether to include your home page in the left Navigation Bar
baseurl : "https://oraios.github.io/serena/"
comments:
hypothesis : false
utterances : false
announcement : "" # A banner announcement at the top of the site.
#######################################################################################
# LaTeX-specific settings
latex:
latex_engine : pdflatex # one of 'pdflatex', 'xelatex' (recommended for unicode), 'luatex', 'platex', 'uplatex'
use_jupyterbook_latex : true # use sphinx-jupyterbook-latex for pdf builds as default
targetname : book.tex
# Add a bibtex file so that we can create citations
#bibtex_bibfiles:
# - refs.bib
#######################################################################################
# Launch button settings
launch_buttons:
notebook_interface : classic # The interface interactive links will activate ["classic", "jupyterlab"]
binderhub_url : "" # The URL of the BinderHub (e.g., https://mybinder.org)
jupyterhub_url : "" # The URL of the JupyterHub (e.g., https://datahub.berkeley.edu)
thebe : false # Add a thebe button to pages (requires the repository to run on Binder)
colab_url : "https://colab.research.google.com"
repository:
url : https://github.com/oraios/serena # The URL to your book's repository
path_to_book : docs # A path to your book's folder, relative to the repository root.
branch : main # Which branch of the repository should be used when creating links
#######################################################################################
# Advanced and power-user settings
sphinx:
extra_extensions :
- sphinx.ext.autodoc
- sphinx.ext.viewcode
- sphinx_toolbox.more_autodoc.sourcelink
#- sphinxcontrib.spelling
local_extensions : # A list of local extensions to load by sphinx specified by "name: path" items
recursive_update : false # A boolean indicating whether to overwrite the Sphinx config (true) or recursively update (false)
config : # key-value pairs to directly over-ride the Sphinx configuration
master_doc: "01-about/000_intro.md"
html_theme_options:
logo:
image_light: ../resources/serena-logo.svg
image_dark: ../resources/serena-logo-dark-mode.svg
autodoc_typehints_format: "short"
autodoc_member_order: "bysource"
autoclass_content: "both"
autodoc_default_options:
show-inheritance: True
autodoc_show_sourcelink: True
add_module_names: False
github_username: oraios
github_repository: serena
nb_execution_mode: "off"
nb_merge_streams: True # This is important for cell outputs to appear as single blocks rather than one block per line
python_use_unqualified_type_names: True
nb_mime_priority_overrides: [
[ 'html', 'application/vnd.jupyter.widget-view+json', 10 ],
[ 'html', 'application/javascript', 20 ],
[ 'html', 'text/html', 30 ],
[ 'html', 'text/latex', 40 ],
[ 'html', 'image/svg+xml', 50 ],
[ 'html', 'image/png', 60 ],
[ 'html', 'image/jpeg', 70 ],
[ 'html', 'text/markdown', 80 ],
[ 'html', 'text/plain', 90 ],
[ 'spelling', 'application/vnd.jupyter.widget-view+json', 10 ],
[ 'spelling', 'application/javascript', 20 ],
[ 'spelling', 'text/html', 30 ],
[ 'spelling', 'text/latex', 40 ],
[ 'spelling', 'image/svg+xml', 50 ],
[ 'spelling', 'image/png', 60 ],
[ 'spelling', 'image/jpeg', 70 ],
[ 'spelling', 'text/markdown', 80 ],
[ 'spelling', 'text/plain', 90 ],
]
mathjax_path: https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js
mathjax3_config:
loader: { load: [ '[tex]/configmacros' ] }
tex:
packages: { '[+]': [ 'configmacros' ] }
macros:
vect: ["{\\mathbf{\\boldsymbol{#1}} }", 1]
E: "{\\mathbb{E}}"
P: "{\\mathbb{P}}"
R: "{\\mathbb{R}}"
abs: ["{\\left| #1 \\right|}", 1]
simpl: ["{\\Delta^{#1} }", 1]
amax: "{\\text{argmax}}"
docs/autogen_rst.py
+155
View File
@@ -0,0 +1,155 @@
import logging
import os
import shutil
from pathlib import Path
from typing import Optional, List
log = logging.getLogger(os.path.basename(__file__))
TOP_LEVEL_PACKAGE = "serena"
PROJECT_NAME = "Serena"
def module_template(module_qualname: str):
module_name = module_qualname.split(".")[-1]
title = module_name.replace("_", r"\_")
return f"""{title}
{"=" * len(title)}
.. automodule:: {module_qualname}
:members:
:undoc-members:
"""
def index_template(package_name: str, doc_references: Optional[List[str]] = None, text_prefix=""):
doc_references = doc_references or ""
if doc_references:
doc_references = "\n" + "\n".join(f"* :doc:`{ref}`" for ref in doc_references) + "\n"
dirname = package_name.split(".")[-1]
title = dirname.replace("_", r"\_")
if title == TOP_LEVEL_PACKAGE:
title = "API Reference"
return f"{title}\n{'=' * len(title)}" + text_prefix + doc_references
def write_to_file(content: str, path: str):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w") as f:
f.write(content)
os.chmod(path, 0o666)
_SUBTITLE = (
f"\n Here is the autogenerated documentation of the {PROJECT_NAME} API. \n \n "
"The Table of Contents to the left has the same structure as the "
"repository's package code. The links at each page point to the submodules and subpackages. \n"
)
def make_rst(src_root, rst_root, clean=False, overwrite=False, package_prefix=""):
"""Creates/updates documentation in form of rst files for modules and packages.
Does not delete any existing rst files. Thus, rst files for packages or modules that have been removed or renamed
should be deleted by hand.
This method should be executed from the project's top-level directory
:param src_root: path to library base directory, typically "src/<library_name>"
:param rst_root: path to the root directory to which .rst files will be written
:param clean: whether to completely clean the target directory beforehand, removing any existing .rst files
:param overwrite: whether to overwrite existing rst files. This should be used with caution as it will delete
all manual changes to documentation files
:package_prefix: a prefix to prepend to each module (for the case where the src_root is not the base package),
which, if not empty, should end with a "."
:return:
"""
rst_root = os.path.abspath(rst_root)
if clean and os.path.isdir(rst_root):
shutil.rmtree(rst_root)
base_package_name = package_prefix + os.path.basename(src_root)
# TODO: reduce duplication with same logic for subpackages below
files_in_dir = os.listdir(src_root)
module_names = [f[:-3] for f in files_in_dir if f.endswith(".py") and not f.startswith("_")]
subdir_refs = [
f"{f}/index"
for f in files_in_dir
if os.path.isdir(os.path.join(src_root, f))
and not f.startswith("_")
and not f.startswith(".")
]
package_index_rst_path = os.path.join(
rst_root,
"index.rst",
)
log.info(f"Writing {package_index_rst_path}")
write_to_file(
index_template(
base_package_name,
doc_references=module_names + subdir_refs,
text_prefix=_SUBTITLE,
),
package_index_rst_path,
)
for root, dirnames, filenames in os.walk(src_root):
if os.path.basename(root).startswith("_"):
continue
base_package_relpath = os.path.relpath(root, start=src_root)
base_package_qualname = package_prefix + os.path.relpath(
root,
start=os.path.dirname(src_root),
).replace(os.path.sep, ".")
for dirname in dirnames:
if dirname.startswith("_"):
log.debug(f"Skipping {dirname}")
continue
files_in_dir = os.listdir(os.path.join(root, dirname))
module_names = [
f[:-3] for f in files_in_dir if f.endswith(".py") and not f.startswith("_")
]
subdir_refs = [
f"{f}/index"
for f in files_in_dir
if os.path.isdir(os.path.join(root, dirname, f)) and not f.startswith("_")
]
package_qualname = f"{base_package_qualname}.{dirname}"
package_index_rst_path = os.path.join(
rst_root,
base_package_relpath,
dirname,
"index.rst",
)
log.info(f"Writing {package_index_rst_path}")
write_to_file(
index_template(package_qualname, doc_references=module_names + subdir_refs),
package_index_rst_path,
)
for filename in filenames:
base_name, ext = os.path.splitext(filename)
if ext == ".py" and not filename.startswith("_"):
module_qualname = f"{base_package_qualname}.{filename[:-3]}"
module_rst_path = os.path.join(rst_root, base_package_relpath, f"{base_name}.rst")
if os.path.exists(module_rst_path) and not overwrite:
log.debug(f"{module_rst_path} already exists, skipping it")
log.info(f"Writing module documentation to {module_rst_path}")
write_to_file(module_template(module_qualname), module_rst_path)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
docs_root = Path(__file__).parent
enable_module_docs = False
if enable_module_docs:
make_rst(
docs_root / ".." / "src" / "serena",
docs_root / "serena",
clean=True,
)
docs/create_toc.py
+9
View File
@@ -0,0 +1,9 @@
import os
from pathlib import Path
# This script provides a platform-independent way of making the jupyter-book call (used in pyproject.toml)
folder = Path(__file__).parent
toc_file = folder / "_toc.yml"
cmd = f"jupyter-book toc from-project docs -e .rst -e .md -e .ipynb >{toc_file}"
print(cmd)
os.system(cmd)
docs/index.md
+2
View File
@@ -0,0 +1,2 @@
<meta http-equiv="refresh" content="0; url=01-about/000_intro.html">
If you are not redirected automatically, [click here](01-about/000_intro.html).
flake.nix
+5 -1
View File
@@ -76,7 +76,11 @@
formatter = pkgs.alejandra;
packages = {
serena = pythonSet.mkVirtualEnv "serena" workspace.deps.default;
serena-env = pythonSet.mkVirtualEnv "serena-env" workspace.deps.default;
serena = pkgs.runCommand "serena" {} ''
mkdir -p $out/bin
ln -s ${packages.serena-env}/bin/serena $out/bin/serena
'';
default = packages.serena;
};
pyproject.toml
+25 -4
View File
@@ -16,6 +16,7 @@ classifiers = [
dependencies = [
"requests>=2.32.3,<3",
"pyright>=1.1.396,<2",
"fortls>=3.2.2",
"overrides>=7.7.0,<8",
"python-dotenv>=1.0.0, <2",
"mcp==1.12.3",
@@ -64,8 +65,15 @@ dev = [
"types-pyyaml>=6.0.12.20241230",
"syrupy>=4.9.1",
"types-requests>=2.32.4.20241230",
# docs
"sphinx>=7,<8",
"sphinx_rtd_theme>=0.5.1",
"sphinx-toolbox==3.7.0",
"jupyter-book>=1,<2",
"nbsphinx",
"pyinstrument",
]
agno = ["agno>=1.2.6", "sqlalchemy>=2.0.40"]
agno = ["agno>=2.2.1", "sqlalchemy>=2.0.40"]
google = ["google-genai>=1.8.0"]
[project.urls]
@@ -113,15 +121,23 @@ PYDEVD_DISABLE_FILE_VALIDATION = "1"
# Uses PYTEST_MARKERS env var for default markers
# For custom markers, one can either adjust the env var or just use -m option in the command line,
# as the second -m option will override the first one.
test = "pytest test -vv -m \"${PYTEST_MARKERS:-not java and not rust and not erlang}\""
test = "pytest test -vv"
_black_check = "black --check src scripts test"
_ruff_check = "ruff check src scripts test"
_black_format = "black src scripts test"
_ruff_format = "ruff check --fix src scripts test"
lint = ["_black_check", "_ruff_check"]
format = ["_ruff_format", "_black_format"]
_mypy = "mypy src/serena"
_mypy = "mypy src/serena src/solidlsp"
type-check = ["_mypy"]
# docs
_autogen_rst = "python docs/autogen_rst.py"
_sphinx_build = "sphinx-build -b html docs docs/_build -W --keep-going"
_jb_generate_toc = "python docs/create_toc.py"
_jb_generate_config = "jupyter-book config sphinx docs/"
doc-clean = "rm -rf docs/_build docs/03_api"
doc-generate-files = ["_autogen_rst", "_jb_generate_toc", "_jb_generate_config"]
doc-build = ["doc-clean", "doc-generate-files", "_sphinx_build"]
[tool.ruff]
target-version = "py311"
@@ -210,7 +226,6 @@ ignore = [
"COM812",
"RET503",
"RET504",
"UP038",
"F403",
"F405",
"C401",
@@ -238,6 +253,7 @@ ignore = [
"C400", # wants to unnecessarily force use of list comprehension
"UP037", # can incorrectly (!) convert quoted type to unquoted type, causing an error
"UP045", # imposes T | None instead of Optional[T]
"UP031", # forbids % operator to format strings
]
unfixable = ["F841", "F601", "F602", "B018"]
extend-fixable = ["F401", "B905", "W291"]
@@ -275,10 +291,15 @@ markers = [
"nix: language server running for Nix",
"dart: language server running for Dart",
"erlang: language server running for Erlang",
"scala: language server running for Scala",
"al: language server running for AL (Microsoft Dynamics 365 Business Central)",
"fsharp: language server running for F#",
"rego: language server running for Rego",
"markdown: language server running for Markdown",
"julia: Julia language server tests",
"fortran: language server running for Fortran",
"haskell: Haskell language server tests",
"yaml: language server running for YAML",
]
[tool.codespell]
repo_dir_sync.py
+363
View File
@@ -0,0 +1,363 @@
# -*- coding: utf-8 -*-
import glob
import os
import shutil
from subprocess import Popen, PIPE
import re
import sys
from typing import List, Optional, Sequence
import platform
def popen(cmd):
shell = platform.system() != "Windows"
p = Popen(cmd, shell=shell, stdin=PIPE, stdout=PIPE)
return p
def call(cmd):
p = popen(cmd)
return p.stdout.read().decode("utf-8")
def execute(cmd, exceptionOnError=True):
"""
:param cmd: the command to execute
:param exceptionOnError: if True, raise on exception on error (return code not 0); if False return
whether the call was successful
:return: True if the call was successful, False otherwise (if exceptionOnError==False)
"""
p = popen(cmd)
p.wait()
success = p.returncode == 0
if exceptionOnError:
if not success:
raise Exception("Command failed: %s" % cmd)
else:
return success
def gitLog(path, arg):
oldPath = os.getcwd()
os.chdir(path)
lg = call("git log --no-merges " + arg)
os.chdir(oldPath)
return lg
def gitCommit(msg):
with open(COMMIT_MSG_FILENAME, "wb") as f:
f.write(msg.encode("utf-8"))
gitCommitWithMessageFromFile(COMMIT_MSG_FILENAME)
def gitCommitWithMessageFromFile(commitMsgFilename):
if not os.path.exists(commitMsgFilename):
raise FileNotFoundError(f"{commitMsgFilename} not found in {os.path.abspath(os.getcwd())}")
os.system(f"git commit --file={commitMsgFilename}")
os.unlink(commitMsgFilename)
COMMIT_MSG_FILENAME = "commitmsg.txt"
class OtherRepo:
SYNC_COMMIT_ID_FILE_LIB_REPO = ".syncCommitId.remote"
SYNC_COMMIT_ID_FILE_THIS_REPO = ".syncCommitId.this"
SYNC_COMMIT_MESSAGE = f"Updated %s sync commit identifiers"
SYNC_BACKUP_DIR = ".syncBackup"
def __init__(self, name, branch, pathToLib):
self.pathToLibInThisRepo = os.path.abspath(pathToLib)
if not os.path.exists(self.pathToLibInThisRepo):
raise ValueError(f"Repository directory '{self.pathToLibInThisRepo}' does not exist")
self.name = name
self.branch = branch
self.libRepo: Optional[LibRepo] = None
def isSyncEstablished(self):
return os.path.exists(os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_LIB_REPO))
def lastSyncIdThisRepo(self):
with open(os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_THIS_REPO), "r") as f:
commitId = f.read().strip()
return commitId
def lastSyncIdLibRepo(self):
with open(os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_LIB_REPO), "r") as f:
commitId = f.read().strip()
return commitId
def gitLogThisRepoSinceLastSync(self):
lg = gitLog(self.pathToLibInThisRepo, '--name-only HEAD "^%s" .' % self.lastSyncIdThisRepo())
lg = re.sub(r'commit [0-9a-z]{8,40}\n.*\n.*\n\s*\n.*\n\s*(\n.*\.syncCommitId\.(this|remote))+', r"", lg, flags=re.MULTILINE) # remove commits with sync commit id update
indent = " "
lg = indent + lg.replace("\n", "\n" + indent)
return lg
def gitLogLibRepoSinceLastSync(self, libRepo: "LibRepo"):
syncIdFile = os.path.join(self.pathToLibInThisRepo, self.SYNC_COMMIT_ID_FILE_LIB_REPO)
if not os.path.exists(syncIdFile):
return ""
with open(syncIdFile, "r") as f:
syncId = f.read().strip()
lg = gitLog(libRepo.libPath, '--name-only HEAD "^%s" .' % syncId)
lg = re.sub(r"Sync (\w+)\n\s*\n", r"Sync\n\n", lg, flags=re.MULTILINE)
indent = " "
lg = indent + lg.replace("\n", "\n" + indent)
return "\n\n" + lg
def _userInputYesNo(self, question) -> bool:
result = None
while result not in ("y", "n"):
result = input(question + " [y|n]: ").strip()
return result == "y"
def pull(self, libRepo: "LibRepo"):
"""
Pulls in changes from this repository into the lib repo
"""
# switch to branch in lib repo
os.chdir(libRepo.rootPath)
execute("git checkout %s" % self.branch)
# check if the branch contains the commit that is referenced as the remote commit
remoteCommitId = self.lastSyncIdLibRepo()
remoteCommitExists = execute("git rev-list HEAD..%s" % remoteCommitId, exceptionOnError=False)
if not remoteCommitExists:
if not self._userInputYesNo(f"\nWARNING: The referenced remote commit {remoteCommitId} does not exist "
f"in your {self.libRepo.name} branch '{self.branch}'!\nSomeone else may have "
f"pulled/pushed in the meantime.\nIt is recommended that you do not continue. "
f"Continue?"):
return
# check if this branch is clean
lgLib = self.gitLogLibRepoSinceLastSync(libRepo).strip()
if lgLib != "":
print(f"The following changes have been added to this branch in the library:\n\n{lgLib}\n\n")
print(f"ERROR: You must push these changes before you can pull or reset this branch to {remoteCommitId}")
sys.exit(1)
# get log with relevant commits in this repo that are to be pulled
lg = self.gitLogThisRepoSinceLastSync()
os.chdir(libRepo.rootPath)
# create commit message file
commitMsg = f"Sync {self.name}\n\n" + lg
with open(COMMIT_MSG_FILENAME, "w") as f:
f.write(commitMsg)
# ask whether to commit these changes
print("Relevant commits:\n\n" + lg + "\n\n")
if not self._userInputYesNo(f"The above changes will be pulled from {self.name}.\n"
f"You may change the commit message by editing {os.path.abspath(COMMIT_MSG_FILENAME)}.\n"
"Continue?"):
os.unlink(COMMIT_MSG_FILENAME)
return
# prepare restoration of ignored files
self.prepare_restoration_of_ignored_files(libRepo.rootPath)
# remove library tree in lib repo
shutil.rmtree(self.libRepo.libDirectory)
# copy tree from this repo to lib repo (but drop the sync commit id files)
shutil.copytree(self.pathToLibInThisRepo, self.libRepo.libDirectory)
for fn in (self.SYNC_COMMIT_ID_FILE_LIB_REPO, self.SYNC_COMMIT_ID_FILE_THIS_REPO):
p = os.path.join(self.libRepo.libDirectory, fn)
if os.path.exists(p):
os.unlink(p)
# restore ignored directories/files
self.restore_ignored_files(libRepo.rootPath)
# make commit in lib repo
os.system("git add %s" % self.libRepo.libDirectory)
gitCommitWithMessageFromFile(COMMIT_MSG_FILENAME)
newSyncCommitIdLibRepo = call("git rev-parse HEAD").strip()
# update commit ids in this repo
os.chdir(self.pathToLibInThisRepo)
newSyncCommitIdThisRepo = call("git rev-parse HEAD").strip()
with open(self.SYNC_COMMIT_ID_FILE_LIB_REPO, "w") as f:
f.write(newSyncCommitIdLibRepo)
with open(self.SYNC_COMMIT_ID_FILE_THIS_REPO, "w") as f:
f.write(newSyncCommitIdThisRepo)
execute('git add %s %s' % (self.SYNC_COMMIT_ID_FILE_LIB_REPO, self.SYNC_COMMIT_ID_FILE_THIS_REPO))
execute(f'git commit -m "{self.SYNC_COMMIT_MESSAGE % self.libRepo.name} (pull)"')
print(f"\n\nIf everything was successful, you should now push your changes to branch "
f"'{self.branch}'\nand get your branch merged into develop (issuing a pull request where appropriate)")
def push(self, libRepo: "LibRepo"):
"""
Pushes changes from the lib repo to this repo
"""
os.chdir(libRepo.rootPath)
# switch to the source repo branch
execute(f"git checkout {self.branch}")
if self.isSyncEstablished():
# check if there are any commits that have not yet been pulled
unpulledCommits = self.gitLogThisRepoSinceLastSync().strip()
if unpulledCommits != "":
print(f"\n{unpulledCommits}\n\n")
if not self._userInputYesNo(f"WARNING: The above changes in repository '{self.name}' have not"
f" yet been pulled.\nYou might want to pull them.\n"
f"If you continue with the push, they will be lost. Continue?"):
return
# get change log in lib repo since last sync
libLogSinceLastSync = self.gitLogLibRepoSinceLastSync(libRepo)
print("Relevant commits:\n\n" + libLogSinceLastSync + "\n\n")
if not self._userInputYesNo("The above changes will be pushed. Continue?"):
return
print()
else:
libLogSinceLastSync = ""
# prepare restoration of ignored files in target repo
base_dir_this_repo = os.path.join(self.pathToLibInThisRepo, "..")
self.prepare_restoration_of_ignored_files(base_dir_this_repo)
# remove the target repo tree and update it with the tree from the source repo
shutil.rmtree(self.pathToLibInThisRepo)
shutil.copytree(libRepo.libPath, self.pathToLibInThisRepo)
# get the commit id of the source repo we just copied
commitId = call("git rev-parse HEAD").strip()
# restore ignored directories and files
self.restore_ignored_files(base_dir_this_repo)
# go to the target repo
os.chdir(self.pathToLibInThisRepo)
# commit new version in this repo
execute("git add .")
with open(self.SYNC_COMMIT_ID_FILE_LIB_REPO, "w") as f:
f.write(commitId)
execute("git add %s" % self.SYNC_COMMIT_ID_FILE_LIB_REPO)
gitCommit(f"{self.libRepo.name} {commitId}" + libLogSinceLastSync)
commitId = call("git rev-parse HEAD").strip()
# update information on the commit id we just added
with open(self.SYNC_COMMIT_ID_FILE_THIS_REPO, "w") as f:
f.write(commitId)
execute("git add %s" % self.SYNC_COMMIT_ID_FILE_THIS_REPO)
execute(f'git commit -m "{self.SYNC_COMMIT_MESSAGE % self.libRepo.name} (push)"')
os.chdir(libRepo.rootPath)
print(f"\n\nIf everything was successful, you should now update the remote branch:\ngit push")
def prepare_restoration_of_ignored_files(self, base_dir: str):
"""
:param base_dir: the directory containing the lib directory, to which ignored paths are relative
"""
cwd = os.getcwd()
os.chdir(base_dir)
# ensure backup dir exists and is empty
if os.path.exists(self.SYNC_BACKUP_DIR):
shutil.rmtree(self.SYNC_BACKUP_DIR)
os.mkdir(self.SYNC_BACKUP_DIR)
# backup ignored, unversioned directories
for d in self.libRepo.fullyIgnoredUnversionedDirectories:
if os.path.exists(d):
shutil.copytree(d, os.path.join(self.SYNC_BACKUP_DIR, d))
os.chdir(cwd)
def restore_ignored_files(self, base_dir: str):
"""
:param base_dir: the directory containing the lib directory, to which ignored paths are relative
"""
cwd = os.getcwd()
os.chdir(base_dir)
# remove fully ignored directories that were overwritten by the sync
for d in self.libRepo.fullyIgnoredVersionedDirectories + self.libRepo.fullyIgnoredUnversionedDirectories:
if os.path.exists(d):
print("Removing overwritten content: %s" % d)
shutil.rmtree(d)
# restore directories and files that can be restored via git
for d in self.libRepo.ignoredDirectories + self.libRepo.fullyIgnoredVersionedDirectories:
restoration_cmd = "git checkout %s" % d
print("Restoring: %s" % restoration_cmd)
os.system(restoration_cmd)
for pattern in self.libRepo.ignoredFileGlobPatterns:
for path in glob.glob(pattern, recursive=True):
print("Restoring via git: %s" % path)
os.system("git checkout %s" % path)
# restore directories that were backed up
for d in self.libRepo.fullyIgnoredUnversionedDirectories:
if os.path.exists(os.path.join(self.SYNC_BACKUP_DIR, d)):
print("Restoring from backup: %s" % d)
shutil.copytree(os.path.join(self.SYNC_BACKUP_DIR, d), d)
# remove backup dir
shutil.rmtree(self.SYNC_BACKUP_DIR)
os.chdir(cwd)
class LibRepo:
def __init__(self, name: str, libDirectory: str,
ignoredDirectories: Sequence[str] = (),
fullyIgnoredVersionedDirectories: Sequence[str] = (),
fullyIgnoredUnversionedDirectories: Sequence[str] = (),
ignoredFileGlobPatterns: Sequence[str] = ()
):
"""
:param name: name of the library being synced
:param libDirectory: relative path to the library directory within this repo
:param ignoredDirectories: ignored directories; existing files in ignored directories will be restored
via 'git checkout' on pull/push, but new files will be added.
This is useful for configuration-like files, where users may have local changes that should not
be overwritten, but new files should still be added.
:param fullyIgnoredVersionedDirectories:
fully ignored versioned directories will be restored to original state after push/pull via git checkout
:param fullyIgnoredUnversionedDirectories:
fully ignored unversioned directories will be backed up and restored to original state after push/pull
:param ignoredFileGlobPatterns: files matching ignored glob patterns will be restored via 'git checkout'
on pull/push
"""
self.name = name
self.rootPath = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
self.libDirectory = libDirectory
self.libPath = os.path.join(self.rootPath, self.libDirectory)
self.ignoredDirectories: List[str] = list(ignoredDirectories)
self.fullyIgnoredVersionedDirectories: List[str] = list(fullyIgnoredVersionedDirectories)
self.fullyIgnoredUnversionedDirectories: List[str] = list(fullyIgnoredUnversionedDirectories)
self.ignoredFileGlobPatterns: List[str] = list(ignoredFileGlobPatterns)
self.otherRepos: List[OtherRepo] = []
def add(self, repo: OtherRepo):
repo.libRepo = self
self.otherRepos.append(repo)
def runMain(self):
repos = self.otherRepos
args = sys.argv[1:]
if len(args) != 2:
print(f"usage: sync.py <{'|'.join([repo.name for repo in repos])}> <push|pull>")
else:
repo = [r for r in repos if r.name == args[0]]
if len(repo) != 1:
raise ValueError(f"Unknown repo '{args[0]}'")
repo = repo[0]
if args[1] == "push":
repo.push(self)
elif args[1] == "pull":
repo.pull(self)
else:
raise ValueError(f"Unknown command '{args[1]}'")
scripts/agno_agent.py
+16 -7
View File
@@ -1,7 +1,6 @@
from agno.models.anthropic.claude import Claude
from agno.models.google.gemini import Gemini
from agno.playground.playground import Playground
from agno.playground.serve import serve_playground_app
from agno.os import AgentOS
from sensai.util import logging
from sensai.util.helper import mark_used
@@ -9,16 +8,26 @@ from serena.agno import SerenaAgnoAgentProvider
mark_used(Gemini, Claude)
# initialize logging (Note: since this module is reimported by serve_playground_app and the logging configuration
# is extended by SerenaAgentProvider, we must handle this here conditionally)
# initialize logging
if __name__ == "__main__":
logging.configure(level=logging.INFO)
# Define the model to use (see Agno documentation for supported models; these are just examples)
# model = Claude(id="claude-3-7-sonnet-20250219")
model = Gemini(id="gemini-2.5-pro-exp-03-25")
model = Gemini(id="gemini-2.5-pro")
app = Playground(agents=[SerenaAgnoAgentProvider.get_agent(model)]).get_app()
# Create the Serena agent using the existing provider
serena_agent = SerenaAgnoAgentProvider.get_agent(model)
# Create AgentOS app with the Serena agent
agent_os = AgentOS(
description="Serena coding assistant powered by AgentOS",
id="serena-agentos",
agents=[serena_agent],
)
app = agent_os.get_app()
if __name__ == "__main__":
serve_playground_app("agno_agent:app", reload=False, log_config=None)
# Start the AgentOS server
agent_os.serve(app="agno_agent:app", reload=False)
scripts/demo_run_tools.py
+5 -4
View File
@@ -7,20 +7,21 @@ import json
from pprint import pprint
from serena.agent import SerenaAgent
from serena.config.serena_config import SerenaConfig
from serena.constants import REPO_ROOT
from serena.tools import FindFileTool, FindReferencingSymbolsTool, GetSymbolsOverviewTool, SearchForPatternTool
from serena.tools import FindFileTool, FindReferencingSymbolsTool, GetSymbolsOverviewTool, JetBrainsFindSymbolTool, SearchForPatternTool
if __name__ == "__main__":
agent = SerenaAgent(project=REPO_ROOT, serena_config=SerenaConfig(gui_log_window_enabled=False, web_dashboard=False))
agent = SerenaAgent(project=REPO_ROOT)
# apply a tool
find_symbol_tool = agent.get_tool(JetBrainsFindSymbolTool)
find_refs_tool = agent.get_tool(FindReferencingSymbolsTool)
find_file_tool = agent.get_tool(FindFileTool)
search_pattern_tool = agent.get_tool(SearchForPatternTool)
overview_tool = agent.get_tool(GetSymbolsOverviewTool)
result = agent.execute_task(
lambda: overview_tool.apply("src/solidlsp/ls.py"),
lambda: find_symbol_tool.apply("displayBasicStats"),
)
pprint(json.loads(result))
# input("Press Enter to continue...")
scripts/profile_tool_call.py
+53
View File
@@ -0,0 +1,53 @@
import cProfile
from pathlib import Path
from typing import Literal
from sensai.util import logging
from sensai.util.logging import LogTime
from sensai.util.profiling import profiled
from serena.agent import SerenaAgent
from serena.config.serena_config import SerenaConfig
from serena.tools import FindSymbolTool
log = logging.getLogger(__name__)
if __name__ == "__main__":
logging.configure()
# The profiler to use:
# Use pyinstrument for hierarchical profiling output
# Use cProfile to determine which functions take the most time overall (and use snakeviz to visualize)
profiler: Literal["pyinstrument", "cprofile"] = "cprofile"
project_path = Path(__file__).parent.parent # Serena root
serena_config = SerenaConfig.from_config_file()
serena_config.log_level = logging.INFO
serena_config.jetbrains = False
serena_config.gui_log_window_enabled = False
serena_config.web_dashboard = False
agent = SerenaAgent(str(project_path), serena_config=serena_config)
# wait for language server to be ready
agent.execute_task(lambda: log.info("Language server is ready."))
def tool_call():
"""This is the function we want to profile."""
# NOTE: We use apply (not apply_ex) to run the tool call directly on the main thread
with LogTime("Tool call"):
result = agent.get_tool(FindSymbolTool).apply(name_path="DQN")
log.info("Tool result:\n%s", result)
if profiler == "pyinstrument":
@profiled(log_to_file=True)
def profiled_tool_call():
tool_call()
profiled_tool_call()
elif profiler == "cprofile":
cProfile.run("tool_call()", "tool_call.pstat")
src/serena/agent.py
+218 -127
View File
@@ -6,13 +6,10 @@ import multiprocessing
import os
import platform
import sys
import threading
import webbrowser
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from logging import Logger
from pathlib import Path
from typing import TYPE_CHECKING, Any, Optional, TypeVar
from typing import TYPE_CHECKING, Optional, TypeVar
from sensai.util import logging
from sensai.util.logging import LogTime
@@ -20,15 +17,17 @@ from sensai.util.logging import LogTime
from interprompt.jinja_template import JinjaTemplate
from serena import serena_version
from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats
from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode
from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, ToolSet, get_serena_managed_in_project_dir
from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition
from serena.dashboard import SerenaDashboardAPI
from serena.ls_manager import LanguageServerManager
from serena.project import Project
from serena.prompt_factory import SerenaPromptFactory
from serena.tools import ActivateProjectTool, GetCurrentConfigTool, Tool, ToolMarker, ToolRegistry
from serena.task_executor import TaskExecutor
from serena.tools import ActivateProjectTool, GetCurrentConfigTool, ReplaceContentTool, Tool, ToolMarker, ToolRegistry
from serena.util.inspection import iter_subclasses
from serena.util.logging import MemoryLogHandler
from solidlsp import SolidLanguageServer
from solidlsp.ls_config import Language
if TYPE_CHECKING:
from serena.gui_log_viewer import GuiLogViewer
@@ -43,40 +42,11 @@ class ProjectNotFoundError(Exception):
pass
class MemoriesManager:
def __init__(self, project_root: str):
self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories"
self._memory_dir.mkdir(parents=True, exist_ok=True)
def _get_memory_file_path(self, name: str) -> Path:
# strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not.
name = name.replace(".md", "")
filename = f"{name}.md"
return self._memory_dir / filename
def load_memory(self, name: str) -> str:
memory_file_path = self._get_memory_file_path(name)
if not memory_file_path.exists():
return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it."
with open(memory_file_path, encoding="utf-8") as f:
return f.read()
def save_memory(self, name: str, content: str) -> str:
memory_file_path = self._get_memory_file_path(name)
with open(memory_file_path, "w", encoding="utf-8") as f:
f.write(content)
return f"Memory {name} written."
def list_memories(self) -> list[str]:
return [f.name.replace(".md", "") for f in self._memory_dir.iterdir() if f.is_file()]
def delete_memory(self, name: str) -> str:
memory_file_path = self._get_memory_file_path(name)
memory_file_path.unlink()
return f"Memory {name} deleted."
class AvailableTools:
"""
Represents the set of available/exposed tools of a SerenaAgent.
"""
def __init__(self, tools: list[Tool]):
"""
:param tools: the list of available tools
@@ -93,6 +63,94 @@ class AvailableTools:
return len(self.tools)
class ToolSet:
"""
Represents a set of tools by their names.
"""
LEGACY_TOOL_NAME_MAPPING = {"replace_regex": ReplaceContentTool.get_name_from_cls()}
"""
maps legacy tool names to their new names for backward compatibility
"""
def __init__(self, tool_names: set[str]) -> None:
self._tool_names = tool_names
@classmethod
def default(cls) -> "ToolSet":
"""
:return: the default tool set, which contains all tools that are enabled by default
"""
from serena.tools import ToolRegistry
return cls(set(ToolRegistry().get_tool_names_default_enabled()))
def apply(self, *tool_inclusion_definitions: "ToolInclusionDefinition") -> "ToolSet":
"""
Applies one or more tool inclusion definitions to this tool set,
resulting in a new tool set.
:param tool_inclusion_definitions: the definitions to apply
:return: a new tool set with the definitions applied
"""
from serena.tools import ToolRegistry
def get_updated_tool_name(tool_name: str) -> str:
"""Retrieves the updated tool name if the provided tool name is deprecated, logging a warning."""
if tool_name in self.LEGACY_TOOL_NAME_MAPPING:
new_tool_name = self.LEGACY_TOOL_NAME_MAPPING[tool_name]
log.warning("Tool name '%s' is deprecated, please use '%s' instead", tool_name, new_tool_name)
return new_tool_name
return tool_name
registry = ToolRegistry()
tool_names = set(self._tool_names)
for definition in tool_inclusion_definitions:
included_tools = []
excluded_tools = []
for included_tool in definition.included_optional_tools:
included_tool = get_updated_tool_name(included_tool)
if not registry.is_valid_tool_name(included_tool):
raise ValueError(f"Invalid tool name '{included_tool}' provided for inclusion")
if included_tool not in tool_names:
tool_names.add(included_tool)
included_tools.append(included_tool)
for excluded_tool in definition.excluded_tools:
excluded_tool = get_updated_tool_name(excluded_tool)
if not registry.is_valid_tool_name(excluded_tool):
raise ValueError(f"Invalid tool name '{excluded_tool}' provided for exclusion")
if excluded_tool in tool_names:
tool_names.remove(excluded_tool)
excluded_tools.append(excluded_tool)
if included_tools:
log.info(f"{definition} included {len(included_tools)} tools: {', '.join(included_tools)}")
if excluded_tools:
log.info(f"{definition} excluded {len(excluded_tools)} tools: {', '.join(excluded_tools)}")
return ToolSet(tool_names)
def without_editing_tools(self) -> "ToolSet":
"""
:return: a new tool set that excludes all tools that can edit
"""
from serena.tools import ToolRegistry
registry = ToolRegistry()
tool_names = set(self._tool_names)
for tool_name in self._tool_names:
if registry.get_tool_class_by_name(tool_name).can_edit():
tool_names.remove(tool_name)
return ToolSet(tool_names)
def get_tool_names(self) -> set[str]:
"""
Returns the names of the tools that are currently included in the tool set.
"""
return self._tool_names
def includes_name(self, tool_name: str) -> bool:
return tool_name in self._tool_names
class SerenaAgent:
def __init__(
self,
@@ -120,12 +178,10 @@ class SerenaAgent:
# project-specific instances, which will be initialized upon project activation
self._active_project: Project | None = None
self.language_server: SolidLanguageServer | None = None
self.memories_manager: MemoriesManager | None = None
# adjust log level
serena_log_level = self.serena_config.log_level
if Logger.root.level > serena_log_level:
if Logger.root.level != serena_log_level:
log.info(f"Changing the root logger level to {serena_log_level}")
Logger.root.setLevel(serena_log_level)
@@ -162,25 +218,9 @@ class SerenaAgent:
if self._gui_log_viewer is not None:
self._gui_log_viewer.set_tool_names(tool_names)
self._tool_usage_stats: ToolUsageStats | None = None
if self.serena_config.record_tool_usage_stats:
token_count_estimator = RegisteredTokenCountEstimator[self.serena_config.token_count_estimator]
log.info(f"Tool usage statistics recording is enabled with token count estimator: {token_count_estimator.name}.")
self._tool_usage_stats = ToolUsageStats(token_count_estimator)
# start the dashboard (web frontend), registering its log handler
if self.serena_config.web_dashboard:
self._dashboard_thread, port = SerenaDashboardAPI(
get_memory_log_handler(), tool_names, agent=self, tool_usage_stats=self._tool_usage_stats
).run_in_thread()
dashboard_url = f"http://127.0.0.1:{port}/dashboard/index.html"
log.info("Serena web dashboard started at %s", dashboard_url)
if self.serena_config.web_dashboard_open_on_launch:
# open the dashboard URL in the default web browser (using a separate process to control
# output redirection)
process = multiprocessing.Process(target=self._open_dashboard, args=(dashboard_url,))
process.start()
process.join(timeout=1)
token_count_estimator = RegisteredTokenCountEstimator[self.serena_config.token_count_estimator]
log.info(f"Will record tool usage statistics with token count estimator: {token_count_estimator.name}.")
self._tool_usage_stats = ToolUsageStats(token_count_estimator)
# log fundamental information
log.info(f"Starting Serena server (version={serena_version()}, process id={os.getpid()}, parent process id={os.getppid()})")
@@ -193,8 +233,8 @@ class SerenaAgent:
# determine the base toolset defining the set of exposed tools (which e.g. the MCP shall see),
# limited by the Serena config, the context (which is fixed for the session) and JetBrains mode
tool_inclusion_definitions: list[ToolInclusionDefinition] = [self.serena_config, self._context]
if self._context.name == RegisteredContext.IDE_ASSISTANT.value:
tool_inclusion_definitions.extend(self._ide_assistant_context_tool_inclusion_definitions(project))
if self._context.single_project:
tool_inclusion_definitions.extend(self._single_project_context_tool_inclusion_definitions(project))
if self.serena_config.jetbrains:
tool_inclusion_definitions.append(SerenaAgentMode.from_name_internal("jetbrains"))
@@ -203,10 +243,8 @@ class SerenaAgent:
log.info(f"Number of exposed tools: {len(self._exposed_tools)}")
# create executor for starting the language server and running tools in another thread
# This executor is used to achieve linear task execution, so it is important to use a single-threaded executor.
self._task_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="SerenaAgentExecutor")
self._task_executor_lock = threading.Lock()
self._task_executor_task_index = 1
# This executor is used to achieve linear task execution
self._task_executor = TaskExecutor("SerenaAgentTaskExecutor")
# Initialize the prompt factory
self.prompt_factory = SerenaPromptFactory()
@@ -227,6 +265,57 @@ class SerenaAgent:
except Exception as e:
log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e)
# start the dashboard (web frontend), registering its log handler
# should be the last thing to happen in the initialization since the dashboard
# may access various parts of the agent
if self.serena_config.web_dashboard:
self._dashboard_thread, port = SerenaDashboardAPI(
get_memory_log_handler(), tool_names, agent=self, tool_usage_stats=self._tool_usage_stats
).run_in_thread()
dashboard_url = f"http://127.0.0.1:{port}/dashboard/index.html"
log.info("Serena web dashboard started at %s", dashboard_url)
if self.serena_config.web_dashboard_open_on_launch:
# open the dashboard URL in the default web browser (using a separate process to control
# output redirection)
process = multiprocessing.Process(target=self._open_dashboard, args=(dashboard_url,))
process.start()
process.join(timeout=1)
# inform the GUI window (if any)
if self._gui_log_viewer is not None:
self._gui_log_viewer.set_dashboard_url(dashboard_url)
def get_current_tasks(self) -> list[TaskExecutor.TaskInfo]:
"""
Gets the list of tasks currently running or queued for execution.
The function returns a list of thread-safe TaskInfo objects (specifically created for the caller).
:return: the list of tasks in the execution order (running task first)
"""
return self._task_executor.get_current_tasks()
def get_last_executed_task(self) -> TaskExecutor.TaskInfo | None:
"""
Gets the last executed task.
:return: the last executed task info or None if no task has been executed yet
"""
return self._task_executor.get_last_executed_task()
def get_language_server_manager(self) -> LanguageServerManager | None:
if self._active_project is not None:
return self._active_project.language_server_manager
return None
def get_language_server_manager_or_raise(self) -> LanguageServerManager:
language_server_manager = self.get_language_server_manager()
if language_server_manager is None:
raise Exception(
"The language server manager is not initialized, indicating a problem during project activation. "
"Inform the user, telling them to inspect Serena's logs in order to determine the issue. "
"IMPORTANT: Wait for further instructions before you continue!"
)
return language_server_manager
def get_context(self) -> SerenaAgentContext:
return self._context
@@ -243,7 +332,7 @@ class SerenaAgent:
os.environ["COMSPEC"] = "" # force use of default shell
log.info("Adjusting COMSPEC environment variable to use the default shell instead of '%s'", comspec)
def _ide_assistant_context_tool_inclusion_definitions(self, project_root_or_name: str | None) -> list[ToolInclusionDefinition]:
def _single_project_context_tool_inclusion_definitions(self, project_root_or_name: str | None) -> list[ToolInclusionDefinition]:
"""
In the IDE assistant context, the agent is assumed to work on a single project, and we thus
want to apply that project's tool exclusions/inclusions from the get-go, limiting the set
@@ -261,6 +350,9 @@ class SerenaAgent:
# and provide responses to the client immediately.
project = self.load_project_from_path_or_name(project_root_or_name, autogenerate=False)
if project is not None:
log.info(
"Applying tool inclusion/exclusion definitions for single-project context based on project '%s'", project.project_name
)
tool_inclusion_definitions.append(
ToolInclusionDefinition(
excluded_tools=[ActivateProjectTool.get_name_from_cls(), GetCurrentConfigTool.get_name_from_cls()]
@@ -269,18 +361,15 @@ class SerenaAgent:
tool_inclusion_definitions.append(project.project_config)
return tool_inclusion_definitions
def record_tool_usage_if_enabled(self, input_kwargs: dict, tool_result: str | dict, tool: Tool) -> None:
def record_tool_usage(self, input_kwargs: dict, tool_result: str | dict, tool: Tool) -> None:
"""
Record the usage of a tool with the given input and output strings if tool usage statistics recording is enabled.
"""
tool_name = tool.get_name()
if self._tool_usage_stats is not None:
input_str = str(input_kwargs)
output_str = str(tool_result)
log.debug(f"Recording tool usage for tool '{tool_name}'")
self._tool_usage_stats.record_tool_usage(tool_name, input_str, output_str)
else:
log.debug(f"Tool usage statistics recording is disabled, not recording usage of '{tool_name}'.")
input_str = str(input_kwargs)
output_str = str(tool_result)
log.debug(f"Recording tool usage for tool '{tool_name}'")
self._tool_usage_stats.record_tool_usage(tool_name, input_str, output_str)
@staticmethod
def _open_dashboard(url: str) -> None:
@@ -359,6 +448,11 @@ class SerenaAgent:
available_tools=self._exposed_tools.tool_names,
available_markers=available_markers,
)
# If a project is active at startup, append its activation message
if self._active_project is not None:
system_prompt += "\n\n" + self._active_project.get_activation_message()
log.info("System prompt:\n%s", system_prompt)
return system_prompt
@@ -382,36 +476,33 @@ class SerenaAgent:
log.info(f"Active tools ({len(self._active_tools)}): {', '.join(self.get_active_tool_names())}")
def issue_task(self, task: Callable[[], Any], name: str | None = None) -> Future:
def issue_task(
self, task: Callable[[], T], name: str | None = None, logged: bool = True, timeout: float | None = None
) -> TaskExecutor.Task[T]:
"""
Issue a task to the executor for asynchronous execution.
It is ensured that tasks are executed in the order they are issued, one after another.
:param task: the task to execute
:param name: the name of the task for logging purposes; if None, use the task function's name
:return: a Future object representing the execution of the task
:param logged: whether to log management of the task; if False, only errors will be logged
:param timeout: the maximum time to wait for task completion in seconds, or None to wait indefinitely
:return: the task object, through which the task's future result can be accessed
"""
with self._task_executor_lock:
task_name = f"Task-{self._task_executor_task_index}[{name or task.__name__}]"
self._task_executor_task_index += 1
return self._task_executor.issue_task(task, name=name, logged=logged, timeout=timeout)
def task_execution_wrapper() -> Any:
with LogTime(task_name, logger=log):
return task()
log.info(f"Scheduling {task_name}")
return self._task_executor.submit(task_execution_wrapper)
def execute_task(self, task: Callable[[], T]) -> T:
def execute_task(self, task: Callable[[], T], name: str | None = None, logged: bool = True, timeout: float | None = None) -> T:
"""
Executes the given task synchronously via the agent's task executor.
This is useful for tasks that need to be executed immediately and whose results are needed right away.
:param task: the task to execute
:param name: the name of the task for logging purposes; if None, use the task function's name
:param logged: whether to log management of the task; if False, only errors will be logged
:param timeout: the maximum time to wait for task completion in seconds, or None to wait indefinitely
:return: the result of the task execution
"""
future = self.issue_task(task)
return future.result()
return self._task_executor.execute_task(task, name=name, logged=logged, timeout=timeout)
def is_using_language_server(self) -> bool:
"""
@@ -424,18 +515,14 @@ class SerenaAgent:
self._active_project = project
self._update_active_tools()
# initialize project-specific instances which do not depend on the language server
self.memories_manager = MemoriesManager(project.project_root)
def init_language_server() -> None:
def init_language_server_manager() -> None:
# start the language server
with LogTime("Language server initialization", logger=log):
self.reset_language_server()
assert self.language_server is not None
self.reset_language_server_manager()
# initialize the language server in the background (if in language server mode)
if self.is_using_language_server():
self.issue_task(init_language_server)
self.issue_task(init_language_server_manager)
if self._project_activation_callback is not None:
self._project_activation_callback()
@@ -542,12 +629,9 @@ class SerenaAgent:
return result_str
def is_language_server_running(self) -> bool:
return self.language_server is not None and self.language_server.is_running()
def reset_language_server(self) -> None:
def reset_language_server_manager(self) -> None:
"""
Starts/resets the language server for the current project
Starts/resets the language server manager for the current project
"""
tool_timeout = self.serena_config.tool_timeout
if tool_timeout is None or tool_timeout < 0:
@@ -557,27 +641,32 @@ class SerenaAgent:
raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds")
ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout
# stop the language server if it is running
if self.is_language_server_running():
assert self.language_server is not None
log.info(f"Stopping the current language server at {self.language_server.repository_root_path} ...")
self.language_server.stop()
self.language_server = None
# instantiate and start the language server
assert self._active_project is not None
self.language_server = self._active_project.create_language_server(
# instantiate and start the necessary language servers
self.get_active_project_or_raise().create_language_server_manager(
log_level=self.serena_config.log_level,
ls_timeout=ls_timeout,
trace_lsp_communication=self.serena_config.trace_lsp_communication,
ls_specific_settings=self.serena_config.ls_specific_settings,
)
log.info(f"Starting the language server for {self._active_project.project_name}")
self.language_server.start()
if not self.language_server.is_running():
raise RuntimeError(
f"Failed to start the language server for {self._active_project.project_name} at {self._active_project.project_root}"
)
def add_language(self, language: Language) -> None:
"""
Adds a new language to the active project, spawning the respective language server and updating the project configuration.
The addition is scheduled via the agent's task executor and executed synchronously, i.e. the method returns
when the addition is complete.
:param language: the language to add
"""
self.execute_task(lambda: self.get_active_project_or_raise().add_language(language), name=f"AddLanguage:{language.value}")
def remove_language(self, language: Language) -> None:
"""
Removes a language from the active project, shutting down the respective language server and updating the project configuration.
The removal is scheduled via the agent's task executor and executed asynchronously.
:param language: the language to remove
"""
self.issue_task(lambda: self.get_active_project_or_raise().remove_language(language), name=f"RemoveLanguage:{language.value}")
def get_tool(self, tool_class: type[TTool]) -> TTool:
return self._all_tools[tool_class] # type: ignore
@@ -586,20 +675,22 @@ class SerenaAgent:
ToolRegistry().print_tool_overview(self._active_tools.values())
def __del__(self) -> None:
self.shutdown()
def shutdown(self, timeout: float = 2.0) -> None:
"""
Destructor to clean up the language server instance and GUI logger
Shuts down the agent, freeing resources and stopping background tasks.
"""
if not hasattr(self, "_is_initialized"):
return
log.info("SerenaAgent is shutting down ...")
if self.is_language_server_running():
log.info("Stopping the language server ...")
assert self.language_server is not None
self.language_server.save_cache()
self.language_server.stop()
if self._active_project is not None:
self._active_project.shutdown(timeout=timeout)
self._active_project = None
if self._gui_log_viewer:
log.info("Stopping the GUI log window ...")
self._gui_log_viewer.stop()
self._gui_log_viewer = None
def get_tool_by_name(self, tool_name: str) -> Tool:
tool_class = ToolRegistry().get_tool_class_by_name(tool_name)
src/serena/agno.py
+10 -12
View File
@@ -6,9 +6,9 @@ from pathlib import Path
from typing import Any
from agno.agent import Agent
from agno.memory import AgentMemory
from agno.db.sqlite import SqliteDb
from agno.memory import MemoryManager
from agno.models.base import Model
from agno.storage.sqlite import SqliteStorage
from agno.tools.function import Function
from agno.tools.toolkit import Toolkit
from dotenv import load_dotenv
@@ -111,10 +111,9 @@ class SerenaAgnoAgentProvider:
raise
# Even though we don't want to keep history between sessions,
# for agno-ui to work as a conversation, we use a persistent storage on disk.
# This storage should be deleted between sessions.
# for agno-ui to work as a conversation, we use a persistent database on disk.
# This database should be deleted between sessions.
# Note that this might collide with custom options for the agent, like adding vector-search based tools.
# See here for an explanation: https://www.reddit.com/r/agno/comments/1jk6qea/regarding_the_built_in_memory/
sql_db_path = (Path("temp") / "agno_agent_storage.db").absolute()
sql_db_path.parent.mkdir(exist_ok=True)
# delete the db file if it exists
@@ -125,19 +124,18 @@ class SerenaAgnoAgentProvider:
agno_agent = Agent(
name="Serena",
model=model,
# See explanation above on why storage is needed
storage=SqliteStorage(table_name="serena_agent_sessions", db_file=str(sql_db_path)),
# See explanation above on why database is needed
db=SqliteDb(db_file=str(sql_db_path)),
description="A fully-featured coding assistant",
tools=[SerenaAgnoToolkit(serena_agent)],
# The tool calls will be shown in the UI anyway since whether to show them is configurable per tool
# Tool calls will be shown in the UI since that's configurable per tool
# To see detailed logs, you should use the serena logger (configure it in the project file path)
show_tool_calls=False,
markdown=True,
system_message=serena_agent.create_system_prompt(),
telemetry=False,
memory=AgentMemory(),
add_history_to_messages=True,
num_history_responses=100, # you might want to adjust this (expense vs. history awareness)
memory_manager=MemoryManager(),
add_history_to_context=True,
num_history_runs=100, # you might want to adjust this (expense vs. history awareness)
)
cls._agent = agno_agent
log.info(f"Agent instantiated: {agno_agent}")
src/serena/analytics.py
+17 -1
View File
@@ -69,12 +69,26 @@ class AnthropicTokenCount(TokenCountEstimator):
return self._send_count_tokens_request(text).input_tokens
class CharCountEstimator(TokenCountEstimator):
"""
A naive character count estimator that estimates tokens based on character count.
"""
def __init__(self, avg_chars_per_token: int = 4):
self._avg_chars_per_token = avg_chars_per_token
def estimate_token_count(self, text: str) -> int:
# Assuming an average of 4 characters per token
return len(text) // self._avg_chars_per_token
_registered_token_estimator_instances_cache: dict[RegisteredTokenCountEstimator, TokenCountEstimator] = {}
class RegisteredTokenCountEstimator(Enum):
TIKTOKEN_GPT4O = "TIKTOKEN_GPT4O"
ANTHROPIC_CLAUDE_SONNET_4 = "ANTHROPIC_CLAUDE_SONNET_4"
CHAR_COUNT = "CHAR_COUNT"
@classmethod
def get_valid_names(cls) -> list[str]:
@@ -89,8 +103,10 @@ class RegisteredTokenCountEstimator(Enum):
return TiktokenCountEstimator(model_name="gpt-4o")
case RegisteredTokenCountEstimator.ANTHROPIC_CLAUDE_SONNET_4:
return AnthropicTokenCount(model_name="claude-sonnet-4-20250514")
case RegisteredTokenCountEstimator.CHAR_COUNT:
return CharCountEstimator(avg_chars_per_token=4)
case _:
raise ValueError(f"Unknown token count estimator: {self.value}")
raise ValueError(f"Unknown token count estimator: {self}")
def load_estimator(self) -> TokenCountEstimator:
estimator_instance = _registered_token_estimator_instances_cache.get(self)
src/serena/cli.py
+132 -60
View File
@@ -1,3 +1,4 @@
import collections
import glob
import json
import os
@@ -11,6 +12,7 @@ from typing import Any, Literal
import click
from sensai.util import logging
from sensai.util.logging import FileLoggerContext, datetime_tag
from sensai.util.string import dict_string
from tqdm import tqdm
from serena.agent import SerenaAgent
@@ -19,14 +21,10 @@ from serena.config.serena_config import ProjectConfig, SerenaConfig, SerenaPaths
from serena.constants import (
DEFAULT_CONTEXT,
DEFAULT_MODES,
PROMPT_TEMPLATES_DIR_IN_USER_HOME,
PROMPT_TEMPLATES_DIR_INTERNAL,
SERENA_LOG_FORMAT,
SERENA_MANAGED_DIR_IN_HOME,
SERENAS_OWN_CONTEXT_YAMLS_DIR,
SERENAS_OWN_MODE_YAMLS_DIR,
USER_CONTEXT_YAMLS_DIR,
USER_MODE_YAMLS_DIR,
)
from serena.mcp import SerenaMCPFactory, SerenaMCPFactorySingleProcess
from serena.project import Project
@@ -268,7 +266,7 @@ class ModeCommands(AutoRegisteringGroup):
if not (name or from_internal):
raise click.UsageError("Provide at least one of --name or --from-internal.")
mode_name = name or from_internal
dest = os.path.join(USER_MODE_YAMLS_DIR, f"{mode_name}.yml")
dest = os.path.join(SerenaPaths().user_modes_dir, f"{mode_name}.yml")
src = (
os.path.join(SERENAS_OWN_MODE_YAMLS_DIR, f"{from_internal}.yml")
if from_internal
@@ -287,7 +285,7 @@ class ModeCommands(AutoRegisteringGroup):
@click.command("edit", help="Edit a custom mode YAML file.")
@click.argument("mode_name")
def edit(mode_name: str) -> None:
path = os.path.join(USER_MODE_YAMLS_DIR, f"{mode_name}.yml")
path = os.path.join(SerenaPaths().user_modes_dir, f"{mode_name}.yml")
if not os.path.exists(path):
if mode_name in SerenaAgentMode.list_registered_mode_names(include_user_modes=False):
click.echo(
@@ -303,7 +301,7 @@ class ModeCommands(AutoRegisteringGroup):
@click.command("delete", help="Delete a custom mode file.")
@click.argument("mode_name")
def delete(mode_name: str) -> None:
path = os.path.join(USER_MODE_YAMLS_DIR, f"{mode_name}.yml")
path = os.path.join(SerenaPaths().user_modes_dir, f"{mode_name}.yml")
if not os.path.exists(path):
click.echo(f"Custom mode '{mode_name}' not found.")
return
@@ -345,7 +343,7 @@ class ContextCommands(AutoRegisteringGroup):
if not (name or from_internal):
raise click.UsageError("Provide at least one of --name or --from-internal.")
ctx_name = name or from_internal
dest = os.path.join(USER_CONTEXT_YAMLS_DIR, f"{ctx_name}.yml")
dest = os.path.join(SerenaPaths().user_contexts_dir, f"{ctx_name}.yml")
src = (
os.path.join(SERENAS_OWN_CONTEXT_YAMLS_DIR, f"{from_internal}.yml")
if from_internal
@@ -364,7 +362,7 @@ class ContextCommands(AutoRegisteringGroup):
@click.command("edit", help="Edit a custom context YAML file.")
@click.argument("context_name")
def edit(context_name: str) -> None:
path = os.path.join(USER_CONTEXT_YAMLS_DIR, f"{context_name}.yml")
path = os.path.join(SerenaPaths().user_contexts_dir, f"{context_name}.yml")
if not os.path.exists(path):
if context_name in SerenaAgentContext.list_registered_context_names(include_user_contexts=False):
click.echo(
@@ -380,7 +378,7 @@ class ContextCommands(AutoRegisteringGroup):
@click.command("delete", help="Delete a custom context file.")
@click.argument("context_name")
def delete(context_name: str) -> None:
path = os.path.join(USER_CONTEXT_YAMLS_DIR, f"{context_name}.yml")
path = os.path.join(SerenaPaths().user_contexts_dir, f"{context_name}.yml")
if not os.path.exists(path):
click.echo(f"Custom context '{context_name}' not found.")
return
@@ -399,10 +397,9 @@ class SerenaConfigCommands(AutoRegisteringGroup):
"edit", help="Edit serena_config.yml in your default editor. Will create a config file from the template if no config is found."
)
def edit() -> None:
config_path = os.path.join(SERENA_MANAGED_DIR_IN_HOME, "serena_config.yml")
if not os.path.exists(config_path):
SerenaConfig.generate_config_file(config_path)
_open_in_editor(config_path)
serena_config = SerenaConfig.from_config_file()
assert serena_config.config_file_path is not None
_open_in_editor(serena_config.config_file_path)
class ProjectCommands(AutoRegisteringGroup):
@@ -414,26 +411,74 @@ class ProjectCommands(AutoRegisteringGroup):
)
@staticmethod
@click.command("generate-yml", help="Generate a project.yml file.")
@click.argument("project_path", type=click.Path(exists=True, file_okay=False), default=os.getcwd())
@click.option("--language", type=str, default=None, help="Programming language; inferred if not specified.")
def generate_yml(project_path: str, language: str | None = None) -> None:
def _create_project(project_path: str, name: str | None, language: tuple[str, ...]) -> ProjectConfig:
"""
Helper method to create a project configuration file.
:param project_path: Path to the project directory
:param name: Optional project name (defaults to directory name if not specified)
:param language: Tuple of language names
:return: The generated ProjectConfig instance
:raises FileExistsError: If project.yml already exists
:raises ValueError: If an unsupported language is specified
"""
yml_path = os.path.join(project_path, ProjectConfig.rel_path_to_project_yml())
if os.path.exists(yml_path):
raise FileExistsError(f"Project file {yml_path} already exists.")
lang_inst = None
languages: list[Language] = []
if language:
try:
lang_inst = Language[language.upper()]
except KeyError:
all_langs = [l.name.lower() for l in Language.iter_all(include_experimental=True)]
raise ValueError(f"Unknown language '{language}'. Supported: {all_langs}")
generated_conf = ProjectConfig.autogenerate(project_root=project_path, project_language=lang_inst)
print(f"Generated project.yml with language {generated_conf.language.value} at {yml_path}.")
for lang in language:
try:
languages.append(Language(lang.lower()))
except ValueError:
all_langs = [l.value for l in Language]
raise ValueError(f"Unknown language '{lang}'. Supported: {all_langs}")
generated_conf = ProjectConfig.autogenerate(
project_root=project_path, project_name=name, languages=languages if languages else None, interactive=True
)
yml_path = ProjectConfig.path_to_project_yml(project_path)
languages_str = ", ".join([lang.value for lang in generated_conf.languages]) if generated_conf.languages else "N/A"
click.echo(f"Generated project with languages {{{languages_str}}} at {yml_path}.")
return generated_conf
@staticmethod
@click.command("index", help="Index a project by saving symbols to the LSP cache.")
@click.command("create", help="Create a new Serena project configuration.")
@click.argument("project_path", type=click.Path(exists=True, file_okay=False), default=os.getcwd())
@click.option("--name", type=str, default=None, help="Project name; defaults to directory name if not specified.")
@click.option(
"--language", type=str, multiple=True, help="Programming language(s); inferred if not specified. Can be passed multiple times."
)
@click.option("--index", is_flag=True, help="Index the project after creation.")
@click.option(
"--log-level",
type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
default="WARNING",
help="Log level for indexing (only used if --index is set).",
)
@click.option("--timeout", type=float, default=10, help="Timeout for indexing a single file (only used if --index is set).")
def create(project_path: str, name: str | None, language: tuple[str, ...], index: bool, log_level: str, timeout: float) -> None:
try:
ProjectCommands._create_project(project_path, name, language)
if index:
click.echo("Indexing project...")
ProjectCommands._index_project(project_path, log_level, timeout=timeout)
except FileExistsError as e:
raise click.ClickException(f"Project already exists: {e}\nUse 'serena project index' to index an existing project.")
except ValueError as e:
raise click.ClickException(str(e))
@staticmethod
@click.command("index", help="Index a project by saving symbols to the LSP cache. Auto-creates project.yml if it doesn't exist.")
@click.argument("project", type=click.Path(exists=True), default=os.getcwd(), required=False)
@click.option("--name", type=str, default=None, help="Project name (only used if auto-creating project.yml).")
@click.option(
"--language",
type=str,
multiple=True,
help="Programming language(s) (only used if auto-creating project.yml). Inferred if not specified.",
)
@click.option(
"--log-level",
type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
@@ -441,7 +486,19 @@ class ProjectCommands(AutoRegisteringGroup):
help="Log level for indexing.",
)
@click.option("--timeout", type=float, default=10, help="Timeout for indexing a single file.")
def index(project: str, log_level: str, timeout: float) -> None:
def index(project: str, name: str | None, language: tuple[str, ...], log_level: str, timeout: float) -> None:
# Check if project.yml exists, if not auto-create it
yml_path = os.path.join(project, ProjectConfig.rel_path_to_project_yml())
if not os.path.exists(yml_path):
click.echo(f"Project configuration not found at {yml_path}. Auto-creating...")
try:
ProjectCommands._create_project(project, name, language)
except FileExistsError:
# Race condition - file was created between check and creation
pass
except ValueError as e:
raise click.ClickException(str(e))
ProjectCommands._index_project(project, log_level, timeout=timeout)
@staticmethod
@@ -459,33 +516,42 @@ class ProjectCommands(AutoRegisteringGroup):
logging.configure(level=lvl)
serena_config = SerenaConfig.from_config_file()
proj = Project.load(os.path.abspath(project))
click.echo(f"Indexing symbols in project {project}…")
ls = proj.create_language_server(log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings)
log_file = os.path.join(project, ".serena", "logs", "indexing.txt")
click.echo(f"Indexing symbols in project {project} …")
ls_mgr = proj.create_language_server_manager(
log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings
)
try:
log_file = os.path.join(project, ".serena", "logs", "indexing.txt")
collected_exceptions: list[Exception] = []
files_failed = []
with ls.start_server():
files = proj.gather_source_files()
collected_exceptions: list[Exception] = []
files_failed = []
language_file_counts: dict[Language, int] = collections.defaultdict(lambda: 0)
for i, f in enumerate(tqdm(files, desc="Indexing")):
try:
ls.request_document_symbols(f, include_body=False)
ls.request_document_symbols(f, include_body=True)
ls = ls_mgr.get_language_server(f)
ls.request_document_symbols(f)
language_file_counts[ls.language] += 1
except Exception as e:
log.error(f"Failed to index {f}, continuing.")
collected_exceptions.append(e)
files_failed.append(f)
if (i + 1) % 10 == 0:
ls.save_cache()
ls.save_cache()
click.echo(f"Symbols saved to {ls.cache_path}")
if len(files_failed) > 0:
os.makedirs(os.path.dirname(log_file), exist_ok=True)
with open(log_file, "w") as f:
for file, exception in zip(files_failed, collected_exceptions, strict=True):
f.write(f"{file}\n")
f.write(f"{exception}\n")
click.echo(f"Failed to index {len(files_failed)} files, see:\n{log_file}")
ls_mgr.save_all_caches()
reported_language_file_counts = {k.value: v for k, v in language_file_counts.items()}
click.echo(f"Indexed files per language: {dict_string(reported_language_file_counts, brackets=None)}")
ls_mgr.save_all_caches()
if len(files_failed) > 0:
os.makedirs(os.path.dirname(log_file), exist_ok=True)
with open(log_file, "w") as f:
for file, exception in zip(files_failed, collected_exceptions, strict=True):
f.write(f"{file}\n")
f.write(f"{exception}\n")
click.echo(f"Failed to index {len(files_failed)} files, see:\n{log_file}")
finally:
ls_mgr.stop_all()
@staticmethod
@click.command("is_ignored_path", help="Check if a path is ignored by the project configuration.")
@@ -522,16 +588,20 @@ class ProjectCommands(AutoRegisteringGroup):
if proj.is_ignored_path(file, ignore_non_source_files=True):
click.echo(f"'{file}' is ignored or declared as non-code file by the project configuration, won't index.")
exit(1)
ls = proj.create_language_server()
with ls.start_server():
symbols, _ = ls.request_document_symbols(file, include_body=False)
ls.request_document_symbols(file, include_body=True)
if verbose:
click.echo(f"Symbols in file '{file}':")
for symbol in symbols:
click.echo(f" - {symbol['name']} at line {symbol['selectionRange']['start']['line']} of kind {symbol['kind']}")
ls.save_cache()
click.echo(f"Successfully indexed file '{file}', {len(symbols)} symbols saved to {ls.cache_path}.")
ls_mgr = proj.create_language_server_manager()
try:
for ls in ls_mgr.iter_language_servers():
click.echo(f"Indexing for language {ls.language.value} …")
document_symbols = ls.request_document_symbols(file)
symbols, _ = document_symbols.get_all_symbols_and_roots()
if verbose:
click.echo(f"Symbols in file '{file}':")
for symbol in symbols:
click.echo(f" - {symbol['name']} at line {symbol['selectionRange']['start']['line']} of kind {symbol['kind']}")
ls.save_cache()
click.echo(f"Successfully indexed file '{file}', {len(symbols)} symbols saved to cache in {ls.cache_dir}.")
finally:
ls_mgr.stop_all()
@staticmethod
@click.command("health-check", help="Perform a comprehensive health check of the project's tools and language server.")
@@ -736,8 +806,9 @@ class PromptCommands(AutoRegisteringGroup):
@staticmethod
def _get_user_prompt_yaml_path(prompt_yaml_name: str) -> str:
os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True)
return os.path.join(PROMPT_TEMPLATES_DIR_IN_USER_HOME, prompt_yaml_name)
templates_dir = SerenaPaths().user_prompt_templates_dir
os.makedirs(templates_dir, exist_ok=True)
return os.path.join(templates_dir, prompt_yaml_name)
@staticmethod
@click.command("list", help="Lists yamls that are used for defining prompts.")
@@ -788,9 +859,10 @@ class PromptCommands(AutoRegisteringGroup):
@staticmethod
@click.command("list-overrides", help="List existing prompt override files")
def list_overrides() -> None:
os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True)
user_templates_dir = SerenaPaths().user_prompt_templates_dir
os.makedirs(user_templates_dir, exist_ok=True)
serena_prompt_yaml_names = [os.path.basename(f) for f in glob.glob(PROMPT_TEMPLATES_DIR_INTERNAL + "/*.yml")]
override_files = glob.glob(os.path.join(PROMPT_TEMPLATES_DIR_IN_USER_HOME, "*.yml"))
override_files = glob.glob(os.path.join(user_templates_dir, "*.yml"))
for file_path in override_files:
if os.path.basename(file_path) in serena_prompt_yaml_names:
click.echo(file_path)
src/serena/code_editor.py
+89 -11
View File
@@ -4,13 +4,15 @@ import os
from abc import ABC, abstractmethod
from collections.abc import Iterable, Iterator, Reversible
from contextlib import contextmanager
from typing import TYPE_CHECKING, Generic, Optional, TypeVar
from typing import TYPE_CHECKING, Generic, Optional, TypeVar, cast
from serena.symbol import JetBrainsSymbol, LanguageServerSymbol, LanguageServerSymbolRetriever, PositionInFile, Symbol
from solidlsp import SolidLanguageServer
from solidlsp import SolidLanguageServer, ls_types
from solidlsp.ls import LSPFileBuffer
from solidlsp.ls_utils import TextUtils
from solidlsp.ls_types import extract_text_edits
from solidlsp.ls_utils import PathUtils, TextUtils
from .constants import DEFAULT_SOURCE_FILE_ENCODING
from .project import Project
from .tools.jetbrains_plugin_client import JetBrainsPluginClient
@@ -27,6 +29,14 @@ class CodeEditor(Generic[TSymbol], ABC):
self.project_root = project_root
self.agent = agent
# set encoding based on active project, if available
encoding = DEFAULT_SOURCE_FILE_ENCODING
if agent is not None:
project = agent.get_active_project()
if project is not None:
encoding = project.project_config.encoding
self.encoding = encoding
class EditedFile(ABC):
@abstractmethod
def get_contents(self) -> str:
@@ -58,7 +68,7 @@ class CodeEditor(Generic[TSymbol], ABC):
yield edited_file
# save the file
abs_path = os.path.join(self.project_root, relative_path)
with open(abs_path, "w", encoding="utf-8") as f:
with open(abs_path, "w", encoding=self.encoding) as f:
f.write(edited_file.get_contents())
@abstractmethod
@@ -207,15 +217,25 @@ class CodeEditor(Generic[TSymbol], ABC):
with self._edited_file_context(relative_file_path) as edited_file:
edited_file.delete_text_between_positions(start_pos, end_pos)
@abstractmethod
def rename_symbol(self, name_path: str, relative_file_path: str, new_name: str) -> str:
"""
Renames the symbol with the given name throughout the codebase.
:param name_path: the name path of the symbol to rename
:param relative_file_path: the relative path of the file containing the symbol
:param new_name: the new name for the symbol
:return: a status message
"""
class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]):
def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, agent: Optional["SerenaAgent"] = None):
super().__init__(project_root=symbol_retriever.get_language_server().repository_root_path, agent=agent)
super().__init__(project_root=symbol_retriever.get_root_path(), agent=agent)
self._symbol_retriever = symbol_retriever
@property
def _lang_server(self) -> SolidLanguageServer:
return self._symbol_retriever.get_language_server()
def _get_language_server(self, relative_path: str) -> SolidLanguageServer:
return self._symbol_retriever.get_language_server(relative_path)
class EditedFile(CodeEditor.EditedFile):
def __init__(self, lang_server: SolidLanguageServer, relative_path: str, file_buffer: LSPFileBuffer):
@@ -232,14 +252,19 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]):
def insert_text_at_position(self, pos: PositionInFile, text: str) -> None:
self._lang_server.insert_text_at_position(self._relative_path, pos.line, pos.col, text)
def apply_text_edits(self, text_edits: list[ls_types.TextEdit]) -> None:
return self._lang_server.apply_text_edits_to_file(self._relative_path, text_edits)
@contextmanager
def _open_file_context(self, relative_path: str) -> Iterator["CodeEditor.EditedFile"]:
with self._lang_server.open_file(relative_path) as file_buffer:
yield self.EditedFile(self._lang_server, relative_path, file_buffer)
lang_server = self._get_language_server(relative_path)
with lang_server.open_file(relative_path) as file_buffer:
yield self.EditedFile(lang_server, relative_path, file_buffer)
def _get_code_file_content(self, relative_path: str) -> str:
"""Get the content of a file using the language server."""
return self._lang_server.language_server.retrieve_full_file_content(relative_path)
lang_server = self._get_language_server(relative_path)
return lang_server.language_server.retrieve_full_file_content(relative_path)
def _find_unique_symbol(self, name_path: str, relative_file_path: str) -> LanguageServerSymbol:
symbol_candidates = self._symbol_retriever.find_by_name(name_path, within_relative_path=relative_file_path)
@@ -252,6 +277,48 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]):
)
return symbol_candidates[0]
def _apply_workspace_edit(self, workspace_edit: ls_types.WorkspaceEdit) -> list[str]:
"""
Apply a WorkspaceEdit by making the changes to files.
:param workspace_edit: The WorkspaceEdit containing the changes to apply
:return: List of relative file paths that were modified
"""
uri_to_edits = extract_text_edits(workspace_edit)
modified_relative_paths = []
# Handle the 'changes' format (URI -> list of TextEdits)
for uri, edits in uri_to_edits.items():
file_path = PathUtils.uri_to_path(uri)
relative_path = os.path.relpath(file_path, self._symbol_retriever.get_root_path())
modified_relative_paths.append(relative_path)
with self._edited_file_context(relative_path) as edited_file:
edited_file = cast(LanguageServerCodeEditor.EditedFile, edited_file)
edited_file.apply_text_edits(edits)
return modified_relative_paths
def rename_symbol(self, name_path: str, relative_file_path: str, new_name: str) -> str:
symbol = self._find_unique_symbol(name_path, relative_file_path)
if not symbol.location.has_position_in_file():
raise ValueError(f"Symbol '{name_path}' does not have a valid position in file for renaming")
# After has_position_in_file check, line and column are guaranteed to be non-None
assert symbol.location.line is not None
assert symbol.location.column is not None
lang_server = self._get_language_server(relative_file_path)
rename_result = lang_server.request_rename_symbol_edit(
relative_file_path=relative_file_path, line=symbol.location.line, column=symbol.location.column, new_name=new_name
)
if rename_result is None:
raise ValueError(
f"Language server for {lang_server.language_id} returned no rename edits for symbol '{name_path}'. "
f"The symbol might not support renaming."
)
modified_files = self._apply_workspace_edit(rename_result)
msg = f"Successfully renamed '{name_path}' to '{new_name}' in {len(modified_files)} file(s)"
return msg
class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]):
def __init__(self, project: Project, agent: Optional["SerenaAgent"] = None) -> None:
@@ -292,3 +359,14 @@ class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]):
"Their locations are: \n " + json.dumps([s["location"] for s in symbols], indent=2)
)
return JetBrainsSymbol(symbols[0], self._project)
def rename_symbol(self, name_path: str, relative_file_path: str, new_name: str) -> str:
with JetBrainsPluginClient.from_project(self._project) as client:
client.rename_symbol(
name_path=name_path,
relative_path=relative_file_path,
new_name=new_name,
rename_in_comments=False,
rename_in_text_occurrences=False,
)
return "Success"
src/serena/config/context_mode.py
+36 -45
View File
@@ -4,7 +4,6 @@ Context and Mode configuration loader
import os
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING, Self
@@ -12,15 +11,14 @@ import yaml
from sensai.util import logging
from sensai.util.string import ToStringMixin
from serena.config.serena_config import ToolInclusionDefinition
from serena.config.serena_config import SerenaPaths, ToolInclusionDefinition
from serena.constants import (
DEFAULT_CONTEXT,
DEFAULT_MODES,
INTERNAL_MODE_YAMLS_DIR,
SERENA_FILE_ENCODING,
SERENAS_OWN_CONTEXT_YAMLS_DIR,
SERENAS_OWN_MODE_YAMLS_DIR,
USER_CONTEXT_YAMLS_DIR,
USER_MODE_YAMLS_DIR,
)
if TYPE_CHECKING:
@@ -56,7 +54,7 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin):
@classmethod
def from_yaml(cls, yaml_path: str | Path) -> Self:
"""Load a mode from a YAML file."""
with open(yaml_path, encoding="utf-8") as f:
with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f:
data = yaml.safe_load(f)
name = data.pop("name", Path(yaml_path).stem)
return cls(name=name, **data)
@@ -65,14 +63,14 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin):
def get_path(cls, name: str) -> str:
"""Get the path to the YAML file for a mode."""
fname = f"{name}.yml"
custom_mode_path = os.path.join(USER_MODE_YAMLS_DIR, fname)
custom_mode_path = os.path.join(SerenaPaths().user_modes_dir, fname)
if os.path.exists(custom_mode_path):
return custom_mode_path
own_yaml_path = os.path.join(SERENAS_OWN_MODE_YAMLS_DIR, fname)
if not os.path.exists(own_yaml_path):
raise FileNotFoundError(
f"Mode {name} not found in {USER_MODE_YAMLS_DIR} or in {SERENAS_OWN_MODE_YAMLS_DIR}."
f"Mode {name} not found in {SerenaPaths().user_modes_dir} or in {SERENAS_OWN_MODE_YAMLS_DIR}."
f"Available modes:\n{cls.list_registered_mode_names()}"
)
return own_yaml_path
@@ -102,7 +100,7 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin):
@classmethod
def list_custom_mode_names(cls) -> list[str]:
"""Names of all custom modes defined by the user."""
return [f.stem for f in Path(USER_MODE_YAMLS_DIR).glob("*.yml")]
return [f.stem for f in Path(SerenaPaths().user_modes_dir).glob("*.yml")]
@classmethod
def load_default_modes(cls) -> list[Self]:
@@ -124,14 +122,29 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin):
"""
name: str
"""the name of the context"""
prompt: str
"""
a Jinja2 template for the generation of the system prompt.
It is formatted by the agent (see SerenaAgent._format_prompt()).
"""
description: str = ""
tool_description_overrides: dict[str, str] = field(default_factory=dict)
"""Maps tool names to custom descriptions, default descriptions are extracted from the tool docstrings."""
"""
maps tool names to custom descriptions, default descriptions are extracted from the tool docstrings.
"""
single_project: bool = False
"""
whether to assume that Serena shall only work on a single project in this context (provided that a project is given
when Serena is started).
If set to true and a project is provided at startup, the set of tools is limited to those required by the project's
concrete configuration, and other tools are excluded completely, allowing the set of tools to be minimal.
The `activate_project` tool will, therefore, be disabled in this case, as project switching is not allowed.
"""
def _tostring_includes(self) -> list[str]:
return ["name"]
@@ -139,7 +152,7 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin):
@classmethod
def from_yaml(cls, yaml_path: str | Path) -> Self:
"""Load a context from a YAML file."""
with open(yaml_path, encoding="utf-8") as f:
with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f:
data = yaml.safe_load(f)
name = data.pop("name", Path(yaml_path).stem)
# Ensure backwards compatibility for tool_description_overrides
@@ -151,14 +164,14 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin):
def get_path(cls, name: str) -> str:
"""Get the path to the YAML file for a context."""
fname = f"{name}.yml"
custom_context_path = os.path.join(USER_CONTEXT_YAMLS_DIR, fname)
custom_context_path = os.path.join(SerenaPaths().user_contexts_dir, fname)
if os.path.exists(custom_context_path):
return custom_context_path
own_yaml_path = os.path.join(SERENAS_OWN_CONTEXT_YAMLS_DIR, fname)
if not os.path.exists(own_yaml_path):
raise FileNotFoundError(
f"Context {name} not found in {USER_CONTEXT_YAMLS_DIR} or in {SERENAS_OWN_CONTEXT_YAMLS_DIR}."
f"Context {name} not found in {SerenaPaths().user_contexts_dir} or in {SERENAS_OWN_CONTEXT_YAMLS_DIR}."
f"Available contexts:\n{cls.list_registered_context_names()}"
)
return own_yaml_path
@@ -166,6 +179,16 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin):
@classmethod
def from_name(cls, name: str) -> Self:
"""Load a registered Serena context."""
legacy_name_mapping = {
"ide-assistant": "claude-code",
}
if name in legacy_name_mapping:
log.warning(
f"Context name '{name}' is deprecated and has been renamed to '{legacy_name_mapping[name]}'. "
f"Please update your configuration; refer to the configuration guide for more details: "
"https://oraios.github.io/serena/02-usage/050_configuration.html#contexts"
)
name = legacy_name_mapping[name]
context_path = cls.get_path(name)
return cls.from_yaml(context_path)
@@ -186,7 +209,7 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin):
@classmethod
def list_custom_context_names(cls) -> list[str]:
"""Names of all custom contexts defined by the user."""
return [f.stem for f in Path(USER_CONTEXT_YAMLS_DIR).glob("*.yml")]
return [f.stem for f in Path(SerenaPaths().user_contexts_dir).glob("*.yml")]
@classmethod
def load_default(cls) -> Self:
@@ -198,35 +221,3 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin):
print(f"{self.name}:\n {self.description}")
if self.excluded_tools:
print(" excluded tools:\n " + ", ".join(sorted(self.excluded_tools)))
class RegisteredContext(Enum):
"""A registered context."""
IDE_ASSISTANT = "ide-assistant"
"""For Serena running within an assistant that already has basic tools, like Claude Code, Cline, Cursor, etc."""
DESKTOP_APP = "desktop-app"
"""For Serena running within Claude Desktop or a similar app which does not have built-in tools for code editing."""
AGENT = "agent"
"""For Serena running as a standalone agent, e.g. through agno."""
def load(self) -> SerenaAgentContext:
"""Load the context."""
return SerenaAgentContext.from_name(self.value)
class RegisteredMode(Enum):
"""A registered mode."""
INTERACTIVE = "interactive"
"""Interactive mode, for multi-turn interactions."""
EDITING = "editing"
"""Editing tools are activated."""
PLANNING = "planning"
"""Editing tools are deactivated."""
ONE_SHOT = "one-shot"
"""Non-interactive mode, where the goal is to finish a task autonomously."""
def load(self) -> SerenaAgentMode:
"""Load the mode."""
return SerenaAgentMode.from_name(self.value)
src/serena/config/serena_config.py
+154 -121
View File
@@ -2,6 +2,7 @@
The Serena Model Context Protocol (MCP) Server
"""
import dataclasses
import os
import shutil
from collections.abc import Iterable
@@ -19,11 +20,11 @@ from sensai.util.logging import LogTime, datetime_tag
from sensai.util.string import ToStringMixin
from serena.constants import (
DEFAULT_ENCODING,
DEFAULT_SOURCE_FILE_ENCODING,
PROJECT_TEMPLATE_FILE,
REPO_ROOT,
SERENA_CONFIG_TEMPLATE_FILE,
SERENA_MANAGED_DIR_IN_HOME,
SERENA_FILE_ENCODING,
SERENA_MANAGED_DIR_NAME,
)
from serena.util.general import load_yaml, save_yaml
@@ -32,6 +33,7 @@ from solidlsp.ls_config import Language
from ..analytics import RegisteredTokenCountEstimator
from ..util.class_decorators import singleton
from ..util.cli_util import ask_yes_no
if TYPE_CHECKING:
from ..project import Project
@@ -39,6 +41,8 @@ if TYPE_CHECKING:
log = logging.getLogger(__name__)
T = TypeVar("T")
DEFAULT_TOOL_TIMEOUT: float = 240
DictType = dict | CommentedMap
TDict = TypeVar("TDict", bound=DictType)
@singleton
@@ -48,9 +52,32 @@ class SerenaPaths:
"""
def __init__(self) -> None:
self.user_config_dir: str = SERENA_MANAGED_DIR_IN_HOME
home_dir = os.getenv("SERENA_HOME")
if home_dir is None or home_dir.strip() == "":
home_dir = str(Path.home() / SERENA_MANAGED_DIR_NAME)
else:
home_dir = home_dir.strip()
self.serena_user_home_dir: str = home_dir
"""
the path to the user's Serena configuration directory, which is typically ~/.serena
the path to the Serena home directory, where the user's configuration/data is stored.
This is ~/.serena by default, but it can be overridden via the SERENA_HOME environment variable.
"""
self.user_prompt_templates_dir: str = os.path.join(self.serena_user_home_dir, "prompt_templates")
"""
directory containing prompt templates defined by the user.
Prompts defined by the user take precedence over Serena's built-in prompt templates.
"""
self.user_contexts_dir: str = os.path.join(self.serena_user_home_dir, "contexts")
"""
directory containing contexts defined by the user.
If a name of a context matches a name of a context in SERENAS_OWN_CONTEXT_YAMLS_DIR,
the user context will override the default context definition.
"""
self.user_modes_dir: str = os.path.join(self.serena_user_home_dir, "modes")
"""
directory containing modes defined by the user.
If a name of a mode matches a name of a mode in SERENAS_OWN_MODES_YAML_DIR,
the user mode will override the default mode definition.
"""
def get_next_log_file_path(self, prefix: str) -> str:
@@ -58,79 +85,13 @@ class SerenaPaths:
:param prefix: the filename prefix indicating the type of the log file
:return: the full path to the log file to use
"""
log_dir = os.path.join(self.user_config_dir, "logs", datetime.now().strftime("%Y-%m-%d"))
log_dir = os.path.join(self.serena_user_home_dir, "logs", datetime.now().strftime("%Y-%m-%d"))
os.makedirs(log_dir, exist_ok=True)
return os.path.join(log_dir, prefix + "_" + datetime_tag() + ".txt")
# TODO: Paths from constants.py should be moved here
class ToolSet:
def __init__(self, tool_names: set[str]) -> None:
self._tool_names = tool_names
@classmethod
def default(cls) -> "ToolSet":
"""
:return: the default tool set, which contains all tools that are enabled by default
"""
from serena.tools import ToolRegistry
return cls(set(ToolRegistry().get_tool_names_default_enabled()))
def apply(self, *tool_inclusion_definitions: "ToolInclusionDefinition") -> "ToolSet":
"""
:param tool_inclusion_definitions: the definitions to apply
:return: a new tool set with the definitions applied
"""
from serena.tools import ToolRegistry
registry = ToolRegistry()
tool_names = set(self._tool_names)
for definition in tool_inclusion_definitions:
included_tools = []
excluded_tools = []
for included_tool in definition.included_optional_tools:
if not registry.is_valid_tool_name(included_tool):
raise ValueError(f"Invalid tool name '{included_tool}' provided for inclusion")
if included_tool not in tool_names:
tool_names.add(included_tool)
included_tools.append(included_tool)
for excluded_tool in definition.excluded_tools:
if not registry.is_valid_tool_name(excluded_tool):
raise ValueError(f"Invalid tool name '{excluded_tool}' provided for exclusion")
if excluded_tool in tool_names:
tool_names.remove(excluded_tool)
excluded_tools.append(excluded_tool)
if included_tools:
log.info(f"{definition} included {len(included_tools)} tools: {', '.join(included_tools)}")
if excluded_tools:
log.info(f"{definition} excluded {len(excluded_tools)} tools: {', '.join(excluded_tools)}")
return ToolSet(tool_names)
def without_editing_tools(self) -> "ToolSet":
"""
:return: a new tool set that excludes all tools that can edit
"""
from serena.tools import ToolRegistry
registry = ToolRegistry()
tool_names = set(self._tool_names)
for tool_name in self._tool_names:
if registry.get_tool_class_by_name(tool_name).can_edit():
tool_names.remove(tool_name)
return ToolSet(tool_names)
def get_tool_names(self) -> set[str]:
"""
Returns the names of the tools that are currently included in the tool set.
"""
return self._tool_names
def includes_name(self, tool_name: str) -> bool:
return tool_name in self._tool_names
@dataclass
class ToolInclusionDefinition:
excluded_tools: Iterable[str] = ()
@@ -161,12 +122,12 @@ def is_running_in_docker() -> bool:
@dataclass(kw_only=True)
class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
project_name: str
language: Language
languages: list[Language]
ignored_paths: list[str] = field(default_factory=list)
read_only: bool = False
ignore_all_files_in_gitignore: bool = True
initial_prompt: str = ""
encoding: str = DEFAULT_ENCODING
encoding: str = DEFAULT_SOURCE_FILE_ENCODING
SERENA_DEFAULT_PROJECT_FILE = "project.yml"
@@ -175,7 +136,12 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
@classmethod
def autogenerate(
cls, project_root: str | Path, project_name: str | None = None, project_language: Language | None = None, save_to_disk: bool = True
cls,
project_root: str | Path,
project_name: str | None = None,
languages: list[Language] | None = None,
save_to_disk: bool = True,
interactive: bool = False,
) -> Self:
"""
Autogenerate a project configuration for a given project root.
@@ -183,8 +149,9 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
:param project_root: the path to the project root
:param project_name: the name of the project; if None, the name of the project will be the name of the directory
containing the project
:param project_language: the programming language of the project; if None, it will be determined automatically
:param languages: the languages of the project; if None, they will be determined automatically
:param save_to_disk: whether to save the project configuration to disk
:param interactive: whether to run in interactive CLI mode, asking the user for input where appropriate
:return: the project configuration
"""
project_root = Path(project_root).resolve()
@@ -192,61 +159,131 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
raise FileNotFoundError(f"Project root not found: {project_root}")
with LogTime("Project configuration auto-generation", logger=log):
project_name = project_name or project_root.name
if project_language is None:
if languages is None:
# determine languages automatically
language_composition = determine_programming_language_composition(str(project_root))
if len(language_composition) == 0:
language_values = ", ".join([lang.value for lang in Language])
raise ValueError(
f"No source files found in {project_root}\n\n"
f"To use Serena with this project, you need to either:\n"
f"1. Add source files in one of the supported languages (Python, JavaScript/TypeScript, Java, C#, Rust, Go, Ruby, C++, PHP, Swift, Elixir, Terraform, Bash)\n"
f"2. Create a project configuration file manually at:\n"
f" {os.path.join(project_root, cls.rel_path_to_project_yml())}\n\n"
f"Example project.yml:\n"
f" project_name: {project_name}\n"
f" language: python # or typescript, java, csharp, rust, go, ruby, cpp, php, swift, elixir, terraform, bash\n"
f"To use Serena with this project, you need to either\n"
f" 1. specify a programming language by adding parameters --language <language>\n"
f" when creating the project via the Serena CLI command OR\n"
f" 2. add source files in one of the supported languages first.\n\n"
f"Supported languages are: {language_values}\n"
f"Read the documentation for more information."
)
# find the language with the highest percentage
dominant_language = max(language_composition.keys(), key=lambda lang: language_composition[lang])
# sort languages by number of files found
languages_and_percentages = sorted(language_composition.items(), key=lambda item: item[1], reverse=True)
# find the language with the highest percentage and enable it
top_language_pair = languages_and_percentages[0]
other_language_pairs = languages_and_percentages[1:]
languages_to_use = [top_language_pair[0]]
# if in interactive mode, ask the user which other languages to enable
if len(other_language_pairs) > 0 and interactive:
print(
"Detected and enabled main language '%s' (%.2f%% of source files)." % (top_language_pair[0], top_language_pair[1])
)
print(f"Additionally detected {len(other_language_pairs)} other language(s).\n")
print("Note: Enable only languages you need symbolic retrieval/editing capabilities for.")
print(" Additional language servers use resources and some languages may require additional")
print(" system-level installations/configuration (see Serena documentation).")
print("\nWhich additional languages do you want to enable?")
for lang, perc in other_language_pairs:
enable = ask_yes_no("Enable %s (%.2f%% of source files)?" % (lang, perc), default=False)
if enable:
languages_to_use.append(lang)
print()
else:
dominant_language = project_language.value
config_with_comments = load_yaml(PROJECT_TEMPLATE_FILE, preserve_comments=True)
languages_to_use = [lang.value for lang in languages]
config_with_comments = cls.load_commented_map(PROJECT_TEMPLATE_FILE)
config_with_comments["project_name"] = project_name
config_with_comments["language"] = dominant_language
config_with_comments["languages"] = languages_to_use
if save_to_disk:
save_yaml(str(project_root / cls.rel_path_to_project_yml()), config_with_comments, preserve_comments=True)
save_yaml(cls.path_to_project_yml(project_root), config_with_comments, preserve_comments=True)
return cls._from_dict(config_with_comments)
@classmethod
def path_to_project_yml(cls, project_root: str | Path) -> str:
return os.path.join(project_root, cls.rel_path_to_project_yml())
@classmethod
def rel_path_to_project_yml(cls) -> str:
return os.path.join(SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE)
@classmethod
def _apply_defaults_to_dict(cls, data: TDict) -> TDict:
# apply defaults for new fields
data["languages"] = data.get("languages", [])
data["ignored_paths"] = data.get("ignored_paths", [])
data["excluded_tools"] = data.get("excluded_tools", [])
data["included_optional_tools"] = data.get("included_optional_tools", [])
data["read_only"] = data.get("read_only", False)
data["ignore_all_files_in_gitignore"] = data.get("ignore_all_files_in_gitignore", True)
data["initial_prompt"] = data.get("initial_prompt", "")
data["encoding"] = data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING)
# backward compatibility: handle single "language" field
if len(data["languages"]) == 0 and "language" in data:
data["languages"] = [data["language"]]
if "language" in data:
del data["language"]
return data
@classmethod
def load_commented_map(cls, yml_path: str) -> CommentedMap:
"""
Load the project configuration as a CommentedMap, preserving comments and ensuring
completeness of the configuration by applying default values for missing fields
and backward compatibility adjustments.
:param yml_path: the path to the project.yml file
:return: a CommentedMap representing a full project configuration
"""
data = load_yaml(yml_path, preserve_comments=True)
return cls._apply_defaults_to_dict(data)
@classmethod
def _from_dict(cls, data: dict[str, Any]) -> Self:
"""
Create a ProjectConfig instance from a configuration dictionary
Create a ProjectConfig instance from a (full) configuration dictionary
"""
language_str = data["language"].lower()
project_name = data["project_name"]
# backwards compatibility
if language_str == "javascript":
log.warning(f"Found deprecated project language `javascript` in project {project_name}, please change to `typescript`")
language_str = "typescript"
try:
language = Language(language_str)
except ValueError as e:
raise ValueError(f"Invalid language: {data['language']}.\nValid languages are: {[l.value for l in Language]}") from e
lang_name_mapping = {"javascript": "typescript"}
languages: list[Language] = []
for language_str in data["languages"]:
orig_language_str = language_str
try:
language_str = language_str.lower()
if language_str in lang_name_mapping:
language_str = lang_name_mapping[language_str]
language = Language(language_str)
languages.append(language)
except ValueError as e:
raise ValueError(
f"Invalid language: {orig_language_str}.\nValid language_strings are: {[l.value for l in Language]}"
) from e
return cls(
project_name=project_name,
language=language,
ignored_paths=data.get("ignored_paths", []),
excluded_tools=data.get("excluded_tools", []),
included_optional_tools=data.get("included_optional_tools", []),
read_only=data.get("read_only", False),
ignore_all_files_in_gitignore=data.get("ignore_all_files_in_gitignore", True),
initial_prompt=data.get("initial_prompt", ""),
encoding=data.get("encoding", DEFAULT_ENCODING),
project_name=data["project_name"],
languages=languages,
ignored_paths=data["ignored_paths"],
excluded_tools=data["excluded_tools"],
included_optional_tools=data["included_optional_tools"],
read_only=data["read_only"],
ignore_all_files_in_gitignore=data["ignore_all_files_in_gitignore"],
initial_prompt=data["initial_prompt"],
encoding=data["encoding"],
)
def to_yaml_dict(self) -> dict:
"""
:return: a yaml-serializable dictionary representation of this configuration
"""
d = dataclasses.asdict(self)
d["languages"] = [lang.value for lang in self.languages]
return d
@classmethod
def load(cls, project_root: Path | str, autogenerate: bool = False) -> Self:
"""
@@ -259,8 +296,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin):
return cls.autogenerate(project_root)
else:
raise FileNotFoundError(f"Project configuration file not found: {yaml_path}")
with open(yaml_path, encoding="utf-8") as f:
yaml_data = yaml.safe_load(f)
yaml_data = cls.load_commented_map(str(yaml_path))
if "project_name" not in yaml_data:
yaml_data["project_name"] = project_root.name
return cls._from_dict(yaml_data)
@@ -339,10 +375,8 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
"""
whether to apply JetBrains mode
"""
record_tool_usage_stats: bool = False
"""Whether to record tool usage statistics, they will be shown in the web dashboard if recording is active.
"""
token_count_estimator: str = RegisteredTokenCountEstimator.TIKTOKEN_GPT4O.name
token_count_estimator: str = RegisteredTokenCountEstimator.CHAR_COUNT.name
"""Only relevant if `record_tool_usage` is True; the name of the token count estimator to use for tool usage statistics.
See the `RegisteredTokenCountEstimator` enum for available options.
@@ -365,7 +399,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
return ["config_file_path"]
@classmethod
def generate_config_file(cls, config_file_path: str) -> None:
def _generate_config_file(cls, config_file_path: str) -> None:
"""
Generates a Serena configuration file at the specified path from the template file.
@@ -383,7 +417,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
if is_running_in_docker():
return os.path.join(REPO_ROOT, cls.CONFIG_FILE_DOCKER)
else:
config_path = os.path.join(SERENA_MANAGED_DIR_IN_HOME, cls.CONFIG_FILE)
config_path = os.path.join(SerenaPaths().serena_user_home_dir, cls.CONFIG_FILE)
# if the config file does not exist, check if we can migrate it from the old location
if not os.path.exists(config_path):
@@ -407,7 +441,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
if not generate_if_missing:
raise FileNotFoundError(f"Serena configuration file not found: {config_file_path}")
log.info(f"Serena configuration file not found at {config_file_path}, autogenerating...")
cls.generate_config_file(config_file_path)
cls._generate_config_file(config_file_path)
# load the configuration
log.info(f"Loading Serena configuration from {config_file_path}")
@@ -456,7 +490,6 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
instance.excluded_tools = loaded_commented_yaml.get("excluded_tools", [])
instance.included_optional_tools = loaded_commented_yaml.get("included_optional_tools", [])
instance.jetbrains = loaded_commented_yaml.get("jetbrains", False)
instance.record_tool_usage_stats = loaded_commented_yaml.get("record_tool_usage_stats", False)
instance.token_count_estimator = loaded_commented_yaml.get(
"token_count_estimator", RegisteredTokenCountEstimator.TIKTOKEN_GPT4O.name
)
@@ -483,11 +516,11 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin):
"""
log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.")
try:
with open(path, encoding="utf-8") as f:
with open(path, encoding=SERENA_FILE_ENCODING) as f:
project_config_data = yaml.safe_load(f)
if "project_name" not in project_config_data:
project_name = path.stem
with open(path, "a", encoding="utf-8") as f:
with open(path, "a", encoding=SERENA_FILE_ENCODING) as f:
f.write(f"\nproject_name: {project_name}")
project_root = project_config_data["project_root"]
shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml()))
src/serena/constants.py
+5 -9
View File
@@ -4,31 +4,27 @@ _repo_root_path = Path(__file__).parent.parent.parent.resolve()
_serena_pkg_path = Path(__file__).parent.resolve()
SERENA_MANAGED_DIR_NAME = ".serena"
_serena_in_home_managed_dir = Path.home() / ".serena"
SERENA_MANAGED_DIR_IN_HOME = str(_serena_in_home_managed_dir)
# TODO: Path-related constants should be moved to SerenaPaths; don't add further constants here.
REPO_ROOT = str(_repo_root_path)
PROMPT_TEMPLATES_DIR_INTERNAL = str(_serena_pkg_path / "resources" / "config" / "prompt_templates")
PROMPT_TEMPLATES_DIR_IN_USER_HOME = str(_serena_in_home_managed_dir / "prompt_templates")
SERENAS_OWN_CONTEXT_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "contexts")
"""The contexts that are shipped with the Serena package, i.e. the default contexts."""
USER_CONTEXT_YAMLS_DIR = str(_serena_in_home_managed_dir / "contexts")
"""Contexts defined by the user. If a name of a context matches a name of a context in SERENAS_OWN_CONTEXT_YAMLS_DIR, the user context will override the default one."""
SERENAS_OWN_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "modes")
"""The modes that are shipped with the Serena package, i.e. the default modes."""
USER_MODE_YAMLS_DIR = str(_serena_in_home_managed_dir / "modes")
"""Modes defined by the user. If a name of a mode matches a name of a mode in SERENAS_OWN_MODE_YAMLS_DIR, the user mode will override the default one."""
INTERNAL_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "internal_modes")
"""Internal modes, never overridden by user modes."""
SERENA_DASHBOARD_DIR = str(_serena_pkg_path / "resources" / "dashboard")
SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons")
DEFAULT_ENCODING = "utf-8"
DEFAULT_SOURCE_FILE_ENCODING = "utf-8"
"""The default encoding assumed for project source files."""
DEFAULT_CONTEXT = "desktop-app"
DEFAULT_MODES = ("interactive", "editing")
SERENA_FILE_ENCODING = "utf-8"
"""The encoding used for Serena's own files, such as configuration files and memories."""
PROJECT_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.template.yml")
SERENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml")
src/serena/dashboard.py
+410 -1
View File
@@ -2,7 +2,7 @@ import os
import socket
import threading
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Self
from flask import Flask, Response, request, send_from_directory
from pydantic import BaseModel
@@ -10,6 +10,7 @@ from sensai.util import logging
from serena.analytics import ToolUsageStats
from serena.constants import SERENA_DASHBOARD_DIR
from serena.task_executor import TaskExecutor
from serena.util.logging import MemoryLogHandler
if TYPE_CHECKING:
@@ -39,6 +40,82 @@ class ResponseToolStats(BaseModel):
stats: dict[str, dict[str, int]]
class ResponseConfigOverview(BaseModel):
active_project: dict[str, str | None]
context: dict[str, str]
modes: list[dict[str, str]]
active_tools: list[str]
tool_stats_summary: dict[str, dict[str, int]]
registered_projects: list[dict[str, str | bool]]
available_tools: list[dict[str, str | bool]]
available_modes: list[dict[str, str | bool]]
available_contexts: list[dict[str, str | bool]]
available_memories: list[str] | None
jetbrains_mode: bool
languages: list[str]
encoding: str | None
class ResponseAvailableLanguages(BaseModel):
languages: list[str]
class RequestAddLanguage(BaseModel):
language: str
class RequestRemoveLanguage(BaseModel):
language: str
class RequestGetMemory(BaseModel):
memory_name: str
class ResponseGetMemory(BaseModel):
content: str
memory_name: str
class RequestSaveMemory(BaseModel):
memory_name: str
content: str
class RequestDeleteMemory(BaseModel):
memory_name: str
class ResponseGetSerenaConfig(BaseModel):
content: str
class RequestSaveSerenaConfig(BaseModel):
content: str
class RequestCancelTaskExecution(BaseModel):
task_id: int
class QueuedExecution(BaseModel):
task_id: int
is_running: bool
name: str
finished_successfully: bool
logged: bool
@classmethod
def from_task_info(cls, task_info: TaskExecutor.TaskInfo) -> Self:
return cls(
task_id=task_info.task_id,
is_running=task_info.is_running,
name=task_info.name,
finished_successfully=task_info.finished_successfully(),
logged=task_info.logged,
)
class SerenaDashboardAPI:
log = logging.getLogger(__qualname__)
@@ -73,6 +150,11 @@ class SerenaDashboardAPI:
return send_from_directory(SERENA_DASHBOARD_DIR, "index.html")
# API routes
@self._app.route("/heartbeat", methods=["GET"])
def get_heartbeat() -> dict[str, Any]:
return {"status": "alive"}
@self._app.route("/get_log_messages", methods=["POST"])
def get_log_messages() -> dict[str, Any]:
request_data = request.get_json()
@@ -104,11 +186,136 @@ class SerenaDashboardAPI:
estimator_name = self._tool_usage_stats.token_estimator_name if self._tool_usage_stats else "unknown"
return {"token_count_estimator_name": estimator_name}
@self._app.route("/get_config_overview", methods=["GET"])
def get_config_overview() -> dict[str, Any]:
result = self._agent.execute_task(self._get_config_overview, logged=False)
return result.model_dump()
@self._app.route("/shutdown", methods=["PUT"])
def shutdown() -> dict[str, str]:
self._shutdown()
return {"status": "shutting down"}
@self._app.route("/get_available_languages", methods=["GET"])
def get_available_languages() -> dict[str, Any]:
result = self._get_available_languages()
return result.model_dump()
@self._app.route("/add_language", methods=["POST"])
def add_language() -> dict[str, str]:
request_data = request.get_json()
if not request_data:
return {"status": "error", "message": "No data provided"}
request_add_language = RequestAddLanguage.model_validate(request_data)
try:
self._add_language(request_add_language)
return {"status": "success", "message": f"Language {request_add_language.language} added successfully"}
except Exception as e:
return {"status": "error", "message": str(e)}
@self._app.route("/remove_language", methods=["POST"])
def remove_language() -> dict[str, str]:
request_data = request.get_json()
if not request_data:
return {"status": "error", "message": "No data provided"}
request_remove_language = RequestRemoveLanguage.model_validate(request_data)
try:
self._remove_language(request_remove_language)
return {"status": "success", "message": f"Language {request_remove_language.language} removed successfully"}
except Exception as e:
return {"status": "error", "message": str(e)}
@self._app.route("/get_memory", methods=["POST"])
def get_memory() -> dict[str, Any]:
request_data = request.get_json()
if not request_data:
return {"status": "error", "message": "No data provided"}
request_get_memory = RequestGetMemory.model_validate(request_data)
try:
result = self._get_memory(request_get_memory)
return result.model_dump()
except Exception as e:
return {"status": "error", "message": str(e)}
@self._app.route("/save_memory", methods=["POST"])
def save_memory() -> dict[str, str]:
request_data = request.get_json()
if not request_data:
return {"status": "error", "message": "No data provided"}
request_save_memory = RequestSaveMemory.model_validate(request_data)
try:
self._save_memory(request_save_memory)
return {"status": "success", "message": f"Memory {request_save_memory.memory_name} saved successfully"}
except Exception as e:
return {"status": "error", "message": str(e)}
@self._app.route("/delete_memory", methods=["POST"])
def delete_memory() -> dict[str, str]:
request_data = request.get_json()
if not request_data:
return {"status": "error", "message": "No data provided"}
request_delete_memory = RequestDeleteMemory.model_validate(request_data)
try:
self._delete_memory(request_delete_memory)
return {"status": "success", "message": f"Memory {request_delete_memory.memory_name} deleted successfully"}
except Exception as e:
return {"status": "error", "message": str(e)}
@self._app.route("/get_serena_config", methods=["GET"])
def get_serena_config() -> dict[str, Any]:
try:
result = self._get_serena_config()
return result.model_dump()
except Exception as e:
return {"status": "error", "message": str(e)}
@self._app.route("/save_serena_config", methods=["POST"])
def save_serena_config() -> dict[str, str]:
request_data = request.get_json()
if not request_data:
return {"status": "error", "message": "No data provided"}
request_save_config = RequestSaveSerenaConfig.model_validate(request_data)
try:
self._save_serena_config(request_save_config)
return {"status": "success", "message": "Serena config saved successfully"}
except Exception as e:
return {"status": "error", "message": str(e)}
@self._app.route("/queued_task_executions", methods=["GET"])
def get_queued_executions() -> dict[str, Any]:
try:
current_executions = self._agent.get_current_tasks()
response = [QueuedExecution.from_task_info(task_info).model_dump() for task_info in current_executions]
return {"queued_executions": response, "status": "success"}
except Exception as e:
return {"status": "error", "message": str(e)}
@self._app.route("/cancel_task_execution", methods=["POST"])
def cancel_task_execution() -> dict[str, Any]:
request_data = request.get_json()
try:
request_cancel_task = RequestCancelTaskExecution.model_validate(request_data)
for task in self._agent.get_current_tasks():
if task.task_id == request_cancel_task.task_id:
task.cancel()
return {"status": "success", "was_cancelled": True}
return {
"status": "success",
"was_cancelled": False,
"message": f"Task with id {request_data.get('task_id')} not found, maybe execution was already finished",
}
except Exception as e:
return {"status": "error", "message": str(e), "was_cancelled": False}
@self._app.route("/last_execution", methods=["GET"])
def get_last_execution() -> dict[str, Any]:
try:
last_execution_info = self._agent.get_last_executed_task()
response = QueuedExecution.from_task_info(last_execution_info).model_dump() if last_execution_info is not None else None
return {"last_execution": response, "status": "success"}
except Exception as e:
return {"status": "error", "message": str(e)}
def _get_log_messages(self, request_log: RequestLog) -> ResponseLog:
all_messages = self._memory_log_handler.get_log_messages()
requested_messages = all_messages[request_log.start_idx :] if request_log.start_idx <= len(all_messages) else []
@@ -129,6 +336,118 @@ class SerenaDashboardAPI:
if self._tool_usage_stats is not None:
self._tool_usage_stats.clear()
def _get_config_overview(self) -> ResponseConfigOverview:
from serena.config.context_mode import SerenaAgentContext, SerenaAgentMode
# Get active project info
project = self._agent.get_active_project()
active_project_name = project.project_name if project else None
project_info = {
"name": active_project_name,
"language": ", ".join([l.value for l in project.project_config.languages]) if project else None,
"path": str(project.project_root) if project else None,
}
# Get context info
context = self._agent.get_context()
context_info = {
"name": context.name,
"description": context.description,
"path": SerenaAgentContext.get_path(context.name),
}
# Get active modes
modes = self._agent.get_active_modes()
modes_info = [{"name": mode.name, "description": mode.description, "path": SerenaAgentMode.get_path(mode.name)} for mode in modes]
active_mode_names = [mode.name for mode in modes]
# Get active tools
active_tools = self._agent.get_active_tool_names()
# Get registered projects
registered_projects: list[dict[str, str | bool]] = []
for proj in self._agent.serena_config.projects:
registered_projects.append(
{
"name": proj.project_name,
"path": str(proj.project_root),
"is_active": proj.project_name == active_project_name,
}
)
# Get all available tools (excluding active ones)
all_tool_names = sorted([tool.get_name_from_cls() for tool in self._agent._all_tools.values()])
available_tools: list[dict[str, str | bool]] = []
for tool_name in all_tool_names:
if tool_name not in active_tools:
available_tools.append(
{
"name": tool_name,
"is_active": False,
}
)
# Get all available modes
all_mode_names = SerenaAgentMode.list_registered_mode_names()
available_modes: list[dict[str, str | bool]] = []
for mode_name in all_mode_names:
available_modes.append(
{
"name": mode_name,
"is_active": mode_name in active_mode_names,
"path": SerenaAgentMode.get_path(mode_name),
}
)
# Get all available contexts
all_context_names = SerenaAgentContext.list_registered_context_names()
available_contexts: list[dict[str, str | bool]] = []
for context_name in all_context_names:
available_contexts.append(
{
"name": context_name,
"is_active": context_name == context.name,
"path": SerenaAgentContext.get_path(context_name),
}
)
# Get basic tool stats (just num_calls for overview)
tool_stats_summary = {}
if self._tool_usage_stats is not None:
full_stats = self._tool_usage_stats.get_tool_stats_dict()
tool_stats_summary = {name: {"num_calls": stats["num_times_called"]} for name, stats in full_stats.items()}
# Get available memories if ReadMemoryTool is active
available_memories = None
if self._agent.tool_is_active("read_memory") and project is not None:
available_memories = project.memories_manager.list_memories()
# Get list of languages for the active project
languages = []
if project is not None:
languages = [lang.value for lang in project.project_config.languages]
# Get file encoding for the active project
encoding = None
if project is not None:
encoding = project.project_config.encoding
return ResponseConfigOverview(
active_project=project_info,
context=context_info,
modes=modes_info,
active_tools=active_tools,
tool_stats_summary=tool_stats_summary,
registered_projects=registered_projects,
available_tools=available_tools,
available_modes=available_modes,
available_contexts=available_contexts,
available_memories=available_memories,
jetbrains_mode=self._agent.serena_config.jetbrains,
languages=languages,
encoding=encoding,
)
def _shutdown(self) -> None:
log.info("Shutting down Serena")
if self._shutdown_callback:
@@ -138,6 +457,96 @@ class SerenaDashboardAPI:
# noinspection PyUnresolvedReferences
os._exit(0)
def _get_available_languages(self) -> ResponseAvailableLanguages:
from solidlsp.ls_config import Language
def run() -> ResponseAvailableLanguages:
all_languages = [lang.value for lang in Language.iter_all(include_experimental=False)]
# Filter out already added languages for the active project
project = self._agent.get_active_project()
if project:
current_languages = [lang.value for lang in project.project_config.languages]
available_languages = [lang for lang in all_languages if lang not in current_languages]
else:
available_languages = all_languages
return ResponseAvailableLanguages(languages=sorted(available_languages))
return self._agent.execute_task(run, logged=False)
def _get_memory(self, request_get_memory: RequestGetMemory) -> ResponseGetMemory:
def run() -> ResponseGetMemory:
project = self._agent.get_active_project()
if project is None:
raise ValueError("No active project")
content = project.memories_manager.load_memory(request_get_memory.memory_name)
return ResponseGetMemory(content=content, memory_name=request_get_memory.memory_name)
return self._agent.execute_task(run, logged=False)
def _save_memory(self, request_save_memory: RequestSaveMemory) -> None:
def run() -> None:
project = self._agent.get_active_project()
if project is None:
raise ValueError("No active project")
project.memories_manager.save_memory(request_save_memory.memory_name, request_save_memory.content)
self._agent.execute_task(run, logged=True, name="SaveMemory")
def _delete_memory(self, request_delete_memory: RequestDeleteMemory) -> None:
def run() -> None:
project = self._agent.get_active_project()
if project is None:
raise ValueError("No active project")
project.memories_manager.delete_memory(request_delete_memory.memory_name)
self._agent.execute_task(run, logged=True, name="DeleteMemory")
def _get_serena_config(self) -> ResponseGetSerenaConfig:
config_path = self._agent.serena_config.config_file_path
if config_path is None or not os.path.exists(config_path):
raise ValueError("Serena config file not found")
with open(config_path, encoding="utf-8") as f:
content = f.read()
return ResponseGetSerenaConfig(content=content)
def _save_serena_config(self, request_save_config: RequestSaveSerenaConfig) -> None:
def run() -> None:
config_path = self._agent.serena_config.config_file_path
if config_path is None:
raise ValueError("Serena config file path not set")
with open(config_path, "w", encoding="utf-8") as f:
f.write(request_save_config.content)
self._agent.execute_task(run, logged=True, name="SaveSerenaConfig")
def _add_language(self, request_add_language: RequestAddLanguage) -> None:
from solidlsp.ls_config import Language
try:
language = Language(request_add_language.language)
except ValueError:
raise ValueError(f"Invalid language: {request_add_language.language}")
# add_language is already thread-safe
self._agent.add_language(language)
def _remove_language(self, request_remove_language: RequestRemoveLanguage) -> None:
from solidlsp.ls_config import Language
try:
language = Language(request_remove_language.language)
except ValueError:
raise ValueError(f"Invalid language: {request_remove_language.language}")
# remove_language is already thread-safe
self._agent.remove_language(language)
@staticmethod
def _find_first_free_port(start_port: int) -> int:
port = start_port
src/serena/gui_log_viewer.py
+16 -4
View File
@@ -55,6 +55,7 @@ class GuiLogViewer:
self.message_queue = queue.Queue()
self.running = False
self.log_thread = None
self.menubar: tk.Menu | None = None
self.tool_names = [] # List to store tool names for highlighting
# Define colors for different log levels
@@ -98,6 +99,17 @@ class GuiLogViewer:
"""
self.tool_names = tool_names
def set_dashboard_url(self, url: str) -> None:
def copy_url():
self.root.clipboard_clear()
self.root.clipboard_append(url)
log.info(f"Copied dashboard URL to clipboard: {url}")
if self.menubar is not None:
dashboard_menu = tk.Menu(self.menubar, tearoff=0)
dashboard_menu.add_command(label="Copy URL", command=copy_url) # type: ignore
self.menubar.add_cascade(label="Dashboard", menu=dashboard_menu)
def add_log(self, message):
"""
Add a log message to the viewer.
@@ -287,11 +299,11 @@ class GuiLogViewer:
# Create menu bar
if self.mode == "dashboard":
menubar = tk.Menu(self.root)
server_menu = tk.Menu(menubar, tearoff=0)
self.menubar = tk.Menu(self.root)
server_menu = tk.Menu(self.menubar, tearoff=0)
server_menu.add_command(label="Shutdown", command=self._shutdown_server) # type: ignore
menubar.add_cascade(label="Server", menu=server_menu)
self.root.config(menu=menubar)
self.menubar.add_cascade(label="Server", menu=server_menu)
self.root.config(menu=self.menubar)
# Configure icons
icon_16 = tk.PhotoImage(file=dashboard_path / "serena-icon-16.png")
src/serena/ls_manager.py
+213
View File
@@ -0,0 +1,213 @@
import logging
import threading
from collections.abc import Iterator
from sensai.util.logging import LogTime
from serena.config.serena_config import SerenaPaths
from serena.constants import SERENA_MANAGED_DIR_NAME
from solidlsp import SolidLanguageServer
from solidlsp.ls_config import Language, LanguageServerConfig
from solidlsp.settings import SolidLSPSettings
log = logging.getLogger(__name__)
class LanguageServerFactory:
def __init__(
self,
project_root: str,
encoding: str,
ignored_patterns: list[str],
ls_timeout: float | None = None,
ls_specific_settings: dict | None = None,
trace_lsp_communication: bool = False,
):
self.project_root = project_root
self.encoding = encoding
self.ignored_patterns = ignored_patterns
self.ls_timeout = ls_timeout
self.ls_specific_settings = ls_specific_settings
self.trace_lsp_communication = trace_lsp_communication
def create_language_server(self, language: Language) -> SolidLanguageServer:
ls_config = LanguageServerConfig(
code_language=language,
ignored_paths=self.ignored_patterns,
trace_lsp_communication=self.trace_lsp_communication,
encoding=self.encoding,
)
log.info(f"Creating language server instance for {self.project_root}, language={language}.")
return SolidLanguageServer.create(
ls_config,
self.project_root,
timeout=self.ls_timeout,
solidlsp_settings=SolidLSPSettings(
solidlsp_dir=SerenaPaths().serena_user_home_dir,
project_data_relative_path=SERENA_MANAGED_DIR_NAME,
ls_specific_settings=self.ls_specific_settings or {},
),
)
class LanguageServerManager:
"""
Manages one or more language servers for a project.
"""
def __init__(
self,
language_servers: dict[Language, SolidLanguageServer],
language_server_factory: LanguageServerFactory | None = None,
) -> None:
"""
:param language_servers: a mapping from language to language server; the servers are assumed to be already started.
The first server in the iteration order is used as the default server.
All servers are assumed to serve the same project root.
:param language_server_factory: factory for language server creation; if None, dynamic (re)creation of language servers
is not supported
"""
self._language_servers = language_servers
self._language_server_factory = language_server_factory
self._default_language_server = next(iter(language_servers.values()))
self._root_path = self._default_language_server.repository_root_path
@staticmethod
def from_languages(languages: list[Language], factory: LanguageServerFactory) -> "LanguageServerManager":
"""
Creates a manager with language servers for the given languages using the given factory.
The language servers are started in parallel threads.
:param languages: the languages for which to spawn language servers
:param factory: the factory for language server creation
:return: the instance
"""
language_servers: dict[Language, SolidLanguageServer] = {}
threads = []
exceptions = {}
lock = threading.Lock()
def start_language_server(language: Language) -> None:
try:
with LogTime(f"Language server startup (language={language.value})"):
language_server = factory.create_language_server(language)
language_server.start()
if not language_server.is_running():
raise RuntimeError(f"Failed to start the language server for language {language.value}")
with lock:
language_servers[language] = language_server
except Exception as e:
log.error(f"Error starting language server for language {language.value}: {e}", exc_info=e)
with lock:
exceptions[language] = e
# start language servers in parallel threads
for language in languages:
thread = threading.Thread(target=start_language_server, args=(language,), name="StartLS:" + language.value)
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
# If any server failed to start up, raise an exception and stop all started language servers
if exceptions:
for ls in language_servers.values():
ls.stop()
failure_messages = "\n".join([f"{lang.value}: {e}" for lang, e in exceptions.items()])
raise Exception(f"Failed to start language servers:\n{failure_messages}")
return LanguageServerManager(language_servers, factory)
def get_root_path(self) -> str:
return self._root_path
def _ensure_functional_ls(self, ls: SolidLanguageServer) -> SolidLanguageServer:
if not ls.is_running():
log.warning(f"Language server for language {ls.language} is not running; restarting ...")
ls = self.restart_language_server(ls.language)
return ls
def get_language_server(self, relative_path: str) -> SolidLanguageServer:
ls: SolidLanguageServer | None = None
if len(self._language_servers) > 1:
for candidate in self._language_servers.values():
if not candidate.is_ignored_path(relative_path, ignore_unsupported_files=True):
ls = candidate
break
if ls is None:
ls = self._default_language_server
return self._ensure_functional_ls(ls)
def _create_and_start_language_server(self, language: Language) -> SolidLanguageServer:
if self._language_server_factory is None:
raise ValueError(f"No language server factory available to create language server for {language}")
language_server = self._language_server_factory.create_language_server(language)
language_server.start()
self._language_servers[language] = language_server
return language_server
def restart_language_server(self, language: Language) -> SolidLanguageServer:
"""
Forces recreation and restart of the language server for the given language.
It is assumed that the language server for the given language is no longer running.
:param language: the language
:return: the newly created language server
"""
if language not in self._language_servers:
raise ValueError(f"No language server for language {language.value} present; cannot restart")
return self._create_and_start_language_server(language)
def add_language_server(self, language: Language) -> SolidLanguageServer:
"""
Dynamically adds a new language server for the given language.
:param language: the language
:param factory: the factory to create the language server
:return: the newly created language server
"""
if language in self._language_servers:
raise ValueError(f"Language server for language {language.value} already present")
return self._create_and_start_language_server(language)
def remove_language_server(self, language: Language, save_cache: bool = False) -> None:
"""
Removes the language server for the given language, stopping it if it is running.
:param language: the language
"""
if language not in self._language_servers:
raise ValueError(f"No language server for language {language.value} present; cannot remove")
ls = self._language_servers.pop(language)
self._stop_language_server(ls, save_cache=save_cache)
@staticmethod
def _stop_language_server(ls: SolidLanguageServer, save_cache: bool = False, timeout: float = 2.0) -> None:
if ls.is_running():
if save_cache:
ls.save_cache()
log.info(f"Stopping language server for language {ls.language} ...")
ls.stop(shutdown_timeout=timeout)
def iter_language_servers(self) -> Iterator[SolidLanguageServer]:
for ls in self._language_servers.values():
yield self._ensure_functional_ls(ls)
def stop_all(self, save_cache: bool = False, timeout: float = 2.0) -> None:
"""
Stops all managed language servers.
:param save_cache: whether to save the cache before stopping
:param timeout: timeout for shutdown of each language server
"""
for ls in self.iter_language_servers():
self._stop_language_server(ls, save_cache=save_cache, timeout=timeout)
def save_all_caches(self) -> None:
"""
Saves the caches of all managed language servers.
"""
for ls in self.iter_language_servers():
if ls.is_running():
ls.save_cache()
src/serena/project.py
+166 -40
View File
@@ -1,27 +1,66 @@
import json
import logging
import os
from pathlib import Path
from typing import Any
import pathspec
from sensai.util.string import ToStringMixin
from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig
from serena.constants import SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME
from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig, get_serena_managed_in_project_dir
from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME
from serena.ls_manager import LanguageServerFactory, LanguageServerManager
from serena.text_utils import MatchedConsecutiveLines, search_files
from serena.util.file_system import GitignoreParser, match_path
from serena.util.general import save_yaml
from solidlsp import SolidLanguageServer
from solidlsp.ls_config import Language, LanguageServerConfig
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.settings import SolidLSPSettings
from solidlsp.ls_config import Language
from solidlsp.ls_utils import FileUtils
log = logging.getLogger(__name__)
class Project:
class MemoriesManager:
def __init__(self, project_root: str):
self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories"
self._memory_dir.mkdir(parents=True, exist_ok=True)
self._encoding = SERENA_FILE_ENCODING
def get_memory_file_path(self, name: str) -> Path:
# strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not.
name = name.replace(".md", "")
filename = f"{name}.md"
return self._memory_dir / filename
def load_memory(self, name: str) -> str:
memory_file_path = self.get_memory_file_path(name)
if not memory_file_path.exists():
return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it."
with open(memory_file_path, encoding=self._encoding) as f:
return f.read()
def save_memory(self, name: str, content: str) -> str:
memory_file_path = self.get_memory_file_path(name)
with open(memory_file_path, "w", encoding=self._encoding) as f:
f.write(content)
return f"Memory {name} written."
def list_memories(self) -> list[str]:
return [f.name.replace(".md", "") for f in self._memory_dir.iterdir() if f.is_file()]
def delete_memory(self, name: str) -> str:
memory_file_path = self.get_memory_file_path(name)
memory_file_path.unlink()
return f"Memory {name} deleted."
class Project(ToStringMixin):
def __init__(self, project_root: str, project_config: ProjectConfig, is_newly_created: bool = False):
self.project_root = project_root
self.project_config = project_config
self.is_newly_created = is_newly_created
self.memories_manager = MemoriesManager(project_root)
self.language_server_manager: LanguageServerManager | None = None
self._is_newly_created = is_newly_created
# create .gitignore file in the project's Serena data folder if not yet present
serena_data_gitignore_path = os.path.join(self.path_to_serena_data_folder(), ".gitignore")
@@ -32,7 +71,7 @@ class Project:
f.write(f"/{SolidLanguageServer.CACHE_FOLDER_NAME}\n")
# gather ignored paths from the project configuration and gitignore files
ignored_patterns = project_config.ignored_paths
ignored_patterns = list(project_config.ignored_paths)
if len(ignored_patterns) > 0:
log.info(f"Using {len(ignored_patterns)} ignored paths from the explicit project configuration.")
log.debug(f"Ignored paths: {ignored_patterns}")
@@ -53,14 +92,16 @@ class Project:
log.debug(f"Processing {len(processed_patterns)} ignored paths")
self._ignore_spec = pathspec.PathSpec.from_lines(pathspec.patterns.GitWildMatchPattern, processed_patterns)
def _tostring_includes(self) -> list[str]:
return []
def _tostring_additional_entries(self) -> dict[str, Any]:
return {"root": self.project_root, "name": self.project_name}
@property
def project_name(self) -> str:
return self.project_config.project_name
@property
def language(self) -> Language:
return self.project_config.language
@classmethod
def load(cls, project_root: str | Path, autogenerate: bool = True) -> "Project":
project_root = Path(project_root).resolve()
@@ -69,12 +110,42 @@ class Project:
project_config = ProjectConfig.load(project_root, autogenerate=autogenerate)
return Project(project_root=str(project_root), project_config=project_config)
def save_config(self) -> None:
"""
Saves the current project configuration to disk.
"""
config_path = os.path.join(self.project_root, self.project_config.rel_path_to_project_yml())
log.info("Saving updated project configuration to %s", config_path)
config_with_comments = ProjectConfig.load_commented_map(config_path)
config_with_comments.update(self.project_config.to_yaml_dict())
save_yaml(config_path, config_with_comments, preserve_comments=True)
def path_to_serena_data_folder(self) -> str:
return os.path.join(self.project_root, SERENA_MANAGED_DIR_NAME)
def path_to_project_yml(self) -> str:
return os.path.join(self.project_root, self.project_config.rel_path_to_project_yml())
def get_activation_message(self) -> str:
"""
:return: a message providing information about the project upon activation (e.g. programming language, memories, initial prompt)
"""
if self._is_newly_created:
msg = f"Created and activated a new project with name '{self.project_name}' at {self.project_root}. "
else:
msg = f"The project with name '{self.project_name}' at {self.project_root} is activated."
languages_str = ", ".join([lang.value for lang in self.project_config.languages])
msg += f"\nProgramming languages: {languages_str}; file encoding: {self.project_config.encoding}"
memories = self.memories_manager.list_memories()
if memories:
msg += (
f"\nAvailable project memories: {json.dumps(memories)}\n"
+ "Use the `read_memory` tool to read these memories later if they are relevant to the task."
)
if self.project_config.initial_prompt:
msg += f"\nAdditional project-specific instructions:\n {self.project_config.initial_prompt}"
return msg
def read_file(self, relative_path: str) -> str:
"""
Reads a file relative to the project root.
@@ -83,9 +154,7 @@ class Project:
:return: the content of the file
"""
abs_path = Path(self.project_root) / relative_path
if not abs_path.exists():
raise FileNotFoundError(f"File not found: {abs_path}")
return abs_path.read_text(encoding=self.project_config.encoding)
return FileUtils.read_file(str(abs_path), self.project_config.encoding)
def get_ignore_spec(self) -> pathspec.PathSpec:
"""
@@ -105,6 +174,12 @@ class Project:
:return: whether the path should be ignored
"""
# special case, never ignore the project root itself
# If the user ignores hidden files, "." might match against the corresponding PathSpec pattern.
# The empty string also points to the project root and should never be ignored.
if str(relative_path) in [".", ""]:
return False
abs_path = os.path.join(self.project_root, relative_path)
if not os.path.exists(abs_path):
raise FileNotFoundError(f"File {abs_path} not found, the ignore check cannot be performed")
@@ -112,8 +187,13 @@ class Project:
# Check file extension if it's a file
is_file = os.path.isfile(abs_path)
if is_file and ignore_non_source_files:
fn_matcher = self.language.get_source_fn_matcher()
if not fn_matcher.is_relevant_filename(abs_path):
is_file_in_supported_language = False
for language in self.project_config.languages:
fn_matcher = language.get_source_fn_matcher()
if fn_matcher.is_relevant_filename(abs_path):
is_file_in_supported_language = True
break
if not is_file_in_supported_language:
return True
# Create normalized path for consistent handling
@@ -249,6 +329,7 @@ class Project:
relative_file_paths,
pattern,
root_path=self.project_root,
file_reader=self.read_file,
context_lines_before=context_lines_before,
context_lines_after=context_lines_after,
paths_include_glob=paths_include_glob,
@@ -277,42 +358,87 @@ class Project:
source_file_path=relative_file_path,
)
def create_language_server(
def create_language_server_manager(
self,
log_level: int = logging.INFO,
ls_timeout: float | None = DEFAULT_TOOL_TIMEOUT - 5,
trace_lsp_communication: bool = False,
ls_specific_settings: dict[Language, Any] | None = None,
) -> SolidLanguageServer:
) -> LanguageServerManager:
"""
Create a language server for a project. Note that you will have to start it
before performing any LS operations.
Creates the language server manager for the project, starting one language server per configured programming language.
:param project: either a path to the project root or a ProjectConfig instance.
If no project.yml is found, the default project configuration will be used.
:param log_level: the log level for the language server
:param ls_timeout: the timeout for the language server
:param trace_lsp_communication: whether to trace LSP communication
:param ls_specific_settings: optional LS specific configuration of the language server,
see docstrings in the inits of subclasses of SolidLanguageServer to see what values may be passed.
:return: the language server
:return: the language server manager, which is also stored in the project instance
"""
ls_config = LanguageServerConfig(
code_language=self.language,
ignored_paths=self._ignored_patterns,
# if there is an existing instance, stop its language servers first
if self.language_server_manager is not None:
log.info("Stopping existing language server manager ...")
self.language_server_manager.stop_all()
self.language_server_manager = None
log.info(f"Creating language server manager for {self.project_root}")
factory = LanguageServerFactory(
project_root=self.project_root,
encoding=self.project_config.encoding,
ignored_patterns=self._ignored_patterns,
ls_timeout=ls_timeout,
ls_specific_settings=ls_specific_settings,
trace_lsp_communication=trace_lsp_communication,
)
ls_logger = LanguageServerLogger(log_level=log_level)
self.language_server_manager = LanguageServerManager.from_languages(self.project_config.languages, factory)
return self.language_server_manager
log.info(f"Creating language server instance for {self.project_root}.")
return SolidLanguageServer.create(
ls_config,
ls_logger,
self.project_root,
timeout=ls_timeout,
solidlsp_settings=SolidLSPSettings(
solidlsp_dir=SERENA_MANAGED_DIR_IN_HOME,
project_data_relative_path=SERENA_MANAGED_DIR_NAME,
ls_specific_settings=ls_specific_settings or {},
),
)
def add_language(self, language: Language) -> None:
"""
Adds a new programming language to the project configuration, starting the corresponding
language server instance if the LS manager is active.
The project configuration is saved to disk after adding the language.
:param language: the programming language to add
"""
if language in self.project_config.languages:
log.info(f"Language {language.value} is already present in the project configuration.")
return
# start the language server (if the LS manager is active)
if self.language_server_manager is None:
log.info("Language server manager is not active; skipping language server startup for the new language.")
else:
log.info("Adding and starting the language server for new language %s ...", language.value)
self.language_server_manager.add_language_server(language)
# update the project configuration
self.project_config.languages.append(language)
self.save_config()
def remove_language(self, language: Language) -> None:
"""
Removes a programming language from the project configuration, stopping the corresponding
language server instance if the LS manager is active.
The project configuration is saved to disk after removing the language.
:param language: the programming language to remove
"""
if language not in self.project_config.languages:
log.info(f"Language {language.value} is not present in the project configuration.")
return
# update the project configuration
self.project_config.languages.remove(language)
self.save_config()
# stop the language server (if the LS manager is active)
if self.language_server_manager is None:
log.info("Language server manager is not active; skipping language server shutdown for the removed language.")
else:
log.info("Removing and stopping the language server for language %s ...", language.value)
self.language_server_manager.remove_language_server(language)
def shutdown(self, timeout: float = 2.0) -> None:
if self.language_server_manager is not None:
self.language_server_manager.stop_all(save_cache=True, timeout=timeout)
self.language_server_manager = None
src/serena/prompt_factory.py
+5 -3
View File
@@ -1,6 +1,7 @@
import os
from serena.constants import PROMPT_TEMPLATES_DIR_IN_USER_HOME, PROMPT_TEMPLATES_DIR_INTERNAL
from serena.config.serena_config import SerenaPaths
from serena.constants import PROMPT_TEMPLATES_DIR_INTERNAL
from serena.generated.generated_prompt_factory import PromptFactory
@@ -10,5 +11,6 @@ class SerenaPromptFactory(PromptFactory):
"""
def __init__(self) -> None:
os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True)
super().__init__(prompts_dir=[PROMPT_TEMPLATES_DIR_IN_USER_HOME, PROMPT_TEMPLATES_DIR_INTERNAL])
user_templates_dir = SerenaPaths().user_prompt_templates_dir
os.makedirs(user_templates_dir, exist_ok=True)
super().__init__(prompts_dir=[user_templates_dir, PROMPT_TEMPLATES_DIR_INTERNAL])
src/serena/resources/config/contexts/agent.yml
+2 -3
View File
@@ -1,7 +1,6 @@
description: All tools except InitialInstructionsTool for agent context
description: Agent context where the system prompt (initial instructions) are provided at startup
prompt: |
You are running in agent context where the system prompt is provided externally. You should use symbolic
tools when possible for code understanding and modification.
You are running in an agent context.
excluded_tools:
- initial_instructions
src/serena/resources/config/contexts/chatgpt.yml
+6 -6
View File
@@ -1,4 +1,4 @@
description: A configuration specific for chatgpt, which has a limit of 30 tools and requires short descriptions.
description: A configuration specific for ChatGPT, which has a limit of 30 tools and requires short descriptions.
prompt: |
You are running in desktop app context where the tools give you access to the code base as well as some
access to the file system, if configured. You interact with the user through a chat interface that is separated
@@ -16,11 +16,11 @@ included_optional_tools:
tool_description_overrides:
find_symbol: |
Retrieves symbols matching `name_path` in a file.
Use `depth > 0` to include children. `name_path` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar"
replace_regex: |
Replaces text using regular expressions. Preferred for smaller edits where symbol-level tools aren't appropriate.
Use wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content.
Retrieves symbols matching `name_path_pattern` in a file.
Use `depth > 0` to include children. `name_path_pattern` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar"
replace_content: |
Replaces content in files. Preferred for smaller edits where symbol-level tools aren't appropriate.
Use mode "regex" with wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content.
Essential for multi-line replacements.
search_for_pattern: |
Flexible pattern search across codebase. Prefer symbolic operations when possible.
src/serena/resources/config/contexts/claude-code.yml
+26
View File
@@ -0,0 +1,26 @@
description: Claude Code (CLI agent where file operations, basic edits, etc. are already covered; single project mode)
prompt: |
You are running in a CLI coding agent context where file operations, basic (line-based) edits and reads
as well as shell commands are handled by your own, internal tools.
If Serena's tools can be used to achieve your task, you should prioritize them.
In particular, it is important that you avoid reading entire source code files unless it is strictly necessary!
Instead, for exploring and reading code in a token-efficient manner, use Serena's overview and symbolic search tools.
For non-code files or for reads where you don't know the symbol's name path, you can use the pattern search tool.
excluded_tools:
- create_text_file
- read_file
- execute_shell_command
- prepare_for_new_conversation
- replace_content
tool_description_overrides: {}
# whether to assume that Serena shall only work on a single project in this context (provided that a project is given
# when Serena is started).
# If set to true and a project is provided at startup, the set of tools is limited to those required by the project's
# concrete configuration, and other tools are excluded completely, allowing the set of tools to be minimal.
# Tools explicitly disabled by the project will not be available at all.
# The `activate_project` tool is always disabled in this case, as project switching cannot be allowed.
single_project: true
src/serena/resources/config/contexts/codex.yml
+9 -16
View File
@@ -1,27 +1,20 @@
description: Non-symbolic editing tools and general shell tool are excluded
description: Codex Non-symbolic editing tools and general shell tool are excluded
prompt: |
You are running in IDE assistant context where file operations, basic (line-based) edits and reads,
and shell commands are handled by your own, internal tools.
The initial instructions and the current config inform you on which tools are available to you,
and how to use them.
Don't attempt to use any excluded tools, instead rely on your own internal tools
for achieving the basic file or shell operations.
You are running in the Codex IDE assistant mode, where file operations, basic (line-based) edits and reads
as well as shell commands are handled by your own, internal tools.
Don't attempt to use any excluded tools; instead, rely on your own internal tools for basic file or shell operations.
If serena's tools can be used for achieving your task,
you should prioritize them. In particular, it is important that you avoid reading entire source code files,
unless it is strictly necessary! Instead, for exploring and reading code in a token-efficient manner,
you should use serena's overview and symbolic search tools. The call of the read_file tool on an entire source code
file should only happen in exceptional cases, usually you should first explore the file (by itself or as part of exploring
the directory containing it) using the symbol_overview tool, and then make targeted reads using find_symbol and other symbolic tools.
For non-code files or for reads where you don't know the symbol's name path you can use the patterns searching tool,
using the read_file as a last resort.
If Serena's tools can be used to achieve your task, you should prioritize them.
In particular, it is important that you avoid reading entire source code files unless it is strictly necessary!
Instead, for exploring and reading code in a token-efficient manner, use Serena's overview and symbolic search tools.
For non-code files or for reads where you don't know the symbol's name path, you can use the pattern search tool.
excluded_tools:
- create_text_file
- read_file
- execute_shell_command
- prepare_for_new_conversation
- replace_regex
- replace_content
tool_description_overrides: {}
src/serena/resources/config/contexts/context.template.yml
+8 -1
View File
@@ -8,4 +8,11 @@ included_optional_tools: []
# mapping of tool names to an override of their descriptions (the default description is the docstring of the Tool's apply method).
# Sometimes, tool descriptions are too long (e.g., for ChatGPT), or users may want to override them for another reason.
tool_description_overrides: {}
tool_description_overrides: {}
# whether to assume that Serena shall only work on a single project in this context (provided that a project is given
# when Serena is started).
# If set to true and a project is provided at startup, the set of tools is limited to those required by the project's
# concrete configuration, and other tools are excluded completely, allowing the set of tools to be minimal.
# The `activate_project` tool will, therefore, be disabled in this case, as project switching is not allowed.
single_project: false
src/serena/resources/config/contexts/desktop-app.yml
+8 -10
View File
@@ -1,15 +1,13 @@
description: All tools included for desktop app context
description: Desktop application context (chat application detached from code) where Serena's full toolset is provided
prompt: |
You are running in desktop app context where the tools give you access to the code base as well as some
access to the file system, if configured. You interact with the user through a chat interface that is separated
from the code base. As a consequence, if you are in interactive mode, your communication with the user should
You are running in a desktop application context.
Serena's tools give you access to the code base as well as some access to the file system (if enabled).
You interact with the user through a chat interface that is separated from the code base.
As a consequence, if you are in interactive mode, your communication with the user should
involve high-level thinking and planning as well as some summarization of any code edits that you make.
For viewing the code edits the user will view them in a separate code editor window, and the back-and-forth
between the chat and the code editor should be minimized as well as facilitated by you.
If complex changes have been made, advise the user on how to review them in the code editor.
If complex relationships that the user asked for should be visualized or explained, consider creating
a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering
options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions.
To view the code edits you make, the user will have switch to a separate application.
To illustrate complex relationships, consider creating diagrams in addition to your text-based communication
(depending on the options for text, html, mermaid diagrams, etc. that you are provided with in your initial instructions).
excluded_tools: []
included_optional_tools:
- switch_modes
src/serena/resources/config/contexts/ide-assistant.yml
-27
View File
@@ -1,27 +0,0 @@
description: Non-symbolic editing tools and general shell tool are excluded
prompt: |
You are running in IDE assistant context where file operations, basic (line-based) edits and reads,
and shell commands are handled by your own, internal tools.
The initial instructions and the current config inform you on which tools are available to you,
and how to use them.
Don't attempt to use any excluded tools, instead rely on your own internal tools
for achieving the basic file or shell operations.
If serena's tools can be used for achieving your task,
you should prioritize them. In particular, it is important that you avoid reading entire source code files,
unless it is strictly necessary! Instead, for exploring and reading code in a token-efficient manner,
you should use serena's overview and symbolic search tools.
Before reading a full file, it is usually best to first explore the file using the symbol_overview tool,
and then make targeted reads using find_symbol and other symbolic tools.
For non-code files or for reads where you don't know the symbol's name path, you can use the pattern searching tool,
and read full files only as a last resort.
excluded_tools:
- create_text_file
- read_file
- execute_shell_command
- prepare_for_new_conversation
- replace_regex
tool_description_overrides: {}
src/serena/resources/config/contexts/ide.yml
+25
View File
@@ -0,0 +1,25 @@
description: Generic IDE coding agent context (basic file operations and shell operations assumed to be covered; single project mode)
prompt: |
You are running in an IDE assistant context where file operations, basic (line-based) edits and reads,
and shell commands are handled by your own, internal tools.
If Serena's tools can be used to achieve your task, you should prioritize them.
In particular, it is important that you avoid reading entire source code files unless it is strictly necessary!
Instead, for exploring and reading code in a token-efficient manner, use Serena's overview and symbolic search tools.
For non-code files or for reads where you don't know the symbol's name path, you can use the pattern search tool.
excluded_tools:
- create_text_file
- read_file
- execute_shell_command
- prepare_for_new_conversation
tool_description_overrides: {}
# whether to assume that Serena shall only work on a single project in this context (provided that a project is given
# when Serena is started).
# If set to true and a project is provided at startup, the set of tools is limited to those required by the project's
# concrete configuration, and other tools are excluded completely, allowing the set of tools to be minimal.
# Tools explicitly disabled by the project will not be available at all.
# The `activate_project` tool is always disabled in this case, as project switching cannot be allowed.
single_project: true
src/serena/resources/config/modes/editing.yml
+22 -97
View File
@@ -1,110 +1,35 @@
description: All tools, with detailed instructions for code editing
prompt: |
You are operating in editing mode. You can edit files with the provided tools
to implement the requested changes to the code base while adhering to the project's code style and patterns.
You are operating in editing mode. You can edit files with the provided tools.
You adhere to the project's code style and patterns.
Use symbolic editing tools whenever possible for precise code modifications.
If no editing task has yet been provided, wait for the user to provide one.
If no explicit editing task has yet been provided, wait for the user to provide one. Do not be overly eager.
When writing new code, think about where it belongs best. Don't generate new files if you don't plan on actually
integrating them into the codebase, instead use the editing tools to insert the code directly into the existing files in that case.
properly integrating them into the codebase.
You have two main approaches for editing code - editing by regex and editing by symbol.
You have two main approaches for editing code: (a) editing at the symbol level and (b) file-based editing.
The symbol-based approach is appropriate if you need to adjust an entire symbol, e.g. a method, a class, a function, etc.
But it is not appropriate if you need to adjust just a few lines of code within a symbol, for that you should
use the regex-based approach that is described below.
It is not appropriate if you need to adjust just a few lines of code within a larger symbol.
Let us first discuss the symbol-based approach.
Symbols are identified by their name path and relative file path, see the description of the `find_symbol` tool for more details
on how the `name_path` matches symbols.
You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file,
or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible
while still solving your task, meaning you only read the bodies when you need to, and after you have found the symbol you want to edit.
Before calling symbolic reading tools, you should have a basic understanding of the repository structure that you can get from memories
or by using the `list_dir` and `find_file` tools (or similar).
For example, if you are working with python code and already know that you need to read the body of the constructor of the class Foo, you can directly
use `find_symbol` with the name path `Foo/__init__` and `include_body=True`. If you don't know yet which methods in `Foo` you need to read or edit,
you can use `find_symbol` with the name path `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding
to read the desired methods with `include_body=True`.
In particular, keep in mind the description of the `replace_symbol_body` tool. If you want to add some new code at the end of the file, you should
use the `insert_after_symbol` tool with the last top-level symbol in the file. If you want to add an import, often a good strategy is to use
`insert_before_symbol` with the first top-level symbol in the file.
You can understand relationships between symbols by using the `find_referencing_symbols` tool. If not explicitly requested otherwise by a user,
you make sure that when you edit a symbol, it is either done in a backward-compatible way, or you find and adjust the references as needed.
The `find_referencing_symbols` tool will give you code snippets around the references, as well as symbolic information.
You will generally be able to use the info from the snippets and the regex-based approach to adjust the references as well.
You can assume that all symbol editing tools are reliable, so you don't need to verify the results if the tool returns without error.
**Symbolic editing**
Use symbolic retrieval tools to identify the symbols you need to edit.
If you need to replace the definition of a symbol, use the `replace_symbol_body` tool.
If you want to add some new code at the end of the file, use the `insert_after_symbol` tool with the last top-level symbol in the file.
Similarly, you can use `insert_before_symbol` with the first top-level symbol in the file to insert code at the beginning of a file.
You can understand relationships between symbols by using the `find_referencing_symbols` tool. If not explicitly requested otherwise by the user,
you make sure that when you edit a symbol, the change is either backward-compatible or you find and update all references as needed.
The `find_referencing_symbols` tool will give you code snippets around the references as well as symbolic information.
You can assume that all symbol editing tools are reliable, so you never need to verify the results if the tools return without error.
{% if 'replace_regex' in available_tools %}
Let us discuss the regex-based approach.
The regex-based approach is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation.
This is the case if you need to adjust just a few lines of code within a method, or a chunk that is much smaller than a whole symbol.
You use other tools to find the relevant content and
then use your knowledge of the codebase to write the regex, if you haven't collected enough information of this content yet.
{% if 'replace_content' in available_tools %}
**File-based editing**
The `replace_content` tool allows you to perform regex-based replacements within files (as well as simple string replacements).
This is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation,
e.g. if you need to adjust just a few lines of code within a method.
You are extremely good at regex, so you never need to check whether the replacement produced the correct result.
In particular, you know what to escape and what not to escape, and you know how to use wildcards.
Also, the regex tool never adds any indentation (contrary to the symbolic editing tools), so you have to take care to add the correct indentation
when using it to insert code.
Moreover, the replacement tool will fail if it can't perform the desired replacement, and this is all the feedback you need.
Your overall goal for replacement operations is to use relatively short regexes, since I want you to minimize the number
of output tokens. For replacements of larger chunks of code, this means you intelligently make use of wildcards for the middle part
and of characteristic snippets for the before/after parts that uniquely identify the chunk.
For small replacements, up to a single line, you follow the following rules:
1. If the snippet to be replaced is likely to be unique within the file, you perform the replacement by directly using the escaped version of the
original.
2. If the snippet is probably not unique, and you want to replace all occurrences, you use the `allow_multiple_occurrences` flag.
3. If the snippet is not unique, and you want to replace a specific occurrence, you make use of the code surrounding the snippet
to extend the regex with content before/after such that the regex will have exactly one match.
4. You generally assume that a snippet is unique, knowing that the tool will return an error on multiple matches. You only read more file content
(for crafvarting a more specific regex) if such a failure unexpectedly occurs.
Examples:
1 Small replacement
You have read code like
```python
...
x = linear(x)
x = relu(x)
return x
...
```
and you want to replace `x = relu(x)` with `x = gelu(x)`.
You first try `replace_regex()` with the regex `x = relu\(x\)` and the replacement `x = gelu(x)`.
If this fails due to multiple matches, you will try `(linear\(x\)\s*)x = relu\(x\)(\s*return)` with the replacement `\1x = gelu(x)\2`.
2 Larger replacement
You have read code like
```python
def my_func():
...
# a comment before the snippet
x = add_fifteen(x)
# beginning of long section within my_func
....
# end of long section
call_subroutine(z)
call_second_subroutine(z)
```
and you want to replace the code starting with `x = add_fifteen(x)` until (including) `call_subroutine(z)`, but not `call_second_subroutine(z)`.
Initially, you assume that the the beginning and end of the chunk uniquely determine it within the file.
Therefore, you perform the replacement by using the regex `x = add_fifteen\(x\)\s*.*?call_subroutine\(z\)`
and the replacement being the new code you want to insert.
If this fails due to multiple matches, you will try to extend the regex with the content before/after the snippet and match groups.
The matching regex becomes:
`(before the snippet\s*)x = add_fifteen\(x\)\s*.*?call_subroutine\(z\)`
and the replacement includes the group as (schematically):
`\1<new_code>`
Generally, I remind you that you rely on the regex tool with providing you the correct feedback, no need for more verification!
IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE LONG REGEXES WITHOUT USING WILDCARDS INSTEAD!
In particular, you know how to use wildcards effectively in order to avoid specifying the full original text to be replaced!
{% endif %}
excluded_tools:
- replace_lines
src/serena/resources/config/modes/interactive.yml
+1 -1
View File
@@ -6,6 +6,6 @@ prompt: |
Break down complex tasks into smaller steps and explain your thinking at each stage. When you're uncertain about
a decision, present options to the user and ask for guidance rather than making assumptions.
Focus on providing informative results for intermediate steps so the user can follow along with your progress and
Focus on providing informative results for intermediate steps, such that the user can follow along with your progress and
provide feedback as needed.
excluded_tools: []
src/serena/resources/config/modes/mode.template.yml
+4 -3
View File
@@ -1,7 +1,8 @@
# See Serena's documentation for more details on concept of modes.
description: Description of the mode, not used in the code.
prompt: Prompt that will form part of the message sent to the model when activating this mode.
description: Description of the mode (meta-information only)
prompt: |
Provide a prompt that will form part of the instructions sent to the model when this mode is activated.
# tools that are to be excluded by this mode
excluded_tools: []
# several tools are excluded by default and have to be explicitly included by the user
included_optional_tools: []
src/serena/resources/config/modes/no-memories.yml
+11
View File
@@ -0,0 +1,11 @@
description: Excludes Serena's memory tools (and onboarding tools, which rely on memory)
prompt: |
Serena's memory tools are not available and the onboarding workflow is not being applied.
excluded_tools:
- write_memory
- read_memory
- delete_memory
- edit_memory
- list_memories
- onboarding
- check_onboarding_performed
src/serena/resources/config/modes/no-onboarding.yml
+2 -4
View File
@@ -1,8 +1,6 @@
description: Onboarding was already performed, exclude all onboarding tools
description: The onboarding process is not used (memories may have been created externally)
prompt: |
You have already performed onboarding, meaning that memories have already been created.
Read a list of available memories using the `list_memories` tool before proceeding with the task.
You don't need to read the actual memories, just remember that they exist and that you can read them later if they are relevant for your task.
The onboarding process is not applied.
excluded_tools:
- onboarding
- check_onboarding_performed
src/serena/resources/config/modes/planning.yml
+2 -3
View File
@@ -1,8 +1,7 @@
description: Only read-only tools, focused on analysis and planning
prompt: |
You are operating in planning mode. Your task is to analyze code but not write any code.
The user may ask you to assist in creating a comprehensive plan, or to learn something about the codebase -
either a small aspect of it or about the whole project.
The user may ask you to assist in creating a comprehensive plan, or to learn something about the codebase.
excluded_tools:
- create_text_file
- replace_symbol_body
@@ -12,4 +11,4 @@ excluded_tools:
- replace_lines
- insert_at_line
- execute_shell_command
- replace_regex
- replace_content
src/serena/resources/config/prompt_templates/system_prompt.yml
+25 -37
View File
@@ -3,64 +3,52 @@
# is encouraged (via the tool description) to call at the beginning of the conversation.
prompts:
system_prompt: |
You are a professional coding agent concerned with one particular codebase. You have
access to semantic coding tools on which you rely heavily for all your work, as well as collection of memory
files containing general information about the codebase. You operate in a resource-efficient and intelligent manner, always
keeping in mind to not read or generate content that is not needed for the task at hand.
You are a professional coding agent.
You have access to semantic coding tools upon which you rely heavily for all your work.
You operate in a resource-efficient and intelligent manner, always keeping in mind to not read or generate
content that is not needed for the task at hand.
When reading code in order to answer a user question or task, you should try reading only the necessary code.
Some tasks may require you to understand the architecture of large parts of the codebase, while for others,
it may be enough to read a small set of symbols or a single file.
Generally, you should avoid reading entire files unless it is absolutely necessary, instead relying on
intelligent step-by-step acquisition of information. {% if 'ToolMarkerSymbolicRead' in available_markers %}However, if you already read a file, it does not make
sense to further analyse it with the symbolic tools (except for the `find_referencing_symbols` tool),
as you already have the information.{% endif %}
You avoid reading entire files unless it is absolutely necessary, instead relying on intelligent step-by-step
acquisition of information. {% if 'ToolMarkerSymbolicRead' in available_markers %}Once you have read a full file, it does not make
sense to analyse it with the symbolic read tools; you already have the information.{% endif %}
I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED!
{% if 'ToolMarkerSymbolicRead' in available_markers %}
CONSIDER INSTEAD USING THE OVERVIEW TOOL AND SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST!
I WILL BE EVEN MORE UPSET IF AFTER HAVING READ AN ENTIRE FILE YOU KEEP READING THE SAME CONTENT WITH THE SYMBOLIC TOOLS!
THE PURPOSE OF THE SYMBOLIC TOOLS IS TO HAVE TO READ LESS CODE, NOT READ THE SAME CONTENT MULTIPLE TIMES!
{% endif %}
You can achieve the intelligent reading of code by using the symbolic tools for getting an overview of symbols and
the relations between them, and then only reading the bodies of symbols that are necessary to answer the question
or complete the task.
You can achieve intelligent reading of code by using the symbolic tools for getting an overview of symbols and
the relations between them, and then only reading the bodies of symbols that are necessary to complete the task at hand.
You can use the standard tools like list_dir, find_file and search_for_pattern if you need to.
When tools allow it, you pass the `relative_path` parameter to restrict the search to a specific file or directory.
For some tools, `relative_path` can only be a file path, so make sure to properly read the tool descriptions.
Where appropriate, you pass the `relative_path` parameter to restrict the search to a specific file or directory.
{% if 'search_for_pattern' in available_tools %}
If you are unsure about a symbol's name or location{% if 'find_symbol' in available_tools %} (to the extent that substring_matching for the symbol name is not enough){% endif %}, you can use the `search_for_pattern` tool, which allows fast
and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %}This way you can first find candidates for symbols or files,
and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %} In this way, you can first find candidates for symbols or files,
and then proceed with the symbolic tools.{% endif %}
{% endif %}
{% if 'ToolMarkerSymbolicRead' in available_markers %}
Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbol` tool for more details
on how the `name_path` matches symbols.
You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file,
or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible
while still solving your task, meaning you only read the bodies when you need to, and after you have found the symbol you want to edit.
For example, if you are working with python code and already know that you need to read the body of the constructor of the class Foo, you can directly
use `find_symbol` with the name path `Foo/__init__` and `include_body=True`. If you don't know yet which methods in `Foo` you need to read or edit,
you can use `find_symbol` with the name path `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding
to read the desired methods with `include_body=True`
Symbols are identified by their `name_path` and `relative_path` (see the description of the `find_symbol` tool).
You can get information about the symbols in a file by using the `get_symbols_overview` tool or use the `find_symbol` to search.
You only read the bodies of symbols when you need to (e.g. if you want to fully understand or edit it).
For example, if you are working with Python code and already know that you need to read the body of the constructor of the class Foo, you can directly
use `find_symbol` with name path pattern `Foo/__init__` and `include_body=True`. If you don't know yet which methods in `Foo` you need to read or edit,
you can use `find_symbol` with name path pattern `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding
to read the desired methods with `include_body=True`.
You can understand relationships between symbols by using the `find_referencing_symbols` tool.
{% endif %}
{% if 'read_memory' in available_tools %}
You generally have access to memories and it may be useful for you to read them, but also only if they help you
to answer the question or complete the task. You can infer which memories are relevant to the current task by reading
the memory names and descriptions.
You generally have access to memories and it may be useful for you to read them.
You infer whether memories are relevant based on their names.
{% endif %}
The context and modes of operation are described below. From them you can infer how to interact with your user
and which tasks and kinds of interactions are expected of you.
The context and modes of operation are described below. These determine how to interact with your user
and which kinds of interactions are expected of you.
Context description:
{{ context_system_prompt }}
Modes descriptions:
{% for prompt in mode_system_prompts %}
- {{ prompt }}
{{ prompt }}
{% endfor %}
You have hereby read the 'Serena Instructions Manual' and do not need to read it again.
src/serena/resources/dashboard/dashboard.css
+1176
View File
File diff suppressed because it is too large. Load diff
src/serena/resources/dashboard/dashboard.js
+1543 -239
View File
File diff suppressed because it is too large. Load diff
src/serena/resources/dashboard/index.html
+288 -261
View File
@@ -8,271 +8,158 @@
<link rel="icon" type="image/png" sizes="16x16" href="serena-icon-16.png">
<link rel="icon" type="image/png" sizes="32x32" href="serena-icon-32.png">
<link rel="icon" type="image/png" sizes="48x48" href="serena-icon-48.png">
<link rel="stylesheet" href="dashboard.css">
<script src="jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-datalabels@2"></script>
<script src="dashboard.js"></script>
<style>
:root {
/* Light theme variables */
--bg-primary: #f5f5f5;
--bg-secondary: #ffffff;
--text-primary: #000000;
--text-secondary: #333333;
--text-muted: #666666;
--border-color: #ddd;
--btn-primary: #eaa45d;
--btn-hover: #dca662;
--btn-disabled: #6c757d;
--shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
--tool-highlight: #ffff00;
--tool-highlight-text: #000000;
--log-debug: #808080;
--log-info: #000000;
--log-warning: #FF8C00;
--log-error: #FF0000;
--stats-header: #f8f9fa;
}
[data-theme="dark"] {
/* Dark theme variables */
--bg-primary: #1a1a1a;
--bg-secondary: #2d2d2d;
--text-primary: #ffffff;
--text-secondary: #e0e0e0;
--text-muted: #b0b0b0;
--border-color: #444;
--btn-primary: #eaa45d;
--btn-hover: #dca662;
--btn-disabled: #6c757d;
--shadow: 0 2px 4px rgba(0, 0, 0, 0.3);
--tool-highlight: #ffd700;
--tool-highlight-text: #000000;
--log-debug: #808080;
--log-info: #ffffff;
--log-warning: #FF8C00;
--log-error: #FF0000;
--stats-header: #3a3a3a;
}
body {
font-family: 'Consolas', 'Monaco', 'Courier New', monospace;
margin: 0;
padding: 20px;
background-color: var(--bg-primary);
color: var(--text-primary);
transition: background-color 0.3s ease, color 0.3s ease;
}
.header {
text-align: center;
margin-bottom: 20px;
}
.log-container {
background-color: var(--bg-secondary);
border: 1px solid var(--border-color);
border-radius: 5px;
height: 600px;
overflow-y: auto;
overflow-x: auto;
padding: 10px;
white-space: pre-wrap;
font-size: 12px;
line-height: 1.4;
color: var(--text-primary);
transition: background-color 0.3s ease, border-color 0.3s ease, color 0.3s ease;
}
.controls {
margin-bottom: 10px;
text-align: center;
display: flex;
justify-content: center;
align-items: center;
gap: 10px;
flex-wrap: wrap;
}
.logo {
margin-bottom: 10px;
text-align: center;
}
.btn {
background-color: var(--btn-primary);
color: white;
border: none;
padding: 8px 16px;
border-radius: 4px;
cursor: pointer;
font-size: 14px;
transition: background-color 0.3s ease;
}
.btn:hover {
background-color: var(--btn-hover);
}
.btn:disabled {
background-color: var(--btn-disabled);
cursor: not-allowed;
}
.theme-toggle {
display: flex;
align-items: center;
gap: 5px;
background-color: var(--bg-secondary);
border: 1px solid var(--border-color);
border-radius: 4px;
padding: 6px 12px;
cursor: pointer;
transition: background-color 0.3s ease, border-color 0.3s ease;
}
.theme-toggle:hover {
background-color: var(--border-color);
}
.theme-toggle .icon {
font-size: 16px;
}
.log-debug {
color: var(--log-debug);
}
.log-info {
color: var(--log-info);
}
.log-warning {
color: var(--log-warning);
}
.log-error {
color: var(--log-error);
}
.log-default {
color: var(--log-info);
}
/* Tool name highlighting */
.tool-name {
background-color: var(--tool-highlight);
color: var(--tool-highlight-text);
font-weight: bold;
}
.loading {
text-align: center;
color: var(--text-muted);
font-style: italic;
}
.error-message {
color: var(--log-error);
text-align: center;
margin: 10px 0;
}
.charts-container {
display: flex;
flex-wrap: wrap;
gap: 15px;
justify-content: space-between;
max-width: 1400px;
margin: 0 auto;
}
.chart-group {
flex: 1;
min-width: 280px;
max-width: 320px;
text-align: center;
}
.chart-wide {
flex: 0 0 100%;
min-width: 100%;
margin-top: 10px;
}
.chart-group h3 {
margin: 0 0 10px 0;
color: var(--text-secondary);
}
.stats-summary {
margin: 0 auto;
border-collapse: collapse;
background: var(--bg-secondary);
border-radius: 5px;
overflow: hidden;
box-shadow: var(--shadow);
transition: background-color 0.3s ease, box-shadow 0.3s ease;
}
.stats-summary th,
.stats-summary td {
padding: 10px 20px;
text-align: left;
border-bottom: 1px solid var(--border-color);
color: var(--text-primary);
transition: border-color 0.3s ease, color 0.3s ease;
}
.stats-summary th {
background-color: var(--stats-header);
font-weight: bold;
transition: background-color 0.3s ease;
}
.stats-summary tr:last-child td {
border-bottom: none;
}
@media (max-width: 768px) {
.charts-container {
flex-direction: column;
}
.chart-group,
.chart-wide {
min-width: auto;
max-width: none;
}
.controls {
flex-direction: column;
gap: 5px;
}
}
</style>
</head>
<body>
<div class="header">
<img id="serena-logo" src="serena-logs.png" alt="Serena" style="max-width: 400px; height: auto;">
</div>
<div id="frame">
<header class="header">
<div class="header-left">
<div class="controls">
<button id="load-logs" class="btn">Reload Log</button>
<button id="shutdown" class="btn">Shutdown Server</button>
<button id="toggle-stats" class="btn">Show Stats</button>
<div id="theme-toggle" class="theme-toggle" title="Toggle theme">
<span class="icon" id="theme-icon">🌙</span>
<span id="theme-text">Dark</span>
<div class="logo-container">
<img id="serena-logo" src="serena-logs.png" alt="Serena">
</div>
<!-- Platinum Sponsors in Header -->
<div id="platinum-sponsors" class="header-sponsor">
</div>
</div>
<div id="error-container"></div>
<div id="log-container" class="log-container"></div>
<nav class="header-nav">
<button id="theme-toggle" class="theme-toggle">
<span id="theme-icon" style="height: 21px">🌙</span>
<span id="theme-text">Dark</span>
</button>
<button id="menu-toggle" class="menu-button">
<span>☰</span>
<span>Menu</span>
</button>
<div id="menu-dropdown" class="menu-dropdown" style="display:none">
<a href="#" data-page="overview" class="active">Overview</a>
<a href="#" data-page="logs">Logs</a>
<a href="#" data-page="stats">Advanced Stats</a>
<hr>
<a href="#" id="menu-shutdown">Shutdown Server</a>
</div>
</nav>
</header>
<div id="stats-section" style="display:none; margin-top:20px;">
<div style="text-align:center; margin-bottom:20px;">
<div class="main">
<!-- Overview Page (Landing) -->
<div id="page-overview" class="page-view">
<div class="overview-container">
<div class="overview-left">
<section class="config-section">
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 15px;">
<h2 style="margin: 0;">Current Configuration</h2>
</div>
<div id="config-display">
<div class="loading">Loading configuration...</div>
</div>
</section>
<section class="basic-stats-section">
<h2>Tool Usage</h2>
<div id="basic-stats-display">
<div class="loading">Loading stats...</div>
</div>
</section>
<section class="executions-section">
<h2>Executions Queue</h2>
<div id="active-executions-display">
<div class="loading">Loading executions...</div>
</div>
</section>
<section class="executions-section">
<h2>Last Execution</h2>
<div id="last-execution-display">
<div class="loading">Loading...</div>
</div>
</section>
<section class="executions-section" style="display:none">
<h2>Cancelled Executions</h2>
<div id="cancelled-executions-display">
<div class="no-stats-message">No cancelled executions.</div>
</div>
</section>
</div>
<div class="overview-right">
<section class="projects-section">
<h2 class="collapsible-header" id="projects-header">
<span>Registered Projects</span>
<span class="toggle-icon">▼</span>
</h2>
<div id="projects-display" class="collapsible-content" style="display:none">
<div class="loading">Loading projects...</div>
</div>
</section>
<section class="projects-section">
<h2 class="collapsible-header" id="available-tools-header">
<span>Available Tools (Disabled)</span>
<span class="toggle-icon">▼</span>
</h2>
<div id="available-tools-display" class="collapsible-content" style="display:none">
<div class="loading">Loading tools...</div>
</div>
</section>
<section class="projects-section">
<h2 class="collapsible-header" id="available-modes-header">
<span>Available Modes</span>
<span class="toggle-icon">▼</span>
</h2>
<div id="available-modes-display" class="collapsible-content" style="display:none">
<div class="loading">Loading modes...</div>
</div>
</section>
<section class="projects-section">
<h2 class="collapsible-header" id="available-contexts-header">
<span>Available Contexts</span>
<span class="toggle-icon">▼</span>
</h2>
<div id="available-contexts-display" class="collapsible-content" style="display:none">
<div class="loading">Loading contexts...</div>
</div>
</section>
<!-- Gold Sponsors -->
<section id="gold-sponsors" class="gold-sponsors-section">
</section>
</div>
</div>
</div>
<!-- Logs Page -->
<div id="page-logs" class="page-view" style="display:none">
<div id="error-container"></div>
<div style="position: relative;">
<button id="copy-logs-btn" class="copy-logs-btn" title="Copy logs">
<svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none"
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
<rect x="9" y="9" width="13" height="13" rx="2" ry="2"></rect>
<path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"></path>
</svg>
<span class="copy-logs-text">copy logs</span>
</button>
<div id="log-container" class="log-container"></div>
</div>
</div>
<!-- Advanced Stats Page -->
<div id="page-stats" class="page-view" style="display:none">
<div class="controls">
<button id="refresh-stats" class="btn">Refresh Stats</button>
<button id="clear-stats" class="btn">Clear Stats</button>
</div>
@@ -280,10 +167,9 @@
<div id="stats-summary" style="margin-bottom:20px; text-align:center;"></div>
<div id="estimator-name" style="text-align:center; margin-bottom:10px;"></div>
<div id="no-stats-message" style="text-align:center; color:var(--text-muted); font-style:italic; display:none;">
No tool stats collected. Have you enabled tool stats collection in the configuration?
No tool stats collected yet.
</div>
<div class="charts-container">
<div class="chart-group">
<h3>Tool Calls</h3>
@@ -304,11 +190,152 @@
</div>
</div>
<script>
$(document).ready(function () {
const dashboard = new Dashboard();
});
</script>
<!-- Add Language Modal -->
<div id="add-language-modal" class="modal" style="display:none">
<div class="modal-content">
<span class="modal-close">&times;</span>
<h3>Add Language</h3>
<p id="modal-project-info" style="margin: 15px 0; color: var(--text-primary); line-height: 1.5;">
Adding a language to serena config of project <strong id="modal-project-name"></strong>.
</p>
<p style="margin: 15px 0; color: var(--text-muted); font-size: 13px; line-height: 1.5;">
Note that this may download some dependencies needed for the language server and then start it,
it may take a few seconds before the LS is responsive.
</p>
<div style="margin: 20px 0;">
<label for="modal-language-select" style="display: block; margin-bottom: 8px; font-weight: 500;">Select
Language:</label>
<select id="modal-language-select" class="language-select"
style="width: 100%; padding: 8px 10px; border-radius: 4px; border: 1px solid var(--border-color); background: var(--bg-primary); color: var(--text-primary); font-size: 14px;">
</select>
</div>
<div style="display: flex; gap: 10px; justify-content: flex-end; margin-top: 20px;">
<button id="modal-cancel-btn" class="btn"
style="background: var(--bg-secondary); color: var(--text-primary); padding: 8px 16px;">Cancel
</button>
<button id="modal-add-btn" class="btn" style="padding: 8px 16px;">Add Language</button>
</div>
</div>
</div>
<!-- Remove Language Confirmation Modal -->
<div id="remove-language-modal" class="modal" style="display:none">
<div class="modal-content">
<span class="modal-close modal-close-remove">&times;</span>
<p style="margin: 20px 0; color: var(--text-primary); line-height: 1.5; font-size: 15px;">
Remove language <strong id="remove-language-name"></strong> from configuration?
</p>
<div style="display: flex; gap: 10px; justify-content: flex-end; margin-top: 20px;">
<button id="remove-modal-cancel-btn" class="btn"
style="background: var(--bg-secondary); color: var(--text-primary); padding: 8px 16px;">Cancel
</button>
<button id="remove-modal-ok-btn" class="btn" style="padding: 8px 16px;">OK</button>
</div>
</div>
</div>
<!-- Edit Memory Modal -->
<div id="edit-memory-modal" class="modal" style="display:none">
<div class="modal-content modal-content-large">
<span class="modal-close modal-close-edit-memory">&times;</span>
<h3 style="margin-bottom: 20px;">Memory: <span id="edit-memory-name"></span></h3>
<textarea id="edit-memory-content" class="memory-editor" rows="20"
style="width: 100%; padding: 10px; border: 1px solid var(--border-color); border-radius: 4px; background: var(--bg-primary); color: var(--text-primary); font-family: 'Courier New', monospace; font-size: 13px; resize: vertical;"></textarea>
<div style="display: flex; gap: 10px; justify-content: flex-end; margin-top: 20px;">
<button id="edit-memory-cancel-btn" class="btn"
style="background: var(--bg-secondary); color: var(--text-primary); padding: 8px 16px;">Cancel
</button>
<button id="edit-memory-save-btn" class="btn" style="padding: 8px 16px;">Save</button>
</div>
</div>
</div>
<!-- Delete Memory Confirmation Modal -->
<div id="delete-memory-modal" class="modal" style="display:none">
<div class="modal-content">
<span class="modal-close modal-close-delete-memory">&times;</span>
<p style="margin: 20px 0; color: var(--text-primary); line-height: 1.5; font-size: 15px;">
Delete memory <strong id="delete-memory-name"></strong>?
</p>
<div style="display: flex; gap: 10px; justify-content: flex-end; margin-top: 20px;">
<button id="delete-memory-cancel-btn" class="btn"
style="background: var(--bg-secondary); color: var(--text-primary); padding: 8px 16px;">Cancel
</button>
<button id="delete-memory-ok-btn" class="btn" style="padding: 8px 16px;">OK</button>
</div>
</div>
</div>
<!-- Create Memory Modal -->
<div id="create-memory-modal" class="modal" style="display:none">
<div class="modal-content">
<span class="modal-close modal-close-create-memory">&times;</span>
<p id="create-memory-project-info" style="margin: 15px 0; color: var(--text-primary); line-height: 1.5;">
Create a new memory for project <strong id="create-memory-project-name"></strong>.
</p>
<p style="margin: 15px 0; color: var(--text-muted); font-size: 13px; line-height: 1.5;">
Memory names should be descriptive and use underscores instead of spaces (e.g., "api_architecture",
"testing_guidelines").
</p>
<div style="margin: 20px 0;">
<label for="create-memory-name-input" style="display: block; margin-bottom: 8px; font-weight: 500;">Memory
Name:</label>
<input type="text" id="create-memory-name-input" class="memory-name-input"
placeholder="e.g., project_overview"
style="width: 100%; padding: 8px 10px; border-radius: 4px; border: 1px solid var(--border-color); background: var(--bg-primary); color: var(--text-primary); font-size: 14px;">
</div>
<div style="display: flex; gap: 10px; justify-content: flex-end; margin-top: 20px;">
<button id="create-memory-cancel-btn" class="btn"
style="background: var(--bg-secondary); color: var(--text-primary); padding: 8px 16px;">Cancel
</button>
<button id="create-memory-create-btn" class="btn" style="padding: 8px 16px;">Create</button>
</div>
</div>
</div>
<!-- Cancel Execution Confirmation Modal -->
<div id="cancel-execution-modal" class="modal" style="display:none">
<div class="modal-content">
<span class="modal-close modal-close-cancel-execution">&times;</span>
<p style="margin: 20px 0; color: var(--text-primary); line-height: 1.5; font-size: 15px;">
Are you sure? The execution will continue running until timeout, it will simply no longer be in the queue.
Abandoning a running execution is only advised as a measure for unblocking Serena.
</p>
<div style="display: flex; gap: 10px; justify-content: flex-end; margin-top: 20px;">
<button id="cancel-execution-cancel-btn" class="btn"
style="background: var(--bg-secondary); color: var(--text-primary); padding: 8px 16px;">Cancel
</button>
<button id="cancel-execution-ok-btn" class="btn" style="padding: 8px 16px;">OK</button>
</div>
</div>
</div>
<!-- Edit Serena Config Modal -->
<div id="edit-serena-config-modal" class="modal" style="display:none">
<div class="modal-content modal-content-large">
<span class="modal-close modal-close-edit-serena-config">&times;</span>
<h3 style="margin-bottom: 10px;">Global Serena Configuration</h3>
<p style="margin: 10px 0 20px 0; color: var(--text-muted); font-size: 13px; line-height: 1.5;">
Note: Changes to the configuration will only take effect after Serena is restarted.
</p>
<textarea id="edit-serena-config-content" class="memory-editor" rows="20"
style="width: 100%; padding: 10px; border: 1px solid var(--border-color); border-radius: 4px; background: var(--bg-primary); color: var(--text-primary); font-family: 'Courier New', monospace; font-size: 13px; resize: vertical;"></textarea>
<div style="display: flex; gap: 10px; justify-content: flex-end; margin-top: 20px;">
<button id="edit-serena-config-cancel-btn" class="btn"
style="background: var(--bg-secondary); color: var(--text-primary); padding: 8px 16px;">Cancel
</button>
<button id="edit-serena-config-save-btn" class="btn" style="padding: 8px 16px;">Save</button>
</div>
</div>
</div>
</div> <!-- End of Main -->
<script>
$(document).ready(function () {
const dashboard = new Dashboard();
});
</script>
</div> <!-- End of Frame -->
</body>
</html>
</html>
src/serena/resources/dashboard/serena-logo-dark-mode.svg
BIN
View File
Binary file not shown.
src/serena/resources/dashboard/serena-logo.svg
BIN
View File
Binary file not shown.
src/serena/resources/project.template.yml
+20 -5
View File
@@ -1,13 +1,28 @@
# language of the project (csharp, python, rust, java, typescript, go, cpp, or ruby)
# * For C, use cpp
# * For JavaScript, use typescript
# list of languages for which language servers are started; choose from:
# al bash clojure cpp csharp csharp_omnisharp
# dart elixir elm erlang fortran go
# haskell java julia kotlin lua markdown
# nix perl php python python_jedi r
# rego ruby ruby_solargraph rust scala swift
# terraform typescript typescript_vts yaml zig
# Note:
# - For C, use cpp
# - For JavaScript, use typescript
# Special requirements:
# * csharp: Requires the presence of a .sln file in the project folder.
language: python
# - csharp: Requires the presence of a .sln file in the project folder.
# When using multiple languages, the first language server that supports a given file will be used for that file.
# The first language is the default language and the respective language server will be used as a fallback.
# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
languages: ["python"]
# the encoding used by text files in the project
# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
encoding: "utf-8"
# whether to use the project's gitignore file to ignore files
# Added on 2025-04-07
ignore_all_files_in_gitignore: true
# list of additional paths to ignore
# same syntax as gitignore, so you can use * and **
# Was previously called `ignored_dirs`, please update your config if you are using that.
src/serena/resources/serena_config.template.yml
+7 -9
View File
@@ -4,7 +4,7 @@ gui_log_window: False
# If you want to see the logs in a web browser, use the `web_dashboard` option instead.
# Limitations: doesn't seem to work with the community version of Claude Desktop for Linux
# Might also cause problems with some MCP clients - if you have any issues, try disabling this
#
# Being able to inspect logs is useful both for troubleshooting and for monitoring the tool calls,
# especially when using the agno playground, since the tool calls are not always shown,
# and the input params are never shown in the agno UI.
@@ -61,16 +61,14 @@ default_max_tool_answer_chars: 150000
# Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default
# through the global configuration.
record_tool_usage_stats: False
# whether to record tool usage statistics, they will be shown in the web dashboard if recording is active.
token_count_estimator: TIKTOKEN_GPT4O
# Only relevant if `record_tool_usage` is True; the name of the token count estimator to use for tool usage statistics.
token_count_estimator: CHAR_COUNT
# the name of the token count estimator to use for tool usage statistics.
# See the `RegisteredTokenCountEstimator` enum for available options.
#
# Note: some token estimators (like tiktoken) may require downloading data files
# on the first run, which can take some time and require internet access. Others, like the Anthropic ones, may require an API key
# and rate limits may apply.
# By default, a very naive character count estimator is used, which simply counts the number of characters.
# You can configure this to TIKTOKEN_GPT4 to use a local tiktoken-based estimator for GPT-4 (will download tiktoken
# data files on first run), or ANTHROPIC_CLAUDE_SONNET_4 which will use the (free of cost) Anthropic API to
# estimate the token count using the Claude Sonnet 4 tokenizer.
# MANAGED BY SERENA, KEEP AT THE BOTTOM OF THE YAML AND DON'T EDIT WITHOUT NEED
src/serena/symbol.py
+104 -61
View File
@@ -4,7 +4,7 @@ import os
from abc import ABC, abstractmethod
from collections.abc import Iterator, Sequence
from dataclasses import asdict, dataclass
from typing import TYPE_CHECKING, Any, Self, Union
from typing import TYPE_CHECKING, Any, NotRequired, Self, TypedDict, Union
from sensai.util.string import ToStringMixin
@@ -12,12 +12,14 @@ from solidlsp import SolidLanguageServer
from solidlsp.ls import ReferenceInSymbol as LSPReferenceInSymbol
from solidlsp.ls_types import Position, SymbolKind, UnifiedSymbolInformation
from .ls_manager import LanguageServerManager
from .project import Project
if TYPE_CHECKING:
from .agent import SerenaAgent
log = logging.getLogger(__name__)
NAME_PATH_SEP = "/"
@dataclass
@@ -77,7 +79,7 @@ class PositionInFile:
return Position(line=self.line, character=self.col)
class Symbol(ABC):
class Symbol(ToStringMixin, ABC):
@abstractmethod
def get_body_start_position(self) -> PositionInFile | None:
pass
@@ -112,45 +114,65 @@ class Symbol(ABC):
"""
class LanguageServerSymbol(Symbol, ToStringMixin):
_NAME_PATH_SEP = "/"
@staticmethod
def match_name_path(
name_path: str,
symbol_name_path_parts: list[str],
substring_matching: bool,
) -> bool:
class NamePathMatcher(ToStringMixin):
def __init__(self, name_path_expr: str, substring_matching: bool) -> None:
"""
Checks if a given `name_path` matches a symbol's qualified name parts.
See docstring of `Symbol.find` for more details.
:param name_path_expr: the name path expression to match against
:param substring_matching: whether to use substring matching for the last segment
"""
assert name_path, "name_path must not be empty"
assert symbol_name_path_parts, "symbol_name_path_parts must not be empty"
name_path_sep = LanguageServerSymbol._NAME_PATH_SEP
assert name_path_expr, "name_path must not be empty"
self._expr = name_path_expr
self._substring_matching = substring_matching
self._is_absolute_pattern = name_path_expr.startswith(NAME_PATH_SEP)
self._pattern_parts = name_path_expr.lstrip(NAME_PATH_SEP).rstrip(NAME_PATH_SEP).split(NAME_PATH_SEP)
is_absolute_pattern = name_path.startswith(name_path_sep)
pattern_parts = name_path.lstrip(name_path_sep).rstrip(name_path_sep).split(name_path_sep)
# extract overload index "[idx]" if present at end of last part
self._overload_idx: int | None = None
last_part = self._pattern_parts[-1]
if last_part.endswith("]") and "[" in last_part:
bracket_idx = last_part.rfind("[")
index_part = last_part[bracket_idx + 1 : -1]
if index_part.isdigit():
self._pattern_parts[-1] = last_part[:bracket_idx]
self._overload_idx = int(index_part)
def _tostring_includes(self) -> list[str]:
return ["_expr"]
def matches_ls_symbol(self, symbol: "LanguageServerSymbol") -> bool:
return self.matches_components(symbol.get_name_path_parts(), symbol.overload_idx)
def matches_components(self, symbol_name_path_parts: list[str], overload_idx: int | None) -> bool:
# filtering based on ancestors
if len(pattern_parts) > len(symbol_name_path_parts):
if len(self._pattern_parts) > len(symbol_name_path_parts):
# can't possibly match if pattern has more parts than symbol
return False
if is_absolute_pattern and len(pattern_parts) != len(symbol_name_path_parts):
if self._is_absolute_pattern and len(self._pattern_parts) != len(symbol_name_path_parts):
# for absolute patterns, the number of parts must match exactly
return False
if symbol_name_path_parts[-len(pattern_parts) : -1] != pattern_parts[:-1]:
if symbol_name_path_parts[-len(self._pattern_parts) : -1] != self._pattern_parts[:-1]:
# ancestors must match
return False
# matching the last part of the symbol name
name_to_match = pattern_parts[-1]
name_to_match = self._pattern_parts[-1]
symbol_name = symbol_name_path_parts[-1]
if substring_matching:
return name_to_match in symbol_name
if self._substring_matching:
if name_to_match not in symbol_name:
return False
else:
return name_to_match == symbol_name
if name_to_match != symbol_name:
return False
# check for matching overload index
if self._overload_idx is not None:
if overload_idx != self._overload_idx:
return False
return True
class LanguageServerSymbol(Symbol, ToStringMixin):
def __init__(self, symbol_root_from_ls: UnifiedSymbolInformation) -> None:
self.symbol_root = symbol_root_from_ls
@@ -172,6 +194,10 @@ class LanguageServerSymbol(Symbol, ToStringMixin):
def symbol_kind(self) -> SymbolKind:
return self.symbol_root["kind"]
@property
def overload_idx(self) -> int | None:
return self.symbol_root.get("overload_idx")
def is_neighbouring_definition_separated_by_empty_line(self) -> bool:
return self.symbol_kind in (SymbolKind.Function, SymbolKind.Method, SymbolKind.Class, SymbolKind.Interface, SymbolKind.Struct)
@@ -255,9 +281,13 @@ class LanguageServerSymbol(Symbol, ToStringMixin):
def get_name_path(self) -> str:
"""
Get the name path of the symbol (e.g. "class/method/inner_function").
Get the name path of the symbol, e.g. "class/method/inner_function" or
"class/method[1]" (overloaded method with identifying index).
"""
return self._NAME_PATH_SEP.join(self.get_name_path_parts())
name_path = NAME_PATH_SEP.join(self.get_name_path_parts())
if "overload_idx" in self.symbol_root:
name_path += f"[{self.symbol_root['overload_idx']}]"
return name_path
def get_name_path_parts(self) -> list[str]:
"""
@@ -320,7 +350,7 @@ class LanguageServerSymbol(Symbol, ToStringMixin):
For example, passing `/class` will match only against top-level symbols like `class` but not against `nested_class/class`.
Passing `/class/method` will match against `class/method` but not `nested_class/class/method` or `method`.
:param name_path: the name path to match against
:param name_path: the name path expression to match against
:param substring_matching: whether to use substring matching (as opposed to exact matching)
of the last segment of `name_path` against the symbol name.
:param include_kinds: an optional sequence of ints representing the LSP symbol kind.
@@ -328,17 +358,14 @@ class LanguageServerSymbol(Symbol, ToStringMixin):
:param exclude_kinds: If provided, symbols of the given kinds will be excluded from the result.
"""
result = []
name_path_matcher = NamePathMatcher(name_path, substring_matching)
def should_include(s: "LanguageServerSymbol") -> bool:
if include_kinds is not None and s.symbol_kind not in include_kinds:
return False
if exclude_kinds is not None and s.symbol_kind in exclude_kinds:
return False
return LanguageServerSymbol.match_name_path(
name_path=name_path,
symbol_name_path_parts=s.get_name_path_parts(),
substring_matching=substring_matching,
)
return name_path_matcher.matches_ls_symbol(s)
def traverse(s: "LanguageServerSymbol") -> None:
if should_include(s):
@@ -441,29 +468,29 @@ class ReferenceInLanguageServerSymbol(ToStringMixin):
class LanguageServerSymbolRetriever:
def __init__(self, lang_server: SolidLanguageServer, agent: Union["SerenaAgent", None] = None) -> None:
def __init__(self, ls: SolidLanguageServer | LanguageServerManager, agent: Union["SerenaAgent", None] = None) -> None:
"""
:param lang_server: the language server to use for symbol retrieval as well as editing operations.
:param ls: the language server or language server manager to use for symbol retrieval and editing operations.
:param agent: the agent to use (only needed for marking files as modified). You can pass None if you don't
need an agent to be aware of file modifications performed by the symbol manager.
"""
self._lang_server = lang_server
if isinstance(ls, SolidLanguageServer):
ls_manager = LanguageServerManager({ls.language: ls})
else:
ls_manager = ls
assert isinstance(ls_manager, LanguageServerManager)
self._ls_manager: LanguageServerManager = ls_manager
self.agent = agent
def set_language_server(self, lang_server: SolidLanguageServer) -> None:
"""
Set the language server to use for symbol retrieval and editing operations.
This is useful if you want to change the language server after initializing the SymbolManager.
"""
self._lang_server = lang_server
def get_root_path(self) -> str:
return self._ls_manager.get_root_path()
def get_language_server(self) -> SolidLanguageServer:
return self._lang_server
def get_language_server(self, relative_path: str) -> SolidLanguageServer:
return self._ls_manager.get_language_server(relative_path)
def find_by_name(
self,
name_path: str,
include_body: bool = False,
include_kinds: Sequence[SymbolKind] | None = None,
exclude_kinds: Sequence[SymbolKind] | None = None,
substring_matching: bool = False,
@@ -475,25 +502,22 @@ class LanguageServerSymbolRetriever:
to symbols within a specific file or directory.
"""
symbols: list[LanguageServerSymbol] = []
symbol_roots = self._lang_server.request_full_symbol_tree(within_relative_path=within_relative_path, include_body=include_body)
for root in symbol_roots:
symbols.extend(
LanguageServerSymbol(root).find(
name_path, include_kinds=include_kinds, exclude_kinds=exclude_kinds, substring_matching=substring_matching
for lang_server in self._ls_manager.iter_language_servers():
symbol_roots = lang_server.request_full_symbol_tree(within_relative_path=within_relative_path)
for root in symbol_roots:
symbols.extend(
LanguageServerSymbol(root).find(
name_path, include_kinds=include_kinds, exclude_kinds=exclude_kinds, substring_matching=substring_matching
)
)
)
return symbols
def get_document_symbols(self, relative_path: str) -> list[LanguageServerSymbol]:
symbol_dicts, _roots = self._lang_server.request_document_symbols(relative_path, include_body=False)
symbols = [LanguageServerSymbol(s) for s in symbol_dicts]
return symbols
def find_by_location(self, location: LanguageServerSymbolLocation) -> LanguageServerSymbol | None:
if location.relative_path is None:
return None
symbol_dicts, _roots = self._lang_server.request_document_symbols(location.relative_path, include_body=False)
for symbol_dict in symbol_dicts:
lang_server = self.get_language_server(location.relative_path)
document_symbols = lang_server.request_document_symbols(location.relative_path)
for symbol_dict in document_symbols.iter_symbols():
symbol = LanguageServerSymbol(symbol_dict)
if symbol.location == location:
return symbol
@@ -561,7 +585,8 @@ class LanguageServerSymbolRetriever:
assert symbol_location.relative_path is not None
assert symbol_location.line is not None
assert symbol_location.column is not None
references = self._lang_server.request_referencing_symbols(
lang_server = self.get_language_server(symbol_location.relative_path)
references = lang_server.request_referencing_symbols(
relative_file_path=symbol_location.relative_path,
line=symbol_location.line,
column=symbol_location.column,
@@ -589,7 +614,8 @@ class LanguageServerSymbolRetriever:
return cls(name_path=symbol.get_name_path(), kind=int(symbol.symbol_kind))
def get_symbol_overview(self, relative_path: str) -> dict[str, list[SymbolOverviewElement]]:
path_to_unified_symbols = self._lang_server.request_overview(relative_path)
lang_server = self.get_language_server(relative_path)
path_to_unified_symbols = lang_server.request_overview(relative_path)
result = {}
for file_path, unified_symbols in path_to_unified_symbols.items():
# TODO: maybe include not just top-level symbols? We could filter by kind to exclude variables
@@ -599,7 +625,15 @@ class LanguageServerSymbolRetriever:
class JetBrainsSymbol(Symbol):
def __init__(self, symbol_dict: dict, project: Project) -> None:
class SymbolDict(TypedDict):
name_path: str
relative_path: str
type: str
text_range: NotRequired[dict]
body: NotRequired[str]
children: NotRequired[list["JetBrainsSymbol.SymbolDict"]]
def __init__(self, symbol_dict: SymbolDict, project: Project) -> None:
"""
:param symbol_dict: dictionary as returned by the JetBrains plugin client.
"""
@@ -609,6 +643,15 @@ class JetBrainsSymbol(Symbol):
self._cached_body_start_position: PositionInFile | None = None
self._cached_body_end_position: PositionInFile | None = None
def _tostring_includes(self) -> list[str]:
return []
def _tostring_additional_entries(self) -> dict[str, Any]:
return dict(name_path=self.get_name_path(), relative_path=self.get_relative_path(), type=self._dict["type"])
def get_name_path(self) -> str:
return self._dict["name_path"]
def get_relative_path(self) -> str:
return self._dict["relative_path"]
src/serena/task_executor.py
+218
View File
@@ -0,0 +1,218 @@
import concurrent.futures
import threading
import time
from collections.abc import Callable
from concurrent.futures import Future
from dataclasses import dataclass
from threading import Thread
from typing import Generic, TypeVar
from sensai.util import logging
from sensai.util.logging import LogTime
from sensai.util.string import ToStringMixin
log = logging.getLogger(__name__)
T = TypeVar("T")
class TaskExecutor:
def __init__(self, name: str):
self._task_executor_lock = threading.Lock()
self._task_executor_queue: list[TaskExecutor.Task] = []
self._task_executor_thread = Thread(target=self._process_task_queue, name=name, daemon=True)
self._task_executor_thread.start()
self._task_executor_task_index = 1
self._task_executor_current_task: TaskExecutor.Task | None = None
self._task_executor_last_executed_task_info: TaskExecutor.TaskInfo | None = None
class Task(ToStringMixin, Generic[T]):
def __init__(self, function: Callable[[], T], name: str, logged: bool = True, timeout: float | None = None):
"""
:param function: the function representing the task to execute
:param name: the name of the task
:param logged: whether to log management of the task; if False, only errors will be logged
:param timeout: the maximum time to wait for task completion in seconds, or None to wait indefinitely
"""
self.name = name
self.future: concurrent.futures.Future = concurrent.futures.Future()
self.logged = logged
self.timeout = timeout
self._function = function
def _tostring_includes(self) -> list[str]:
return ["name"]
def start(self) -> None:
"""
Executes the task in a separate thread, setting the result or exception on the future.
"""
def run_task() -> None:
try:
if self.future.done():
if self.logged:
log.info(f"Task {self.name} was already completed/cancelled; skipping execution")
return
with LogTime(self.name, logger=log, enabled=self.logged):
result = self._function()
if not self.future.done():
self.future.set_result(result)
except Exception as e:
if not self.future.done():
log.error(f"Error during execution of {self.name}: {e}", exc_info=e)
self.future.set_exception(e)
thread = Thread(target=run_task, name=self.name)
thread.start()
def is_done(self) -> bool:
"""
:return: whether the task has completed (either successfully, with failure, or via cancellation)
"""
return self.future.done()
def result(self, timeout: float | None = None) -> T:
"""
Blocks until the task is done or the timeout is reached, and returns the result.
If an exception occurred during task execution, it is raised here.
If the timeout is reached, a TimeoutError is raised (but the task is not cancelled).
If the task is cancelled, a CancelledError is raised.
:param timeout: the maximum time to wait in seconds; if None, use the task's own timeout
(which may be None to wait indefinitely)
:return: True if the task is done, False if the timeout was reached
"""
return self.future.result(timeout=timeout)
def cancel(self) -> None:
"""
Cancels the task. If it has not yet started, it will not be executed.
If it has already started, its future will be marked as cancelled and will raise a CancelledError
when its result is requested.
"""
self.future.cancel()
def wait_until_done(self, timeout: float | None = None) -> None:
"""
Waits until the task is done or the timeout is reached.
The task is done if it either completed successfully, failed with an exception, or was cancelled.
:param timeout: the maximum time to wait in seconds; if None, use the task's own timeout
(which may be None to wait indefinitely)
"""
try:
self.future.result(timeout=timeout)
except:
pass
def _process_task_queue(self) -> None:
while True:
# obtain task from the queue
task: TaskExecutor.Task | None = None
with self._task_executor_lock:
if len(self._task_executor_queue) > 0:
task = self._task_executor_queue.pop(0)
if task is None:
time.sleep(0.1)
continue
# start task execution asynchronously
with self._task_executor_lock:
self._task_executor_current_task = task
if task.logged:
log.info("Starting execution of %s", task.name)
task.start()
# wait for task completion
task.wait_until_done(timeout=task.timeout)
with self._task_executor_lock:
self._task_executor_current_task = None
if task.logged:
self._task_executor_last_executed_task_info = self.TaskInfo.from_task(task, is_running=False)
@dataclass
class TaskInfo:
name: str
is_running: bool
future: Future
"""
future for accessing the task's result
"""
task_id: int
"""
unique identifier of the task
"""
logged: bool
def finished_successfully(self) -> bool:
return self.future.done() and not self.future.cancelled() and self.future.exception() is None
@staticmethod
def from_task(task: "TaskExecutor.Task", is_running: bool) -> "TaskExecutor.TaskInfo":
return TaskExecutor.TaskInfo(name=task.name, is_running=is_running, future=task.future, task_id=id(task), logged=task.logged)
def cancel(self) -> None:
self.future.cancel()
def get_current_tasks(self) -> list[TaskInfo]:
"""
Gets the list of tasks currently running or queued for execution.
The function returns a list of thread-safe TaskInfo objects (specifically created for the caller).
:return: the list of tasks in the execution order (running task first)
"""
tasks = []
with self._task_executor_lock:
if self._task_executor_current_task is not None:
tasks.append(self.TaskInfo.from_task(self._task_executor_current_task, True))
for task in self._task_executor_queue:
if not task.is_done():
tasks.append(self.TaskInfo.from_task(task, False))
return tasks
def issue_task(self, task: Callable[[], T], name: str | None = None, logged: bool = True, timeout: float | None = None) -> Task[T]:
"""
Issue a task to the executor for asynchronous execution.
It is ensured that tasks are executed in the order they are issued, one after another.
:param task: the task to execute
:param name: the name of the task for logging purposes; if None, use the task function's name
:param logged: whether to log management of the task; if False, only errors will be logged
:param timeout: the maximum time to wait for task completion in seconds, or None to wait indefinitely
:return: the task object, through which the task's future result can be accessed
"""
with self._task_executor_lock:
if logged:
task_prefix_name = f"Task-{self._task_executor_task_index}"
self._task_executor_task_index += 1
else:
task_prefix_name = "BackgroundTask"
task_name = f"{task_prefix_name}:{name or task.__name__}"
if logged:
log.info(f"Scheduling {task_name}")
task_obj = self.Task(function=task, name=task_name, logged=logged, timeout=timeout)
self._task_executor_queue.append(task_obj)
return task_obj
def execute_task(self, task: Callable[[], T], name: str | None = None, logged: bool = True, timeout: float | None = None) -> T:
"""
Executes the given task synchronously via the agent's task executor.
This is useful for tasks that need to be executed immediately and whose results are needed right away.
:param task: the task to execute
:param name: the name of the task for logging purposes; if None, use the task function's name
:param logged: whether to log management of the task; if False, only errors will be logged
:param timeout: the maximum time to wait for task completion in seconds, or None to wait indefinitely
:return: the result of the task execution
"""
task_obj = self.issue_task(task, name=name, logged=logged, timeout=timeout)
return task_obj.result()
def get_last_executed_task(self) -> TaskInfo | None:
"""
Gets information about the last executed task.
:return: TaskInfo of the last executed task, or None if no task has been executed yet.
"""
with self._task_executor_lock:
return self._task_executor_last_executed_task_info
src/serena/text_utils.py
+4 -2
View File
@@ -9,6 +9,8 @@ from typing import Any, Self
from joblib import Parallel, delayed
from serena.constants import DEFAULT_SOURCE_FILE_ENCODING
log = logging.getLogger(__name__)
@@ -236,8 +238,8 @@ def search_text(
def default_file_reader(file_path: str) -> str:
"""Reads using utf-8 encoding."""
with open(file_path, encoding="utf-8") as f:
"""Reads using the default encoding."""
with open(file_path, encoding=DEFAULT_SOURCE_FILE_ENCODING) as f:
return f.read()
src/serena/tools/config_tools.py
+5 -21
View File
@@ -1,38 +1,22 @@
import json
from serena.config.context_mode import SerenaAgentMode
from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional
class ActivateProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject):
"""
Activates a project by name.
Activates a project based on the project name or path.
"""
def apply(self, project: str) -> str:
"""
Activates the project with the given name.
Activates the project with the given name or path.
:param project: the name of a registered project to activate or a path to a project directory
"""
active_project = self.agent.activate_project_from_path_or_name(project)
if active_project.is_newly_created:
result_str = (
f"Created and activated a new project with name '{active_project.project_name}' at {active_project.project_root}, language: {active_project.project_config.language.value}. "
"You can activate this project later by name.\n"
f"The project's Serena configuration is in {active_project.path_to_project_yml()}. In particular, you may want to edit the project name and the initial prompt."
)
else:
result_str = f"Activated existing project with name '{active_project.project_name}' at {active_project.project_root}, language: {active_project.project_config.language.value}"
if active_project.project_config.initial_prompt:
result_str += f"\nAdditional project information:\n {active_project.project_config.initial_prompt}"
result_str += (
f"\nAvailable memories:\n {json.dumps(list(self.memories_manager.list_memories()))}"
+ "You should not read these memories directly, but rather use the `read_memory` tool to read them later if needed for the task."
)
result_str += f"\nAvailable tools:\n {json.dumps(self.agent.get_active_tool_names())}"
return result_str
result = active_project.get_activation_message()
result += "\nIMPORTANT: If you have not yet read the 'Serena Instructions Manual', do it now before continuing!"
return result
class RemoveProjectTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional):
src/serena/tools/file_tools.py
+67 -27
View File
@@ -6,12 +6,12 @@ File and file system-related tools, specifically for
* editing at the file level
"""
import json
import os
import re
from collections import defaultdict
from fnmatch import fnmatch
from pathlib import Path
from typing import Literal
from serena.text_utils import search_files
from serena.tools import SUCCESS_RESULT, EditedFileContext, Tool, ToolMarkerCanEdit, ToolMarkerOptional
@@ -59,7 +59,7 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit):
Write a new file or overwrite an existing file.
:param relative_path: the relative path to the file to create
:param content: the (utf-8-encoded) content to write to the file
:param content: the (appropriately encoded) content to write to the file
:return: a message indicating success or failure
"""
project_root = self.get_project_root()
@@ -74,11 +74,11 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit):
), f"Cannot create file outside of the project directory, got {relative_path=}"
abs_path.parent.mkdir(parents=True, exist_ok=True)
abs_path.write_text(content, encoding="utf-8")
abs_path.write_text(content, encoding=self.project.project_config.encoding)
answer = f"File created: {relative_path}."
if will_overwrite_existing:
answer += " Overwrote existing file."
return json.dumps(answer)
return answer
class ListDirTool(Tool):
@@ -105,7 +105,7 @@ class ListDirTool(Tool):
"project_root": self.get_project_root(),
"hint": "Check if the path is correct relative to the project root",
}
return json.dumps(error_info)
return self._to_json(error_info)
self.project.validate_relative_path(relative_path, require_not_ignored=skip_ignored_files)
@@ -117,7 +117,7 @@ class ListDirTool(Tool):
is_ignored_file=self.project.is_ignored_path if skip_ignored_files else None,
)
result = json.dumps({"dirs": dirs, "files": files})
result = self._to_json({"dirs": dirs, "files": files})
return self._limit_length(result, max_answer_chars)
@@ -153,57 +153,96 @@ class FindFileTool(Tool):
relative_to=self.get_project_root(),
)
result = json.dumps({"files": files})
result = self._to_json({"files": files})
return result
class ReplaceRegexTool(Tool, ToolMarkerCanEdit):
class ReplaceContentTool(Tool, ToolMarkerCanEdit):
"""
Replaces content in a file by using regular expressions.
Replaces content in a file (optionally using regular expressions).
"""
def apply(
self,
relative_path: str,
regex: str,
needle: str,
repl: str,
mode: Literal["literal", "regex"],
allow_multiple_occurrences: bool = False,
) -> str:
r"""
Replaces one or more occurrences of the given regular expression.
Replaces one or more occurrences of a given pattern in a file with new content.
This is the preferred way to replace content in a file whenever the symbol-level
tools are not appropriate.
Even large sections of code can be replaced by providing a concise regular expression of
the form "beginning.*?end-of-text-to-be-replaced".
Always try to use wildcards to avoid specifying the exact content of the code to be replaced,
especially if it spans several lines.
IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE UNNECESSARILY LONG REGEXES WITHOUT USING WILDCARDS!
VERY IMPORTANT: The "regex" mode allows very large sections of code to be replaced without fully quoting them!
Use a regex of the form "beginning.*?end-of-text-to-be-replaced" to be faster and more economical!
ALWAYS try to use wildcards to avoid specifying the exact content to be replaced,
especially if it spans several lines. Note that you cannot make mistakes, because if the regex should match
multiple occurrences while you disabled `allow_multiple_occurrences`, an error will be returned, and you can retry
with a revised regex.
Therefore, using regex mode with suitable wildcards is usually the best choice!
:param relative_path: the relative path to the file
:param regex: a Python-style regular expression, matches of which will be replaced.
Dot matches all characters, multi-line matching is enabled.
:param repl: the string to replace the matched content with, which may contain
backreferences like \1, \2, etc.
IMPORTANT: Make sure to escape special characters appropriately!
Use "\n" to insert a newline, but use "\\n" to insert the string "\n" within a string literal.
:param needle: the string or regex pattern to search for.
If `mode` is "literal", this string will be matched exactly.
If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module,
with flags DOTALL and MULTILINE enabled).
:param repl: the replacement string (verbatim).
If mode is "regex", the string can contain backreferences to matched groups in the needle regex,
specified using the syntax $!1, $!2, etc. for groups 1, 2, etc.
:param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted.
:param allow_multiple_occurrences: if True, the regex may match multiple occurrences in the file
and all of them will be replaced.
If this is set to False and the regex matches multiple occurrences, an error will be returned
(and you may retry with a revised, more specific regex).
"""
self.project.validate_relative_path(relative_path, require_not_ignored=True)
return self.replace_content(
relative_path, needle, repl, mode=mode, allow_multiple_occurrences=allow_multiple_occurrences, require_not_ignored=True
)
def replace_content(
self,
relative_path: str,
needle: str,
repl: str,
mode: Literal["literal", "regex"],
allow_multiple_occurrences: bool = False,
require_not_ignored: bool = True,
) -> str:
"""
Performs the replacement, with additional options not exposed in the tool.
This function can be used internally by other tools.
"""
self.project.validate_relative_path(relative_path, require_not_ignored=require_not_ignored)
with EditedFileContext(relative_path, self.agent) as context:
original_content = context.get_original_content()
if mode == "literal":
regex = re.escape(needle)
elif mode == "regex":
regex = needle
else:
raise ValueError(f"Invalid mode: '{mode}', expected 'literal' or 'regex'.")
# escape backslashes in repl
repl = repl.replace("\\", "\\\\")
# convert $!1, $!2, etc. to \1, \2, etc. for backreferences
repl = re.sub(r"\$!(\d+)", "\\\\1", repl)
# perform replacement
updated_content, n = re.subn(regex, repl, original_content, flags=re.DOTALL | re.MULTILINE)
if n == 0:
return f"Error: No matches found for regex '{regex}' in file '{relative_path}'."
return f"Error: No matches of search expression found in file '{relative_path}'."
if not allow_multiple_occurrences and n > 1:
return (
f"Error: Regex '{regex}' matches {n} occurrences in file '{relative_path}'. "
"Please revise the regex to be more specific or enable allow_multiple_occurrences if this is expected."
f"Error: Expression matches {n} occurrences in file '{relative_path}'. "
"Please revise the expression to be more specific or enable allow_multiple_occurrences if this is expected."
)
context.set_updated_content(updated_content)
return SUCCESS_RESULT
@@ -390,6 +429,7 @@ class SearchForPatternTool(Tool):
matches = search_files(
rel_paths_to_search,
substring_pattern,
file_reader=self.project.read_file,
root_path=self.get_project_root(),
paths_include_glob=paths_include_glob,
paths_exclude_glob=paths_exclude_glob,
@@ -399,5 +439,5 @@ class SearchForPatternTool(Tool):
for match in matches:
assert match.source_file_path is not None
file_to_matches[match.source_file_path].append(match.to_display_string())
result = json.dumps(file_to_matches)
result = self._to_json(file_to_matches)
return self._limit_length(result, max_answer_chars)
src/serena/tools/jetbrains_plugin_client.py
+23 -2
View File
@@ -136,7 +136,7 @@ class JetBrainsPluginClient(ToStringMixin):
self, name_path: str, relative_path: str | None = None, include_body: bool = False, depth: int = 0, include_location: bool = False
) -> dict[str, Any]:
"""
Find symbols by name.
Finds symbols by name.
:param name_path: the name path to match
:param relative_path: the relative path to which to restrict the search
@@ -156,7 +156,7 @@ class JetBrainsPluginClient(ToStringMixin):
def find_references(self, name_path: str, relative_path: str) -> dict[str, Any]:
"""
Find references to a symbol.
Finds references to a symbol.
:param name_path: the name path of the symbol
:param relative_path: the relative path
@@ -172,6 +172,27 @@ class JetBrainsPluginClient(ToStringMixin):
request_data = {"relativePath": relative_path}
return self._make_request("POST", "/getSymbolsOverview", request_data)
def rename_symbol(
self, name_path: str, relative_path: str, new_name: str, rename_in_comments: bool, rename_in_text_occurrences: bool
) -> None:
"""
Renames a symbol.
:param name_path: the name path of the symbol
:param relative_path: the relative path
:param new_name: the new name for the symbol
:param rename_in_comments: whether to rename in comments
:param rename_in_text_occurrences: whether to rename in text occurrences
"""
request_data = {
"namePath": name_path,
"relativePath": relative_path,
"newName": new_name,
"renameInComments": rename_in_comments,
"renameInTextOccurrences": rename_in_text_occurrences,
}
self._make_request("POST", "/renameSymbol", request_data)
def is_service_available(self) -> bool:
try:
self.project_root()
src/serena/tools/jetbrains_tools.py
+25 -39
View File
@@ -1,56 +1,42 @@
import json
from serena.tools import Tool, ToolMarkerOptional, ToolMarkerSymbolicRead
from serena.tools.jetbrains_plugin_client import JetBrainsPluginClient
class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
"""
Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
Performs a global (or local) search for symbols using the JetBrains backend
"""
def apply(
self,
name_path: str,
name_path_pattern: str,
depth: int = 0,
relative_path: str | None = None,
include_body: bool = False,
max_answer_chars: int = -1,
) -> str:
"""
Retrieves information on all symbols/code entities (classes, methods, etc.) based on the given `name_path`,
which represents a pattern for the symbol's path within the symbol tree of a single file.
The returned symbol location can be used for edits or further queries.
Retrieves information on all symbols/code entities (classes, methods, etc.) based on the given name path pattern.
The returned symbol information can be used for edits or further queries.
Specify `depth > 0` to retrieve children (e.g., methods of a class).
The matching behavior is determined by the structure of `name_path`, which can
either be a simple name (e.g. "method") or a name path like "class/method" (relative name path)
or "/class/method" (absolute name path).
Note that the name path is not a path in the file system but rather a path in the symbol tree
**within a single file**. Thus, file or directory names should never be included in the `name_path`.
For restricting the search to a single file or directory, pass the `relative_path` parameter.
The retrieved symbols' `name_path` attribute will always be composed of symbol names, never file
or directory names.
A name path is a path in the symbol tree *within a source file*.
For example, the method `my_method` defined in class `MyClass` would have the name path `MyClass/my_method`.
If a symbol is overloaded (e.g., in Java), a 0-based index is appended (e.g. "MyClass/my_method[0]") to
uniquely identify it.
Key aspects of the name path matching behavior:
- The name of the retrieved symbols will match the last segment of `name_path`, while preceding segments
will restrict the search to symbols that have a desired sequence of ancestors.
- If there is no `/` in `name_path`, there is no restriction on the ancestor symbols.
For example, passing `method` will match against all symbols with name paths like `method`,
`class/method`, `class/nested_class/method`, etc.
- If `name_path` contains at least one `/`, the matching is restricted to symbols
with the respective ancestors. For example, passing `class/method` will match against
`class/method` as well as `nested_class/class/method` but not `other_class/method`.
- If `name_path` starts with a `/`, it will be treated as an absolute name path pattern, i.e.
all ancestors are provided and must match.
For example, passing `/class` will match only against top-level symbols named `class` but
will not match `nested_class/class`. Passing `/class/method` will match `class/method` but
not `outer_class/class/method`.
To search for a symbol, you provide a name path pattern that is used to match against name paths.
It can be
* a simple name (e.g. "method"), which will match any symbol with that name
* a relative path like "class/method", which will match any symbol with that name path suffix
* an absolute name path "/class/method" (absolute name path), which requires an exact match of the full name path within the source file.
Append an index `[i]` to match a specific overload only, e.g. "MyClass/my_method[1]".
:param name_path: The name path pattern to search for, see above for details.
:param depth: Depth to retrieve descendants (e.g., 1 for class methods/attributes).
:param relative_path: Optional. Restrict search to this file or directory.
If None, searches entire codebase.
:param name_path_pattern: the name path matching pattern (see above)
:param depth: depth up to which descendants shall be retrieved (e.g. use 1 to also retrieve immediate children;
for the case where the symbol is a class, this will return its methods).
Default 0.
:param relative_path: Optional. Restrict search to this file or directory. If None, searches entire codebase.
If a directory is passed, the search will be restricted to the files in that directory.
If a file is passed, the search will be restricted to that file.
If you have some knowledge about the codebase, you should use this parameter, as it will significantly
@@ -62,18 +48,18 @@ class JetBrainsFindSymbolTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
"""
with JetBrainsPluginClient.from_project(self.project) as client:
response_dict = client.find_symbol(
name_path=name_path,
name_path=name_path_pattern,
relative_path=relative_path,
depth=depth,
include_body=include_body,
)
result = json.dumps(response_dict)
result = self._to_json(response_dict)
return self._limit_length(result, max_answer_chars)
class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
"""
Finds symbols that reference the given symbol
Finds symbols that reference the given symbol using the JetBrains backend
"""
def apply(
@@ -98,13 +84,13 @@ class JetBrainsFindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead, ToolMark
name_path=name_path,
relative_path=relative_path,
)
result = json.dumps(response_dict)
result = self._to_json(response_dict)
return self._limit_length(result, max_answer_chars)
class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOptional):
"""
Retrieves an overview of the top-level symbols within a specified file
Retrieves an overview of the top-level symbols within a specified file using the JetBrains backend
"""
def apply(
@@ -128,5 +114,5 @@ class JetBrainsGetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead, ToolMarkerOp
response_dict = client.get_symbols_overview(
relative_path=relative_path,
)
result = json.dumps(response_dict)
result = self._to_json(response_dict)
return self._limit_length(result, max_answer_chars)
src/serena/tools/memory_tools.py
+33 -7
View File
@@ -1,6 +1,6 @@
import json
from typing import Literal
from serena.tools import Tool
from serena.tools import ReplaceContentTool, Tool
class WriteMemoryTool(Tool):
@@ -8,19 +8,21 @@ class WriteMemoryTool(Tool):
Writes a named memory (for future reference) to Serena's project-specific memory store.
"""
def apply(self, memory_name: str, content: str, max_answer_chars: int = -1) -> str:
def apply(self, memory_file_name: str, content: str, max_answer_chars: int = -1) -> str:
"""
Write some information about this project that can be useful for future tasks to a memory in md format.
Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format.
The memory name should be meaningful.
"""
# NOTE: utf-8 encoding is configured in the MemoriesManager
if max_answer_chars == -1:
max_answer_chars = self.agent.serena_config.default_max_tool_answer_chars
if len(content) > max_answer_chars:
raise ValueError(
f"Content for {memory_name} is too long. Max length is {max_answer_chars} characters. " + "Please make the content shorter."
f"Content for {memory_file_name} is too long. Max length is {max_answer_chars} characters. "
+ "Please make the content shorter."
)
return self.memories_manager.save_memory(memory_name, content)
return self.memories_manager.save_memory(memory_file_name, content)
class ReadMemoryTool(Tool):
@@ -47,7 +49,7 @@ class ListMemoriesTool(Tool):
"""
List available memories. Any memory can be read using the `read_memory` tool.
"""
return json.dumps(self.memories_manager.list_memories())
return self._to_json(self.memories_manager.list_memories())
class DeleteMemoryTool(Tool):
@@ -62,3 +64,27 @@ class DeleteMemoryTool(Tool):
or no longer relevant for the project.
"""
return self.memories_manager.delete_memory(memory_file_name)
class EditMemoryTool(Tool):
def apply(
self,
memory_file_name: str,
needle: str,
repl: str,
mode: Literal["literal", "regex"],
) -> str:
r"""
Replaces content matching a regular expression in a memory.
:param memory_file_name: the name of the memory
:param needle: the string or regex pattern to search for.
If `mode` is "literal", this string will be matched exactly.
If `mode` is "regex", this string will be treated as a regular expression (syntax of Python's `re` module,
with flags DOTALL and MULTILINE enabled).
:param repl: the replacement string (verbatim).
:param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted.
"""
replace_content_tool = self.agent.get_tool(ReplaceContentTool)
rel_path = self.memories_manager.get_memory_file_path(memory_file_name).relative_to(self.get_project_root())
return replace_content_tool.replace_content(str(rel_path), needle, repl, mode=mode, require_not_ignored=False)
src/serena/tools/symbol_tools.py
+56 -41
View File
@@ -3,7 +3,6 @@ Language server-related tools
"""
import dataclasses
import json
import os
from collections.abc import Sequence
from copy import copy
@@ -42,7 +41,7 @@ class RestartLanguageServerTool(Tool, ToolMarkerOptional):
"""Use this tool only on explicit user request or after confirmation.
It may be necessary to restart the language server if it hangs.
"""
self.agent.reset_language_server()
self.agent.reset_language_server_manager()
return SUCCESS_RESULT
@@ -73,18 +72,19 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead):
if os.path.isdir(file_path):
raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ")
result = symbol_retriever.get_symbol_overview(relative_path)[relative_path]
result_json_str = json.dumps([dataclasses.asdict(i) for i in result])
result_json_str = self._to_json([dataclasses.asdict(i) for i in result])
return self._limit_length(result_json_str, max_answer_chars)
class FindSymbolTool(Tool, ToolMarkerSymbolicRead):
"""
Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
Performs a global (or local) search using the language server backend.
"""
# noinspection PyDefaultArgument
def apply(
self,
name_path: str,
name_path_pattern: str,
depth: int = 0,
relative_path: str = "",
include_body: bool = False,
@@ -94,38 +94,26 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead):
max_answer_chars: int = -1,
) -> str:
"""
Retrieves information on all symbols/code entities (classes, methods, etc.) based on the given `name_path`,
which represents a pattern for the symbol's path within the symbol tree of a single file.
The returned symbol location can be used for edits or further queries.
Specify `depth > 0` to retrieve children (e.g., methods of a class).
Retrieves information on all symbols/code entities (classes, methods, etc.) based on the given name path pattern.
The returned symbol information can be used for edits or further queries.
Specify `depth > 0` to also retrieve children/descendants (e.g., methods of a class).
The matching behavior is determined by the structure of `name_path`, which can
either be a simple name (e.g. "method") or a name path like "class/method" (relative name path)
or "/class/method" (absolute name path). Note that the name path is not a path in the file system
but rather a path in the symbol tree **within a single file**. Thus, file or directory names should never
be included in the `name_path`. For restricting the search to a single file or directory,
the `within_relative_path` parameter should be used instead. The retrieved symbols' `name_path` attribute
will always be composed of symbol names, never file or directory names.
A name path is a path in the symbol tree *within a source file*.
For example, the method `my_method` defined in class `MyClass` would have the name path `MyClass/my_method`.
If a symbol is overloaded (e.g., in Java), a 0-based index is appended (e.g. "MyClass/my_method[0]") to
uniquely identify it.
Key aspects of the name path matching behavior:
- Trailing slashes in `name_path` play no role and are ignored.
- The name of the retrieved symbols will match (either exactly or as a substring)
the last segment of `name_path`, while other segments will restrict the search to symbols that
have a desired sequence of ancestors.
- If there is no starting or intermediate slash in `name_path`, there is no
restriction on the ancestor symbols. For example, passing `method` will match
against symbols with name paths like `method`, `class/method`, `class/nested_class/method`, etc.
- If `name_path` contains a `/` but doesn't start with a `/`, the matching is restricted to symbols
with the same ancestors as the last segment of `name_path`. For example, passing `class/method` will match against
`class/method` as well as `nested_class/class/method` but not `method`.
- If `name_path` starts with a `/`, it will be treated as an absolute name path pattern, meaning
that the first segment of it must match the first segment of the symbol's name path.
For example, passing `/class` will match only against top-level symbols like `class` but not against `nested_class/class`.
Passing `/class/method` will match against `class/method` but not `nested_class/class/method` or `method`.
To search for a symbol, you provide a name path pattern that is used to match against name paths.
It can be
* a simple name (e.g. "method"), which will match any symbol with that name
* a relative path like "class/method", which will match any symbol with that name path suffix
* an absolute name path "/class/method" (absolute name path), which requires an exact match of the full name path within the source file.
Append an index `[i]` to match a specific overload only, e.g. "MyClass/my_method[1]".
:param name_path: The name path pattern to search for, see above for details.
:param depth: Depth to retrieve descendants (e.g., 1 for class methods/attributes).
:param name_path_pattern: the name path matching pattern (see above)
:param depth: depth up to which descendants shall be retrieved (e.g. use 1 to also retrieve immediate children;
for the case where the symbol is a class, this will return its methods).
Default 0.
:param relative_path: Optional. Restrict search to this file or directory. If None, searches entire codebase.
If a directory is passed, the search will be restricted to the files in that directory.
If a file is passed, the search will be restricted to that file.
@@ -139,7 +127,8 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead):
If not provided, all kinds are included.
:param exclude_kinds: Optional. List of LSP symbol kind integers to exclude. Takes precedence over `include_kinds`.
If not provided, no kinds are excluded.
:param substring_matching: If True, use substring matching for the last segment of `name`.
:param substring_matching: If True, use substring matching for the last element of the pattern, such that
"Foo/get" would match "Foo/getValue" and "Foo/getData".
:param max_answer_chars: Max characters for the JSON result. If exceeded, no content is returned.
-1 means the default value from the config will be used.
:return: a list of symbols (with locations) matching the name.
@@ -148,23 +137,23 @@ class FindSymbolTool(Tool, ToolMarkerSymbolicRead):
parsed_exclude_kinds: Sequence[SymbolKind] | None = [SymbolKind(k) for k in exclude_kinds] if exclude_kinds else None
symbol_retriever = self.create_language_server_symbol_retriever()
symbols = symbol_retriever.find_by_name(
name_path,
include_body=include_body,
name_path_pattern,
include_kinds=parsed_include_kinds,
exclude_kinds=parsed_exclude_kinds,
substring_matching=substring_matching,
within_relative_path=relative_path,
)
symbol_dicts = [_sanitize_symbol_dict(s.to_dict(kind=True, location=True, depth=depth, include_body=include_body)) for s in symbols]
result = json.dumps(symbol_dicts)
result = self._to_json(symbol_dicts)
return self._limit_length(result, max_answer_chars)
class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead):
"""
Finds symbols that reference the symbol at the given location (optionally filtered by type).
Finds symbols that reference the given symbol using the language server backend
"""
# noinspection PyDefaultArgument
def apply(
self,
name_path: str,
@@ -208,13 +197,13 @@ class FindReferencingSymbolsTool(Tool, ToolMarkerSymbolicRead):
)
ref_dict["content_around_reference"] = content_around_ref.to_display_string()
reference_dicts.append(ref_dict)
result = json.dumps(reference_dicts)
result = self._to_json(reference_dicts)
return self._limit_length(result, max_answer_chars)
class ReplaceSymbolBodyTool(Tool, ToolMarkerSymbolicEdit):
"""
Replaces the full definition of a symbol.
Replaces the full definition of a symbol using the language server backend.
"""
def apply(
@@ -293,3 +282,29 @@ class InsertBeforeSymbolTool(Tool, ToolMarkerSymbolicEdit):
code_editor = self.create_code_editor()
code_editor.insert_before_symbol(name_path, relative_file_path=relative_path, body=body)
return SUCCESS_RESULT
class RenameSymbolTool(Tool, ToolMarkerSymbolicEdit):
"""
Renames a symbol throughout the codebase using language server refactoring capabilities.
"""
def apply(
self,
name_path: str,
relative_path: str,
new_name: str,
) -> str:
"""
Renames the symbol with the given `name_path` to `new_name` throughout the entire codebase.
Note: for languages with method overloading, like Java, name_path may have to include a method's
signature to uniquely identify a method.
:param name_path: name path of the symbol to rename (definitions in the `find_symbol` tool apply)
:param relative_path: the relative path to the file containing the symbol to rename
:param new_name: the new name for the symbol
:return: result summary indicating success or failure
"""
code_editor = self.create_code_editor()
status_message = code_editor.rename_symbol(name_path, relative_file_path=relative_path, new_name=new_name)
return status_message
src/serena/tools/tools_base.py
+36 -20
View File
@@ -1,4 +1,5 @@
import inspect
import json
import os
from abc import ABC
from collections.abc import Iterable
@@ -10,7 +11,7 @@ from mcp.server.fastmcp.utilities.func_metadata import FuncMetadata, func_metada
from sensai.util import logging
from sensai.util.string import dict_string
from serena.project import Project
from serena.project import MemoriesManager, Project
from serena.prompt_factory import PromptFactory
from serena.symbol import LanguageServerSymbolRetriever
from serena.util.class_decorators import singleton
@@ -18,7 +19,7 @@ from serena.util.inspection import iter_subclasses
from solidlsp.ls_exceptions import SolidLSPException
if TYPE_CHECKING:
from serena.agent import MemoriesManager, SerenaAgent
from serena.agent import SerenaAgent
from serena.code_editor import CodeEditor
log = logging.getLogger(__name__)
@@ -42,15 +43,13 @@ class Component(ABC):
@property
def memories_manager(self) -> "MemoriesManager":
assert self.agent.memories_manager is not None
return self.agent.memories_manager
return self.project.memories_manager
def create_language_server_symbol_retriever(self) -> LanguageServerSymbolRetriever:
if not self.agent.is_using_language_server():
raise Exception("Cannot create LanguageServerSymbolRetriever; agent is not in language server mode.")
language_server = self.agent.language_server
assert language_server is not None
return LanguageServerSymbolRetriever(language_server, agent=self.agent)
language_server_manager = self.agent.get_language_server_manager_or_raise()
return LanguageServerSymbolRetriever(language_server_manager, agent=self.agent)
@property
def project(self) -> Project:
@@ -240,33 +239,39 @@ class Tool(Component):
try:
# check whether the tool requires an active project and language server
if not isinstance(self, ToolMarkerDoesNotRequireActiveProject):
if self.agent._active_project is None:
if self.agent.get_active_project() is None:
return (
"Error: No active project. Ask the user to provide the project path or to select a project from this list of known projects: "
+ f"{self.agent.serena_config.project_names}"
)
if self.agent.is_using_language_server() and not self.agent.is_language_server_running():
log.info("Language server is not running. Starting it ...")
self.agent.reset_language_server()
# apply the actual tool
try:
result = apply_fn(**kwargs)
except SolidLSPException as e:
if e.is_language_server_terminated():
log.error(f"Language server terminated while executing tool ({e}). Restarting the language server and retrying ...")
self.agent.reset_language_server()
result = apply_fn(**kwargs)
affected_language = e.get_affected_language()
if affected_language is not None:
log.error(
f"Language server terminated while executing tool ({e}). Restarting the language server and retrying ..."
)
self.agent.get_language_server_manager_or_raise().restart_language_server(affected_language)
result = apply_fn(**kwargs)
else:
log.error(
f"Language server terminated while executing tool ({e}), but affected language is unknown. Not retrying."
)
raise
else:
raise
# record tool usage
self.agent.record_tool_usage_if_enabled(kwargs, result, self)
self.agent.record_tool_usage(kwargs, result, self)
except Exception as e:
if not catch_exceptions:
raise
msg = f"Error executing tool: {e}"
msg = f"Error executing tool: {e.__class__.__name__} - {e}"
log.error(f"Error executing tool: {e}", exc_info=e)
result = msg
@@ -274,15 +279,26 @@ class Tool(Component):
log.info(f"Result: {result}")
try:
if self.agent.language_server is not None:
self.agent.language_server.save_cache()
ls_manager = self.agent.get_language_server_manager()
if ls_manager is not None:
ls_manager.save_all_caches()
except Exception as e:
log.error(f"Error saving language server cache: {e}")
return result
future = self.agent.issue_task(task, name=self.__class__.__name__)
return future.result(timeout=self.agent.serena_config.tool_timeout)
# execute the tool in the agent's task executor, with timeout
try:
task_exec = self.agent.issue_task(task, name=self.__class__.__name__)
return task_exec.result(timeout=self.agent.serena_config.tool_timeout)
except Exception as e: # typically TimeoutError (other exceptions caught in task)
msg = f"Error: {e.__class__.__name__} - {e}"
log.error(msg)
return msg
@staticmethod
def _to_json(x: Any) -> str:
return json.dumps(x, ensure_ascii=False)
class EditedFileContext:
src/serena/tools/workflow_tools.py
+9 -10
View File
@@ -16,8 +16,7 @@ class CheckOnboardingPerformedTool(Tool):
def apply(self) -> str:
"""
Checks whether project onboarding was already performed.
You should always call this tool before beginning to actually work on the project/after activating a project,
but after calling the initial instructions tool.
You should always call this tool before beginning to actually work on the project/after activating a project.
"""
from .memory_tools import ListMemoriesTool
@@ -122,18 +121,18 @@ class PrepareForNewConversationTool(Tool):
return self.prompt_factory.create_prepare_for_new_conversation()
class InitialInstructionsTool(Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional):
class InitialInstructionsTool(Tool, ToolMarkerDoesNotRequireActiveProject):
"""
Gets the initial instructions for the current project.
Should only be used in settings where the system prompt cannot be set,
e.g. in clients you have no control over, like Claude Desktop.
Provides instructions on how to use the Serena toolbox.
Should only be used in settings where the system prompt is not read automatically by the client.
NOTE: Some MCP clients (including Claude Desktop) do not read the system prompt automatically!
"""
def apply(self) -> str:
"""
Get the initial instructions for the current coding project.
If you haven't received instructions on how to use Serena's tools in the system prompt,
you should always call this tool before starting to work (including using any other tool) on any programming task,
the only exception being when you are asked to call `activate_project`, which you should then call before.
Provides the 'Serena Instructions Manual', which contains essential information on how to use the Serena toolbox.
IMPORTANT: If you have not yet read the manual, call this tool immediately after you are given your task by the user,
as it will critically inform you!
"""
return self.agent.create_system_prompt()
src/serena/util/cli_util.py
+12
View File
@@ -0,0 +1,12 @@
def ask_yes_no(question: str, default: bool | None = None) -> bool:
default_prompt = "Y/n" if default else "y/N"
while True:
answer = input(f"{question} [{default_prompt}] ").strip().lower()
if answer == "" and default is not None:
return default
if answer in ("y", "yes"):
return True
if answer in ("n", "no"):
return False
print("Please answer yes/y or no/n.")
src/serena/util/general.py
+4 -2
View File
@@ -4,6 +4,8 @@ from typing import Literal, overload
from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap
from serena.constants import SERENA_FILE_ENCODING
def _create_YAML(preserve_comments: bool = False) -> YAML:
"""
@@ -20,7 +22,7 @@ def load_yaml(path: str, preserve_comments: Literal[False]) -> dict: ...
@overload
def load_yaml(path: str, preserve_comments: Literal[True]) -> CommentedMap: ...
def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap:
with open(path, encoding="utf-8") as f:
with open(path, encoding=SERENA_FILE_ENCODING) as f:
yaml = _create_YAML(preserve_comments)
return yaml.load(f)
@@ -28,5 +30,5 @@ def load_yaml(path: str, preserve_comments: bool = False) -> dict | CommentedMap
def save_yaml(path: str, data: dict | CommentedMap, preserve_comments: bool = False) -> None:
yaml = _create_YAML(preserve_comments)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
with open(path, "w", encoding=SERENA_FILE_ENCODING) as f:
yaml.dump(data, f)
src/serena/util/logging.py
+42
View File
@@ -1,11 +1,14 @@
import queue
import threading
from collections.abc import Callable
from typing import Optional
from sensai.util import logging
from serena.constants import SERENA_LOG_FORMAT
lg = logging
class MemoryLogHandler(logging.Handler):
def __init__(self, level: int = logging.NOTSET) -> None:
@@ -65,3 +68,42 @@ class LogBuffer:
def get_log_messages(self) -> list[str]:
with self._lock:
return self._log_messages.copy()
class SuspendedLoggersContext:
"""A context manager that provides an isolated logging environment.
Temporarily removes all root log handlers upon entry, providing a clean slate
for defining new log handlers within the context. Upon exit, restores the original
logging configuration. This is useful when you need to temporarily configure
an isolated logging setup with well-defined log handlers.
The context manager:
- Removes all existing (root) log handlers on entry
- Allows defining new temporary handlers within the context
- Restores the original configuration (handlers and root log level) on exit
Example:
>>> with SuspendedLoggersContext():
... # No handlers are active here (configure your own and set desired log level)
... pass
>>> # Original log handlers are restored here
"""
def __init__(self) -> None:
self.saved_root_handlers: list = []
self.saved_root_level: Optional[int] = None
def __enter__(self) -> "SuspendedLoggersContext":
root_logger = lg.getLogger()
self.saved_root_handlers = root_logger.handlers.copy()
self.saved_root_level = root_logger.level
root_logger.handlers.clear()
return self
def __exit__(self, exc_type, exc_val, exc_tb) -> None: # type: ignore
root_logger = lg.getLogger()
root_logger.handlers = self.saved_root_handlers
if self.saved_root_level is not None:
root_logger.setLevel(self.saved_root_level)
src/solidlsp/language_servers/al_language_server.py
+93 -103
View File
@@ -15,11 +15,13 @@ from overrides import override
from solidlsp.language_servers.common import quote_windows_path
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.ls_types import SymbolKind, UnifiedSymbolInformation
from solidlsp.lsp_protocol_handler.lsp_types import Definition, DefinitionParams, LocationLink
from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
from solidlsp.settings import SolidLSPSettings
log = logging.getLogger(__name__)
class ALLanguageServer(SolidLanguageServer):
"""
@@ -36,9 +38,7 @@ class ALLanguageServer(SolidLanguageServer):
- File opening requirement before symbol retrieval
"""
def __init__(
self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings
):
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""
Initialize the AL Language Server.
@@ -57,7 +57,7 @@ class ALLanguageServer(SolidLanguageServer):
"""
# Setup runtime dependencies and get the language server command
# This will download the AL extension if needed
cmd = self._setup_runtime_dependencies(logger, config, solidlsp_settings)
cmd = self._setup_runtime_dependencies(config, solidlsp_settings)
self._project_load_check_supported: bool = True
"""Whether the AL server supports the project load status check request.
@@ -67,17 +67,10 @@ class ALLanguageServer(SolidLanguageServer):
request fails, preventing repeated unsuccessful attempts.
"""
super().__init__(
config,
logger,
repository_root_path,
ProcessLaunchInfo(cmd=cmd, cwd=repository_root_path),
"al", # Language ID for LSP
solidlsp_settings,
)
super().__init__(config, repository_root_path, ProcessLaunchInfo(cmd=cmd, cwd=repository_root_path), "al", solidlsp_settings)
@classmethod
def _download_al_extension(cls, logger: LanguageServerLogger, url: str, target_dir: str) -> bool:
def _download_al_extension(cls, url: str, target_dir: str) -> bool:
"""
Download and extract the AL extension from VS Code marketplace.
@@ -98,7 +91,7 @@ class ALLanguageServer(SolidLanguageServer):
"""
try:
logger.log(f"Downloading AL extension from {url}", logging.INFO)
log.info(f"Downloading AL extension from {url}")
# Create target directory for the extension
os.makedirs(target_dir, exist_ok=True)
@@ -117,7 +110,7 @@ class ALLanguageServer(SolidLanguageServer):
temp_file = os.path.join(target_dir, "al_extension_temp.vsix")
total_size = int(response.headers.get("content-length", 0))
logger.log(f"Downloading {total_size / 1024 / 1024:.1f} MB...", logging.INFO)
log.info(f"Downloading {total_size / 1024 / 1024:.1f} MB...")
with open(temp_file, "wb") as f:
downloaded = 0
@@ -127,9 +120,9 @@ class ALLanguageServer(SolidLanguageServer):
downloaded += len(chunk)
if total_size > 0 and downloaded % (10 * 1024 * 1024) == 0: # Log progress every 10MB
progress = (downloaded / total_size) * 100
logger.log(f"Download progress: {progress:.1f}%", logging.INFO)
log.info(f"Download progress: {progress:.1f}%")
logger.log("Download complete, extracting...", logging.INFO)
log.info("Download complete, extracting...")
# Extract VSIX file (VSIX files are just ZIP archives with a different extension)
# This will extract the extension folder containing the language server binaries
@@ -139,17 +132,15 @@ class ALLanguageServer(SolidLanguageServer):
# Clean up temp file
os.remove(temp_file)
logger.log("AL extension extracted successfully", logging.INFO)
log.info("AL extension extracted successfully")
return True
except Exception as e:
logger.log(f"Error downloading/extracting AL extension: {e}", logging.ERROR)
log.error(f"Error downloading/extracting AL extension: {e}")
return False
@classmethod
def _setup_runtime_dependencies(
cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings
) -> str:
def _setup_runtime_dependencies(cls, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings) -> str:
"""
Setup runtime dependencies for AL Language Server and return the command to start the server.
@@ -167,10 +158,10 @@ class ALLanguageServer(SolidLanguageServer):
system = platform.system()
# Find existing extension or download if needed
extension_path = cls._find_al_extension(logger, solidlsp_settings)
extension_path = cls._find_al_extension(solidlsp_settings)
if extension_path is None:
logger.log("AL extension not found on disk, attempting to download...", logging.INFO)
extension_path = cls._download_and_install_al_extension(logger, solidlsp_settings)
log.info("AL extension not found on disk, attempting to download...")
extension_path = cls._download_and_install_al_extension(solidlsp_settings)
if extension_path is None:
raise RuntimeError(
@@ -187,10 +178,10 @@ class ALLanguageServer(SolidLanguageServer):
raise RuntimeError(f"AL Language Server executable not found at: {executable_path}")
# Prepare and return the executable command
return cls._prepare_executable(executable_path, system, logger)
return cls._prepare_executable(executable_path, system)
@classmethod
def _find_al_extension(cls, logger: LanguageServerLogger, solidlsp_settings: SolidLSPSettings) -> str | None:
def _find_al_extension(cls, solidlsp_settings: SolidLSPSettings) -> str | None:
"""
Find AL extension in various locations.
@@ -206,28 +197,28 @@ class ALLanguageServer(SolidLanguageServer):
# Check environment variable
env_path = os.environ.get("AL_EXTENSION_PATH")
if env_path and os.path.exists(env_path):
logger.log(f"Found AL extension via AL_EXTENSION_PATH: {env_path}", logging.DEBUG)
log.debug(f"Found AL extension via AL_EXTENSION_PATH: {env_path}")
return env_path
elif env_path:
logger.log(f"AL_EXTENSION_PATH set but directory not found: {env_path}", logging.WARNING)
log.warning(f"AL_EXTENSION_PATH set but directory not found: {env_path}")
# Check default download location
default_path = os.path.join(cls.ls_resources_dir(solidlsp_settings), "al-extension", "extension")
if os.path.exists(default_path):
logger.log(f"Found AL extension in default location: {default_path}", logging.DEBUG)
log.debug(f"Found AL extension in default location: {default_path}")
return default_path
# Search VS Code extensions
vscode_path = cls._find_al_extension_in_vscode(logger)
vscode_path = cls._find_al_extension_in_vscode()
if vscode_path:
logger.log(f"Found AL extension in VS Code: {vscode_path}", logging.DEBUG)
log.debug(f"Found AL extension in VS Code: {vscode_path}")
return vscode_path
logger.log("AL extension not found in any known location", logging.DEBUG)
log.debug("AL extension not found in any known location")
return None
@classmethod
def _download_and_install_al_extension(cls, logger: LanguageServerLogger, solidlsp_settings: SolidLSPSettings) -> str | None:
def _download_and_install_al_extension(cls, solidlsp_settings: SolidLSPSettings) -> str | None:
"""
Download and install AL extension from VS Code marketplace.
@@ -241,17 +232,17 @@ class ALLanguageServer(SolidLanguageServer):
AL_VERSION = "latest"
url = f"https://marketplace.visualstudio.com/_apis/public/gallery/publishers/ms-dynamics-smb/vsextensions/al/{AL_VERSION}/vspackage"
logger.log(f"Downloading AL extension from: {url}", logging.INFO)
log.info(f"Downloading AL extension from: {url}")
if cls._download_al_extension(logger, url, al_extension_dir):
if cls._download_al_extension(url, al_extension_dir):
extension_path = os.path.join(al_extension_dir, "extension")
if os.path.exists(extension_path):
logger.log("AL extension downloaded and installed successfully", logging.INFO)
log.info("AL extension downloaded and installed successfully")
return extension_path
else:
logger.log(f"Download completed but extension not found at: {extension_path}", logging.ERROR)
log.error(f"Download completed but extension not found at: {extension_path}")
else:
logger.log("Failed to download AL extension from marketplace", logging.ERROR)
log.error("Failed to download AL extension from marketplace")
return None
@@ -278,7 +269,7 @@ class ALLanguageServer(SolidLanguageServer):
raise RuntimeError(f"Unsupported platform: {system}")
@classmethod
def _prepare_executable(cls, executable_path: str, system: str, logger: LanguageServerLogger) -> str:
def _prepare_executable(cls, executable_path: str, system: str) -> str:
"""
Prepare the executable by setting permissions and handling path quoting.
@@ -295,16 +286,16 @@ class ALLanguageServer(SolidLanguageServer):
if system in ["Linux", "Darwin"]:
st = os.stat(executable_path)
os.chmod(executable_path, st.st_mode | stat.S_IEXEC)
logger.log(f"Set execute permission on: {executable_path}", logging.DEBUG)
log.debug(f"Set execute permission on: {executable_path}")
logger.log(f"Using AL Language Server executable: {executable_path}", logging.INFO)
log.info(f"Using AL Language Server executable: {executable_path}")
# The AL Language Server uses stdio for LSP communication by default
# Use the utility function to handle Windows path quoting
return quote_windows_path(executable_path)
@classmethod
def _get_language_server_command_fallback(cls, logger: LanguageServerLogger) -> str:
def _get_language_server_command_fallback(cls) -> str:
"""
Get the command to start the AL language server.
@@ -332,10 +323,10 @@ class ALLanguageServer(SolidLanguageServer):
if potential_extension:
al_extension_path = str(potential_extension)
logger.log(f"Found AL extension in current directory: {al_extension_path}", logging.DEBUG)
log.debug(f"Found AL extension in current directory: {al_extension_path}")
else:
# Try to find in common VS Code extension locations
al_extension_path = cls._find_al_extension_in_vscode(logger)
al_extension_path = cls._find_al_extension_in_vscode()
if not al_extension_path:
raise RuntimeError(
@@ -367,14 +358,14 @@ class ALLanguageServer(SolidLanguageServer):
st = os.stat(executable)
os.chmod(executable, st.st_mode | stat.S_IEXEC)
logger.log(f"Using AL Language Server executable: {executable}", logging.INFO)
log.info(f"Using AL Language Server executable: {executable}")
# The AL Language Server uses stdio for LSP communication (no --stdio flag needed)
# Use the utility function to handle Windows path quoting
return quote_windows_path(executable)
@classmethod
def _find_al_extension_in_vscode(cls, logger: LanguageServerLogger) -> str | None:
def _find_al_extension_in_vscode(cls) -> str | None:
"""
Try to find AL extension in common VS Code extension locations.
@@ -406,11 +397,11 @@ class ALLanguageServer(SolidLanguageServer):
for base_path in possible_paths:
if base_path.exists():
logger.log(f"Searching for AL extension in: {base_path}", logging.DEBUG)
log.debug(f"Searching for AL extension in: {base_path}")
# Look for AL extension directories
for item in base_path.iterdir():
if item.is_dir() and item.name.startswith("ms-dynamics-smb.al-"):
logger.log(f"Found AL extension at: {item}", logging.DEBUG)
log.debug(f"Found AL extension at: {item}")
return str(item)
return None
@@ -485,7 +476,7 @@ class ALLanguageServer(SolidLanguageServer):
return initialize_params
@override
def _start_server(self):
def _start_server(self) -> None:
"""
Starts the AL Language Server process and initializes it.
@@ -495,21 +486,21 @@ class ALLanguageServer(SolidLanguageServer):
"""
# Set up event handlers
def do_nothing(params):
def do_nothing(params: str) -> None:
return
def window_log_message(msg):
self.logger.log(f"AL LSP: window/logMessage: {msg}", logging.INFO)
def window_log_message(msg: dict) -> None:
log.info(f"AL LSP: window/logMessage: {msg}")
def publish_diagnostics(params):
def publish_diagnostics(params: dict) -> None:
# AL server publishes diagnostics during initialization
uri = params.get("uri", "")
diagnostics = params.get("diagnostics", [])
self.logger.log(f"AL LSP: Diagnostics for {uri}: {len(diagnostics)} issues", logging.DEBUG)
log.debug(f"AL LSP: Diagnostics for {uri}: {len(diagnostics)} issues")
def handle_al_notifications(params):
def handle_al_notifications(params: dict) -> None:
# AL server sends custom notifications during project loading
self.logger.log("AL LSP: Notification received", logging.DEBUG)
log.debug("AL LSP: Notification received")
# Register handlers for AL-specific notifications
# These notifications are sent by the AL server during initialization and operation
@@ -519,27 +510,24 @@ class ALLanguageServer(SolidLanguageServer):
self.server.on_notification("al/refreshExplorerObjects", handle_al_notifications) # AL-specific object updates
# Start the server process
self.logger.log("Starting AL Language Server process", logging.INFO)
log.info("Starting AL Language Server process")
self.server.start()
# Send initialize request
initialize_params = self._get_initialize_params(self.repository_root_path)
self.logger.log(
"Sending initialize request from LSP client to AL LSP server and awaiting response",
logging.INFO,
)
log.info("Sending initialize request from LSP client to AL LSP server and awaiting response")
# Send initialize and wait for response
resp = self.server.send_request("initialize", initialize_params)
if resp is None:
raise RuntimeError("AL Language Server initialization failed - no response")
self.logger.log("AL Language Server initialized successfully", logging.INFO)
log.info("AL Language Server initialized successfully")
# Send initialized notification
self.server.send_notification("initialized", {})
self.logger.log("Sent initialized notification", logging.INFO)
log.info("Sent initialized notification")
@override
def start(self) -> "ALLanguageServer":
@@ -597,9 +585,9 @@ class ALLanguageServer(SolidLanguageServer):
}
},
)
self.logger.log("Sent workspace configuration", logging.DEBUG)
log.debug("Sent workspace configuration")
except Exception as e:
self.logger.log(f"Failed to send workspace config: {e}", logging.WARNING)
log.warning(f"Failed to send workspace config: {e}")
# Check if app.json exists and open it
# app.json is the AL project manifest file (similar to package.json for Node.js)
@@ -619,9 +607,9 @@ class ALLanguageServer(SolidLanguageServer):
{"textDocument": {"uri": app_json_uri, "languageId": "json", "version": 1, "text": app_json_content}},
)
self.logger.log(f"Opened app.json: {app_json_uri}", logging.DEBUG)
log.debug(f"Opened app.json: {app_json_uri}")
except Exception as e:
self.logger.log(f"Failed to open app.json: {e}", logging.WARNING)
log.warning(f"Failed to open app.json: {e}")
# Try to set active workspace (AL-specific custom LSP request)
# This is optional and may not be supported by all AL server versions
@@ -635,13 +623,13 @@ class ALLanguageServer(SolidLanguageServer):
"workspacePath": self.repository_root_path,
"setActiveWorkspace": True,
},
"timeout": 2, # Quick timeout since this is optional
},
timeout=2, # Quick timeout since this is optional
)
self.logger.log(f"Set active workspace result: {result}", logging.DEBUG)
log.debug(f"Set active workspace result: {result}")
except Exception as e:
# This is a custom AL request, not critical if it fails
self.logger.log(f"Failed to set active workspace (non-critical): {e}", logging.DEBUG)
log.debug(f"Failed to set active workspace (non-critical): {e}")
# Check if project supports load status check (optional)
# Many AL server versions don't support this, so we use a short timeout
@@ -680,7 +668,7 @@ class ALLanguageServer(SolidLanguageServer):
return super().is_ignored_dirname(dirname) or dirname in al_ignore_dirs
@override
def request_full_symbol_tree(self, within_relative_path: str | None = None, include_body: bool = False) -> list[dict]:
def request_full_symbol_tree(self, within_relative_path: str | None = None) -> list[UnifiedSymbolInformation]:
"""
Override to handle AL's requirement of opening files before requesting symbols.
@@ -703,7 +691,7 @@ class ALLanguageServer(SolidLanguageServer):
Full symbol tree with all AL symbols from opened files organized by directory
"""
self.logger.log("AL: Starting request_full_symbol_tree with file opening", logging.DEBUG)
log.debug("AL: Starting request_full_symbol_tree with file opening")
# Determine the root path for scanning
if within_relative_path is not None:
@@ -713,7 +701,7 @@ class ALLanguageServer(SolidLanguageServer):
if os.path.isfile(within_abs_path):
# Single file case - use parent class implementation
_, root_nodes = self.request_document_symbols(within_relative_path, include_body=include_body)
root_nodes = self.request_document_symbols(within_relative_path).root_symbols
return root_nodes
# Directory case - scan within this directory
@@ -742,26 +730,27 @@ class ALLanguageServer(SolidLanguageServer):
# File is outside repository root, skip it
continue
self.logger.log(f"AL: Found {len(al_files)} AL files", logging.DEBUG)
log.debug(f"AL: Found {len(al_files)} AL files")
if not al_files:
self.logger.log("AL: No AL files found in repository", logging.WARNING)
log.warning("AL: No AL files found in repository")
return []
# Collect all symbols from all files
all_file_symbols = []
all_file_symbols: list[UnifiedSymbolInformation] = []
file_symbol: UnifiedSymbolInformation
for file_path, relative_path in al_files:
try:
# Use our overridden request_document_symbols which handles opening
self.logger.log(f"AL: Getting symbols for {relative_path}", logging.DEBUG)
all_syms, root_syms = self.request_document_symbols(relative_path, include_body=include_body)
log.debug(f"AL: Getting symbols for {relative_path}")
all_syms, root_syms = self.request_document_symbols(relative_path).get_all_symbols_and_roots()
if root_syms:
# Create a file-level symbol containing the document symbols
file_symbol = {
"name": file_path.stem, # Just the filename without extension
"kind": 1, # File
"kind": SymbolKind.File,
"children": root_syms,
"location": {
"uri": file_path.as_uri(),
@@ -771,12 +760,12 @@ class ALLanguageServer(SolidLanguageServer):
},
}
all_file_symbols.append(file_symbol)
self.logger.log(f"AL: Added {len(root_syms)} symbols from {relative_path}", logging.DEBUG)
log.debug(f"AL: Added {len(root_syms)} symbols from {relative_path}")
elif all_syms:
# If we only got all_syms but not root, use all_syms
file_symbol = {
"name": file_path.stem,
"kind": 1, # File
"kind": SymbolKind.File,
"children": all_syms,
"location": {
"uri": file_path.as_uri(),
@@ -786,19 +775,20 @@ class ALLanguageServer(SolidLanguageServer):
},
}
all_file_symbols.append(file_symbol)
self.logger.log(f"AL: Added {len(all_syms)} symbols from {relative_path}", logging.DEBUG)
log.debug(f"AL: Added {len(all_syms)} symbols from {relative_path}")
except Exception as e:
self.logger.log(f"AL: Failed to get symbols for {relative_path}: {e}", logging.WARNING)
log.warning(f"AL: Failed to get symbols for {relative_path}: {e}")
if all_file_symbols:
self.logger.log(f"AL: Returning symbols from {len(all_file_symbols)} files", logging.DEBUG)
log.debug(f"AL: Returning symbols from {len(all_file_symbols)} files")
# Group files by directory
directory_structure = {}
directory_structure: dict[str, list] = {}
for file_symbol in all_file_symbols:
rel_path = file_symbol["location"]["relativePath"]
assert rel_path is not None
path_parts = rel_path.split("/")
if len(path_parts) > 1:
@@ -824,7 +814,7 @@ class ALLanguageServer(SolidLanguageServer):
# Create directory symbol
dir_symbol = {
"name": Path(dir_path).name,
"kind": 4, # Package/Directory
"kind": SymbolKind.Package, # Package/Directory
"children": file_symbols,
"location": {
"relativePath": dir_path,
@@ -836,7 +826,7 @@ class ALLanguageServer(SolidLanguageServer):
return result
else:
self.logger.log("AL: No symbols found in any files", logging.WARNING)
log.warning("AL: No symbols found in any files")
return []
# ===== Phase 1: Custom AL Command Implementations =====
@@ -859,10 +849,10 @@ class ALLanguageServer(SolidLanguageServer):
try:
# Use custom AL command instead of standard LSP
response = self.server.send_request("al/gotodefinition", al_params)
self.logger.log(f"AL gotodefinition response: {response}", logging.DEBUG)
return response
log.debug(f"AL gotodefinition response: {response}")
return response # type: ignore[return-value]
except Exception as e:
self.logger.log(f"Failed to use al/gotodefinition, falling back to standard: {e}", logging.WARNING)
log.warning(f"Failed to use al/gotodefinition, falling back to standard: {e}")
# Fallback to standard LSP method if custom command fails
return super()._send_definition_request(definition_params)
@@ -880,7 +870,7 @@ class ALLanguageServer(SolidLanguageServer):
"""
if not hasattr(self, "server") or not self.server_started:
self.logger.log("Cannot check project load - server not started", logging.DEBUG)
log.debug("Cannot check project load - server not started")
return False
# Check if we've already determined this request isn't supported
@@ -889,7 +879,7 @@ class ALLanguageServer(SolidLanguageServer):
try:
# Use a very short timeout since this is just a status check
response = self.server.send_request("al/hasProjectClosureLoadedRequest", {}, timeout=1)
response = self.server.send_request("al/hasProjectClosureLoadedRequest", {"timeout": 1})
# Response can be boolean directly, dict with 'loaded' field, or None
if isinstance(response, bool):
return response
@@ -897,15 +887,15 @@ class ALLanguageServer(SolidLanguageServer):
return response.get("loaded", False)
elif response is None:
# None typically means the project is still loading
self.logger.log("Project load check returned None", logging.DEBUG)
log.debug("Project load check returned None")
return False
else:
self.logger.log(f"Unexpected response type for project load check: {type(response)}", logging.DEBUG)
log.debug(f"Unexpected response type for project load check: {type(response)}")
return False
except Exception as e:
# Mark as unsupported to avoid repeated failed attempts
self._project_load_check_supported = False
self.logger.log(f"Project load check not supported by this AL server version: {e}", logging.DEBUG)
log.debug(f"Project load check not supported by this AL server version: {e}")
# Assume loaded if we can't check
return True
@@ -925,16 +915,16 @@ class ALLanguageServer(SolidLanguageServer):
"""
start_time = time.time()
self.logger.log(f"Checking AL project load status (timeout: {timeout}s)...", logging.DEBUG)
log.debug(f"Checking AL project load status (timeout: {timeout}s)...")
while time.time() - start_time < timeout:
if self.check_project_loaded():
elapsed = time.time() - start_time
self.logger.log(f"AL project fully loaded after {elapsed:.1f}s", logging.INFO)
log.info(f"AL project fully loaded after {elapsed:.1f}s")
return True
time.sleep(0.5)
self.logger.log(f"Project load check timed out after {timeout}s (non-critical)", logging.DEBUG)
log.debug(f"Project load check timed out after {timeout}s (non-critical)")
return False
def set_active_workspace(self, workspace_uri: str | None = None) -> None:
@@ -953,7 +943,7 @@ class ALLanguageServer(SolidLanguageServer):
"""
if not hasattr(self, "server") or not self.server_started:
self.logger.log("Cannot set active workspace - server not started", logging.DEBUG)
log.debug("Cannot set active workspace - server not started")
return
if workspace_uri is None:
@@ -963,7 +953,7 @@ class ALLanguageServer(SolidLanguageServer):
try:
self.server.send_request("al/setActiveWorkspace", params)
self.logger.log(f"Set active workspace to: {workspace_uri}", logging.INFO)
log.info(f"Set active workspace to: {workspace_uri}")
except Exception as e:
self.logger.log(f"Failed to set active workspace: {e}", logging.WARNING)
log.warning(f"Failed to set active workspace: {e}")
# Non-critical error, continue operation
src/solidlsp/language_servers/bash_language_server.py
+38 -61
View File
@@ -9,15 +9,15 @@ import pathlib
import shutil
import threading
from solidlsp import ls_types
from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls import DocumentSymbols, LSPFileBuffer, SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
from solidlsp.settings import SolidLSPSettings
log = logging.getLogger(__name__)
class BashLanguageServer(SolidLanguageServer):
"""
@@ -25,17 +25,14 @@ class BashLanguageServer(SolidLanguageServer):
Contains various configurations and settings specific to Bash scripting.
"""
def __init__(
self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings
):
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""
Creates a BashLanguageServer instance. This class is not meant to be instantiated directly.
Use LanguageServer.create() instead.
"""
bash_lsp_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings)
bash_lsp_executable_path = self._setup_runtime_dependencies(config, solidlsp_settings)
super().__init__(
config,
logger,
repository_root_path,
ProcessLaunchInfo(cmd=bash_lsp_executable_path, cwd=repository_root_path),
"bash",
@@ -45,9 +42,7 @@ class BashLanguageServer(SolidLanguageServer):
self.initialize_searcher_command_available = threading.Event()
@classmethod
def _setup_runtime_dependencies(
cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings
) -> str:
def _setup_runtime_dependencies(cls, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings) -> str:
"""
Setup runtime dependencies for Bash Language Server and return the command to start the server.
"""
@@ -77,9 +72,9 @@ class BashLanguageServer(SolidLanguageServer):
bash_executable_path += ".cmd"
if not os.path.exists(bash_executable_path):
logger.log(f"Bash Language Server executable not found at {bash_executable_path}. Installing...", logging.INFO)
deps.install(logger, bash_ls_dir)
logger.log("Bash language server dependencies installed successfully", logging.INFO)
log.info(f"Bash Language Server executable not found at {bash_executable_path}. Installing...")
deps.install(bash_ls_dir)
log.info("Bash language server dependencies installed successfully")
if not os.path.exists(bash_executable_path):
raise FileNotFoundError(
@@ -126,32 +121,32 @@ class BashLanguageServer(SolidLanguageServer):
}
],
}
return initialize_params
return initialize_params # type: ignore
def _start_server(self):
def _start_server(self) -> None:
"""
Starts the Bash Language Server, waits for the server to be ready and yields the LanguageServer instance.
"""
def register_capability_handler(params):
def register_capability_handler(params: dict) -> None:
assert "registrations" in params
for registration in params["registrations"]:
if registration["method"] == "workspace/executeCommand":
self.initialize_searcher_command_available.set()
return
def execute_client_command_handler(params):
def execute_client_command_handler(params: dict) -> list:
return []
def do_nothing(params):
def do_nothing(params: dict) -> None:
return
def window_log_message(msg):
self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO)
def window_log_message(msg: dict) -> None:
log.info(f"LSP: window/logMessage: {msg}")
# Check for bash-language-server ready signals
message_text = msg.get("message", "")
if "Analyzing" in message_text or "analysis complete" in message_text.lower():
self.logger.log("Bash language server analysis signals detected", logging.INFO)
log.info("Bash language server analysis signals detected")
self.server_ready.set()
self.completions_available.set()
@@ -161,16 +156,13 @@ class BashLanguageServer(SolidLanguageServer):
self.server.on_notification("$/progress", do_nothing)
self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
self.logger.log("Starting Bash server process", logging.INFO)
log.info("Starting Bash server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
self.logger.log(
"Sending initialize request from LSP client to LSP server and awaiting response",
logging.INFO,
)
log.info("Sending initialize request from LSP client to LSP server and awaiting response")
init_response = self.server.send.initialize(initialize_params)
self.logger.log(f"Received initialize response from bash server: {init_response}", logging.DEBUG)
log.debug(f"Received initialize response from bash server: {init_response}")
# Enhanced capability checks for bash-language-server 5.6.0
assert init_response["capabilities"]["textDocumentSync"] in [1, 2] # Full or Incremental
@@ -178,53 +170,38 @@ class BashLanguageServer(SolidLanguageServer):
# Verify document symbol support is available
if "documentSymbolProvider" in init_response["capabilities"]:
self.logger.log("Bash server supports document symbols", logging.INFO)
log.info("Bash server supports document symbols")
else:
self.logger.log("Warning: Bash server does not report document symbol support", logging.WARNING)
log.warning("Warning: Bash server does not report document symbol support")
self.server.notify.initialized({})
# Wait for server readiness with timeout
self.logger.log("Waiting for Bash language server to be ready...", logging.INFO)
log.info("Waiting for Bash language server to be ready...")
if not self.server_ready.wait(timeout=3.0):
# Fallback: assume server is ready after timeout
self.logger.log("Timeout waiting for bash server ready signal, proceeding anyway", logging.WARNING)
# This is common. bash-language-server doesn't always send explicit ready signals. Log as info
log.info("Timeout waiting for bash server ready signal, proceeding anyway")
self.server_ready.set()
self.completions_available.set()
else:
self.logger.log("Bash server initialization complete", logging.INFO)
log.info("Bash server initialization complete")
def request_document_symbols(
self, relative_file_path: str, include_body: bool = False
) -> tuple[list[ls_types.UnifiedSymbolInformation], list[ls_types.UnifiedSymbolInformation]]:
"""
Request document symbols from bash-language-server via LSP.
def request_document_symbols(self, relative_file_path: str, file_buffer: LSPFileBuffer | None = None) -> DocumentSymbols:
# Uses the standard LSP documentSymbol request which provides reliable function detection
# for all bash function syntaxes including:
# - function name() { ... } (with function keyword)
# - name() { ... } (traditional syntax)
# - Functions with various indentation levels
# - Functions with comments before/after/inside
Uses the standard LSP documentSymbol request which provides reliable function detection
for all bash function syntaxes including:
- function name() { ... } (with function keyword)
- name() { ... } (traditional syntax)
- Functions with various indentation levels
- Functions with comments before/after/inside
Args:
relative_file_path: Path to the bash file relative to repository root
include_body: Whether to include function bodies in symbol information
Returns:
Tuple of (all_symbols, root_symbols) detected by the LSP server
"""
self.logger.log(f"Requesting document symbols via LSP for {relative_file_path}", logging.DEBUG)
log.debug(f"Requesting document symbols via LSP for {relative_file_path}")
# Use the standard LSP approach - bash-language-server handles all function syntaxes correctly
all_symbols, root_symbols = super().request_document_symbols(relative_file_path, include_body)
document_symbols = super().request_document_symbols(relative_file_path, file_buffer=file_buffer)
# Log detection results for debugging
functions = [s for s in all_symbols if s.get("kind") == 12]
self.logger.log(
f"LSP function detection for {relative_file_path}: Found {len(functions)} functions",
logging.INFO,
)
functions = [s for s in document_symbols.iter_symbols() if s.get("kind") == 12]
log.info(f"LSP function detection for {relative_file_path}: Found {len(functions)} functions")
return all_symbols, root_symbols
return document_symbols
src/solidlsp/language_servers/clangd_language_server.py
+51 -43
View File
@@ -6,16 +6,18 @@ import logging
import os
import pathlib
import threading
from typing import Any, cast
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
from solidlsp.settings import SolidLSPSettings
from .common import RuntimeDependency, RuntimeDependencyCollection
log = logging.getLogger(__name__)
class ClangdLanguageServer(SolidLanguageServer):
"""
@@ -24,20 +26,13 @@ class ClangdLanguageServer(SolidLanguageServer):
Also make sure compile_commands.json is created at root of the source directory. Check clangd test case for example.
"""
def __init__(
self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings
):
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""
Creates a ClangdLanguageServer instance. This class is not meant to be instantiated directly. Use LanguageServer.create() instead.
"""
clangd_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings)
clangd_executable_path = self._setup_runtime_dependencies(config, solidlsp_settings)
super().__init__(
config,
logger,
repository_root_path,
ProcessLaunchInfo(cmd=clangd_executable_path, cwd=repository_root_path),
"cpp",
solidlsp_settings,
config, repository_root_path, ProcessLaunchInfo(cmd=clangd_executable_path, cwd=repository_root_path), "cpp", solidlsp_settings
)
self.server_ready = threading.Event()
self.service_ready_event = threading.Event()
@@ -45,12 +40,12 @@ class ClangdLanguageServer(SolidLanguageServer):
self.resolve_main_method_available = threading.Event()
@classmethod
def _setup_runtime_dependencies(
cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings
) -> str:
def _setup_runtime_dependencies(cls, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings) -> str:
"""
Setup runtime dependencies for ClangdLanguageServer and return the command to start the server.
"""
import shutil
deps = RuntimeDependencyCollection(
[
RuntimeDependency(
@@ -89,21 +84,37 @@ class ClangdLanguageServer(SolidLanguageServer):
)
clangd_ls_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "clangd")
dep = deps.get_single_dep_for_current_platform()
clangd_executable_path = deps.binary_path(clangd_ls_dir)
if not os.path.exists(clangd_executable_path):
logger.log(
f"Clangd executable not found at {clangd_executable_path}. Downloading from {dep.url}",
logging.INFO,
)
deps.install(logger, clangd_ls_dir)
if not os.path.exists(clangd_executable_path):
raise FileNotFoundError(
f"Clangd executable not found at {clangd_executable_path}.\n"
"Make sure you have installed clangd. See https://clangd.llvm.org/installation"
)
os.chmod(clangd_executable_path, 0o755)
try:
dep = deps.get_single_dep_for_current_platform()
except RuntimeError:
dep = None
if dep is None:
# No prebuilt binary available, look for system-installed clangd
clangd_executable_path = shutil.which("clangd")
if not clangd_executable_path:
raise FileNotFoundError(
"Clangd is not installed on your system.\n"
+ "Please install clangd using your system package manager:\n"
+ " Ubuntu/Debian: sudo apt-get install clangd\n"
+ " Fedora/RHEL: sudo dnf install clang-tools-extra\n"
+ " Arch Linux: sudo pacman -S clang\n"
+ "See https://clangd.llvm.org/installation for more details."
)
log.info(f"Using system-installed clangd at {clangd_executable_path}")
else:
# Standard download and install for platforms with prebuilt binaries
clangd_executable_path = deps.binary_path(clangd_ls_dir)
if not os.path.exists(clangd_executable_path):
log.info(f"Clangd executable not found at {clangd_executable_path}. Downloading from {dep.url}")
_ = deps.install(clangd_ls_dir)
if not os.path.exists(clangd_executable_path):
raise FileNotFoundError(
f"Clangd executable not found at {clangd_executable_path}.\n"
+ "Make sure you have installed clangd. See https://clangd.llvm.org/installation"
)
os.chmod(clangd_executable_path, 0o755)
return clangd_executable_path
@staticmethod
@@ -133,9 +144,9 @@ class ClangdLanguageServer(SolidLanguageServer):
],
}
return initialize_params
return cast(InitializeParams, initialize_params)
def _start_server(self):
def _start_server(self) -> None:
"""
Starts the Clangd Language Server, waits for the server to be ready and yields the LanguageServer instance.
@@ -149,7 +160,7 @@ class ClangdLanguageServer(SolidLanguageServer):
# LanguageServer has been shutdown
"""
def register_capability_handler(params):
def register_capability_handler(params: Any) -> None:
assert "registrations" in params
for registration in params["registrations"]:
if registration["method"] == "workspace/executeCommand":
@@ -157,25 +168,25 @@ class ClangdLanguageServer(SolidLanguageServer):
self.resolve_main_method_available.set()
return
def lang_status_handler(params):
def lang_status_handler(params: Any) -> None:
# TODO: Should we wait for
# server -> client: {'jsonrpc': '2.0', 'method': 'language/status', 'params': {'type': 'ProjectStatus', 'message': 'OK'}}
# Before proceeding?
if params["type"] == "ServiceReady" and params["message"] == "ServiceReady":
self.service_ready_event.set()
def execute_client_command_handler(params):
def execute_client_command_handler(params: Any) -> list:
return []
def do_nothing(params):
def do_nothing(params: Any) -> None:
return
def check_experimental_status(params):
def check_experimental_status(params: Any) -> None:
if params["quiescent"] == True:
self.server_ready.set()
def window_log_message(msg):
self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO)
def window_log_message(msg: dict) -> None:
log.info(f"LSP: window/logMessage: {msg}")
self.server.on_request("client/registerCapability", register_capability_handler)
self.server.on_notification("language/status", lang_status_handler)
@@ -186,16 +197,13 @@ class ClangdLanguageServer(SolidLanguageServer):
self.server.on_notification("language/actionableNotification", do_nothing)
self.server.on_notification("experimental/serverStatus", check_experimental_status)
self.logger.log("Starting Clangd server process", logging.INFO)
log.info("Starting Clangd server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
self.logger.log(
"Sending initialize request from LSP client to LSP server and awaiting response",
logging.INFO,
)
log.info("Sending initialize request from LSP client to LSP server and awaiting response")
init_response = self.server.send.initialize(initialize_params)
assert init_response["capabilities"]["textDocumentSync"]["change"] == 2
assert init_response["capabilities"]["textDocumentSync"]["change"] == 2 # type: ignore
assert "completionProvider" in init_response["capabilities"]
assert init_response["capabilities"]["completionProvider"] == {
"triggerCharacters": [".", "<", ">", ":", '"', "/", "*"],
src/solidlsp/language_servers/clojure_lsp.py
+24 -30
View File
@@ -8,16 +8,18 @@ import pathlib
import shutil
import subprocess
import threading
from typing import cast
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
from solidlsp.settings import SolidLSPSettings
from .common import RuntimeDependency, RuntimeDependencyCollection
log = logging.getLogger(__name__)
def run_command(cmd: list, capture_output: bool = True) -> subprocess.CompletedProcess:
return subprocess.run(
@@ -25,7 +27,7 @@ def run_command(cmd: list, capture_output: bool = True) -> subprocess.CompletedP
)
def verify_clojure_cli():
def verify_clojure_cli() -> None:
install_msg = "Please install the official Clojure CLI from:\n https://clojure.org/guides/getting_started"
if shutil.which("clojure") is None:
raise FileNotFoundError("`clojure` not found.\n" + install_msg)
@@ -85,16 +87,13 @@ class ClojureLSP(SolidLanguageServer):
]
)
def __init__(
self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings
):
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""
Creates a ClojureLSP instance. This class is not meant to be instantiated directly. Use LanguageServer.create() instead.
"""
clojure_lsp_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings)
clojure_lsp_executable_path = self._setup_runtime_dependencies(config, solidlsp_settings)
super().__init__(
config,
logger,
repository_root_path,
ProcessLaunchInfo(cmd=clojure_lsp_executable_path, cwd=repository_root_path),
"clojure",
@@ -106,9 +105,7 @@ class ClojureLSP(SolidLanguageServer):
self.service_ready_event = threading.Event()
@classmethod
def _setup_runtime_dependencies(
cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings
) -> str:
def _setup_runtime_dependencies(cls, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings) -> str:
"""Setup runtime dependencies for clojure-lsp and return the command to start the server."""
verify_clojure_cli()
deps = ClojureLSP.runtime_dependencies
@@ -117,11 +114,10 @@ class ClojureLSP(SolidLanguageServer):
clojurelsp_ls_dir = cls.ls_resources_dir(solidlsp_settings)
clojurelsp_executable_path = deps.binary_path(clojurelsp_ls_dir)
if not os.path.exists(clojurelsp_executable_path):
logger.log(
log.info(
f"Downloading and extracting clojure-lsp from {dependency.url} to {clojurelsp_ls_dir}",
logging.INFO,
)
deps.install(logger, clojurelsp_ls_dir)
deps.install(clojurelsp_ls_dir)
if not os.path.exists(clojurelsp_executable_path):
raise FileNotFoundError(f"Download failed? Could not find clojure-lsp executable at {clojurelsp_executable_path}")
os.chmod(clojurelsp_executable_path, 0o755)
@@ -131,7 +127,7 @@ class ClojureLSP(SolidLanguageServer):
def _get_initialize_params(repository_absolute_path: str) -> InitializeParams:
"""Returns the init params for clojure-lsp."""
root_uri = pathlib.Path(repository_absolute_path).as_uri()
return { # type: ignore
result = { # type: ignore
"processId": os.getpid(),
"rootPath": repository_absolute_path,
"rootUri": root_uri,
@@ -159,9 +155,10 @@ class ClojureLSP(SolidLanguageServer):
"trace": "off",
"workspaceFolders": [{"uri": root_uri, "name": os.path.basename(repository_absolute_path)}],
}
return cast(InitializeParams, result)
def _start_server(self):
def register_capability_handler(params):
def _start_server(self) -> None:
def register_capability_handler(params: dict) -> None:
assert "registrations" in params
for registration in params["registrations"]:
if registration["method"] == "workspace/executeCommand":
@@ -169,25 +166,25 @@ class ClojureLSP(SolidLanguageServer):
self.resolve_main_method_available.set()
return
def lang_status_handler(params):
def lang_status_handler(params: dict) -> None:
# TODO: Should we wait for
# server -> client: {'jsonrpc': '2.0', 'method': 'language/status', 'params': {'type': 'ProjectStatus', 'message': 'OK'}}
# Before proceeding?
if params["type"] == "ServiceReady" and params["message"] == "ServiceReady":
self.service_ready_event.set()
def execute_client_command_handler(params):
def execute_client_command_handler(params: dict) -> list:
return []
def do_nothing(params):
def do_nothing(params: dict) -> None:
return
def check_experimental_status(params):
if params["quiescent"] == True:
def check_experimental_status(params: dict) -> None:
if params["quiescent"] is True:
self.server_ready.set()
def window_log_message(msg):
self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO)
def window_log_message(msg: dict) -> None:
log.info(f"LSP: window/logMessage: {msg}")
self.server.on_request("client/registerCapability", register_capability_handler)
self.server.on_notification("language/status", lang_status_handler)
@@ -198,21 +195,18 @@ class ClojureLSP(SolidLanguageServer):
self.server.on_notification("language/actionableNotification", do_nothing)
self.server.on_notification("experimental/serverStatus", check_experimental_status)
self.logger.log("Starting clojure-lsp server process", logging.INFO)
log.info("Starting clojure-lsp server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
self.logger.log(
"Sending initialize request from LSP client to LSP server and awaiting response",
logging.INFO,
)
log.info("Sending initialize request from LSP client to LSP server and awaiting response")
init_response = self.server.send.initialize(initialize_params)
assert init_response["capabilities"]["textDocumentSync"]["change"] == 2
assert init_response["capabilities"]["textDocumentSync"]["change"] == 2 # type: ignore
assert "completionProvider" in init_response["capabilities"]
# Clojure-lsp completion provider capabilities are more flexible than other servers'
completion_provider = init_response["capabilities"]["completionProvider"]
assert completion_provider["resolveProvider"] == True
assert completion_provider["resolveProvider"] is True
assert "triggerCharacters" in completion_provider
self.server.notify.initialized({})
# after initialize, Clojure-lsp is ready to serve
src/solidlsp/language_servers/common.py
+10 -9
View File
@@ -8,7 +8,6 @@ from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass, replace
from typing import Any, cast
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.ls_utils import FileUtils, PlatformUtils
from solidlsp.util.subprocess_util import subprocess_kwargs
@@ -79,7 +78,7 @@ class RuntimeDependencyCollection:
return target_dir
return os.path.join(target_dir, dep.binary_name)
def install(self, logger: LanguageServerLogger, target_dir: str) -> dict[str, str]:
def install(self, target_dir: str) -> dict[str, str]:
"""Install all dependencies for the current platform into *target_dir*.
Returns a mapping from dependency id to the resolved binary path.
@@ -88,7 +87,7 @@ class RuntimeDependencyCollection:
results: dict[str, str] = {}
for dep in self.get_dependencies_for_current_platform():
if dep.url:
self._install_from_url(dep, logger, target_dir)
self._install_from_url(dep, target_dir)
if dep.command:
self._run_command(dep.command, target_dir)
if dep.binary_name:
@@ -103,7 +102,7 @@ class RuntimeDependencyCollection:
if not PlatformUtils.get_platform_id().is_windows():
import pwd
kwargs["user"] = pwd.getpwuid(os.getuid()).pw_name
kwargs["user"] = pwd.getpwuid(os.getuid()).pw_name # type: ignore
is_windows = platform.system() == "Windows"
if not isinstance(command, str) and not is_windows:
@@ -121,7 +120,7 @@ class RuntimeDependencyCollection:
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**kwargs,
)
) # type: ignore
if completed_process.returncode != 0:
log.warning("Command '%s' failed with return code %d", command, completed_process.returncode)
log.warning("Command output:\n%s", completed_process.stdout)
@@ -131,13 +130,15 @@ class RuntimeDependencyCollection:
)
@staticmethod
def _install_from_url(dep: RuntimeDependency, logger: LanguageServerLogger, target_dir: str) -> None:
assert dep.url is not None
def _install_from_url(dep: RuntimeDependency, target_dir: str) -> None:
if not dep.url:
raise ValueError(f"Dependency {dep.id} has no URL")
if dep.archive_type in ("gz", "binary") and dep.binary_name:
dest = os.path.join(target_dir, dep.binary_name)
FileUtils.download_and_extract_archive(logger, dep.url, dest, dep.archive_type)
FileUtils.download_and_extract_archive(dep.url, dest, dep.archive_type)
else:
FileUtils.download_and_extract_archive(logger, dep.url, target_dir, dep.archive_type or "zip")
FileUtils.download_and_extract_archive(dep.url, target_dir, dep.archive_type or "zip")
def quote_windows_path(path: str) -> str:
src/solidlsp/language_servers/csharp_language_server.py
+70 -93
View File
@@ -12,6 +12,7 @@ import tarfile
import threading
import urllib.request
import zipfile
from collections.abc import Iterable
from pathlib import Path
from typing import Any, cast
@@ -20,15 +21,16 @@ from overrides import override
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.ls_exceptions import SolidLSPException
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.ls_utils import PathUtils
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams, InitializeResult
from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
from solidlsp.settings import SolidLSPSettings
from solidlsp.util.zip import SafeZipExtractor
from .common import RuntimeDependency, RuntimeDependencyCollection
log = logging.getLogger(__name__)
_RUNTIME_DEPENDENCIES = [
RuntimeDependency(
id="CSharpLanguageServer",
@@ -133,7 +135,7 @@ _RUNTIME_DEPENDENCIES = [
]
def breadth_first_file_scan(root_dir):
def breadth_first_file_scan(root_dir: str) -> Iterable[str]:
"""
Perform a breadth-first scan of files in the given directory.
Yields file paths in breadth-first order.
@@ -155,7 +157,7 @@ def breadth_first_file_scan(root_dir):
pass
def find_solution_or_project_file(root_dir) -> str | None:
def find_solution_or_project_file(root_dir: str) -> str | None:
"""
Find the first .sln file in breadth-first order.
If no .sln file is found, look for a .csproj file.
@@ -197,14 +199,12 @@ class CSharpLanguageServer(SolidLanguageServer):
See the `_RUNTIME_DEPENDENCIES` variable above for the available dependency ids and platform_ids.
"""
def __init__(
self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings
):
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""
Creates a CSharpLanguageServer instance. This class is not meant to be instantiated directly.
Use LanguageServer.create() instead.
"""
dotnet_path, language_server_path = self._ensure_server_installed(logger, config, solidlsp_settings)
dotnet_path, language_server_path = self._ensure_server_installed(config, solidlsp_settings)
# Find solution or project file
solution_or_project = find_solution_or_project_file(repository_root_path)
@@ -218,20 +218,13 @@ class CSharpLanguageServer(SolidLanguageServer):
# The language server will discover the solution/project from the workspace root
if solution_or_project:
logger.log(f"Found solution/project file: {solution_or_project}", logging.INFO)
log.info(f"Found solution/project file: {solution_or_project}")
else:
logger.log("No .sln or .csproj file found, language server will attempt auto-discovery", logging.WARNING)
log.warning("No .sln or .csproj file found, language server will attempt auto-discovery")
logger.log(f"Language server command: {' '.join(cmd)}", logging.DEBUG)
log.debug(f"Language server command: {' '.join(cmd)}")
super().__init__(
config,
logger,
repository_root_path,
ProcessLaunchInfo(cmd=cmd, cwd=repository_root_path),
"csharp",
solidlsp_settings,
)
super().__init__(config, repository_root_path, ProcessLaunchInfo(cmd=cmd, cwd=repository_root_path), "csharp", solidlsp_settings)
self.initialization_complete = threading.Event()
@@ -240,47 +233,35 @@ class CSharpLanguageServer(SolidLanguageServer):
return super().is_ignored_dirname(dirname) or dirname in ["bin", "obj", "packages", ".vs"]
@classmethod
def _ensure_server_installed(
cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings
) -> tuple[str, str]:
def _ensure_server_installed(cls, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings) -> tuple[str, str]:
"""
Ensure .NET runtime and Microsoft.CodeAnalysis.LanguageServer are available.
Returns a tuple of (dotnet_path, language_server_dll_path).
"""
language_enum = cls.get_language_enum_instance()
ls_specific_settings = solidlsp_settings.ls_specific_settings or {}
language_specific_config = ls_specific_settings.get(language_enum, {})
language_specific_config = solidlsp_settings.get_ls_specific_settings(cls.get_language_enum_instance())
runtime_dependency_overrides = cast(list[dict[str, Any]], language_specific_config.get("runtime_dependencies", []))
logger.log("Resolving runtime dependencies", logging.DEBUG)
if language_specific_config:
logger.log(
f"Language-specific config for {language_enum}: {language_specific_config}",
logging.DEBUG,
)
log.debug("Resolving runtime dependencies")
runtime_dependencies = RuntimeDependencyCollection(
_RUNTIME_DEPENDENCIES,
overrides=runtime_dependency_overrides,
)
logger.log(
log.debug(
f"Available runtime dependencies: {runtime_dependencies.get_dependencies_for_current_platform}",
logging.DEBUG,
)
# Find the dependencies for our platform
lang_server_dep = runtime_dependencies.get_single_dep_for_current_platform("CSharpLanguageServer")
dotnet_runtime_dep = runtime_dependencies.get_single_dep_for_current_platform("DotNetRuntime")
dotnet_path = CSharpLanguageServer._ensure_dotnet_runtime(logger, dotnet_runtime_dep, solidlsp_settings)
server_dll_path = CSharpLanguageServer._ensure_language_server(logger, lang_server_dep, solidlsp_settings)
dotnet_path = CSharpLanguageServer._ensure_dotnet_runtime(dotnet_runtime_dep, solidlsp_settings)
server_dll_path = CSharpLanguageServer._ensure_language_server(lang_server_dep, solidlsp_settings)
return dotnet_path, server_dll_path
@classmethod
def _ensure_dotnet_runtime(
cls, logger: LanguageServerLogger, dotnet_runtime_dep: RuntimeDependency, solidlsp_settings: SolidLSPSettings
) -> str:
def _ensure_dotnet_runtime(cls, dotnet_runtime_dep: RuntimeDependency, solidlsp_settings: SolidLSPSettings) -> str:
"""Ensure .NET runtime is available and return the dotnet executable path."""
# TODO: use RuntimeDependency util methods instead of custom validation/download logic
@@ -291,32 +272,33 @@ class CSharpLanguageServer(SolidLanguageServer):
try:
result = subprocess.run([system_dotnet, "--list-runtimes"], capture_output=True, text=True, check=True)
if "Microsoft.NETCore.App 9." in result.stdout:
logger.log("Found system .NET 9 runtime", logging.INFO)
log.info("Found system .NET 9 runtime")
return system_dotnet
except subprocess.CalledProcessError:
pass
# Download .NET 9 runtime using config
return cls._ensure_dotnet_runtime_from_config(logger, dotnet_runtime_dep, solidlsp_settings)
return cls._ensure_dotnet_runtime_from_config(dotnet_runtime_dep, solidlsp_settings)
@classmethod
def _ensure_language_server(
cls, logger: LanguageServerLogger, lang_server_dep: RuntimeDependency, solidlsp_settings: SolidLSPSettings
) -> str:
def _ensure_language_server(cls, lang_server_dep: RuntimeDependency, solidlsp_settings: SolidLSPSettings) -> str:
"""Ensure language server is available and return the DLL path."""
package_name = lang_server_dep.package_name
package_version = lang_server_dep.package_version
server_dir = Path(cls.ls_resources_dir(solidlsp_settings)) / f"{package_name}.{package_version}"
assert lang_server_dep.binary_name is not None
server_dll = server_dir / lang_server_dep.binary_name
if server_dll.exists():
logger.log(f"Using cached Microsoft.CodeAnalysis.LanguageServer from {server_dll}", logging.INFO)
log.info(f"Using cached Microsoft.CodeAnalysis.LanguageServer from {server_dll}")
return str(server_dll)
# Download and install the language server
logger.log(f"Downloading {package_name} version {package_version}...", logging.INFO)
package_path = cls._download_nuget_package_direct(logger, package_name, package_version, solidlsp_settings)
log.info(f"Downloading {package_name} version {package_version}...")
assert package_version is not None
assert package_name is not None
package_path = cls._download_nuget_package_direct(package_name, package_version, solidlsp_settings)
# Extract and install
cls._extract_language_server(lang_server_dep, package_path, server_dir)
@@ -328,7 +310,7 @@ class CSharpLanguageServer(SolidLanguageServer):
if platform.system().lower() != "windows":
server_dll.chmod(0o755)
logger.log(f"Successfully installed Microsoft.CodeAnalysis.LanguageServer to {server_dll}", logging.INFO)
log.info(f"Successfully installed Microsoft.CodeAnalysis.LanguageServer to {server_dll}")
return str(server_dll)
@staticmethod
@@ -355,9 +337,7 @@ class CSharpLanguageServer(SolidLanguageServer):
shutil.copytree(source_dir, server_dir, dirs_exist_ok=True)
@classmethod
def _download_nuget_package_direct(
cls, logger: LanguageServerLogger, package_name: str, package_version: str, solidlsp_settings: SolidLSPSettings
) -> Path:
def _download_nuget_package_direct(cls, package_name: str, package_version: str, solidlsp_settings: SolidLSPSettings) -> Path:
"""
Download a NuGet package directly from the Azure NuGet feed.
Returns the path to the extracted package directory.
@@ -370,7 +350,7 @@ class CSharpLanguageServer(SolidLanguageServer):
try:
# First, get the service index from the Azure feed
logger.log("Fetching NuGet service index from Azure feed...", logging.DEBUG)
log.debug("Fetching NuGet service index from Azure feed...")
with urllib.request.urlopen(azure_feed_url) as response:
service_index = json.loads(response.read().decode())
@@ -389,7 +369,7 @@ class CSharpLanguageServer(SolidLanguageServer):
package_version_lower = package_version.lower()
package_url = f"{package_base_address.rstrip('/')}/{package_id_lower}/{package_version_lower}/{package_id_lower}.{package_version_lower}.nupkg"
logger.log(f"Downloading package from: {package_url}", logging.DEBUG)
log.debug(f"Downloading package from: {package_url}")
# Download the .nupkg file
nupkg_file = temp_dir / f"{package_name}.{package_version}.nupkg"
@@ -406,7 +386,7 @@ class CSharpLanguageServer(SolidLanguageServer):
# Clean up the nupkg file
nupkg_file.unlink()
logger.log(f"Successfully downloaded and extracted {package_name} version {package_version}", logging.INFO)
log.info(f"Successfully downloaded and extracted {package_name} version {package_version}")
return package_extract_dir
except Exception as e:
@@ -415,9 +395,7 @@ class CSharpLanguageServer(SolidLanguageServer):
) from e
@classmethod
def _ensure_dotnet_runtime_from_config(
cls, logger: LanguageServerLogger, dotnet_runtime_dep: RuntimeDependency, solidlsp_settings: SolidLSPSettings
) -> str:
def _ensure_dotnet_runtime_from_config(cls, dotnet_runtime_dep: RuntimeDependency, solidlsp_settings: SolidLSPSettings) -> str:
"""
Ensure .NET 9 runtime is available using runtime dependency configuration.
Returns the path to the dotnet executable.
@@ -431,28 +409,28 @@ class CSharpLanguageServer(SolidLanguageServer):
try:
result = subprocess.run([system_dotnet, "--list-runtimes"], capture_output=True, text=True, check=True)
if "Microsoft.NETCore.App 9." in result.stdout:
logger.log("Found system .NET 9 runtime", logging.INFO)
log.info("Found system .NET 9 runtime")
return system_dotnet
except subprocess.CalledProcessError:
pass
# Download .NET 9 runtime using config
dotnet_dir = Path(cls.ls_resources_dir(solidlsp_settings)) / "dotnet-runtime-9.0"
assert dotnet_runtime_dep.binary_name is not None, "Runtime dependency must have a binary_name"
dotnet_exe = dotnet_dir / dotnet_runtime_dep.binary_name
if dotnet_exe.exists():
logger.log(f"Using cached .NET runtime from {dotnet_exe}", logging.INFO)
log.info(f"Using cached .NET runtime from {dotnet_exe}")
return str(dotnet_exe)
# Download .NET runtime
logger.log("Downloading .NET 9 runtime...", logging.INFO)
log.info("Downloading .NET 9 runtime...")
dotnet_dir.mkdir(parents=True, exist_ok=True)
custom_dotnet_runtime_url = solidlsp_settings.ls_specific_settings.get(cls.get_language_enum_instance(), {}).get(
"dotnet_runtime_url"
)
custom_settings = solidlsp_settings.get_ls_specific_settings(cls.get_language_enum_instance())
custom_dotnet_runtime_url = custom_settings.get("dotnet_runtime_url")
if custom_dotnet_runtime_url is not None:
logger.log(f"Using custom .NET runtime url: {custom_dotnet_runtime_url}", logging.INFO)
log.info(f"Using custom .NET runtime url: {custom_dotnet_runtime_url}")
url = custom_dotnet_runtime_url
else:
url = dotnet_runtime_dep.url
@@ -462,7 +440,7 @@ class CSharpLanguageServer(SolidLanguageServer):
# Download the runtime
download_path = dotnet_dir / f"dotnet-runtime.{archive_type}"
try:
logger.log(f"Downloading from {url}", logging.DEBUG)
log.debug(f"Downloading from {url}")
urllib.request.urlretrieve(url, download_path)
# Extract the archive
@@ -481,7 +459,7 @@ class CSharpLanguageServer(SolidLanguageServer):
if platform.system().lower() != "windows":
dotnet_exe.chmod(0o755)
logger.log(f"Successfully installed .NET 9 runtime to {dotnet_exe}", logging.INFO)
log.info(f"Successfully installed .NET 9 runtime to {dotnet_exe}")
return str(dotnet_exe)
except Exception as e:
@@ -542,11 +520,11 @@ class CSharpLanguageServer(SolidLanguageServer):
},
)
def _start_server(self):
def do_nothing(params):
def _start_server(self) -> None:
def do_nothing(params: dict) -> None:
return
def window_log_message(msg):
def window_log_message(msg: dict) -> None:
"""Log messages from the language server."""
message_text = msg.get("message", "")
level = msg.get("type", 4) # Default to Log level
@@ -554,15 +532,15 @@ class CSharpLanguageServer(SolidLanguageServer):
# Map LSP message types to Python logging levels
level_map = {1: logging.ERROR, 2: logging.WARNING, 3: logging.INFO, 4: logging.DEBUG} # Error # Warning # Info # Log
self.logger.log(f"LSP: {message_text}", level_map.get(level, logging.DEBUG))
log.info(f"LSP: {message_text}", level_map.get(level, logging.DEBUG))
def handle_progress(params):
def handle_progress(params: dict) -> None:
"""Handle progress notifications from the language server."""
token = params.get("token", "")
value = params.get("value", {})
# Log raw progress for debugging
self.logger.log(f"Progress notification received: {params}", logging.DEBUG)
log.debug(f"Progress notification received: {params}")
# Handle different progress notification types
kind = value.get("kind")
@@ -573,27 +551,27 @@ class CSharpLanguageServer(SolidLanguageServer):
percentage = value.get("percentage")
if percentage is not None:
self.logger.log(f"Progress [{token}]: {title} - {message} ({percentage}%)", logging.INFO)
log.debug(f"Progress [{token}]: {title} - {message} ({percentage}%)")
else:
self.logger.log(f"Progress [{token}]: {title} - {message}", logging.INFO)
log.info(f"Progress [{token}]: {title} - {message}")
elif kind == "report":
message = value.get("message", "")
percentage = value.get("percentage")
if percentage is not None:
self.logger.log(f"Progress [{token}]: {message} ({percentage}%)", logging.INFO)
log.info(f"Progress [{token}]: {message} ({percentage}%)")
elif message:
self.logger.log(f"Progress [{token}]: {message}", logging.INFO)
log.info(f"Progress [{token}]: {message}")
elif kind == "end":
message = value.get("message", "Operation completed")
self.logger.log(f"Progress [{token}]: {message}", logging.INFO)
log.info(f"Progress [{token}]: {message}")
def handle_workspace_configuration(params):
def handle_workspace_configuration(params: dict) -> list:
"""Handle workspace/configuration requests from the server."""
items = params.get("items", [])
result = []
result: list[Any] = []
for item in items:
section = item.get("section", "")
@@ -636,17 +614,17 @@ class CSharpLanguageServer(SolidLanguageServer):
return result
def handle_work_done_progress_create(params):
def handle_work_done_progress_create(params: dict) -> None:
"""Handle work done progress create requests."""
# Just acknowledge the request
return
def handle_register_capability(params):
def handle_register_capability(params: dict) -> None:
"""Handle client/registerCapability requests."""
# Just acknowledge the request - we don't need to track these for now
return
def handle_project_needs_restore(params):
def handle_project_needs_restore(params: dict) -> None:
return
# Set up notification handlers
@@ -658,21 +636,21 @@ class CSharpLanguageServer(SolidLanguageServer):
self.server.on_request("client/registerCapability", handle_register_capability)
self.server.on_request("workspace/_roslyn_projectNeedsRestore", handle_project_needs_restore)
self.logger.log("Starting Microsoft.CodeAnalysis.LanguageServer process", logging.INFO)
log.info("Starting Microsoft.CodeAnalysis.LanguageServer process")
try:
self.server.start()
except Exception as e:
self.logger.log(f"Failed to start language server process: {e}", logging.ERROR)
log.info(f"Failed to start language server process: {e}", logging.ERROR)
raise SolidLSPException(f"Failed to start C# language server: {e}")
# Send initialization
initialize_params = self._get_initialize_params()
self.logger.log("Sending initialize request to language server", logging.INFO)
log.info("Sending initialize request to language server")
try:
init_response = self.server.send.initialize(initialize_params)
self.logger.log(f"Received initialize response: {init_response}", logging.DEBUG)
log.info(f"Received initialize response: {init_response}")
except Exception as e:
raise SolidLSPException(f"Failed to initialize C# language server for {self.repository_root_path}: {e}") from e
@@ -703,20 +681,19 @@ class CSharpLanguageServer(SolidLanguageServer):
self.initialization_complete.set()
self.completions_available.set()
self.logger.log(
log.info(
"Microsoft.CodeAnalysis.LanguageServer initialized and ready\n"
"Waiting for language server to index project files...\n"
"This may take a while for large projects",
logging.INFO,
"This may take a while for large projects"
)
def _force_pull_diagnostics(self, init_response: dict) -> None:
def _force_pull_diagnostics(self, init_response: dict | InitializeResult) -> None:
"""
Apply the diagnostic capabilities hack.
Forces the server to support pull diagnostics.
"""
capabilities = init_response.get("capabilities", {})
diagnostic_provider = capabilities.get("diagnosticProvider", {})
diagnostic_provider: Any = capabilities.get("diagnosticProvider", {})
# Add the diagnostic capabilities hack
if isinstance(diagnostic_provider, dict):
@@ -727,7 +704,7 @@ class CSharpLanguageServer(SolidLanguageServer):
"workspaceDiagnostics": True,
}
)
self.logger.log("Applied diagnostic capabilities hack for better C# diagnostics", logging.DEBUG)
log.debug("Applied diagnostic capabilities hack for better C# diagnostics")
def _open_solution_and_projects(self) -> None:
"""
@@ -744,7 +721,7 @@ class CSharpLanguageServer(SolidLanguageServer):
if solution_file:
solution_uri = PathUtils.path_to_uri(solution_file)
self.server.notify.send_notification("solution/open", {"solution": solution_uri})
self.logger.log(f"Opened solution file: {solution_file}", logging.INFO)
log.debug(f"Opened solution file: {solution_file}")
# Find and open project files
project_files = []
@@ -756,7 +733,7 @@ class CSharpLanguageServer(SolidLanguageServer):
if project_files:
project_uris = [PathUtils.path_to_uri(project_file) for project_file in project_files]
self.server.notify.send_notification("project/open", {"projects": project_uris})
self.logger.log(f"Opened project files: {project_files}", logging.DEBUG)
log.debug(f"Opened project files: {project_files}")
@override
def _get_wait_time_for_cross_file_referencing(self) -> float:
src/solidlsp/language_servers/dart_language_server.py
+22 -29
View File
@@ -1,36 +1,35 @@
import logging
import os
import pathlib
from typing import cast
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
from solidlsp.settings import SolidLSPSettings
from ..ls_config import LanguageServerConfig
from ..lsp_protocol_handler.lsp_types import InitializeParams
from .common import RuntimeDependency, RuntimeDependencyCollection
log = logging.getLogger(__name__)
class DartLanguageServer(SolidLanguageServer):
"""
Provides Dart specific instantiation of the LanguageServer class. Contains various configurations and settings specific to Dart.
"""
def __init__(self, config, logger, repository_root_path, solidlsp_settings: SolidLSPSettings):
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings) -> None:
"""
Creates a DartServer instance. This class is not meant to be instantiated directly. Use LanguageServer.create() instead.
"""
executable_path = self._setup_runtime_dependencies(logger, solidlsp_settings)
executable_path = self._setup_runtime_dependencies(solidlsp_settings)
super().__init__(
config,
logger,
repository_root_path,
ProcessLaunchInfo(cmd=executable_path, cwd=repository_root_path),
"dart",
solidlsp_settings,
config, repository_root_path, ProcessLaunchInfo(cmd=executable_path, cwd=repository_root_path), "dart", solidlsp_settings
)
@classmethod
def _setup_runtime_dependencies(cls, logger: "LanguageServerLogger", solidlsp_settings: SolidLSPSettings) -> str:
def _setup_runtime_dependencies(cls, solidlsp_settings: SolidLSPSettings) -> str:
deps = RuntimeDependencyCollection(
[
RuntimeDependency(
@@ -80,7 +79,7 @@ class DartLanguageServer(SolidLanguageServer):
dart_executable_path = deps.binary_path(dart_ls_dir)
if not os.path.exists(dart_executable_path):
deps.install(logger, dart_ls_dir)
deps.install(dart_ls_dir)
assert os.path.exists(dart_executable_path)
os.chmod(dart_executable_path, 0o755)
@@ -88,7 +87,7 @@ class DartLanguageServer(SolidLanguageServer):
return f"{dart_executable_path} language-server --client-id multilspy.dart --client-version 1.2"
@staticmethod
def _get_initialize_params(repository_absolute_path: str):
def _get_initialize_params(repository_absolute_path: str) -> InitializeParams:
"""
Returns the initialize params for the Dart Language Server.
"""
@@ -114,24 +113,24 @@ class DartLanguageServer(SolidLanguageServer):
],
}
return initialize_params
return cast(InitializeParams, initialize_params)
def _start_server(self):
def _start_server(self) -> None:
"""
Start the language server and yield when the server is ready.
"""
def execute_client_command_handler(params):
def execute_client_command_handler(params: dict) -> list:
return []
def do_nothing(params):
def do_nothing(params: dict) -> None:
return
def check_experimental_status(params):
def check_experimental_status(params: dict) -> None:
pass
def window_log_message(msg):
self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO)
def window_log_message(msg: dict) -> None:
log.info(f"LSP: window/logMessage: {msg}")
self.server.on_request("client/registerCapability", do_nothing)
self.server.on_notification("language/status", do_nothing)
@@ -142,17 +141,11 @@ class DartLanguageServer(SolidLanguageServer):
self.server.on_notification("language/actionableNotification", do_nothing)
self.server.on_notification("experimental/serverStatus", check_experimental_status)
self.logger.log("Starting dart-language-server server process", logging.INFO)
log.info("Starting dart-language-server server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
self.logger.log(
"Sending initialize request to dart-language-server",
logging.DEBUG,
)
init_response = self.server.send_request("initialize", initialize_params)
self.logger.log(
f"Received initialize response from dart-language-server: {init_response}",
logging.INFO,
)
log.debug("Sending initialize request to dart-language-server")
init_response = self.server.send_request("initialize", initialize_params) # type: ignore
log.info(f"Received initialize response from dart-language-server: {init_response}")
self.server.notify.initialized({})
src/solidlsp/language_servers/eclipse_jdtls.py
+154 -73
View File
@@ -10,17 +10,20 @@ import shutil
import threading
import uuid
from pathlib import PurePath
from typing import cast
from overrides import override
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls import LSPFileBuffer, SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.ls_types import UnifiedSymbolInformation
from solidlsp.ls_utils import FileUtils, PlatformUtils
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.lsp_protocol_handler.lsp_types import DocumentSymbol, InitializeParams, SymbolInformation
from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
from solidlsp.settings import SolidLSPSettings
log = logging.getLogger(__name__)
@dataclasses.dataclass
class RuntimeDependencyPaths:
@@ -39,18 +42,32 @@ class RuntimeDependencyPaths:
class EclipseJDTLS(SolidLanguageServer):
"""
r"""
The EclipseJDTLS class provides a Java specific implementation of the LanguageServer class
You can configure the following options in ls_specific_settings (in serena_config.yml):
- maven_user_settings: Path to Maven settings.xml file (default: ~/.m2/settings.xml)
- gradle_user_home: Path to Gradle user home directory (default: ~/.gradle)
Note: Gradle wrapper is disabled by default. Projects will use the bundled Gradle distribution.
Example configuration in ~/.serena/serena_config.yml:
```yaml
ls_specific_settings:
java:
maven_user_settings: "/home/user/.m2/settings.xml" # Unix/Linux/Mac
# maven_user_settings: 'C:\\Users\\YourName\\.m2\\settings.xml' # Windows (use single quotes!)
gradle_user_home: "/home/user/.gradle" # Unix/Linux/Mac
# gradle_user_home: 'C:\\Users\\YourName\\.gradle' # Windows (use single quotes!)
```
"""
def __init__(
self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings
):
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""
Creates a new EclipseJDTLS instance initializing the language server settings appropriately.
This class is not meant to be instantiated directly. Use LanguageServer.create() instead.
"""
runtime_dependency_paths = self._setupRuntimeDependencies(logger, config, solidlsp_settings)
runtime_dependency_paths = self._setupRuntimeDependencies(config, solidlsp_settings)
self.runtime_dependency_paths = runtime_dependency_paths
# ws_dir is the workspace directory for the EclipseJDTLS server
@@ -93,48 +110,46 @@ class EclipseJDTLS(SolidLanguageServer):
# TODO: Add "self.runtime_dependency_paths.jre_home_path"/bin to $PATH as well
proc_env = {"syntaxserver": "false", "JAVA_HOME": self.runtime_dependency_paths.jre_home_path}
proc_cwd = repository_root_path
cmd = " ".join(
[
jre_path,
"--add-modules=ALL-SYSTEM",
"--add-opens",
"java.base/java.util=ALL-UNNAMED",
"--add-opens",
"java.base/java.lang=ALL-UNNAMED",
"--add-opens",
"java.base/sun.nio.fs=ALL-UNNAMED",
"-Declipse.application=org.eclipse.jdt.ls.core.id1",
"-Dosgi.bundles.defaultStartLevel=4",
"-Declipse.product=org.eclipse.jdt.ls.core.product",
"-Djava.import.generatesMetadataFilesAtProjectRoot=false",
"-Dfile.encoding=utf8",
"-noverify",
"-XX:+UseParallelGC",
"-XX:GCTimeRatio=4",
"-XX:AdaptiveSizePolicyWeight=90",
"-Dsun.zip.disableMemoryMapping=true",
"-Djava.lsp.joinOnCompletion=true",
"-Xmx3G",
"-Xms100m",
"-Xlog:disable",
"-Dlog.level=ALL",
f'"-javaagent:{lombok_jar_path}"',
f'"-Djdt.core.sharedIndexLocation={shared_cache_location}"',
"-jar",
f'"{jdtls_launcher_jar}"',
"-configuration",
f'"{jdtls_config_path}"',
"-data",
f'"{data_dir}"',
]
)
cmd = [
jre_path,
"--add-modules=ALL-SYSTEM",
"--add-opens",
"java.base/java.util=ALL-UNNAMED",
"--add-opens",
"java.base/java.lang=ALL-UNNAMED",
"--add-opens",
"java.base/sun.nio.fs=ALL-UNNAMED",
"-Declipse.application=org.eclipse.jdt.ls.core.id1",
"-Dosgi.bundles.defaultStartLevel=4",
"-Declipse.product=org.eclipse.jdt.ls.core.product",
"-Djava.import.generatesMetadataFilesAtProjectRoot=false",
"-Dfile.encoding=utf8",
"-noverify",
"-XX:+UseParallelGC",
"-XX:GCTimeRatio=4",
"-XX:AdaptiveSizePolicyWeight=90",
"-Dsun.zip.disableMemoryMapping=true",
"-Djava.lsp.joinOnCompletion=true",
"-Xmx3G",
"-Xms100m",
"-Xlog:disable",
"-Dlog.level=ALL",
f"-javaagent:{lombok_jar_path}",
f"-Djdt.core.sharedIndexLocation={shared_cache_location}",
"-jar",
f"{jdtls_launcher_jar}",
"-configuration",
f"{jdtls_config_path}",
"-data",
f"{data_dir}",
]
self.service_ready_event = threading.Event()
self.intellicode_enable_command_available = threading.Event()
self.initialize_searcher_command_available = threading.Event()
super().__init__(
config, logger, repository_root_path, ProcessLaunchInfo(cmd, proc_env, proc_cwd), "java", solidlsp_settings=solidlsp_settings
config, repository_root_path, ProcessLaunchInfo(cmd, proc_env, proc_cwd), "java", solidlsp_settings=solidlsp_settings
)
@override
@@ -156,9 +171,7 @@ class EclipseJDTLS(SolidLanguageServer):
]
@classmethod
def _setupRuntimeDependencies(
cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings
) -> RuntimeDependencyPaths:
def _setupRuntimeDependencies(cls, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings) -> RuntimeDependencyPaths:
"""
Setup runtime dependencies for EclipseJDTLS and return the paths.
"""
@@ -250,7 +263,6 @@ class EclipseJDTLS(SolidLanguageServer):
if not os.path.exists(gradle_path):
FileUtils.download_and_extract_archive(
logger,
runtime_dependencies["gradle"]["platform-agnostic"]["url"],
str(PurePath(gradle_path).parent),
runtime_dependencies["gradle"]["platform-agnostic"]["archiveType"],
@@ -276,7 +288,7 @@ class EclipseJDTLS(SolidLanguageServer):
os.path.exists(jdtls_readonly_config_path),
]
):
FileUtils.download_and_extract_archive(logger, dependency["url"], vscode_java_path, dependency["archiveType"])
FileUtils.download_and_extract_archive(dependency["url"], vscode_java_path, dependency["archiveType"])
os.chmod(jre_path, 0o755)
@@ -299,7 +311,7 @@ class EclipseJDTLS(SolidLanguageServer):
os.path.exists(intellisense_members_path),
]
):
FileUtils.download_and_extract_archive(logger, dependency["url"], intellicode_directory_path, dependency["archiveType"])
FileUtils.download_and_extract_archive(dependency["url"], intellicode_directory_path, dependency["archiveType"])
assert os.path.exists(intellicode_directory_path)
assert os.path.exists(intellicode_jar_path)
@@ -326,6 +338,52 @@ class EclipseJDTLS(SolidLanguageServer):
repository_absolute_path = os.path.abspath(repository_absolute_path)
repo_uri = pathlib.Path(repository_absolute_path).as_uri()
# Load user's Maven and Gradle configuration paths from ls_specific_settings["java"]
# Maven settings: default to ~/.m2/settings.xml
default_maven_settings_path = os.path.join(os.path.expanduser("~"), ".m2", "settings.xml")
custom_maven_settings_path = self._custom_settings.get("maven_user_settings")
if custom_maven_settings_path is not None:
# User explicitly provided a path
if not os.path.exists(custom_maven_settings_path):
error_msg = (
f"Provided maven settings file not found: {custom_maven_settings_path}. "
f"Fix: create the file, update path in ~/.serena/serena_config.yml (ls_specific_settings -> java -> maven_user_settings), "
f"or remove the setting to use default ({default_maven_settings_path})"
)
log.error(error_msg)
raise FileNotFoundError(error_msg)
maven_settings_path = custom_maven_settings_path
log.info(f"Using Maven settings from custom location: {maven_settings_path}")
elif os.path.exists(default_maven_settings_path):
maven_settings_path = default_maven_settings_path
log.info(f"Using Maven settings from default location: {maven_settings_path}")
else:
maven_settings_path = None
log.info(f"Maven settings not found at default location ({default_maven_settings_path}), will use JDTLS defaults")
# Gradle user home: default to ~/.gradle
default_gradle_home = os.path.join(os.path.expanduser("~"), ".gradle")
custom_gradle_home = self._custom_settings.get("gradle_user_home")
if custom_gradle_home is not None:
# User explicitly provided a path
if not os.path.exists(custom_gradle_home):
error_msg = (
f"Gradle user home directory not found: {custom_gradle_home}. "
f"Fix: create the directory, update path in ~/.serena/serena_config.yml (ls_specific_settings -> java -> gradle_user_home), "
f"or remove the setting to use default (~/.gradle)"
)
log.error(error_msg)
raise FileNotFoundError(error_msg)
gradle_user_home = custom_gradle_home
log.info(f"Using Gradle user home from custom location: {gradle_user_home}")
elif os.path.exists(default_gradle_home):
gradle_user_home = default_gradle_home
log.info(f"Using Gradle user home from default location: {gradle_user_home}")
else:
gradle_user_home = None
log.info(f"Gradle user home not found at default location ({default_gradle_home}), will use JDTLS defaults")
initialize_params = {
"locale": "en",
"rootPath": repository_absolute_path,
@@ -512,7 +570,7 @@ class EclipseJDTLS(SolidLanguageServer):
"checkProjectSettingsExclusions": False,
"updateBuildConfiguration": "interactive",
"maven": {
"userSettings": None,
"userSettings": maven_settings_path,
"globalSettings": None,
"notCoveredPluginExecutionSeverity": "warning",
"defaultMojoExecutionAction": "ignore",
@@ -538,7 +596,7 @@ class EclipseJDTLS(SolidLanguageServer):
"offline": {"enabled": False},
"arguments": None,
"jvmArguments": None,
"user": {"home": None},
"user": {"home": gradle_user_home},
"annotationProcessing": {"enabled": True},
},
"exclusions": [
@@ -549,7 +607,9 @@ class EclipseJDTLS(SolidLanguageServer):
],
"generatesMetadataFilesAtProjectRoot": False,
},
"maven": {"downloadSources": True, "updateSnapshots": True},
# Set updateSnapshots to False to improve performance and avoid unnecessary network calls
# Snapshots will only be updated when explicitly requested by the user
"maven": {"downloadSources": True, "updateSnapshots": False},
"eclipse": {"downloadSources": True},
"signatureHelp": {"enabled": True, "description": {"enabled": True}},
"implementationsCodeLens": {"enabled": True},
@@ -624,29 +684,29 @@ class EclipseJDTLS(SolidLanguageServer):
],
}
initialize_params["initializationOptions"]["workspaceFolders"] = [repo_uri]
initialize_params["initializationOptions"]["workspaceFolders"] = [repo_uri] # type: ignore
bundles = [self.runtime_dependency_paths.intellicode_jar_path]
initialize_params["initializationOptions"]["bundles"] = bundles
initialize_params["initializationOptions"]["settings"]["java"]["configuration"]["runtimes"] = [
initialize_params["initializationOptions"]["bundles"] = bundles # type: ignore
initialize_params["initializationOptions"]["settings"]["java"]["configuration"]["runtimes"] = [ # type: ignore
{"name": "JavaSE-21", "path": self.runtime_dependency_paths.jre_home_path, "default": True}
]
for runtime in initialize_params["initializationOptions"]["settings"]["java"]["configuration"]["runtimes"]:
for runtime in initialize_params["initializationOptions"]["settings"]["java"]["configuration"]["runtimes"]: # type: ignore
assert "name" in runtime
assert "path" in runtime
assert os.path.exists(runtime["path"]), f"Runtime required for eclipse_jdtls at path {runtime['path']} does not exist"
gradle_settings = initialize_params["initializationOptions"]["settings"]["java"]["import"]["gradle"]
gradle_settings = initialize_params["initializationOptions"]["settings"]["java"]["import"]["gradle"] # type: ignore
gradle_settings["home"] = self.runtime_dependency_paths.gradle_path
gradle_settings["java"]["home"] = self.runtime_dependency_paths.jre_path
return initialize_params
return cast(InitializeParams, initialize_params)
def _start_server(self):
def _start_server(self) -> None:
"""
Starts the Eclipse JDTLS Language Server
"""
def register_capability_handler(params):
def register_capability_handler(params: dict) -> None:
assert "registrations" in params
for registration in params["registrations"]:
if registration["method"] == "textDocument/completion":
@@ -664,22 +724,22 @@ class EclipseJDTLS(SolidLanguageServer):
self.intellicode_enable_command_available.set()
return
def lang_status_handler(params):
def lang_status_handler(params: dict) -> None:
# TODO: Should we wait for
# server -> client: {'jsonrpc': '2.0', 'method': 'language/status', 'params': {'type': 'ProjectStatus', 'message': 'OK'}}
# Before proceeding?
if params["type"] == "ServiceReady" and params["message"] == "ServiceReady":
self.service_ready_event.set()
def execute_client_command_handler(params):
def execute_client_command_handler(params: dict) -> list:
assert params["command"] == "_java.reloadBundles.command"
assert params["arguments"] == []
return []
def window_log_message(msg):
self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO)
def window_log_message(msg: dict) -> None:
log.info(f"LSP: window/logMessage: {msg}")
def do_nothing(params):
def do_nothing(params: dict) -> None:
return
self.server.on_request("client/registerCapability", register_capability_handler)
@@ -690,22 +750,19 @@ class EclipseJDTLS(SolidLanguageServer):
self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
self.server.on_notification("language/actionableNotification", do_nothing)
self.logger.log("Starting EclipseJDTLS server process", logging.INFO)
log.info("Starting EclipseJDTLS server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
self.logger.log(
"Sending initialize request from LSP client to LSP server and awaiting response",
logging.INFO,
)
log.info("Sending initialize request from LSP client to LSP server and awaiting response")
init_response = self.server.send.initialize(initialize_params)
assert init_response["capabilities"]["textDocumentSync"]["change"] == 2
assert init_response["capabilities"]["textDocumentSync"]["change"] == 2 # type: ignore
assert "completionProvider" not in init_response["capabilities"]
assert "executeCommandProvider" not in init_response["capabilities"]
self.server.notify.initialized({})
self.server.notify.workspace_did_change_configuration({"settings": initialize_params["initializationOptions"]["settings"]})
self.server.notify.workspace_did_change_configuration({"settings": initialize_params["initializationOptions"]["settings"]}) # type: ignore
self.intellicode_enable_command_available.wait()
@@ -721,3 +778,27 @@ class EclipseJDTLS(SolidLanguageServer):
# TODO: Add comments about why we wait here, and how this can be optimized
self.service_ready_event.wait()
def _request_document_symbols(
self, relative_file_path: str, file_data: LSPFileBuffer | None
) -> list[SymbolInformation] | list[DocumentSymbol] | None:
result = super()._request_document_symbols(relative_file_path, file_data=file_data)
if result is None:
return None
# JDTLS sometimes returns symbol names with type information to handle overloads,
# e.g. "myMethod(int) <T>", but we want overloads to be handled via overload_idx,
# which requires the name to be just "myMethod".
def fix_name(symbol: SymbolInformation | DocumentSymbol | UnifiedSymbolInformation) -> None:
if "(" in symbol["name"]:
symbol["name"] = symbol["name"][: symbol["name"].index("(")]
children = symbol.get("children")
if children:
for child in children: # type: ignore
fix_name(child)
for root_symbol in result:
fix_name(root_symbol)
return result
src/solidlsp/language_servers/elixir_tools/elixir_tools.py
+58 -48
View File
@@ -4,26 +4,32 @@ import pathlib
import stat
import subprocess
import threading
import time
from typing import Any, cast
from overrides import override
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.ls_utils import FileUtils, PlatformId, PlatformUtils
from solidlsp.lsp_protocol_handler import lsp_types
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
from solidlsp.settings import SolidLSPSettings
from ..common import RuntimeDependency
log = logging.getLogger(__name__)
class ElixirTools(SolidLanguageServer):
"""
Provides Elixir specific instantiation of the LanguageServer class using Next LS from elixir-tools.
"""
@override
def _get_wait_time_for_cross_file_referencing(self) -> float:
return 10.0 # Elixir projects need a lot of time to compile and index before cross-file references work
@override
def is_ignored_dirname(self, dirname: str) -> bool:
# For Elixir projects, we should ignore:
@@ -34,7 +40,17 @@ class ElixirTools(SolidLanguageServer):
# - cover: coverage reports
return super().is_ignored_dirname(dirname) or dirname in ["_build", "deps", "node_modules", ".elixir_ls", "cover"]
def _is_next_ls_internal_file(self, abs_path: str) -> bool:
@override
def is_ignored_path(self, relative_path: str, ignore_unsupported_files: bool = True) -> bool:
"""Check if a path should be ignored for symbol indexing."""
if relative_path.endswith("mix.exs"):
# These are project configuration files, not source code with symbols to index
return True
return super().is_ignored_path(relative_path, ignore_unsupported_files)
@staticmethod
def _is_next_ls_internal_file(abs_path: str) -> bool:
"""Check if an absolute path is a Next LS internal file that should be ignored."""
return any(
pattern in abs_path
@@ -47,7 +63,7 @@ class ElixirTools(SolidLanguageServer):
)
@override
def _send_references_request(self, relative_file_path: str, line: int, column: int):
def _send_references_request(self, relative_file_path: str, line: int, column: int) -> list[lsp_types.Location] | None:
"""Override to filter out Next LS internal files from references."""
from solidlsp.ls_utils import PathUtils
@@ -63,14 +79,14 @@ class ElixirTools(SolidLanguageServer):
if isinstance(item, dict) and "uri" in item:
abs_path = PathUtils.uri_to_path(item["uri"])
if self._is_next_ls_internal_file(abs_path):
self.logger.log(f"Filtering out Next LS internal file: {abs_path}", logging.DEBUG)
log.debug(f"Filtering out Next LS internal file: {abs_path}")
continue
filtered_response.append(item)
return filtered_response
@classmethod
def _get_elixir_version(cls):
def _get_elixir_version(cls) -> str | None:
"""Get the installed Elixir version or None if not found."""
try:
result = subprocess.run(["elixir", "--version"], capture_output=True, text=True, check=False)
@@ -81,9 +97,7 @@ class ElixirTools(SolidLanguageServer):
return None
@classmethod
def _setup_runtime_dependencies(
cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings
) -> str:
def _setup_runtime_dependencies(cls, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings) -> str:
"""
Setup runtime dependencies for Next LS.
Downloads the Next LS binary for the current platform and returns the path to the executable.
@@ -95,7 +109,7 @@ class ElixirTools(SolidLanguageServer):
"Elixir is not installed. Please install Elixir from https://elixir-lang.org/install.html and make sure it is added to your PATH."
)
logger.log(f"Found Elixir: {elixir_version}", logging.INFO)
log.info(f"Found Elixir: {elixir_version}")
platform_id = PlatformUtils.get_platform_id()
@@ -156,11 +170,13 @@ class ElixirTools(SolidLanguageServer):
dependency = runtime_deps[platform_id]
executable_path = os.path.join(next_ls_dir, "nextls")
assert dependency.binary_name is not None
binary_path = os.path.join(next_ls_dir, dependency.binary_name)
if not os.path.exists(executable_path):
logger.log(f"Downloading Next LS binary from {dependency.url}", logging.INFO)
FileUtils.download_file(logger, dependency.url, binary_path)
log.info(f"Downloading Next LS binary from {dependency.url}")
assert dependency.url is not None
FileUtils.download_file(dependency.url, binary_path)
# Make the binary executable on Unix-like systems
os.chmod(binary_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
@@ -173,17 +189,14 @@ class ElixirTools(SolidLanguageServer):
assert os.path.exists(executable_path), f"Next LS executable not found at {executable_path}"
logger.log(f"Next LS binary ready at: {executable_path}", logging.INFO)
log.info(f"Next LS binary ready at: {executable_path}")
return executable_path
def __init__(
self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings
):
nextls_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings)
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
nextls_executable_path = self._setup_runtime_dependencies(config, solidlsp_settings)
super().__init__(
config,
logger,
repository_root_path,
ProcessLaunchInfo(cmd=f'"{nextls_executable_path}" --stdio', cwd=repository_root_path),
"elixir",
@@ -250,33 +263,38 @@ class ElixirTools(SolidLanguageServer):
"didChangeConfiguration": {"dynamicRegistration": True},
"executeCommand": {"dynamicRegistration": True},
},
"window": {
"showMessage": {"messageActionItem": {"additionalPropertiesSupport": True}},
"showDocument": {"support": True},
"workDoneProgress": True,
},
},
"workspaceFolders": [{"uri": root_uri, "name": os.path.basename(repository_absolute_path)}],
}
return initialize_params
return cast(InitializeParams, initialize_params)
def _start_server(self):
def _start_server(self) -> None:
"""Start Next LS server process"""
def register_capability_handler(params):
def register_capability_handler(params: Any) -> None:
return
def window_log_message(msg):
def window_log_message(msg: Any) -> None:
"""Handle window/logMessage notifications from Next LS"""
message_text = msg.get("message", "")
self.logger.log(f"LSP: window/logMessage: {message_text}", logging.INFO)
log.info(f"LSP: window/logMessage: {message_text}")
# Check for the specific Next LS readiness signal
# Based on Next LS source: "Runtime for folder #{name} is ready..."
if "Runtime for folder" in message_text and "is ready..." in message_text:
self.logger.log("Next LS runtime is ready based on official log message", logging.INFO)
log.info("Next LS runtime is ready based on official log message")
self.server_ready.set()
def do_nothing(params):
def do_nothing(params: Any) -> None:
return
def check_server_ready(params):
def check_server_ready(params: Any) -> None:
"""
Handle $/progress notifications from Next LS.
Keep as fallback for error detection, but primary readiness detection
@@ -288,47 +306,45 @@ class ElixirTools(SolidLanguageServer):
if value.get("kind") == "end":
message = value.get("message", "")
if "has initialized!" in message:
self.logger.log("Next LS initialization progress completed", logging.INFO)
log.info("Next LS initialization progress completed")
# Note: We don't set server_ready here - we wait for the log message
def work_done_progress(params):
def work_done_progress(params: Any) -> None:
"""
Handle $/workDoneProgress notifications from Next LS.
Keep for completeness but primary readiness detection is via window/logMessage.
"""
value = params.get("value", {})
if value.get("kind") == "end":
self.logger.log("Next LS work done progress completed", logging.INFO)
log.info("Next LS work done progress completed")
# Note: We don't set server_ready here - we wait for the log message
self.server.on_request("client/registerCapability", register_capability_handler)
self.server.on_notification("window/logMessage", window_log_message)
self.server.on_notification("$/progress", check_server_ready)
self.server.on_notification("window/workDoneProgress/create", do_nothing)
self.server.on_request("window/workDoneProgress/create", do_nothing)
self.server.on_notification("$/workDoneProgress", work_done_progress)
self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
self.logger.log("Starting Next LS server process", logging.INFO)
log.info("Starting Next LS server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
self.logger.log(
"Sending initialize request from LSP client to LSP server and awaiting response",
logging.INFO,
)
log.info("Sending initialize request from LSP client to LSP server and awaiting response")
init_response = self.server.send.initialize(initialize_params)
# Verify server capabilities - be more lenient with Next LS
self.logger.log(f"Next LS capabilities: {list(init_response['capabilities'].keys())}", logging.INFO)
log.info(f"Next LS capabilities: {list(init_response['capabilities'].keys())}")
# Next LS may not provide all capabilities immediately, so we check for basic ones
assert "textDocumentSync" in init_response["capabilities"], f"Missing textDocumentSync in {init_response['capabilities']}"
# Some capabilities might be optional or provided later
# Some capabilities might be optional or provided later. This is expected, so we log as info
if "completionProvider" not in init_response["capabilities"]:
self.logger.log("Warning: completionProvider not available in initial capabilities", logging.WARNING)
log.info("completionProvider not available in initial capabilities")
if "definitionProvider" not in init_response["capabilities"]:
self.logger.log("Warning: definitionProvider not available in initial capabilities", logging.WARNING)
log.info("definitionProvider not available in initial capabilities")
self.server.notify.initialized({})
self.completions_available.set()
@@ -336,18 +352,12 @@ class ElixirTools(SolidLanguageServer):
# Wait for Next LS to send the specific "Runtime for folder X is ready..." log message
# This is the authoritative signal that Next LS is truly ready for requests
ready_timeout = 180.0
self.logger.log(f"Waiting up to {ready_timeout} seconds for Next LS runtime readiness...", logging.INFO)
log.info(f"Waiting up to {ready_timeout} seconds for Next LS runtime readiness...")
if self.server_ready.wait(timeout=ready_timeout):
self.logger.log("Next LS is ready and available for requests", logging.INFO)
log.info("Next LS is ready and available for requests")
# Add a small settling period to ensure background indexing is complete
# Next LS often continues compilation/indexing in background after ready signal
settling_time = 120.0
self.logger.log(f"Allowing {settling_time} seconds for Next LS background indexing to complete...", logging.INFO)
time.sleep(settling_time)
self.logger.log("Next LS settling period complete", logging.INFO)
else:
error_msg = f"Next LS failed to initialize within {ready_timeout} seconds. This may indicate a problem with the Elixir installation, project compilation, or Next LS itself."
self.logger.log(error_msg, logging.ERROR)
log.error(error_msg)
raise RuntimeError(error_msg)
src/solidlsp/language_servers/elm_language_server.py
+19 -30
View File
@@ -13,33 +13,27 @@ from sensai.util.logging import LogTime
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.ls_logger import LanguageServerLogger
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
from solidlsp.settings import SolidLSPSettings
from .common import RuntimeDependency, RuntimeDependencyCollection
log = logging.getLogger(__name__)
class ElmLanguageServer(SolidLanguageServer):
"""
Provides Elm specific instantiation of the LanguageServer class. Contains various configurations and settings specific to Elm.
"""
def __init__(
self, config: LanguageServerConfig, logger: LanguageServerLogger, repository_root_path: str, solidlsp_settings: SolidLSPSettings
):
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""
Creates an ElmLanguageServer instance. This class is not meant to be instantiated directly. Use LanguageServer.create() instead.
"""
elm_lsp_executable_path = self._setup_runtime_dependencies(logger, config, solidlsp_settings)
elm_lsp_executable_path = self._setup_runtime_dependencies(config, solidlsp_settings)
super().__init__(
config,
logger,
repository_root_path,
ProcessLaunchInfo(cmd=elm_lsp_executable_path, cwd=repository_root_path),
"elm",
solidlsp_settings,
config, repository_root_path, ProcessLaunchInfo(cmd=elm_lsp_executable_path, cwd=repository_root_path), "elm", solidlsp_settings
)
self.server_ready = threading.Event()
@@ -53,16 +47,14 @@ class ElmLanguageServer(SolidLanguageServer):
]
@classmethod
def _setup_runtime_dependencies(
cls, logger: LanguageServerLogger, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings
) -> list[str]:
def _setup_runtime_dependencies(cls, config: LanguageServerConfig, solidlsp_settings: SolidLSPSettings) -> list[str]:
"""
Setup runtime dependencies for Elm Language Server and return the command to start the server.
"""
# Check if elm-language-server is already installed globally
system_elm_ls = shutil.which("elm-language-server")
if system_elm_ls:
logger.log(f"Found system-installed elm-language-server at {system_elm_ls}", logging.INFO)
log.info(f"Found system-installed elm-language-server at {system_elm_ls}")
return [system_elm_ls, "--stdio"]
# Verify node and npm are installed
@@ -86,9 +78,9 @@ class ElmLanguageServer(SolidLanguageServer):
elm_ls_dir = os.path.join(cls.ls_resources_dir(solidlsp_settings), "elm-lsp")
elm_ls_executable_path = os.path.join(elm_ls_dir, "node_modules", ".bin", "elm-language-server")
if not os.path.exists(elm_ls_executable_path):
logger.log(f"Elm Language Server executable not found at {elm_ls_executable_path}. Installing...", logging.INFO)
with LogTime("Installation of Elm language server dependencies", logger=logger.logger):
deps.install(logger, elm_ls_dir)
log.info(f"Elm Language Server executable not found at {elm_ls_executable_path}. Installing...")
with LogTime("Installation of Elm language server dependencies", logger=log):
deps.install(elm_ls_dir)
if not os.path.exists(elm_ls_executable_path):
raise FileNotFoundError(
@@ -143,31 +135,28 @@ class ElmLanguageServer(SolidLanguageServer):
}
],
}
return initialize_params
return initialize_params # type: ignore[return-value]
def _start_server(self):
def _start_server(self) -> None:
"""
Starts the Elm Language Server, waits for the server to be ready and yields the LanguageServer instance.
"""
def do_nothing(params):
def do_nothing(params: dict) -> None:
return
def window_log_message(msg):
self.logger.log(f"LSP: window/logMessage: {msg}", logging.INFO)
def window_log_message(msg: dict) -> None:
log.info(f"LSP: window/logMessage: {msg}")
self.server.on_notification("window/logMessage", window_log_message)
self.server.on_notification("$/progress", do_nothing)
self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
self.logger.log("Starting Elm server process", logging.INFO)
log.info("Starting Elm server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
self.logger.log(
"Sending initialize request from LSP client to LSP server and awaiting response",
logging.INFO,
)
log.info("Sending initialize request from LSP client to LSP server and awaiting response")
init_response = self.server.send.initialize(initialize_params)
# Elm-specific capability checks
@@ -178,11 +167,11 @@ class ElmLanguageServer(SolidLanguageServer):
assert "documentSymbolProvider" in init_response["capabilities"]
self.server.notify.initialized({})
self.logger.log("Elm server initialized successfully, waiting for workspace scan...", logging.INFO)
log.info("Elm server initialized successfully, waiting for workspace scan...")
self.server_ready.set()
self.completions_available.set()
self.logger.log("Elm server ready", logging.INFO)
log.info("Elm server ready")
@override
def _get_wait_time_for_cross_file_referencing(self) -> float:
Loaded 100 of 214 files, more files were not shown because too many files have changed in this diff. Show more