tiennm99/serena
1
0 Fork 0
mirror of https://github.com/tiennm99/serena.git synced 2026-10-13 04:15:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into pywebview

Browse Source 
Conflicts:
	pyproject.toml
	src/serena/agent.py
	src/serena/cli.py
	src/serena/dashboard.py
	uv.lock
This commit is contained in:
Dominik Jain authored and Dominik Jain committed 2026-03-26 13:39:41 +01:00
2 parents 161c820909 483538a293
commit 971e2a5d28
172 files changed
+7873 -1863
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/pytest.yml
+29 -2
View File
@@ -139,7 +139,10 @@ jobs:
- name: Setup .NET SDK (for F# and C# languages)
uses: actions/setup-dotnet@v4
with:
dotnet-version: '8.0.x'
dotnet-version: '10.0.x'
- name: List .NET runtimes
shell: bash
run: dotnet --list-runtimes
- name: Install Terraform
uses: hashicorp/setup-terraform@v3
with:
@@ -196,6 +199,23 @@ jobs:
- name: Install Ruby language server
shell: bash
run: gem install ruby-lsp
- name: Install OCaml and opam
uses: ocaml/setup-ocaml@v3
with:
ocaml-compiler: ${{ runner.os == 'Windows' && '4.14' || '5.3.x' }}
dune-cache: true
opam-repositories: |
${{ runner.os == 'Windows' && 'opam-repository-mingw: https://github.com/ocaml-opam/opam-repository-mingw.git#sunset' || '' }}
default: https://github.com/ocaml/opam-repository.git
- name: Install OCaml packages
shell: bash
run: |
if [ "$RUNNER_OS" = "Windows" ]; then
opam install -y dune ocaml-lsp-server
else
# Require ocaml-lsp-server >= 1.23.0 for cross-file reference support
opam install -y dune 'ocaml-lsp-server>=1.23.0'
fi
- name: Install R
uses: r-lib/actions/setup-r@v2
with:
@@ -363,6 +383,9 @@ jobs:
echo "PERL_MB_OPT=--install_base \"$HOME/perl5\"" >> $GITHUB_ENV
echo "PERL_MM_OPT=INSTALL_BASE=$HOME/perl5" >> $GITHUB_ENV
echo "$HOME/perl5/bin" >> $GITHUB_PATH
- name: Install ansible-core and ansible-lint (for Ansible language server tests)
shell: bash
run: uv run pip install ansible-core ansible-lint
- name: Install Elm
shell: bash
run: npm install -g elm@0.19.1-6
@@ -394,7 +417,7 @@ jobs:
- name: Install Regal (Rego Language Server)
shell: bash
run: |
REGAL_VERSION="0.36.1"
REGAL_VERSION="0.39.0"
if [[ "${{ runner.os }}" == "Linux" ]]; then
if [[ "$(uname -m)" == "x86_64" ]]; then
@@ -524,6 +547,10 @@ jobs:
echo "ERROR: FPCDIR does not exist: $FPCDIR"
exit 1
fi
- name: Build Lean 4 test project
uses: leanprover/lean-action@v1
with:
lake-package-directory: test/resources/repos/lean4/test_repo
- name: Cache language servers
id: cache-language-servers
uses: actions/cache@v3
.gitignore
+13 -1
View File
@@ -226,13 +226,22 @@ tmp/
# Exception: Don't ignore Nix test repository lib directory (contains source code)
!/test/resources/repos/nix/test_repo/lib
# Exception: Don't ignore OCaml test repository lib directory (contains source code)
!/test/resources/repos/ocaml/test_repo/lib
# Exception: Don't ignore Julia test repository lib directory (contains source code)
!/test/resources/repos/julia/test_repo/lib
# Exception: Don't ignore Solidity test repository lib directory (contains source code)
!/test/resources/repos/solidity/test_repo/contracts/lib
# Swift
/test/resources/repos/swift/test_repo/.build
/test/resources/repos/swift/test_repo/.swiftpm
# OCaml
/test/resources/repos/ocaml/test_repo/_build
# Elm
/test/resources/repos/elm/test_repo/.elm/
/test/resources/repos/elm/test_repo/elm-stuff/
@@ -253,5 +262,8 @@ dist-newstyle/
cabal.project.local*
.ghc.environment.*
# Lean 4
.lake/
zz-misc/
vue-implementation/
vue-implementation/
.serena/.gitignore
+1
View File
@@ -1 +1,2 @@
/cache
/project.local.yml
.serena/project.yml
+23 -1
View File
@@ -90,8 +90,22 @@ included_optional_tools: []
# This cannot be combined with non-empty excluded_tools or included_optional_tools.
# initial prompt for the project. It will always be given to the LLM upon activating the project
# (contrary to the memories, which are loaded on demand).
initial_prompt: ""
initial_prompt: |
IMPORTANT: You use an idiomatic, object-oriented style.
In particular, this implies that, for any non-trivial interfaces, you use interfaces that expect explicitly typed abstractions
rather than mere functions (i.e. use the strategy pattern, for example).
You avoid the use of low-level data structures in all cases where an object-oriented abstraction would be more appropriate.
For simple data storage, you use dataclasses instead of dictionaries or tuples.
You structure function implementations into functional blocks that are separated by blank lines.
Atop each functional block, you write an elliptical phrase (starting with lower-case letter) that describes the purpose of the
block in a concise manner.
Docstrings: You consistently use reStructuredText.
Comments:
When describing parameters, methods/functions and classes, you use a precise style, where the initial (elliptical) phrase
clearly defines *what* it is. Any details then follow in subsequent sentences.
# the encoding used by text files in the project
# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
encoding: utf-8
@@ -128,3 +142,11 @@ symbol_info_budget:
# Note: the backend is fixed at startup. If a project with a different backend
# is activated post-init, an error will be returned.
language_backend:
# list of regex patterns which, when matched, mark a memory entry as read‑only.
# Extends the list from the global configuration, merging the two lists.
read_only_memory_patterns: []
# line ending convention for file writes: unset (use global setting, "lf", "crlf", or "native" (platform default)
# If not undefined, overrides the global setting in serena_config.yml
line_ending:
CHANGELOG.md
+35 -9
View File
@@ -1,13 +1,30 @@
# latest
# Latest
Status of the `main` branch. Changes prior to the next official version change will appear here.
* Memories:
* Add `ignored_memory_patterns` configuration option (regex-based) to completely exclude
matching memories from listing, reading, and writing. Configurable at both global
(`serena_config.yml`) and project (`project.yml`) level, merged additively.
Useful for projects with large numbers of archived memory files that inflate
`activate_project` output.
* New language support:
* Add Solidity language server support (`Language.SOLIDITY`) using the
Nomic Foundation `@nomicfoundation/solidity-language-server`. Automatically
installed via npm. Supports `.sol` files with go-to-definition, find references,
document symbols, hover, and diagnostics. Works best with a `foundry.toml` or
`hardhat.config.js` in the project root.
* General:
* Add monorepo/multi-language support
* Project configuration files (`project.yml`) can now define multiple languages.
Auto-detection adds only the most prominent language by default.
* Additional languages can be conveniently added via the Dashboard while a project is already activated.
* The current project can be activated automatically even when the MCP configuration is global (through the --project-from-cwd flag)
* Add support for querying projects other than the currently active one via new tools `QueryProjectTool` and `ListQueryableProjectsTool`.
The `QueryProjectTool` allows Serena tools to be called on other projects.
* For the LSP backend, calling symbolic tools require a project server to be spawned that will launch the respective language servers
* For the JetBrains backend, all projects for which IDE instances are open can directly be queried
* Support overloaded symbols in `FindSymbolTool` and related tools
* Name paths of overloaded symbols now include an index (e.g., `myOverloadedFunction[2]`)
* Responses of the Java language server, which handled this in its own way, are now adapted accordingly,
@@ -19,26 +36,35 @@ Status of the `main` branch. Changes prior to the next official version change w
* View tool usage statistics
* View and create memories and edit the serena configuration file
* Log page now has save (downloads a snapshot) and clear (resets log view) buttons alongside the existing copy button
* New two-tier caching of language server document symbols and considerable performance improvements surrounding symbol retrieval/indexing
* Language server backend:
* New two-tier caching of language server document symbols and considerable performance improvements surrounding symbol retrieval/indexing
* Allow passing language server specific settings through `ls_specific_settings` field (in `serena_config.yml`)
* Add the JetBrains language backend as an alternative to language servers
* Improve management of the Serena projects
* Facilitate project activation based on the current directory (through the `--project-from-cwd` parameter)
* Add notion of a "single-project context" (flag `single_project`), allowing user-defined contexts to behave
like the built-in `ide-assistant` context (where the available tools are restricted to ones required by the active
project and project changes are disabled)
* The location of Serena's project-specific data folder can now be flexibly configured, allowing, in particular,
locations outside of the project folder, thus improving support for read-only projects.
* Add support for `project.local.yml` for local overrides that should not be versioned
* Various fixes related to indexing, special paths and determination of ignored paths
* Decreased `TOOL_DEFAULT_MAX_ANSWER_LENGTH` to be in accordance with (below) typical max-tokens configurations
* Allow passing language server specific settings through `ls_specific_settings` field (in `serena_config.yml`)
* Add notion of a "single-project context" (flag `single_project`), allowing user-defined contexts to behave
like the built-in `ide-assistant` context (where the available tools are restricted to ones required by the active
project and project changes are disabled)
* Client support:
* New mode `oaicompat-agent` and extensions enhancing OpenAI tool compatibility, permitting Serena to work with llama.cpp
* Tools:
* *New tool*: `jet_brains_type_hierarchy`
* Symbol information (hover, docstring, quick-info) is now provided as part of `find_symbol` and related tool responses.
* Added `QueryProjectTool` and `ListQueryableProjectTool` (see above)
* Added `RenameSymbolTool` for renaming symbols across the codebase (if LS supports this operation).
* Replaced `ReplaceRegexTool` with `ReplaceContentTool`, which supports both plain text and regex-based replacements
(and which requires no escaping in the replacement text, making it more robust)
* Decreased `TOOL_DEFAULT_MAX_ANSWER_LENGTH` to be in accordance with (below) typical max-tokens configurations
* Language support:
* **Add support for Lean 4** via built-in `lean --server` with cross-file reference support (requires `lean` and `lake` via [elan](https://github.com/leanprover/elan))
* **Add support for OCaml** via ocaml-lsp-server with cross-file reference support on OCaml 5.2+ (requires opam; see [setup guide](docs/03-special-guides/ocaml_setup_guide_for_serena.md))
* **Add Phpactor as alternative PHP language server** (specify `php_phpactor` as language; requires PHP 8.1+)
* **Add support for Fortran** via fortls language server (requires `pip install fortls`)
* **Add partial support for Groovy** requires user-provided Groovy language server JAR (see [setup guide](docs/03-special-guides/groovy_setup_guide_for_serena.md))
CLAUDE.md
+2 -2
View File
@@ -5,7 +5,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
## Development Commands
**Essential Commands (use these exact commands):**
- `uv run poe format` - Format code (BLACK + RUFF) - ONLY allowed formatting command
- `uv run poe format` - Format code (RUFF) - ONLY allowed formatting command
- `uv run poe type-check` - Run mypy type checking - ONLY allowed type checking command
- `uv run poe test` - Run tests with default markers (excludes java/rust by default)
- `uv run poe test -m "python or go"` - Run specific language tests
@@ -108,7 +108,7 @@ Configuration is loaded from (in order of precedence):
## Working with the Codebase
- Project uses Python 3.11 with `uv` for dependency management
- Strict typing with mypy, formatted with black + ruff
- Strict typing with mypy, formatted with ruff
- Language servers run as separate processes with LSP communication
- Memory system enables persistent project knowledge
- Context/mode system allows workflow customization
CONTRIBUTING.md
+3
View File
@@ -11,6 +11,9 @@ The following types of contributions can be submitted directly via pull requests
For other changes, please open an issue first to discuss your ideas with the maintainers.
When submitting a PR, ensure a well-defined scope.
Every PR should cover a single logical change or a set of closely related changes.
### Adding Support for a New Language Server
See the corresponding [memory](.serena/memories/adding_new_language_support_guide.md).
README.md
+6 -8
View File
@@ -18,7 +18,6 @@ Instead, it can use code-centric tools like `find_symbol`, `find_referencing_sym
<p align="center">
<a href="CHANGELOG.md"><img src="https://img.shields.io/badge/Updates-1e293b?style=flat&logo=rss&logoColor=white&labelColor=1e293b" alt="Changelog" /></a>
<a href="roadmap.md"><img src="https://img.shields.io/badge/Roadmap-14532d?style=flat&logo=target&logoColor=white&labelColor=14532d" alt="Roadmap" /></a>
<a href="lessons_learned.md"><img src="https://img.shields.io/badge/Lessons-Learned-7c4700?style=flat&logo=readthedocs&logoColor=white&labelColor=7c4700" alt="Lessons Learned" /></a>
</p>
@@ -81,8 +80,8 @@ Serena incorporates a powerful abstraction layer for the integration of language
that implement the language server protocol (LSP).
The underlying language servers are typically open-source projects (like Serena) or at least freely available for use.
With Serena's LSP library, we provide **support for over 30 programming languages**, including
AL, Bash, C#, C/C++, Clojure, Dart, Elixir, Elm, Erlang, Fortran, Go, Groovy (partial support), Haskell, Java, Javascript, Julia, Kotlin, Lua, Markdown, MATLAB, Nix, Perl, PHP, PowerShell, Python, R, Ruby, Rust, Scala, Swift, TOML, TypeScript, YAML, and Zig.
With Serena's LSP library, we provide **support for over 40 programming languages**, including
AL, Ansible, Bash, C#, C/C++, Clojure, Dart, Elixir, Elm, Erlang, Fortran, F# (currently with some bugs), GLSL, Go, Groovy (partial support), Haskell, HLSL, Java, Javascript, Julia, Kotlin, Lean 4, Lua, Luau, Markdown, MATLAB, Nix, OCaml, Perl, PHP, PowerShell, Python, R, Ruby, Rust, Scala, Solidity, Swift, TOML, TypeScript, WGSL, YAML, and Zig.
> [!IMPORTANT]
> Some language servers require additional dependencies to be installed; see the [Language Support](https://oraios.github.io/serena/01-about/020_programming-languages.html) page for details.
@@ -90,14 +89,13 @@ AL, Bash, C#, C/C++, Clojure, Dart, Elixir, Elm, Erlang, Fortran, Go, Groovy (pa
### The Serena JetBrains Plugin
As an alternative to language servers, the [Serena JetBrains Plugin](https://plugins.jetbrains.com/plugin/28946-serena/)
leverages the powerful code analysis capabilities of your JetBrains IDE.
The plugin naturally supports all programming languages and frameworks that are supported by JetBrains IDEs,
including IntelliJ IDEA, PyCharm, Android Studio, WebStorm, PhpStorm, RubyMine, GoLand, CLion, and others.
Only Rider is not supported.
leverages the powerful code analysis capabilities of your JetBrains IDE.
The plugin naturally supports all programming languages and frameworks that are supported by JetBrains IDEs,
including IntelliJ IDEA, PyCharm, Android Studio, WebStorm, PhpStorm, RubyMine, GoLand, and potentially others (Rider and CLion are unsupported though).
<a href="https://plugins.jetbrains.com/plugin/28946-serena/"><img src="docs/_static/images/jetbrains-marketplace-button.png"></a>
The plugin offers the most robust and most powerful Serena experience.
The plugin offers the most robust and most powerful Serena experience.
See our [documentation page](https://oraios.github.io/serena/02-usage/025_jetbrains_plugin.html) for further details and instructions.
## Quick Start
docs/01-about/.gitignore
+1
View File
@@ -0,0 +1 @@
/035_tools.md
docs/01-about/020_programming-languages.md
+27 -6
View File
@@ -28,9 +28,15 @@ We currently provide direct, out-of-the-box support for the programming language
Some languages require additional installations or setup steps, as noted.
* **AL**
* **Ansible**
(experimental; requires Node.js and npm; automatically installs `@ansible/ansible-language-server`;
must be explicitly specified in the `languages` entry in the `project.yml`; requires `ansible` in PATH for full functionality)
the upstream `@ansible/ansible-language-server@1.2.3` supports hover, completion, definition,
semantic tokens, and validation; document symbols, workspace symbols, references, and rename
are not supported by this version)
* **Bash**
* **C#**
(by default, uses the Roslyn language server (language `csharp`), requires `pwsh` ([PowerShell 7+](https://learn.microsoft.com/en-us/powershell/scripting/install/install-powershell-on-windows?view=powershell-7.5)) on Windows;
(by default, uses the Roslyn language server (language `csharp`), requiring [.NET v10+](https://dotnet.microsoft.com/en-us/download/dotnet) and, on Windows, `pwsh` ([PowerShell 7+](https://learn.microsoft.com/en-us/powershell/scripting/install/install-powershell-on-windows?view=powershell-7.5));
set language to `csharp_omnisharp` to use OmiSharp instead)
* **C/C++**
(by default, uses the clangd language server (language `cpp`) but we also support ccls (language `cpp_ccls`);
@@ -45,25 +51,36 @@ Some languages require additional installations or setup steps, as noted.
* **Erlang**
(requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls); experimental, might be slow or hang)
* **F#**
(requires .NET SDK 8.0+; uses FsAutoComplete/Ionide, which is auto-installed; for Homebrew .NET on macOS, set DOTNET_ROOT in your environment)
(requires [.NET v8.0+](https://dotnet.microsoft.com/en-us/download/dotnet); uses FsAutoComplete/Ionide, which is auto-installed; for Homebrew .NET on macOS, set DOTNET_ROOT in your environment)
* **Fortran**
(requires installation of fortls: `pip install fortls`)
* **Go**
(requires installation of `gopls`)
* **Groovy**
(requires local groovy-language-server.jar setup via `GROOVY_LS_JAR_PATH` or configuration)
* **Haskell**
* **Haskell**
(automatically locates HLS via ghcup, stack, or system PATH; supports Stack and Cabal projects)
* **HLSL / GLSL / WGSL**
(uses [shader-language-server](https://github.com/antaalt/shader-sense) (language `hlsl`); automatically downloaded;
on macOS, requires Rust toolchain for building from source;
note: reference search is not supported by this language server)
* **Java**
* **JavaScript**
* **JavaScript**
(supported via the TypeScript language server, i.e. use language `typescript` for both JavaScript and TypeScript)
* **Julia**
* **Kotlin**
(uses the pre-alpha [official kotlin LS](https://github.com/Kotlin/kotlin-lsp), some issues may appear)
* **Lean 4**
(requires `lean` and `lake` installed via [elan](https://github.com/leanprover/elan); uses the built-in Lean 4 LSP;
the project must be a Lake project with `lake build` run before use)
* **Lua**
* **Luau**
* **Markdown**
(must explicitly enable language `markdown`, primarily useful for documentation-heavy projects)
* **Nix**
(requires nixd installation)
* **OCaml**
(requires opam and ocaml-lsp-server to be installed manually; see the [OCaml Setup Guide](../03-special-guides/ocaml_setup_guide_for_serena.md))
* **Pascal**
(uses Pascal/Lazarus, which is automatically downloaded; set `PP` and `FPCDIR` environment variables for source navigation)
* **Perl**
@@ -80,6 +97,9 @@ Some languages require additional installations or setup steps, as noted.
(requires [rustup](https://rustup.rs/) - uses rust-analyzer from your toolchain)
* **Scala**
(requires some [manual setup](../03-special-guides/scala_setup_guide_for_serena); uses Metals LSP)
* **Solidity**
(experimental; requires Node.js and npm; automatically installs `@nomicfoundation/solidity-language-server`;
works best with a `foundry.toml` or `hardhat.config.js` in the project root)
* **Swift**
* **TypeScript**
* **Vue**
@@ -96,7 +116,8 @@ see Serena's [memory on that](https://github.com/oraios/serena/blob/main/.serena
As an alternative to language servers, the [Serena JetBrains Plugin](https://plugins.jetbrains.com/plugin/28946-serena/)
leverages the powerful code analysis capabilities of JetBrains IDEs.
The plugin naturally supports all programming languages and frameworks that are supported by JetBrains IDEs,
including IntelliJ IDEA, PyCharm, Android Studio, AppCode, WebStorm, PhpStorm, RubyMine, GoLand, AppCode, CLion, and others.
including IntelliJ IDEA, PyCharm, Android Studio, WebStorm, PhpStorm, RubyMine, GoLand, and potentially others
(Rider and CLion are unsupported though).
When using the plugin, Serena connects to an instance of your JetBrains IDE via the plugin. For users who already
work in a JetBrains IDE, this means Serena seamlessly integrates with the IDE instance you typically have open anyway,
@@ -120,4 +141,4 @@ of choice.
</p>
```
See the [JetBrains Plugin documentation](../02-usage/025_jetbrains_plugin) for usage details.
See the [JetBrains Plugin documentation](../02-usage/025_jetbrains_plugin) for usage details.
docs/01-about/035_tools.md
-49
View File
@@ -1,49 +0,0 @@
# List of Tools
Find the full list of Serena's tools below (output of `<serena> tools list --all`).
Note that in most configurations, only a subset of these tools will be enabled simultaneously (see the section on [configuration](../02-usage/050_configuration) for details).
* `activate_project`: Activates a project based on the project name or path.
* `check_onboarding_performed`: Checks whether project onboarding was already performed.
* `create_text_file`: Creates/overwrites a file in the project directory.
* `delete_lines`: Deletes a range of lines within a file.
* `delete_memory`: Deletes a memory from Serena's project-specific memory store.
* `edit_memory`:
* `execute_shell_command`: Executes a shell command.
* `find_file`: Finds files in the given relative paths
* `find_referencing_symbols`: Finds symbols that reference the given symbol using the language server backend
* `find_symbol`: Performs a global (or local) search using the language server backend.
* `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes.
* `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
* `initial_instructions`: Provides instructions on how to use the Serena toolbox.
Should only be used in settings where the system prompt is not read automatically by the client.
NOTE: Some MCP clients (including Claude Desktop) do not read the system prompt automatically!
* `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
* `insert_at_line`: Inserts content at a given line in a file.
* `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
* `jet_brains_find_referencing_symbols`: Finds symbols that reference the given symbol using the JetBrains backend
* `jet_brains_find_symbol`: Performs a global (or local) search for symbols using the JetBrains backend
* `jet_brains_get_symbols_overview`: Retrieves an overview of the top-level symbols within a specified file using the JetBrains backend
* `jet_brains_type_hierarchy`: Retrieves the type hierarchy (supertypes and/or subtypes) of a symbol using the JetBrains backend
* `list_dir`: Lists files and directories in the given directory (optionally with recursion).
* `list_memories`: Lists memories in Serena's project-specific memory store.
* `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
* `open_dashboard`: Opens the Serena web dashboard in the default web browser.
The dashboard provides logs, session information, and tool usage statistics.
* `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context).
* `read_file`: Reads a file within the project directory.
* `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
* `remove_project`: Removes a project from the Serena configuration.
* `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities.
* `replace_content`: Replaces content in a file (optionally using regular expressions).
* `replace_lines`: Replaces a range of lines within a file with new content.
* `replace_symbol_body`: Replaces the full definition of a symbol using the language server backend.
* `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
* `search_for_pattern`: Performs a search for a pattern in the project.
* `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
* `switch_modes`: Activates modes by providing a list of their names
* `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
* `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
* `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
* `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
docs/02-usage/020_running.md
+9 -2
View File
@@ -132,7 +132,7 @@ to use such a launch command.
(streamable-http)=
### Streamable HTTP Mode
When using instead the *Streamable HTTP* mode, you control the server lifecycle yourself,
When using *Streamable HTTP* mode, you control the server lifecycle yourself,
i.e. you start the server and provide the client with the URL to connect to it.
Simply provide `start-mcp-server` with the `--transport streamable-http` option and optionally provide the desired port
@@ -147,7 +147,14 @@ you would run
and then configure your client to connect to `http://localhost:9121/mcp`.
Note that while the legacy SSE transport is also supported (via `--transport sse` with corresponding /sse endpoint), its use is discouraged.
**When to use.** Note that Serena is a stateful MCP server, and only one coding project can be active at a time.
Therefore, starting a single Serena instance and connecting it to multiple clients is only
appropriate if all clients will be working on the same project.
If you want several agents to work on different projects, making each client/agent start its own server
in stdio mode is likely the best option.
See section [The Project Workflow](040_workflow) for more information on how to manage projects in Serena.
The legacy SSE transport is also supported (via `--transport sse` with corresponding /sse endpoint), its use is discouraged.
(mcp-args)=
### MCP Server Command-Line Arguments
docs/02-usage/040_workflow.md
+68 -17
View File
@@ -36,7 +36,7 @@ For instance, when using `uvx`, run
* For an existing project, the main programming language will be detected automatically,
but you can choose to explicitly specify multiple languages by passing the `--language` parameter
multiple times (e.g. `--language python --language typescript`).
* You can optionally specify a custom project name with `--name "My Project"`.
* You can optionally specify a custom project name with `--name my-name`.
* You can immediately index the project after creation with `--index`.
(project-config)=
@@ -57,7 +57,17 @@ The file allows you to configure ...
* the set of tools and modes to use by default
For detailed information on the parameters and possible settings, see the
[template file](https://github.com/oraios/serena/blob/main/src/serena/resources/project.template.yml).
[template file](https://github.com/oraios/serena/blob/main/src/serena/resources/project.template.yml).
:::{note}
Many settings in project.yml *extend* or *override* settings in the global configuration file `serena_config.yml`.
So use the project configuration specifically for aspects that apply only to the particular project.
:::
**Local Overrides**. The project.yml file is intended to be versioned together with the project.
You can specify local overrides for the settings in a `project.local.yml` file in the same directory
(which, by default, is ignored by git).
Any keys defined therein will override the respective key in `project.yml`.
(indexing)=
### Indexing
@@ -103,22 +113,12 @@ By default, Serena will perform an **onboarding process** when
it is started for the first time for a project.
The goal of the onboarding is for Serena to get familiar with the project
and to store memories, which it can then draw upon in future interactions.
If an LLM should fail to complete the onboarding and does not actually write the
respective memories to disk, you may need to ask it to do so explicitly.
The onboarding will usually read a lot of content from the project, thus filling
up the context. It can therefore be advisable to switch to another conversation
once the onboarding is complete.
After the onboarding, we recommend that you have a quick look at the memories and,
if necessary, edit them or add additional ones.
In general, **memories** provide a way for Serena to store and retrieve
information about the project, relevant conventions, and other relevant aspects.
**Memories** are files stored in `.serena/memories/` in the project directory,
which the agent can choose to read in subsequent interactions.
Feel free to read and adjust them as needed; you can also add new ones manually.
Every file in the `.serena/memories/` directory is a memory file.
Whenever Serena starts working on a project, the list of memories is
provided, and the agent can decide to read them.
We found that memories can significantly improve the user experience with Serena.
For more information on this, including how to manage
or disable these features, see [Memories & Onboarding](045_memories).
## Preparing Your Project
@@ -163,4 +163,55 @@ linting results, and test results to assess the correctness of its actions.
Therefore, software that is designed to meaningful interpretable outputs (e.g. log messages)
and that has a good test coverage is much easier to work with for Serena.
We generally recommend to start an editing task from a state where all linting checks and tests pass.
We generally recommend to start an editing task from a state where all linting checks and tests pass.
## Multiple Projects, Multiple Agents
There are several ways in which you might want to work with multiple projects simultaneously.
### A Single Agent Editing Multiple Projects Simultaneously
If fulfilling a task requires a single agent to edit code in multiple projects, the recommended approach is to create a **monorepo folder**,
i.e. a folder that contains all the projects as sub-folders, and open that monorepo folder as a project in Serena.
You may also use symbolic links to create a monorepo folder if the projects are located in different places on your filesystem.
If several languages are used across the projects, specify all of them as needed when using the LSP backend;
For JetBrains mode, make sure that your IDE is configured to work with all the languages used across the projects (e.g. by installing the respective language plugins).
(query-projects)=
### Reading from External Projects
If, while working on a project, you want Serena to be able to read code or other information from another project (e.g. a library or otherwise related project),
this can be enabled via the `query_project` tool.
Provided that the project you want to query is known to Serena (i.e. you have created it as described above),
the `query_project` tool allows the agent to query files and symbolic information from that project.
To enable this tool, [activate the mode](modes) `query-projects`.
This also enables a second tool for listing projects that can be queried.
Depending on the language backend being used, the management of resources for the external projects varies:
* When using the JetBrains backend, make sure that every project for which you want symbolic queries to work is open in an IDE instance.
* When using the LSP backend, executing symbolic tools via the query tool requires that Serena's **Project Server** be started,
which will automatically spawn the necessary language servers for the projects that are queried.
To start the server, run
<serena> start-project-server
where `<serena>` is your way of running Serena. For example, when using `uvx`, run
uvx --from git+https://github.com/oraios/serena serena start-project-server
### Multiple Agents Accessing a Single Serena Instance
If you want multiple agents to access the same project via a single Serena instance,
i.e. you do not want several instances of Serena (including its language servers) to be running,
you can achieve this by [starting the Serena MCP server in HTTP mode](streamable-http)
and connecting all client agents to the same HTTP endpoint.
The agents will then share the resources of the single Serena instance.
### Multiple Agents Working on Different Projects
For this use case, simply run a separate instance of Serena for each project, which naturally
occurs when Serena is started by the MCP client in stdio mode.
docs/02-usage/045_memories.md
+101
View File
@@ -0,0 +1,101 @@
# Memories & Onboarding
Serena provides the functionality of a fully featured agent, and a useful aspect of this is Serena's memory system.
Despite its simplicity, we received positive feedback from many users who tend to combine it with their
agent's internal memory management (e.g., `AGENTS.md` files).
## Memories
Memories are simple, human-readable Markdown files that both you and
your agent can create, read, and edit.
Serena differentiates between
* **project-specific memories**, which are stored in the `.serena/memories/` directory within your project folder, and
* **global memories**, which are shared across all projects and, by default, are stored in `~/.serena/memories/global/`
The LLM is informed about the existence of memories and instructed to read them when appropriate,
inferring appropriateness from the file name.
When the agent starts working on a project, it receives the list of available memories.
The agent should be instructed to update memories by the user when appropriate.
### Organizing Memories
Memories can be organized into **topics** by using `/` in the memory name (e.g. `modules/frontend`).
The structure is mapped to the file system, where topics correspond to subdirectories.
The `list_memories` tool can filter by topic, allowing the agent to explore even large numbers of memories in a structured way.
(global-memories)=
### Global Memories
Global memories use the top-level topic `global`, i.e. whenever a memory name starts with `global/`,
it is stored in the global memories directory and is shared across all projects.
By default, deletion and editing of global memories is allowed.
If you want to protect them from accidental modification by the agent,
you can add regex patterns to `read_only_memory_patterns` in your global or
project-level [configuration](050_configuration). For example, setting "global/.*" will mark all global memories as read-only. The agent will be informed which memories are read-only.
Since global memories are not versioned alongside your project files,
it can be helpful to track global memories with git (i.e. to make `~/.serena/memories/` a git repository)
in order to have a history of changes and the possibility to revert them if needed.
### Ignoring Memories
Projects that accumulate large numbers of archived memory files can use `ignored_memory_patterns`
to exclude them from `list_memories` and `activate_project` output. Add regex patterns to the
global or project-level [configuration](050_configuration):
```yaml
ignored_memory_patterns: ["_archive/.*", "_episodes/.*"]
```
Ignored memories are completely excluded — they cannot be accessed via `read_memory`, `write_memory`,
or any other memory tool. To read an ignored memory file, use the `read_file` tool on the raw file path
(e.g., `.serena/memories/_archive/2026-03/some-topic.md`).
Like `read_only_memory_patterns`, patterns from the global and project-level configurations are merged additively.
### Manually Editing Memories
You may edit memories directly in the file system, using your preferred text editor or IDE.
Alternatively, access them via the [Serena Dashboard](060_dashboard), which provides a graphical interface for
viewing, creating, editing, and deleting memories while Serena is running.
(onboarding)=
## Onboarding
By default, Serena performs an **onboarding process** when it encounters a project
for the first time (i.e., when no project memories exist yet).
The goal of the onboarding is for Serena to get familiar with the project —
its structure, build system, testing setup, and other essential aspects —
and to store this knowledge as memories for future interactions.
In further project activations, Serena will check whether onboarding was already
performed by looking for existing project memories and will skip the onboarding
process if memories are found.
### How Onboarding Works
1. When a project is activated, Serena checks whether onboarding was already
performed (by checking if any memories exist).
2. If no memories are found, Serena triggers the onboarding process, which
reads key files and directories to understand the project.
3. The gathered information is written into project-specific memory files (see above).
### Tips for Onboarding
- **Context usage**: The onboarding process will read a lot of content from the project,
filling up the context window. It is therefore advisable to **switch to a new conversation**
once the onboarding is complete.
- **LLM failures**: If an LLM fails to complete the onboarding and does not actually
write the respective memories to disk, you may need to ask it to do so explicitly.
- **Review the results**: After onboarding, we recommend having a quick look at the
generated memories and editing them or adding new ones as needed.
## Disabling Memories and Onboarding
If you do not require the functionality described in this section, you can selectively disable it.
* To disable all memory related tools (including onboarding), adding `no-memories` to the `base_modes`
in Serena's [global configuration](050_configuration).
* Similarly, to disable only onboarding, add `no-onboarding` to the `base_modes`.
docs/02-usage/050_configuration.md
+62 -3
View File
@@ -33,6 +33,9 @@ Some of the configurable settings include:
* logging settings
* advanced settings specific to individual language servers (see [below](ls-specific-settings))
The global configuration settings apply to all projects.
Some of the settings it contains can, however, be *extended* or *overridden* in project-specific settings, contexts and modes.
For detailed information on the parameters and possible settings, see the
[template file](https://github.com/oraios/serena/blob/main/src/serena/resources/serena_config.template.yml).
@@ -113,7 +116,8 @@ Examples of built-in modes include:
* `one-shot`: Configures Serena for tasks that should be completed in a single response, often used with `planning` for generating reports or initial plans.
* `no-onboarding`: Skips the initial onboarding process if it's not needed for a particular session but retains the memory tools (assuming initial memories were created externally).
* `onboarding`: Focuses on the project onboarding process.
* `no-memories`: Disables all memory tools (and tools building on memories such as onboarding tools)
* `no-memories`: Disables all memory tools (and tools building on memories such as onboarding tools)
* `query-projects`: Enables tools for querying other Serena projects (without activating them); see section [Reading from External Projects](query-projects)
Find the concrete definitions of these modes [here](https://github.com/oraios/serena/tree/main/src/serena/resources/config/modes).
@@ -172,6 +176,35 @@ For advanced users, Serena's configuration can be further customized.
The Serena user data directory (where configuration, language server files, logs, etc. are stored) defaults to `~/.serena`.
You can change this location by setting the `SERENA_HOME` environment variable to your desired path.
### Per-Project Serena Folder Location
By default, each project stores its Serena data (memories, caches, etc.) in a `.serena` folder inside the project root.
You can customize this location globally via the `project_serena_folder_location` setting in `serena_config.yml`.
The setting supports two placeholders:
| Placeholder | Description |
|----------------------|-------------------------------------------------|
| `$projectDir` | The absolute path to the project root directory |
| `$projectFolderName` | The name of the project folder |
**Examples:**
```yaml
# Default: data stored inside the project directory
project_serena_folder_location: "$projectDir/.serena"
# Central location: all project data under a shared directory
project_serena_folder_location: "/projects-metadata/$projectFolderName/.serena"
```
When a project is loaded, Serena uses the following fallback logic:
1. Check if a `.serena` folder exists at the configured path.
2. If not, check if one exists in the project root (default/legacy location).
3. If neither exists, create the folder at the configured path.
This ensures backward compatibility: existing projects that already have a `.serena` folder in the project root will continue to work, even after changing the `project_serena_folder_location` setting.
(ls-specific-settings)=
### Language Server-Specific Settings
@@ -180,8 +213,10 @@ You can change this location by setting the `SERENA_HOME` environment variable t
Most users will not need to adjust these settings.
:::
Under the key `ls_specific_settings` in `serena_config.yml`, you can you pass per-language,
language server-specific configuration.
Under the key `ls_specific_settings` in `serena_config.yml`, you can you pass global per-language,
language server-specific configuration. You can use the same key in the project configuration files (`project.yml`
and `project.local.yml` ) to override or extend the global settings for a specific project.
The settings are merged on top-level, meaning that project-level settings for a language will replace global settings for the same language.
Structure:
@@ -197,6 +232,7 @@ Most settings are currently undocumented. Please refer to the
implementation to determine supported settings.
:::
(override-ls-path)=
#### Overriding the Language Server Path
Some language servers, particularly those that use a single core path for the language server (e.g. the main executable),
@@ -322,6 +358,29 @@ ls_specific_settings:
jvm_options: "-Xmx4G -XX:+UseG1GC" # JVM options (default: -Xmx2G). Set to "" to disable.
```
#### Luau
Serena uses [`luau-lsp`](https://github.com/JohnnyMorganz/luau-lsp) for Luau support.
**Runtime Requirements:**
- `luau-lsp` is used from PATH if available.
- Otherwise, Serena downloads the pinned `luau-lsp` release for the current platform.
**Configuration:**
```yaml
ls_specific_settings:
luau:
ls_path: "/path/to/luau-lsp" # Optional: override the language server executable
platform: "roblox" # "roblox" (default) or "standard"
roblox_security_level: "PluginSecurity" # Roblox only: None, PluginSecurity, LocalUserSecurity, RobloxScriptSecurity
```
Notes:
- In `roblox` mode, Serena downloads Roblox definitions and Roblox API docs and passes them to `luau-lsp`.
- In `standard` mode, Serena skips Roblox definitions and only downloads the standard Luau docs bundle.
#### Pascal (`pasls`)
Serena uses [pasls](https://github.com/genericptr/pascal-language-server) (Pascal Language Server) for Pascal/Free Pascal support.
docs/03-special-guides/cpp_setup.md
+3 -1
View File
@@ -15,6 +15,8 @@ for cross-file reference finding, see below for details.
Your project must have a `compile_commands.json` file at the repository root.
This file is essential for correct parsing and cross-file reference finding.
You can use a specific clangd or ccls installation (e.g., a custom build or a version provided by your project),
by specifying the path in your configuration, see [ls-specific-settings](ls-specific-settings) for details.
## compile_commands.json Requirements
@@ -27,7 +29,7 @@ For reliable cross-file reference finding with clangd, your `compile_commands.js
### With clangd
Serena automatically downloads and manages clangd. Since clangd does not properly work with relative paths in `compile_commands.json`,
By default, Serena automatically downloads and manages clangd. Since clangd does not properly work with relative paths in `compile_commands.json`,
Serena will detect them and transform them into absolute paths automatically (writing a new `compile_commands.json` file), if needed.
#### Customizing the Compilation Database Location
docs/03-special-guides/ocaml_setup_guide_for_serena.md
+88
View File
@@ -0,0 +1,88 @@
# OCaml Setup Guide for Serena
This guide explains how to set up an OCaml project so that Serena can provide code intelligence via ocaml-lsp-server (ocamllsp).
Unlike some other languages, Serena does not download the OCaml language server automatically. You must install it yourself via opam, as OCaml tooling is compiled from source against your specific environment.
---
## Prerequisites
Install the following on your system and ensure they are available on `PATH`:
- **opam** (OCaml package manager)
- macOS: `brew install opam`
- Ubuntu/Debian: `sudo apt install opam`
- Fedora: `sudo dnf install opam`
- Other: https://opam.ocaml.org/doc/Install.html
- **OCaml compiler** (via opam)
- OCaml < 5.1 or >= 5.1.1 (OCaml 5.1.0 is **not supported** by ocaml-lsp-server)
- Recommended: OCaml 4.14.x (stable) or 5.2+ (for cross-file references)
- **ocaml-lsp-server** (via opam)
- **dune** (build system, via opam)
---
## Installation
1. Initialize opam if you haven't already:
```bash
opam init
eval $(opam env)
```
2. Create an opam switch with a compatible OCaml version:
```bash
# For cross-file reference support (recommended)
opam switch create serena-ocaml ocaml-base-compiler.5.2.1
eval $(opam env)
# Or for stable OCaml 4.14.x
opam switch create serena-ocaml ocaml-base-compiler.4.14.2
eval $(opam env)
```
3. Install the language server and build tools:
```bash
opam install ocaml-lsp-server dune
```
4. Verify the installation:
```bash
opam exec -- ocamllsp --version
opam exec -- ocaml -version
```
---
## Cross-File References
Cross-file reference support (finding all usages of a symbol across your project) requires:
- OCaml >= 5.2
- ocaml-lsp-server >= 1.23.0
- dune >= 3.16.0
When these requirements are met, Serena automatically builds the cross-file index during startup via `dune build @ocaml-index`. Without these versions, references are limited to the current file.
---
## Using Serena with OCaml
- Serena automatically detects OCaml files (`*.ml`, `*.mli`) and Reason files (`*.re`, `*.rei`).
- The language server is started via `opam exec -- ocamllsp`, so your opam environment must be configured.
- Ensure your project builds successfully with `dune build` before using Serena for best results.
---
## Troubleshooting
| Problem | Solution |
|---------|----------|
| "opam not found" | Install opam and add it to PATH |
| "OCaml 5.1.0 is incompatible" | Create a new switch: `opam switch create <name> ocaml-base-compiler.5.2.1` |
| "ocaml-lsp-server not found" | `opam install ocaml-lsp-server` |
| Cross-file refs not working | Ensure OCaml >= 5.2 and ocaml-lsp-server >= 1.23.0; run `dune build` first |
| Stale index | Rebuild with `dune build @ocaml-index` |
---
## Reference
- opam: https://opam.ocaml.org
- ocaml-lsp-server: https://github.com/ocaml/ocaml-lsp
- Project-wide occurrences: https://discuss.ocaml.org/t/ann-project-wide-occurrences-in-merlin-and-lsp/14847
docs/_config.yml
+1 -1
View File
@@ -12,7 +12,7 @@ title : Serena Documentation # The title of the book. Wil
author : Oraios AI & Oraios Software # The author of the book
copyright : "2025 by Serena contributors" # Copyright year to be placed in the footer
# Patterns to skip when building the book. Can be glob-style (e.g. "*skip.ipynb")
exclude_patterns : ['**.ipynb_checkpoints', '.DS_Store', 'Thumbs.db', '_build', 'jupyter_execute', '.jupyter_cache', '.pytest_cache', 'docs/autogen_rst.py', 'docs/create_toc.py']
exclude_patterns : ['**.ipynb_checkpoints', '.DS_Store', 'Thumbs.db', '_build', 'jupyter_execute', '.jupyter_cache', '.pytest_cache', 'docs/autogen_docs.py', 'docs/create_toc.py']
# Auto-exclude files not in the toc
only_build_toc_files : true
docs/autogen_rst.py → docs/autogen_docs.py
+30
View File
@@ -4,6 +4,8 @@ import shutil
from pathlib import Path
from typing import Optional, List
from sensai.util.string import TextBuilder
log = logging.getLogger(os.path.basename(__file__))
TOP_LEVEL_PACKAGE = "serena"
@@ -143,10 +145,38 @@ def make_rst(src_root, rst_root, clean=False, overwrite=False, package_prefix=""
write_to_file(module_template(module_qualname), module_rst_path)
def autogen_tool_list(target_filename = "01-about/035_tools.md"):
from serena.tools import ToolRegistry
target_file = Path(__file__).parent / target_filename
with open(target_file, "w") as f:
f.write("<!-- This file is auto-generated by docs/autogen_docs.py. Do not edit it manually. -->\n\n")
f.write("# Tools\n\n")
f.write("Find the full list of Serena's tools below. \n")
f.write("Note that in most configurations, only a subset of these tools will be enabled simultaneously.\n")
f.write("Tools marked as optional are disabled by default.\n\n")
tools_by_module = ToolRegistry().get_registered_tools_by_module()
priority_modules = {"serena.tools.symbol_tools": 1, "serena.tools.jetbrains_tools": 2}
text = TextBuilder()
sorted_modules = sorted(tools_by_module.keys(), key=lambda m: (priority_modules.get(m, 3), m))
for module in sorted_modules:
tools = tools_by_module[module]
module = module.replace("serena.tools.", "")
text.with_line(f"* **{module}**")
for tool in tools:
info = " *(optional)*" if tool.is_optional else ""
text.with_line(f"* `{tool.tool_name}`{info}: {tool.class_docstring}", indent=2)
f.write(text.build())
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
docs_root = Path(__file__).parent
enable_module_docs = False
autogen_tool_list()
if enable_module_docs:
make_rst(
docs_root / ".." / "src" / "serena",
flake.nix
+10 -1
View File
@@ -77,7 +77,16 @@
packages = {
serena-env = pythonSet.mkVirtualEnv "serena-env" workspace.deps.default;
serena = pkgs.runCommand "serena" {} ''
serena = pkgs.runCommand "serena" {
meta = {
description = "A powerful coding agent toolkit providing semantic retrieval and editing capabilities (MCP server & Agno integration)";
homepage = "https://oraios.github.io/serena";
changelog = "https://github.com/oraios/serena/blob/main/CHANGELOG.md";
mainProgram = "serena";
license = pkgs.lib.licenses.mit;
platforms = lib.platforms.all;
};
} ''
mkdir -p $out/bin
ln -s ${packages.serena-env}/bin/serena $out/bin/serena
'';
pyproject.toml
+69 -58
View File
@@ -8,36 +8,44 @@ version = "0.1.4"
description = ""
authors = [{ name = "Oraios AI", email = "info@oraios-ai.de" }]
readme = "README.md"
requires-python = ">=3.11, <3.12"
requires-python = ">=3.11, <3.15"
classifiers = [
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3.11",
]
dependencies = [
"requests>=2.32.3,<3",
"pyright>=1.1.396,<2",
"fortls>=3.2.2",
"overrides>=7.7.0,<8",
"python-dotenv>=1.0.0, <2",
"mcp==1.23.0",
"flask>=3.0.0",
"sensai-utils>=1.5.0",
"pydantic>=2.10.6",
"types-pyyaml>=6.0.12.20241230",
"pyyaml>=6.0.2",
"requests==2.32.5",
"pyright==1.1.403",
"fortls==3.2.2",
"overrides==7.7.0",
"python-dotenv==1.2.1",
"mcp==1.26.0",
"flask==3.1.3", # bumped from 3.1.1 for CVE fix (also fixes werkzeug alert)
"sensai-utils==1.5.0",
"pydantic==2.12.5",
"types-pyyaml==6.0.12.20250516",
"pyyaml==6.0.2",
"ruamel.yaml==0.18.14",
"jinja2>=3.1.6",
"dotenv>=0.9.9",
"pathspec>=0.12.1",
"psutil>=7.0.0",
"docstring_parser>=0.16",
"joblib>=1.5.1",
"tqdm>=4.67.1",
"tiktoken>=0.9.0",
"anthropic>=0.54.0",
"beautifulsoup4>=4.14.2",
"jinja2==3.1.6",
"dotenv==0.9.9",
"pathspec==0.12.1",
"psutil==7.0.0",
"docstring_parser==0.17.0",
"joblib==1.5.1",
"tqdm==4.67.1",
"tiktoken==0.12.0",
"anthropic==0.59.0",
"beautifulsoup4==4.14.2",
"pywebview",
"pystray>=0.19.5",
# Transitive deps pinned for security (dependabot alerts).
# Exact pins because uvx installs from git, ignoring the lock file.
"urllib3==2.6.3",
"werkzeug==3.1.7",
"starlette==1.0.0",
"python-multipart==0.0.22",
"filelock==3.25.2",
"cryptography==46.0.5",
]
[tool.uv.sources]
@@ -59,29 +67,35 @@ text = "MIT"
[project.optional-dependencies]
dev = [
"black[jupyter]>=23.7.0, <26", # black 26 is incompatible with our pathspec version
"jinja2",
"jinja2==3.1.6",
# In version 1.0.4 we get a NoneType error related to some config conversion (yml_analytics is None and should be a list)
"mypy>=1.16.1",
"poethepoet>=0.20.0",
"pytest>=8.0.2",
"pytest-xdist>=3.5.0",
"ruff>=0.0.285",
"toml-sort>=0.24.2",
"types-pyyaml>=6.0.12.20241230",
"syrupy>=4.9.1",
"types-requests>=2.32.4.20241230",
"mypy==1.17.0",
"poethepoet==0.36.0",
"pytest==8.4.1",
"pytest-xdist==3.8.0",
"ruff==0.12.5",
"toml-sort==0.24.2",
"types-pyyaml==6.0.12.20250516",
"syrupy==4.9.1",
"types-requests==2.32.4.20250809",
# docs
"sphinx>=7,<8",
"sphinx_rtd_theme>=0.5.1",
"sphinx==7.4.7",
"sphinx_rtd_theme==2.0.0",
"sphinx-toolbox==3.7.0",
"jupyter-book>=1,<2",
"nbsphinx",
"pyinstrument",
"pytest-timeout>=2.4.0",
"jupyter-book==1.0.4.post1",
"nbsphinx==0.9.7",
"pyinstrument==5.1.1",
"pytest-timeout==2.4.0",
# Transitive dev deps pinned for security (dependabot alerts).
# Exact pins because uvx installs from git, ignoring the lock file.
"tornado==6.5.5",
"nbconvert==7.17.0",
"wheel==0.46.3",
"pyasn1==0.6.3",
"PyJWT==2.10.1",
]
agno = ["agno>=2.2.1", "sqlalchemy>=2.0.40"]
google = ["google-genai>=1.8.0"]
agno = ["agno==2.5.10", "sqlalchemy==2.0.41"] # agno bumped for session state overwrite CVE
google = ["google-genai==1.27.0"]
[project.urls]
Homepage = "https://github.com/oraios/serena"
@@ -89,15 +103,6 @@ Homepage = "https://github.com/oraios/serena"
[tool.hatch.build.targets.wheel]
packages = ["src/serena", "src/interprompt", "src/solidlsp"]
[tool.black]
line-length = 140
target-version = ["py311"]
exclude = '''
/(
src/solidlsp/language_servers/.*/static|src/multilspy
)/
'''
[tool.doc8]
max-line-length = 1000
@@ -140,22 +145,22 @@ type = "simple"
# For custom markers, one can either adjust the env var or just use -m option in the command line,
# as the second -m option will override the first one.
test = "pytest test -vv"
_black_check = "black --check src scripts test"
_ruff_check = "ruff check src scripts test"
_black_format = "black src scripts test"
_ruff_format = "ruff check --fix src scripts test"
lint = ["_black_check", "_ruff_check"]
format = ["_ruff_format", "_black_format"]
_ruff_format_check = "ruff format --check src scripts test"
_ruff_fix = "ruff check --fix src scripts test"
_ruff_format = "ruff format src scripts test"
lint = ["_ruff_format_check", "_ruff_check"]
format = ["_ruff_fix", "_ruff_format"]
_mypy_core = "mypy src/serena src/solidlsp"
_mypy_test = "mypy --disable-error-code no-untyped-def test"
type-check = ["_mypy_core", "_mypy_test"]
# docs
_autogen_rst = "python docs/autogen_rst.py"
_autogen_docs = "python docs/autogen_docs.py"
_sphinx_build = "sphinx-build -b html docs docs/_build -W --keep-going"
_jb_generate_toc = "python docs/create_toc.py"
_jb_generate_config = "jupyter-book config sphinx docs/"
doc-clean = "rm -rf docs/_build docs/03_api"
doc-generate-files = ["_autogen_rst", "_jb_generate_toc", "_jb_generate_config"]
doc-generate-files = ["_autogen_docs", "_jb_generate_toc", "_jb_generate_config"]
doc-build = ["doc-clean", "doc-generate-files", "_sphinx_build"]
[tool.ruff]
@@ -312,9 +317,11 @@ markers = [
"ruby: language server running for Ruby (uses ruby-lsp)",
"zig: language server running for Zig",
"lua: language server running for Lua",
"luau: language server running for Luau",
"nix: language server running for Nix",
"dart: language server running for Dart",
"erlang: language server running for Erlang",
"ocaml: language server running for OCaml and Reason",
"scala: language server running for Scala",
"al: language server running for AL (Microsoft Dynamics 365 Business Central)",
"fsharp: language server running for F#",
@@ -331,11 +338,15 @@ markers = [
"toml: language server running for TOML",
"matlab: language server running for MATLAB (requires MATLAB R2021b+)",
"systemverilog: language server running for SystemVerilog (uses verible-verilog-ls)",
"hlsl: language server running for HLSL shaders (uses shader-language-server)",
"lean4: language server running for Lean 4",
"solidity: language server running for Solidity (uses @nomicfoundation/solidity-language-server)",
"ansible: language server running for Ansible (uses @ansible/ansible-language-server)",
]
[tool.codespell]
# Ref: https://github.com/codespell-project/codespell#using-a-config-file
skip = '.git*,*.svg,*.lock,*.min.*'
check-hidden = true
# ignore-regex = ''
ignore-regex = '\.\w+'
ignore-words-list = 'paket'
roadmap.md
-63
View File
@@ -1,63 +0,0 @@
# Roadmap
This document gives an overview of the ongoing and future development of Serena.
If you have a proposal or want to discuss something, feel free to open a discussion
on Github. For a summary of the past development, see the [changelog](/CHANGELOG.md).
Want to see us reach our goals faster? You can help out with an issue, start a discussion, or
inform us about funding opportunities so that we can devote more time to the project.
## Overall Goals
Serena has the potential to be the go-to tool for most LLM coding tasks, since it is
unique in its ability to be used as MCP Server in any kind of environment
while still being a capable agent. We want to achieve the following goals in terms of functionality:
1. Top performance (comparable to API-based coding agents) when used through official (free) clients like Claude Desktop.
1. Lowering API costs and potentially improving performance of coding clients (Claude Code, Codex, Cline, Roo, Cursor/Windsurf/VSCode etc).
1. Transparency and simplicity of use. Achieved through the dashboard/logging GUI.
1. Integrations with major frameworks that don't accept MCP. Usable as a library.
Apart from the functional goals, we have the goal of having great code design, so that Serena can be viewed
as a reference for how to implement MCP Servers. Such projects are an emerging technology, and
best practices are yet to be determined. We will share our experiences in [lessons learned](/lessons_learned.md).
## Immediate/Ongoing
- Support for projects using multiple programming languages.
- Evaluate whether `ReplaceLinesTool` can be removed in favor of a more reliable and performant editing approach.
- Generally experiment with various approaches to editing tools
- Manual evaluation on selected tasks from SWE-verified
- Manual evaluation of cost-lowering and performance when used within popular non-MCP agents
- Improvements in prompts, in particular giving examples and extending modes and contexts
## Upcoming
- Publishing Serena as a package that can also be used as library
- Use linting and type-hierarchy from the LSP in tools
- Tools for refactoring (rename, move) - speculative, maybe won't do this.
- Tracking edits and rolling them back with the dashboard
- Improve configurability and safety of shell tool. Maybe autogeneration of tools from a list of commands and descriptions.
- Transparent comparison with DesktopCommander and ...
- Automatic evaluation using OpenHands, submission to SWE-Bench
- Evaluation whether incorporating other MCPs increases performance or usability (memory bank is a candidate)
- More documentation and best practices
## Stretch
- Allow for sandboxing and parallel instances of Serena, maybe use openhands or codex for that
- Incorporate a verifier model or generally a second model (maybe for applying edits) as a tool.
- Building on the above, allow for the second model itself to be reachable through an MCP server, so it can be used for free
- Tracking edits performed with shell tools
## Beyond Serena
The technologies and approaches taken in Serena can be used for various research and service ideas. Some thought that we had are:
- PR and issue assistant working with GitHub, similar to how [OpenHands](https://github.com/All-Hands-AI/OpenHands)
and [qodo](https://github.com/qodo-ai/pr-agent) operate. Should be callable through @serena
- Tuning a coding LLM with Serena's tools with RL on one-shot tasks. We would need compute-funding for that
- Develop a web app to quantitatively compare the performance of various agents by scraping PRs and manually crafted metadata.
The main metric for coding agents should be *developer experience*, and that is hard to grasp and is poorly correlated with
performance on current benchmarks.
scripts/print_language_list.py
+1 -1
View File
@@ -7,7 +7,7 @@ from solidlsp.ls_config import Language
if __name__ == "__main__":
lang_strings = sorted([l.value for l in Language])
max_len = max(len(s) for s in lang_strings)
fmt = f"%-{max_len+2}s"
fmt = f"%-{max_len + 2}s"
for i, l in enumerate(lang_strings):
if i % 5 == 0:
print("\n# ", end="")
src/serena/agent.py
+38 -49
View File
@@ -7,14 +7,15 @@ import os
import platform
import subprocess
import sys
from collections.abc import Callable, Sequence
from collections.abc import Callable, Iterator, Sequence
from contextlib import contextmanager
from logging import Logger
from typing import TYPE_CHECKING, Optional, TypeVar
import webview
from sensai.util import logging
from sensai.util.logging import LogTime
from sensai.util.string import TextBuilder
from sensai.util.string import dict_string
from interprompt.jinja_template import JinjaTemplate
from serena import serena_version
@@ -30,8 +31,8 @@ from serena.config.serena_config import (
ToolInclusionDefinition,
)
from serena.dashboard import SerenaDashboardAPI, SerenaDashboardViewer
from serena.ls_manager import LanguageServerManager, LanguageServerManagerInitialisationError
from serena.project import Project
from serena.ls_manager import LanguageServerManager
from serena.project import MemoriesManager, Project
from serena.prompt_factory import SerenaPromptFactory
from serena.task_executor import TaskExecutor
from serena.tools import ActivateProjectTool, GetCurrentConfigTool, OpenDashboardTool, ReplaceContentTool, Tool, ToolMarker, ToolRegistry
@@ -256,7 +257,6 @@ class SerenaAgent:
# project-specific instances, which will be initialized upon project activation
self._active_project: Project | None = None
self._lsp_init_error: LanguageServerManagerInitialisationError | None = None
# determine registered project to be activated (if any)
registered_project_to_activate: RegisteredProject | None = (
@@ -315,7 +315,7 @@ class SerenaAgent:
# log fundamental information
log.info(
f"Starting Serena server (version={serena_version()}, process id={os.getpid()}, parent process id={os.getppid()}; "
f"language backend={self.serena_config.language_backend.name})"
f"language backend={self.serena_config.language_backend.name}); Python version={platform.python_version()}, platform={platform.platform()}"
)
log.info("Configuration file: %s", self.serena_config.config_file_path)
log.info("Available projects: {}".format(", ".join(self.serena_config.project_names)))
@@ -475,17 +475,8 @@ class SerenaAgent:
return None
def get_language_server_manager_or_raise(self) -> LanguageServerManager:
language_server_manager = self.get_language_server_manager()
if language_server_manager is None:
msg = TextBuilder("The language server manager is not initialized, indicating a problem during project initialisation.")
if self._lsp_init_error is not None:
msg.with_text(str(self._lsp_init_error))
msg.with_text("For details, please check the logs. " + self.get_log_inspection_instructions())
msg.with_text(
"IMPORTANT: Stop, do not attempt workarounds. Inform the user and wait for further instructions before you continue!"
)
raise Exception(msg.build())
return language_server_manager
active_project = self.get_active_project_or_raise()
return active_project.get_language_server_manager_or_raise()
def get_log_inspection_instructions(self) -> str:
if self.serena_config.web_dashboard:
@@ -493,7 +484,7 @@ class SerenaAgent:
else:
log_path = SerenaPaths().last_returned_log_file_path
if log_path is not None:
return f"Find the current log file here: f{log_path}"
return f"Find the current log file here: {log_path}"
else:
return "Unfortunately, logs are not available. We recommend enabling the web dashboard/logging in general."
@@ -569,15 +560,6 @@ class SerenaAgent:
start_new_session=True, # Detach from parent process
)
def get_project_root(self) -> str:
"""
:return: the root directory of the active project (if any); raises a ValueError if there is no active project
"""
project = self.get_active_project()
if project is None:
raise ValueError("Cannot get project root if no project is active.")
return project.project_root
def get_exposed_tool_instances(self) -> list["Tool"]:
"""
:return: the tool instances which are exposed (e.g. to the MCP client).
@@ -629,12 +611,17 @@ class SerenaAgent:
def create_system_prompt(self) -> str:
available_tools = self._active_tools
available_markers = available_tools.tool_marker_names
global_memories = MemoriesManager(
serena_data_folder=None, read_only_memory_patterns=self.serena_config.read_only_memory_patterns
).list_global_memories()
global_memories_str = dict_string(global_memories.to_dict()) if len(global_memories) > 0 else ""
log.info("Generating system prompt with available_tools=(see active tools), available_markers=%s", available_markers)
system_prompt = self.prompt_factory.create_system_prompt(
context_system_prompt=self._format_prompt(self._context.prompt),
mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self.get_active_modes()],
available_tools=available_tools.tool_names,
available_markers=available_markers,
global_memories_list=global_memories_str,
)
# If a project is active at startup, append its activation message
@@ -730,7 +717,13 @@ class SerenaAgent:
f"(2) Configure one MCP server per backend in your client."
)
# shut down the previously active project to release its language server processes
if self._active_project is not None:
log.info(f"Shutting down previously active project '{self._active_project.project_name}' before switching")
self._active_project.shutdown()
self._active_project = project
project.set_agent(self)
if update_active_modes:
self._update_active_modes()
@@ -744,7 +737,7 @@ class SerenaAgent:
self.reset_language_server_manager()
# initialize the language server in the background (if in language server mode)
if self.is_using_language_server():
if self.get_language_backend().is_lsp():
self.issue_task(init_language_server_manager)
if self._project_activation_callback is not None:
@@ -841,26 +834,7 @@ class SerenaAgent:
"""
Starts/resets the language server manager for the current project
"""
tool_timeout = self.serena_config.tool_timeout
if tool_timeout is None or tool_timeout < 0:
ls_timeout = None
else:
if tool_timeout < 10:
raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds")
ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout
# instantiate and start the necessary language servers
try:
self._lsp_init_error = None
self.get_active_project_or_raise().create_language_server_manager(
log_level=self.serena_config.log_level,
ls_timeout=ls_timeout,
trace_lsp_communication=self.serena_config.trace_lsp_communication,
ls_specific_settings=self.serena_config.ls_specific_settings,
)
except LanguageServerManagerInitialisationError as e:
self._lsp_init_error = e
raise
self.get_active_project_or_raise().create_language_server_manager()
def add_language(self, language: Language) -> None:
"""
@@ -894,7 +868,8 @@ class SerenaAgent:
"""
Shuts down the agent, freeing resources and stopping background tasks.
"""
if not hasattr(self, "_is_initialized"):
# guard against __del__ being called on a partially constructed instance
if not hasattr(self, "_active_project"):
return
log.info("SerenaAgent is shutting down ...")
if self._active_project is not None:
@@ -914,3 +889,17 @@ class SerenaAgent:
if ls_manager is None:
return []
return ls_manager.get_active_languages()
@contextmanager
def active_project_context(self, project: Project) -> Iterator[None]:
"""
Context manager for temporarily setting/overriding the active project
:param project: the project to be active
"""
original_project = self._active_project
self._active_project = project
try:
yield
finally:
self._active_project = original_project
src/serena/cli.py
+70 -15
View File
@@ -350,6 +350,61 @@ class TopLevelCommands(AutoRegisteringGroup):
else:
print(f"{prefix}\n{instr}\n{postfix}")
@staticmethod
@click.command(
"start-project-server",
help="Starts the Serena project server, which exposes project querying capabilities via HTTP.",
context_settings={"max_content_width": _MAX_CONTENT_WIDTH},
)
@click.option(
"--host",
type=str,
default="127.0.0.1",
show_default=True,
help="Listen address for the project server.",
)
@click.option(
"--port",
type=int,
default=None,
help="Listen port for the project server (default: ProjectServer.PORT).",
)
@click.option(
"--log-level",
type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]),
default=None,
help="Override log level in config.",
)
def start_project_server(
host: str,
port: int | None,
log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None,
) -> None:
from serena.project_server import ProjectServer
# initialize logging
Logger.root.setLevel(logging.INFO)
formatter = logging.Formatter(SERENA_LOG_FORMAT)
stderr_handler = logging.StreamHandler(stream=sys.stderr)
stderr_handler.formatter = formatter
Logger.root.addHandler(stderr_handler)
log_path = SerenaPaths().get_next_log_file_path("project-server")
file_handler = logging.FileHandler(log_path, mode="w")
file_handler.formatter = formatter
Logger.root.addHandler(file_handler)
if log_level is not None:
Logger.root.setLevel(logging.getLevelNamesMapping()[log_level])
log.info("Starting Serena project server")
log.info("Storing logs in %s", log_path)
server = ProjectServer()
run_kwargs: dict[str, Any] = {"host": host}
if port is not None:
run_kwargs["port"] = port
server.run(**run_kwargs)
@staticmethod
@click.command(
"dashboard-viewer",
@@ -560,7 +615,8 @@ class ProjectCommands(AutoRegisteringGroup):
:return: the RegisteredProject instance
"""
project_root = Path(project_path).resolve()
yml_path = ProjectConfig.path_to_project_yml(project_root)
serena_config = SerenaConfig.from_config_file()
yml_path = serena_config.get_project_yml_location(str(project_root))
if os.path.exists(yml_path):
raise FileExistsError(f"Project file {yml_path} already exists.")
@@ -574,14 +630,14 @@ class ProjectCommands(AutoRegisteringGroup):
raise ValueError(f"Unknown language '{lang}'. Supported: {all_langs}")
generated_conf = ProjectConfig.autogenerate(
project_root=project_path, project_name=name, languages=languages if languages else None, interactive=True
project_root=project_path,
serena_config=serena_config,
project_name=name,
languages=languages if languages else None,
interactive=True,
)
yml_path = ProjectConfig.path_to_project_yml(project_path)
languages_str = ", ".join([lang.value for lang in generated_conf.languages]) if generated_conf.languages else "N/A"
click.echo(f"Generated project with languages {{{languages_str}}} at {yml_path}.")
# add to SerenaConfig's list of registered projects
serena_config = SerenaConfig.from_config_file()
registered_project = serena_config.get_registered_project(str(project_root))
if registered_project is None:
registered_project = RegisteredProject(str(project_root), generated_conf)
@@ -656,9 +712,7 @@ class ProjectCommands(AutoRegisteringGroup):
serena_config = SerenaConfig.from_config_file()
proj = registered_project.get_project_instance(serena_config=serena_config)
click.echo(f"Indexing symbols in {proj} …")
ls_mgr = proj.create_language_server_manager(
log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings
)
ls_mgr = proj.create_language_server_manager()
try:
log_file = os.path.join(proj.project_root, ".serena", "logs", "indexing.txt")
@@ -769,6 +823,9 @@ class ProjectCommands(AutoRegisteringGroup):
logging.configure(level=logging.INFO)
project_path = os.path.abspath(project)
serena_config = SerenaConfig.from_config_file()
serena_config.language_backend = LanguageBackend.LSP
serena_config.gui_log_window = False
serena_config.web_dashboard = False
proj = Project.load(project_path, serena_config=serena_config)
# Create log file with timestamp
@@ -783,9 +840,7 @@ class ProjectCommands(AutoRegisteringGroup):
try:
# Create SerenaAgent with dashboard disabled
log.info("Creating SerenaAgent with disabled dashboard...")
serena_config = SerenaConfig.from_config_file()
serena_config.gui_log_window = False
serena_config.web_dashboard = False
agent = SerenaAgent(project=project_path, serena_config=serena_config)
log.info("SerenaAgent created successfully")
@@ -819,7 +874,7 @@ class ProjectCommands(AutoRegisteringGroup):
# Test 1: Get symbols overview
log.info("Testing GetSymbolsOverviewTool on file: %s", target_file)
overview_data = agent.execute_task(lambda: overview_tool.get_symbol_overview(target_file))
log.info("GetSymbolsOverviewTool returned %d symbols", len(overview_data))
log.info(f"GetSymbolsOverviewTool returned: {overview_data}")
if not overview_data:
log.error("No symbols found in file %s", target_file)
@@ -840,8 +895,8 @@ class ProjectCommands(AutoRegisteringGroup):
selected_symbol = overview_data[0]
log.info("No class or function found, using first available symbol")
symbol_name = selected_symbol.get("name_path", "unknown")
symbol_kind = selected_symbol.get("kind", "unknown")
symbol_name = selected_symbol["name"]
symbol_kind = selected_symbol["kind"]
log.info("Using symbol for testing: %s (kind: %s)", symbol_name, symbol_kind)
# Test 2: FindSymbolTool
src/serena/code_editor.py
+10 -22
View File
@@ -4,7 +4,7 @@ import os
from abc import ABC, abstractmethod
from collections.abc import Iterable, Iterator, Reversible
from contextlib import contextmanager
from typing import TYPE_CHECKING, Generic, Optional, TypeVar, cast
from typing import Generic, TypeVar, cast
from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient
from serena.symbol import JetBrainsSymbol, LanguageServerSymbol, LanguageServerSymbolRetriever, PositionInFile, Symbol
@@ -12,29 +12,17 @@ from solidlsp import SolidLanguageServer, ls_types
from solidlsp.ls import LSPFileBuffer
from solidlsp.ls_utils import PathUtils, TextUtils
from .constants import DEFAULT_SOURCE_FILE_ENCODING
from .project import Project
if TYPE_CHECKING:
from .agent import SerenaAgent
log = logging.getLogger(__name__)
TSymbol = TypeVar("TSymbol", bound=Symbol)
class CodeEditor(Generic[TSymbol], ABC):
def __init__(self, project_root: str, agent: Optional["SerenaAgent"] = None) -> None:
self.project_root = project_root
self.agent = agent
# set encoding based on active project, if available
encoding = DEFAULT_SOURCE_FILE_ENCODING
if agent is not None:
project = agent.get_active_project()
if project is not None:
encoding = project.project_config.encoding
self.encoding = encoding
def __init__(self, project: Project) -> None:
self.project_root = project.project_root
self.encoding = project.project_config.encoding
self.newline = project.line_ending.newline_str
class EditedFile(ABC):
def __init__(self, relative_path: str) -> None:
@@ -82,7 +70,7 @@ class CodeEditor(Generic[TSymbol], ABC):
def _save_edited_file(self, edited_file: "CodeEditor.EditedFile") -> None:
abs_path = os.path.join(self.project_root, edited_file.relative_path)
new_contents = edited_file.get_contents()
with open(abs_path, "w", encoding=self.encoding) as f:
with open(abs_path, "w", encoding=self.encoding, newline=self.newline) as f:
f.write(new_contents)
@abstractmethod
@@ -244,8 +232,8 @@ class CodeEditor(Generic[TSymbol], ABC):
class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]):
def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, agent: Optional["SerenaAgent"] = None):
super().__init__(project_root=symbol_retriever.get_root_path(), agent=agent)
def __init__(self, symbol_retriever: LanguageServerSymbolRetriever):
super().__init__(project=symbol_retriever.project)
self._symbol_retriever = symbol_retriever
def _get_language_server(self, relative_path: str) -> SolidLanguageServer:
@@ -379,9 +367,9 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]):
class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]):
def __init__(self, project: Project, agent: Optional["SerenaAgent"] = None) -> None:
def __init__(self, project: Project) -> None:
self._project = project
super().__init__(project_root=project.project_root, agent=agent)
super().__init__(project)
class EditedFile(CodeEditor.EditedFile):
def __init__(self, relative_path: str, project: Project):
src/serena/config/serena_config.py
+345 -95
View File
@@ -4,6 +4,7 @@ The Serena Model Context Protocol (MCP) Server
import dataclasses
import os
import re
import shutil
from collections.abc import Iterator, Sequence
from copy import deepcopy
@@ -22,6 +23,7 @@ from sensai.util.string import ToStringMixin
from serena.constants import (
DEFAULT_SOURCE_FILE_ENCODING,
PROJECT_LOCAL_TEMPLATE_FILE,
PROJECT_TEMPLATE_FILE,
REPO_ROOT,
SERENA_CONFIG_TEMPLATE_FILE,
@@ -85,6 +87,12 @@ class SerenaPaths:
"""
file containing the ID of the last read news snippet
"""
global_memories_path = Path(os.path.join(self.serena_user_home_dir, "memories", "global"))
global_memories_path.mkdir(parents=True, exist_ok=True)
self.global_memories_path = global_memories_path
"""
directory where global memories are stored, i.e. memories that are available across all projects
"""
self.last_returned_log_file_path: str | None = None
"""
the path to the last log file returned by `get_next_log_file_path`. If this is not None, the logs
@@ -98,7 +106,7 @@ class SerenaPaths:
"""
log_dir = os.path.join(self.serena_user_home_dir, "logs", datetime.now().strftime("%Y-%m-%d"))
os.makedirs(log_dir, exist_ok=True)
self.last_returned_log_file_path = os.path.join(log_dir, prefix + "_" + datetime_tag() + ".txt")
self.last_returned_log_file_path = os.path.join(log_dir, prefix + "_" + datetime_tag() + f"_{os.getpid()}" + ".txt")
return self.last_returned_log_file_path
# TODO: Paths from constants.py should be moved here
@@ -167,6 +175,41 @@ class LanguageBackend(Enum):
return backend
raise ValueError(f"Unknown language backend '{backend_str}': valid values are {[b.value for b in LanguageBackend]}")
def is_lsp(self) -> bool:
return self == LanguageBackend.LSP
def is_jetbrains(self) -> bool:
return self == LanguageBackend.JETBRAINS
class LineEnding(Enum):
"""Line ending convention for file writes."""
LF = "lf"
CRLF = "crlf"
NATIVE = "native"
@property
def newline_str(self) -> str | None:
"""The newline parameter value for :func:`open` and :meth:`Path.write_text`.
Returns ``None`` for native mode (platform default).
"""
if self is LineEnding.LF:
return "\n"
elif self is LineEnding.CRLF:
return "\r\n"
return None
@classmethod
def from_str(cls, value: str) -> "LineEnding":
"""Parse a string value into a :class:`LineEnding`."""
try:
return cls(value.lower())
except ValueError as e:
valid = [le.value for le in cls]
raise ValueError(f"Invalid line_ending: {value!r}. Valid values are: {valid}") from e
@dataclass
class SharedConfig(ModeSelectionDefinition, ToolInclusionDefinition, ToStringMixin):
@@ -177,14 +220,22 @@ class SharedConfig(ModeSelectionDefinition, ToolInclusionDefinition, ToStringMix
symbol_info_budget: float | None = None
language_backend: LanguageBackend | None = None
line_ending: LineEnding | None = None
read_only_memory_patterns: list[str] = field(default_factory=list)
ignored_memory_patterns: list[str] = field(default_factory=list)
ls_specific_settings: dict = field(default_factory=dict)
"""Advanced configuration option allowing to configure language server implementation specific options, see SolidLSPSettings for more info."""
class SerenaConfigError(Exception):
pass
def get_serena_managed_in_project_dir(project_root: str | Path) -> str:
return os.path.join(project_root, SERENA_MANAGED_DIR_NAME)
DEFAULT_PROJECT_SERENA_FOLDER_LOCATION = "$projectDir/" + SERENA_MANAGED_DIR_NAME
"""
The default template for the project Serena folder location.
Uses $projectDir and $projectFolderName as placeholders.
"""
@dataclass(kw_only=True)
@@ -197,7 +248,12 @@ class ProjectConfig(SharedConfig):
initial_prompt: str = ""
encoding: str = DEFAULT_SOURCE_FILE_ENCODING
SERENA_DEFAULT_PROJECT_FILE = "project.yml"
# internal fields which are not mapped to/from the configuration file (must start with "_")
_local_override_keys: list[str] = field(default_factory=list)
# class-level constants
SERENA_PROJECT_FILE = "project.yml"
SERENA_LOCAL_PROJECT_FILE = "project.local.yml"
FIELDS_WITHOUT_DEFAULTS = {"project_name", "languages"}
YAML_COMMENT_NORMALISATION = YamlCommentNormalisation.LEADING
"""
@@ -212,6 +268,7 @@ class ProjectConfig(SharedConfig):
def autogenerate(
cls,
project_root: str | Path,
serena_config: "SerenaConfig",
project_name: str | None = None,
languages: list[Language] | None = None,
save_to_disk: bool = True,
@@ -221,6 +278,7 @@ class ProjectConfig(SharedConfig):
Autogenerate a project configuration for a given project root.
:param project_root: the path to the project root
:param serena_config: the global Serena configuration
:param project_name: the name of the project; if None, the name of the project will be the name of the directory
containing the project
:param languages: the languages of the project; if None, they will be determined automatically
@@ -233,70 +291,79 @@ class ProjectConfig(SharedConfig):
raise FileNotFoundError(f"Project root not found: {project_root}")
with LogTime("Project configuration auto-generation", logger=log):
log.info("Project root: %s", project_root)
project_name = project_name or project_root.name
project_folder_name = project_root.name
project_name = project_name or project_folder_name
if languages is None:
# determine languages automatically
log.info("Determining programming languages used in the project")
language_composition = determine_programming_language_composition(str(project_root))
log.info("Language composition: %s", language_composition)
if len(language_composition) == 0:
language_values = ", ".join([lang.value for lang in Language])
raise ValueError(
f"No source files found in {project_root}\n\n"
f"To use Serena with this project, you need to either\n"
f" 1. specify a programming language by adding parameters --language <language>\n"
f" when creating the project via the Serena CLI command OR\n"
f" 2. add source files in one of the supported languages first.\n\n"
f"Supported languages are: {language_values}\n"
f"Read the documentation for more information."
log.warning(
"No source files for supported language servers were found in %s. "
"Creating project with no configured languages. "
"Symbol-related tools (e.g. find_symbol, get_symbols_overview) will not work "
"when using the LSP backend. You can add languages later via the Serena dashboard "
"or by manually editing the project configuration.",
project_root,
)
# sort languages by number of files found
languages_and_percentages = sorted(
language_composition.items(), key=lambda item: (item[1], item[0].get_priority()), reverse=True
)
# find the language with the highest percentage and enable it
top_language_pair = languages_and_percentages[0]
other_language_pairs = languages_and_percentages[1:]
languages_to_use: list[str] = [top_language_pair[0].value]
# if in interactive mode, ask the user which other languages to enable
if len(other_language_pairs) > 0 and interactive:
print(
"Detected and enabled main language '%s' (%.2f%% of source files)."
% (top_language_pair[0].value, top_language_pair[1])
languages_to_use: list[str] = []
else:
# sort languages by number of files found
languages_and_percentages = sorted(
language_composition.items(), key=lambda item: (item[1], item[0].get_priority()), reverse=True
)
print(f"Additionally detected {len(other_language_pairs)} other language(s).\n")
print("Note: Enable only languages you need symbolic retrieval/editing capabilities for.")
print(" Additional language servers use resources and some languages may require additional")
print(" system-level installations/configuration (see Serena documentation).")
print("\nWhich additional languages do you want to enable?")
for lang, perc in other_language_pairs:
enable = ask_yes_no("Enable %s (%.2f%% of source files)?" % (lang.value, perc), default=False)
if enable:
languages_to_use.append(lang.value)
print()
# find the language with the highest percentage and enable it
top_language_pair = languages_and_percentages[0]
other_language_pairs = languages_and_percentages[1:]
languages_to_use = [top_language_pair[0].value]
# if in interactive mode, ask the user which other languages to enable
if len(other_language_pairs) > 0 and interactive:
print(
"Detected and enabled main language '%s' (%.2f%% of source files)."
% (top_language_pair[0].value, top_language_pair[1])
)
print(f"Additionally detected {len(other_language_pairs)} other language(s).\n")
print("Note: Enable only languages you need symbolic retrieval/editing capabilities for.")
print(" Additional language servers use resources and some languages may require additional")
print(" system-level installations/configuration (see Serena documentation).")
print("\nWhich additional languages do you want to enable?")
for lang, perc in other_language_pairs:
enable = ask_yes_no("Enable %s (%.2f%% of source files)?" % (lang.value, perc), default=False)
if enable:
languages_to_use.append(lang.value)
print()
log.info("Using languages: %s", languages_to_use)
else:
languages_to_use = [lang.value for lang in languages]
config_with_comments, _ = cls._load_yaml(PROJECT_TEMPLATE_FILE)
config_with_comments, _ = cls._load_yaml_dict(PROJECT_TEMPLATE_FILE)
config_with_comments["project_name"] = project_name
config_with_comments["languages"] = languages_to_use
if save_to_disk:
project_yml_path = cls.path_to_project_yml(project_root)
project_yml_path = serena_config.get_project_yml_location(str(project_root))
log.info("Saving project configuration to %s", project_yml_path)
save_yaml(project_yml_path, config_with_comments)
return cls._from_dict(config_with_comments)
project_local_yml_path = os.path.join(os.path.dirname(project_yml_path), cls.SERENA_LOCAL_PROJECT_FILE)
shutil.copy(PROJECT_LOCAL_TEMPLATE_FILE, project_local_yml_path)
return cls._from_dict(config_with_comments, local_override_keys=[])
@classmethod
def path_to_project_yml(cls, project_root: str | Path) -> str:
return os.path.join(project_root, cls.rel_path_to_project_yml())
def default_project_yml_path(cls, project_root: str | Path) -> str:
"""
:return: the default path to the project.yml file (inside ``$projectDir/.serena/``).
This is suitable as a fallback when no ``SerenaConfig`` is available to resolve
a potentially customised location.
"""
return os.path.join(str(project_root), SERENA_MANAGED_DIR_NAME, cls.SERENA_PROJECT_FILE)
@classmethod
def rel_path_to_project_yml(cls) -> str:
return os.path.join(SERENA_MANAGED_DIR_NAME, cls.SERENA_DEFAULT_PROJECT_FILE)
@classmethod
def _load_yaml(
cls, yml_path: str, comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE
def _load_yaml_dict(
cls,
yml_path: str,
comment_normalisation: YamlCommentNormalisation = YamlCommentNormalisation.NONE,
apply_defaults: bool = True,
) -> tuple[CommentedMap, bool]:
"""
Load the project configuration as a CommentedMap, preserving comments and ensuring
@@ -304,24 +371,33 @@ class ProjectConfig(SharedConfig):
and backward compatibility adjustments.
:param yml_path: the path to the project.yml file
:param comment_normalisation: the strategy to use for normalising comments in the loaded YAML
:param apply_defaults: whether to apply default values for missing fields
:return: a tuple `(dict, was_complete)` where dict is a CommentedMap representing a
full project configuration and `was_complete` indicates whether the loaded configuration
was complete (i.e., did not require any default values to be applied)
was complete (i.e., did not require any default values to be applied) for the case where
`apply_defaults` is True; If `apply_defaults` is False, the returned dict may be incomplete
and `was_complete` will always be True.
"""
data = load_yaml(yml_path, comment_normalisation=comment_normalisation)
# apply defaults
was_complete = True
for field_info in dataclasses.fields(cls):
key = field_info.name
if key in cls.FIELDS_WITHOUT_DEFAULTS:
continue
if key not in data:
was_complete = False
default_value = get_dataclass_default(cls, key)
data.setdefault(key, default_value)
if apply_defaults:
for field_info in dataclasses.fields(cls):
key = field_info.name
if key.startswith("_"):
continue
if key in cls.FIELDS_WITHOUT_DEFAULTS:
continue
if key not in data:
was_complete = False
default_value = get_dataclass_default(cls, key)
data.setdefault(key, default_value)
# backward compatibility: handle single "language" field
# backward compatibility
# NOTE: This must also work for project.local.yml files, which may be highly incomplete
# * handle single "language" field
if "languages" not in data and "language" in data:
data["languages"] = [data["language"]]
del data["language"]
@@ -329,9 +405,13 @@ class ProjectConfig(SharedConfig):
return data, was_complete
@classmethod
def _from_dict(cls, data: dict[str, Any]) -> Self:
def _from_dict(cls, data: dict[str, Any], local_override_keys: list[str]) -> Self:
"""
Create a ProjectConfig instance from a (full) configuration dictionary
:param data: the configuration dictionary; must contain all required fields and use the same field names as
the ProjectConfig dataclass
:param local_override_keys: the list of keys that have been overridden from project.local.yml
"""
lang_name_mapping = {"javascript": "typescript"}
languages: list[Language] = []
@@ -362,6 +442,9 @@ class ProjectConfig(SharedConfig):
language_backend_value = data.get("language_backend")
language_backend = LanguageBackend.from_str(language_backend_value) if language_backend_value else None
line_ending_value = data.get("line_ending")
line_ending = LineEnding.from_str(line_ending_value) if line_ending_value else None
return cls(
project_name=data["project_name"],
languages=languages,
@@ -370,13 +453,18 @@ class ProjectConfig(SharedConfig):
fixed_tools=data["fixed_tools"],
included_optional_tools=data["included_optional_tools"],
read_only=data["read_only"],
read_only_memory_patterns=data.get("read_only_memory_patterns", []),
ignored_memory_patterns=data.get("ignored_memory_patterns", []),
ignore_all_files_in_gitignore=data["ignore_all_files_in_gitignore"],
initial_prompt=data["initial_prompt"],
encoding=data["encoding"],
line_ending=line_ending,
language_backend=language_backend,
base_modes=data["base_modes"],
default_modes=data["default_modes"],
symbol_info_budget=symbol_info_budget,
ls_specific_settings=data.get("ls_specific_settings", {}),
_local_override_keys=local_override_keys,
)
def _to_yaml_dict(self) -> dict:
@@ -384,59 +472,117 @@ class ProjectConfig(SharedConfig):
:return: a yaml-serializable dictionary representation of this configuration
"""
d = dataclasses.asdict(self)
# drop internal fields starting with underscore
keys = list(d.keys())
for k in keys:
if k.startswith("_"):
del d[k]
# map fields using non-primitive types to a YAML-compatible representation
d["languages"] = [lang.value for lang in self.languages]
d["language_backend"] = self.language_backend.value if self.language_backend is not None else None
d["line_ending"] = self.line_ending.value if self.line_ending is not None else None
return d
@classmethod
def load(cls, project_root: Path | str, autogenerate: bool = False) -> Self:
def _project_local_yml_path(cls, project_yml_path: str) -> str:
return os.path.join(os.path.dirname(project_yml_path), cls.SERENA_LOCAL_PROJECT_FILE)
@classmethod
def load(cls, project_root: Path | str, serena_config: "SerenaConfig", autogenerate: bool = False) -> Self:
"""
Load a ProjectConfig instance from the path to the project root.
:param project_root: the path to the project root
:param serena_config: the global Serena configuration
:param autogenerate: whether to auto-generate the configuration if it does not exist
"""
project_root = Path(project_root)
yaml_path = project_root / cls.rel_path_to_project_yml()
project_folder_name = project_root.name
yaml_path = serena_config.get_project_yml_location(project_root)
log.debug("Loading project configuration from %s", yaml_path)
# auto-generate if necessary
if not yaml_path.exists():
if not os.path.exists(yaml_path):
if autogenerate:
return cls.autogenerate(project_root)
return cls.autogenerate(project_root, serena_config)
else:
raise FileNotFoundError(f"Project configuration file not found: {yaml_path}")
# load the configuration dictionary
yaml_data, was_complete = cls._load_yaml(str(yaml_path))
yaml_data, was_complete = cls._load_yaml_dict(str(yaml_path))
if "project_name" not in yaml_data:
yaml_data["project_name"] = project_root.name
yaml_data["project_name"] = project_folder_name
# apply overrides from project.local.yml, if present
local_yaml_path = cls._project_local_yml_path(str(yaml_path))
local_override_keys = []
if os.path.exists(local_yaml_path):
local_yaml_data, _ = cls._load_yaml_dict(local_yaml_path, apply_defaults=False)
if local_yaml_data:
local_override_keys = list(local_yaml_data.keys())
log.debug(
"Applying project configuration overrides from %s with keys %s",
local_yaml_path,
local_override_keys,
)
yaml_data.update(local_yaml_data)
# instantiate the ProjectConfig
project_config = cls._from_dict(yaml_data)
project_config = cls._from_dict(yaml_data, local_override_keys=local_override_keys)
# if the configuration was incomplete, re-save it to disk
if not was_complete:
log.info("Project configuration in %s was incomplete, re-saving with default values for missing fields", yaml_path)
project_config.save(project_root)
project_config.save(str(yaml_path), save_project_local_yml=False)
return project_config
def save(self, project_root: Path | str) -> None:
def save(self, project_yml_path: str, save_project_local_yml: bool = True) -> None:
"""
Saves the project configuration to disk.
Saves the project configuration to disk, updating both the project.yml file and, optionally,
the project.local.yml file to reflect overridden keys.
:param project_root: the root directory of the project
Keys that are overridden by project.local.yml are not updated in project.yml.
Only keys that are overridden are updated in project.local.yml.
:param project_yml_path: the path to the project.yml file
:param save_project_local_yml: whether to also update the project.local.yml file to reflect overridden keys
"""
config_path = self.path_to_project_yml(project_root)
config_path = project_yml_path
log.info("Saving updated project configuration to %s", config_path)
# load original commented map and update it with current values
config_with_comments, _ = self._load_yaml(config_path, self.YAML_COMMENT_NORMALISATION)
config_with_comments.update(self._to_yaml_dict())
# get the current configuration as a dictionary
cur_dict = self._to_yaml_dict()
# load commented map from the original file and update all non-overridden keys
config_with_comments, _ = self._load_yaml_dict(config_path, self.YAML_COMMENT_NORMALISATION)
for key in cur_dict:
if key not in self._local_override_keys:
config_with_comments[key] = cur_dict[key]
# transfer missing comments from the template file
template_config, _ = self._load_yaml(PROJECT_TEMPLATE_FILE, self.YAML_COMMENT_NORMALISATION)
template_config, _ = self._load_yaml_dict(PROJECT_TEMPLATE_FILE, self.YAML_COMMENT_NORMALISATION)
transfer_missing_yaml_comments(template_config, config_with_comments, self.YAML_COMMENT_NORMALISATION)
# save project.yml
save_yaml(config_path, config_with_comments)
# update project.local.yml to reflect overridden keys if necessary
if save_project_local_yml:
project_local_yml_path = self._project_local_yml_path(project_yml_path)
if self._local_override_keys and os.path.exists(project_local_yml_path):
log.info("Saving updated local project configuration to %s", project_local_yml_path)
local_config_with_comments, _ = self._load_yaml_dict(
project_local_yml_path, comment_normalisation=YamlCommentNormalisation.NONE, apply_defaults=False
)
for key in self._local_override_keys:
if key in cur_dict:
local_config_with_comments[key] = cur_dict[key]
save_yaml(project_local_yml_path, local_config_with_comments)
class RegisteredProject(ToStringMixin):
def __init__(
@@ -472,8 +618,8 @@ class RegisteredProject(ToStringMixin):
)
@classmethod
def from_project_root(cls, project_root: str | Path) -> "RegisteredProject":
project_config = ProjectConfig.load(project_root)
def from_project_root(cls, project_root: str | Path, serena_config: "SerenaConfig") -> "RegisteredProject":
project_config = ProjectConfig.load(project_root, serena_config=serena_config)
return RegisteredProject(
project_root=str(project_root),
project_config=project_config,
@@ -488,7 +634,7 @@ class RegisteredProject(ToStringMixin):
"""
return self.project_root.samefile(Path(path).resolve())
def get_project_instance(self, serena_config: "SerenaConfig | None") -> "Project":
def get_project_instance(self, serena_config: "SerenaConfig") -> "Project":
"""
Returns the project instance for this registered project, loading it if necessary.
"""
@@ -524,11 +670,6 @@ class SerenaConfig(SharedConfig):
jetbrains_plugin_server_address: str = "127.0.0.1"
tool_timeout: float = DEFAULT_TOOL_TIMEOUT
language_backend: LanguageBackend = LanguageBackend.LSP
"""
the language backend to use for code understanding features
"""
token_count_estimator: str = RegisteredTokenCountEstimator.CHAR_COUNT.name
"""Only relevant if `record_tool_usage` is True; the name of the token count estimator to use for tool usage statistics.
See the `RegisteredTokenCountEstimator` enum for available options.
@@ -542,15 +683,29 @@ class SerenaConfig(SharedConfig):
Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default
through the global configuration.
"""
ls_specific_settings: dict = field(default_factory=dict)
"""Advanced configuration option allowing to configure language server implementation specific options, see SolidLSPSettings for more info."""
ignored_paths: list[str] = field(default_factory=list)
"""List of paths to ignore across all projects. Same syntax as gitignore, so you can use * and **.
These patterns are merged additively with each project's own ignored_paths."""
project_serena_folder_location: str = DEFAULT_PROJECT_SERENA_FOLDER_LOCATION
"""
Template for the location of the per-project .serena data folder (memories, caches, etc.).
Supports the following placeholders:
- $projectDir: the absolute path to the project root directory
- $projectFolderName: the name of the project folder
Examples:
- "$projectDir/.serena" (default, stores data inside the project)
- "/projects-metadata/$projectFolderName/.serena" (stores data in a central location)
"""
# settings with overridden defaults
language_backend: LanguageBackend = LanguageBackend.LSP
"""
the language backend to use for code understanding features
"""
default_modes: Sequence[str] | None = ("interactive", "editing")
line_ending: LineEnding = LineEnding.NATIVE
symbol_info_budget: float = 10.0
"""
Time budget (seconds) for requests when tools request include_info (currently
@@ -559,7 +714,6 @@ class SerenaConfig(SharedConfig):
If the budget is exceeded, Serena stops issuing further requests and returns partial info results.
0 disables the budget (no early stopping). Negative values are invalid.
"""
# *** fields that are NOT mapped to/from the configuration file ***
_loaded_commented_yaml: CommentedMap | None = None
@@ -572,7 +726,7 @@ class SerenaConfig(SharedConfig):
# *** static members ***
CONFIG_FILE = "serena_config.yml"
CONFIG_FIELDS_WITH_TYPE_CONVERSION = {"projects", "language_backend"}
CONFIG_FIELDS_WITH_TYPE_CONVERSION = {"projects", "language_backend", "line_ending"}
# *** methods ***
@classmethod
@@ -675,15 +829,20 @@ class SerenaConfig(SharedConfig):
instance.projects = []
for path in loaded_commented_yaml["projects"]:
path = Path(path).resolve()
if not path.exists() or (path.is_dir() and not (path / ProjectConfig.rel_path_to_project_yml()).exists()):
log.warning(f"Project path {path} does not exist or does not contain a project configuration file, skipping.")
try:
path_exists = path.exists()
except OSError as e:
log.warning(f"Project path {path} is not accessible ({e}), skipping.")
continue
if not path_exists or (path.is_dir() and not os.path.isfile(instance.get_project_yml_location(str(path)))):
log.warning(f"Project path {path} does not exist or no associated project configuration file found, skipping.")
continue
if path.is_file():
path = cls._migrate_out_of_project_config_file(path)
if path is None:
continue
num_migrations += 1
project_config = ProjectConfig.load(path)
project_config = ProjectConfig.load(path, serena_config=instance) # instance is sufficiently populated
project = RegisteredProject(
project_root=str(path),
project_config=project_config,
@@ -704,6 +863,14 @@ class SerenaConfig(SharedConfig):
del loaded_commented_yaml["jetbrains"]
instance.language_backend = language_backend
# determine line ending
line_ending_value = loaded_commented_yaml.get("line_ending")
if line_ending_value:
instance.line_ending = LineEnding.from_str(line_ending_value)
else:
num_migrations += 1
instance.line_ending = get_dataclass_default(SerenaConfig, "line_ending")
# migrate deprecated "gui_log_level" field if necessary
if "gui_log_level" in loaded_commented_yaml:
num_migrations += 1
@@ -711,6 +878,14 @@ class SerenaConfig(SharedConfig):
instance.log_level = loaded_commented_yaml["gui_log_level"]
del loaded_commented_yaml["gui_log_level"]
# migrate "edit_global_memories"
if "edit_global_memories" in loaded_commented_yaml:
num_migrations += 1
edit_global_memories = loaded_commented_yaml["edit_global_memories"]
if not edit_global_memories:
instance.read_only_memory_patterns.append("global/.*")
del loaded_commented_yaml["edit_global_memories"]
# re-save the configuration file if any migrations were performed
if num_migrations > 0:
log.info("Legacy configuration was migrated; re-saving configuration file")
@@ -736,7 +911,7 @@ class SerenaConfig(SharedConfig):
with open(path, "a", encoding=SERENA_FILE_ENCODING) as f:
f.write(f"\nproject_name: {project_name}")
project_root = project_config_data["project_root"]
shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml()))
shutil.move(str(path), ProjectConfig.default_project_yml_path(project_root))
return Path(project_root).resolve()
except Exception as e:
log.error(f"Error migrating configuration file: {e}")
@@ -765,7 +940,7 @@ class SerenaConfig(SharedConfig):
return project_candidates[0]
elif len(project_candidates) > 1:
raise ValueError(
f"Multiple projects found with name '{project_root_or_name}'. Please activate it by location instead. "
f"Multiple projects found with name '{project_root_or_name}'. Please reference it by location instead. "
f"Locations: {[p.project_root for p in project_candidates]}"
)
# no project found by name; check if it's a path
@@ -775,9 +950,9 @@ class SerenaConfig(SharedConfig):
return project
# no registered project found; auto-register if project configuration exists
if autoregister:
config_path = ProjectConfig.path_to_project_yml(project_root_or_name)
config_path = self.get_project_yml_location(project_root_or_name)
if os.path.isfile(config_path):
registered_project = RegisteredProject.from_project_root(project_root_or_name)
registered_project = RegisteredProject.from_project_root(project_root_or_name, serena_config=self)
self.add_registered_project(registered_project)
return registered_project
# nothing found
@@ -820,7 +995,7 @@ class SerenaConfig(SharedConfig):
f"Project with path {project_root} was already added with name '{already_registered_project.project_name}'."
)
project_config = ProjectConfig.load(project_root, autogenerate=True)
project_config = ProjectConfig.load(project_root, serena_config=self, autogenerate=True)
new_project = Project(
project_root=str(project_root),
@@ -863,6 +1038,9 @@ class SerenaConfig(SharedConfig):
# convert language backend to string
commented_yaml["language_backend"] = self.language_backend.value
# convert line ending to string
commented_yaml["line_ending"] = self.line_ending.value
# transfer comments from the template file
# NOTE: The template file now uses leading comments, but we previously used trailing comments,
# so we apply a conversion, which detects the old style and transforms it.
@@ -873,6 +1051,78 @@ class SerenaConfig(SharedConfig):
save_yaml(self.config_file_path, commented_yaml)
@staticmethod
def _resolve_serena_folder_location(template: str, placeholders: dict[str, str]) -> str:
"""
Resolves a folder location template by replacing known ``$placeholder`` tokens
and raising on any unrecognised ones.
:param template: the template string (e.g. ``"$projectDir/.serena"``)
:param placeholders: mapping from placeholder name (without ``$``) to replacement value
:return: the resolved absolute path
:raises SerenaConfigError: if the template contains an unknown ``$placeholder``
"""
def _replace(match: re.Match[str]) -> str:
name = match.group(1)
if name not in placeholders:
raise SerenaConfigError(
f"Unknown placeholder '${name}' in project_serena_folder_location. "
f"Supported placeholders: {', '.join('$' + k for k in placeholders)}"
)
return placeholders[name]
result = re.sub(r"\$([A-Za-z_]\w*)", _replace, template)
return os.path.abspath(result)
def get_configured_project_serena_folder(self, project_root: str | Path) -> str:
"""
Returns the resolved absolute path to the .serena data folder for a project,
applying placeholder substitution to ``project_serena_folder_location``
without any fallback logic.
:param project_root: the absolute path to the project root directory
:return: the resolved absolute path to the project's .serena folder
:raises SerenaConfigError: if the template contains an unknown placeholder
"""
project_folder_name = Path(project_root).name
placeholders = {
"projectDir": str(project_root),
"projectFolderName": project_folder_name,
}
return self._resolve_serena_folder_location(self.project_serena_folder_location, placeholders)
def get_project_serena_folder(self, project_root: str | Path) -> str:
"""
Resolves the location of the project's .serena data folder using fallback logic:
1. If the folder exists at the configured path (``project_serena_folder_location``), use it.
2. Otherwise, if it exists at the default location inside the project root, use that.
3. If neither exists, return the configured path (for creation).
:param project_root: the absolute path to the project root directory
:return: the resolved absolute path to the .serena data folder
:raises SerenaConfigError: if the configured template contains an unknown placeholder
"""
configured_path = self.get_configured_project_serena_folder(project_root)
if os.path.isdir(configured_path):
return configured_path
default_path = os.path.join(str(project_root), SERENA_MANAGED_DIR_NAME)
if configured_path != default_path and os.path.isdir(default_path):
return default_path
return configured_path
def get_project_yml_location(self, project_root: str | Path) -> str:
"""
Returns the resolved absolute path to the project.yml configuration file,
based on the resolved .serena data folder (with fallback logic).
:param project_root: the absolute path to the project root directory
:return: the resolved absolute path to the project's project.yml file
"""
serena_folder = self.get_project_serena_folder(project_root)
return os.path.join(serena_folder, ProjectConfig.SERENA_PROJECT_FILE)
def propagate_settings(self) -> None:
"""
Propagate settings from this configuration to individual components that are statically configured
src/serena/constants.py
+1
View File
@@ -25,6 +25,7 @@ SERENA_FILE_ENCODING = "utf-8"
"""The encoding used for Serena's own files, such as configuration files and memories."""
PROJECT_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.template.yml")
PROJECT_LOCAL_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.local.template.yml")
SERENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml")
SERENA_LOG_FORMAT = "%(levelname)-5s %(asctime)-15s [%(threadName)s] %(name)s:%(funcName)s:%(lineno)d - %(message)s"
src/serena/dashboard.py
+38 -7
View File
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Self
import pystray
import webview
from flask import Flask, Response, request, send_from_directory
from flask import Flask, Response, redirect, request, send_from_directory
from PIL import Image
from pydantic import BaseModel
from pystray import MenuItem as Item
@@ -94,6 +94,11 @@ class RequestDeleteMemory(BaseModel):
memory_name: str
class RequestRenameMemory(BaseModel):
old_name: str
new_name: str
class ResponseGetSerenaConfig(BaseModel):
content: str
@@ -148,6 +153,10 @@ class SerenaDashboardAPI:
return self._memory_log_handler
def _setup_routes(self) -> None:
@self._app.route("/")
def redirect_to_dashboard() -> Response:
return redirect("/dashboard/") # type: ignore[return-value]
# Static files
@self._app.route("/dashboard/<path:filename>")
def serve_dashboard(filename: str) -> Response:
@@ -274,6 +283,18 @@ class SerenaDashboardAPI:
except Exception as e:
return {"status": "error", "message": str(e)}
@self._app.route("/rename_memory", methods=["POST"])
def rename_memory() -> dict[str, str]:
request_data = request.get_json()
if not request_data:
return {"status": "error", "message": "No data provided"}
request_rename_memory = RequestRenameMemory.model_validate(request_data)
try:
result_message = self._rename_memory(request_rename_memory)
return {"status": "success", "message": result_message}
except Exception as e:
return {"status": "error", "message": str(e)}
@self._app.route("/get_serena_config", methods=["GET"])
def get_serena_config() -> dict[str, Any]:
try:
@@ -489,7 +510,7 @@ class SerenaDashboardAPI:
# Get available memories if ReadMemoryTool is active
available_memories = None
if self._agent.tool_is_active("read_memory") and project is not None:
available_memories = project.memories_manager.list_memories()
available_memories = project.memories_manager.list_memories().get_full_list()
# Get list of languages for the active project
languages = []
@@ -512,7 +533,7 @@ class SerenaDashboardAPI:
available_modes=available_modes,
available_contexts=available_contexts,
available_memories=available_memories,
jetbrains_mode=not self._agent.is_using_language_server(),
jetbrains_mode=self._agent.get_language_backend().is_jetbrains(),
languages=languages,
encoding=encoding,
current_client=Tool.get_last_tool_call_client_str(),
@@ -561,8 +582,7 @@ class SerenaDashboardAPI:
project = self._agent.get_active_project()
if project is None:
raise ValueError("No active project")
project.memories_manager.save_memory(request_save_memory.memory_name, request_save_memory.content)
project.memories_manager.save_memory(request_save_memory.memory_name, request_save_memory.content, is_tool_context=False)
self._agent.execute_task(run, logged=True, name="SaveMemory")
@@ -571,11 +591,22 @@ class SerenaDashboardAPI:
project = self._agent.get_active_project()
if project is None:
raise ValueError("No active project")
project.memories_manager.delete_memory(request_delete_memory.memory_name)
project.memories_manager.delete_memory(request_delete_memory.memory_name, is_tool_context=False)
self._agent.execute_task(run, logged=True, name="DeleteMemory")
def _rename_memory(self, request_rename_memory: RequestRenameMemory) -> str:
def run() -> str:
project = self._agent.get_active_project()
if project is None:
raise ValueError("No active project")
return project.memories_manager.move_memory(
request_rename_memory.old_name, request_rename_memory.new_name, is_tool_context=False
)
return self._agent.execute_task(run, logged=True, name="RenameMemory")
def _get_serena_config(self) -> ResponseGetSerenaConfig:
config_path = self._agent.serena_config.config_file_path
if config_path is None or not os.path.exists(config_path):
src/serena/generated/generated_prompt_factory.py
+7 -1
View File
@@ -33,6 +33,12 @@ class PromptFactory(PromptFactoryBase):
return self._render_prompt("prepare_for_new_conversation", locals())
def create_system_prompt(
self, *, available_markers: Any, available_tools: Any, context_system_prompt: Any, mode_system_prompts: Any
self,
*,
available_markers: Any,
available_tools: Any,
context_system_prompt: Any,
global_memories_list: Any,
mode_system_prompts: Any,
) -> str:
return self._render_prompt("system_prompt", locals())
src/serena/jetbrains/jetbrains_plugin_client.py
+50
View File
@@ -8,6 +8,7 @@ import logging
import re
import threading
from concurrent.futures.thread import ThreadPoolExecutor
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Literal, Optional, Self, TypeVar, cast
@@ -16,6 +17,7 @@ from requests import Response
from sensai.util.string import ToStringMixin
import serena.jetbrains.jetbrains_types as jb
from serena.config.serena_config import RegisteredProject
from serena.jetbrains.jetbrains_types import PluginStatusDTO
from serena.project import Project
from serena.util.class_decorators import singleton
@@ -63,6 +65,12 @@ class ServerNotFoundError(Exception):
"""Raised when the plugin's service is not found."""
@dataclass
class MatchedClient:
client: "JetBrainsPluginClient"
registered_project: RegisteredProject
@singleton
class JetBrainsPluginClientManager:
"""
@@ -73,6 +81,7 @@ class JetBrainsPluginClientManager:
def __init__(self) -> None:
self._clients: dict[int, "JetBrainsPluginClient"] = {}
self._matched_clients: list[MatchedClient] = []
self._lock = threading.Lock()
def _submit_scan(self) -> list[concurrent.futures.Future["JetBrainsPluginClient"]]:
@@ -116,6 +125,47 @@ class JetBrainsPluginClientManager:
"with the Serena plugin installed and running!"
)
def match_clients(self, registered_projects: list[RegisteredProject]) -> list[MatchedClient]:
"""
Scans for plugin instances and matches them against the given registered projects.
:param registered_projects: the list of registered projects to match plugin instances against
:return: the list of matched clients with their corresponding registered project
"""
matched_clients = []
for future in self._submit_scan():
client = future.result()
if client.project_root is not None:
for rp in registered_projects:
if client.matches(Path(rp.project_root)):
matched_clients.append(MatchedClient(client, rp))
break
self._matched_clients = matched_clients
return matched_clients
def get_matched_client(
self, registered_project: RegisteredProject, registered_projects: list[RegisteredProject]
) -> Optional["JetBrainsPluginClient"]:
"""
Gets the matched client for a given registered project, if any.
:param registered_project: the registered project to get the matched client for
:param registered_projects: the list of all registered projects (used to perform matching of all clients
if no match is found for the given project)
:return: the matched client or None if no match is found
"""
def find_match() -> Optional["JetBrainsPluginClient"]:
for matched_client in self._matched_clients:
if matched_client.registered_project.project_root == registered_project.project_root:
return matched_client.client
return None
match = find_match()
if match is None:
self.match_clients(registered_projects)
return find_match()
class JetBrainsPluginClient(ToStringMixin):
"""
src/serena/ls_manager.py
+20 -11
View File
@@ -6,7 +6,6 @@ from collections.abc import Iterator
from sensai.util.logging import LogTime
from serena.config.serena_config import SerenaPaths
from serena.constants import SERENA_MANAGED_DIR_NAME
from solidlsp import SolidLanguageServer
from solidlsp.ls_config import Language, LanguageServerConfig
from solidlsp.settings import SolidLSPSettings
@@ -23,6 +22,7 @@ class LanguageServerFactory:
def __init__(
self,
project_root: str,
project_data_path: str,
encoding: str,
ignored_patterns: list[str],
ls_timeout: float | None = None,
@@ -30,6 +30,7 @@ class LanguageServerFactory:
trace_lsp_communication: bool = False,
):
self.project_root = project_root
self.project_data_path = project_data_path
self.encoding = encoding
self.ignored_patterns = ignored_patterns
self.ls_timeout = ls_timeout
@@ -51,7 +52,7 @@ class LanguageServerFactory:
timeout=self.ls_timeout,
solidlsp_settings=SolidLSPSettings(
solidlsp_dir=SerenaPaths().serena_user_home_dir,
project_data_relative_path=SERENA_MANAGED_DIR_NAME,
project_data_path=self.project_data_path,
ls_specific_settings=self.ls_specific_settings or {},
),
)
@@ -76,8 +77,12 @@ class LanguageServerManager:
"""
self._language_servers = language_servers
self._language_server_factory = language_server_factory
self._default_language_server = next(iter(language_servers.values()))
self._root_path = self._default_language_server.repository_root_path
@property
def _default_language_server(self) -> SolidLanguageServer:
if len(self._language_servers) == 0:
raise ValueError("No language servers available in the manager")
return next(iter(self._language_servers.values()))
@staticmethod
def from_languages(languages: list[Language], factory: LanguageServerFactory) -> "LanguageServerManager":
@@ -138,25 +143,26 @@ class LanguageServerManager:
return LanguageServerManager(language_servers, factory)
def get_root_path(self) -> str:
return self._root_path
def _ensure_functional_ls(self, ls: SolidLanguageServer) -> SolidLanguageServer:
if not ls.is_running():
log.warning(f"Language server for language {ls.language} is not running; restarting ...")
ls = self.restart_language_server(ls.language)
return ls
def _get_suitable_language_server(self, relative_path: str) -> SolidLanguageServer | None:
""":param relative_path: relative path to a file"""
for candidate in self._language_servers.values():
if not candidate.is_ignored_path(relative_path, ignore_unsupported_files=True):
return candidate
return None
def get_language_server(self, relative_path: str) -> SolidLanguageServer:
""":param relative_path: relative path to a file"""
ls: SolidLanguageServer | None = None
if len(self._language_servers) > 1:
if os.path.isdir(relative_path):
raise ValueError(f"Expected a file path, but got a directory: {relative_path}")
for candidate in self._language_servers.values():
if not candidate.is_ignored_path(relative_path, ignore_unsupported_files=True):
ls = candidate
break
ls = self._get_suitable_language_server(relative_path)
if ls is None:
ls = self._default_language_server
return self._ensure_functional_ls(ls)
@@ -241,3 +247,6 @@ class LanguageServerManager:
for ls in self.iter_language_servers():
if ls.is_running():
ls.save_cache()
def has_suitable_ls_for_file(self, relative_file_path: str) -> bool:
return self._get_suitable_language_server(relative_file_path) is not None
src/serena/mcp.py
+6 -2
View File
@@ -341,8 +341,12 @@ class SerenaMCPFactory:
openai_tool_compatible = self.context.name in ["chatgpt", "codex", "oaicompat-agent"]
self._set_mcp_tools(mcp_server, openai_tool_compatible=openai_tool_compatible)
log.info("MCP server lifetime setup complete")
yield
log.info("MCP server shutting down")
try:
yield
finally:
log.info("MCP server shutting down")
if self.agent is not None:
self.agent.shutdown()
def _get_initial_instructions(self) -> str:
assert self.agent is not None
src/serena/project.py
+265 -78
View File
@@ -1,103 +1,228 @@
import json
import logging
import os
import re
import shutil
import threading
from collections.abc import Sequence
from pathlib import Path
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Literal, Optional
import pathspec
from sensai.util.logging import LogTime
from sensai.util.string import ToStringMixin
from sensai.util.string import TextBuilder, ToStringMixin
from serena.config.serena_config import (
DEFAULT_TOOL_TIMEOUT,
ProjectConfig,
get_serena_managed_in_project_dir,
SerenaConfig,
SerenaPaths,
)
from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_NAME
from serena.constants import SERENA_FILE_ENCODING
from serena.ls_manager import LanguageServerFactory, LanguageServerManager
from serena.util.file_system import GitignoreParser, match_path
from serena.util.text_utils import MatchedConsecutiveLines, search_files
from serena.util.text_utils import ContentReplacer, MatchedConsecutiveLines, search_files
from solidlsp import SolidLanguageServer
from solidlsp.ls_config import Language
from solidlsp.ls_utils import FileUtils
if TYPE_CHECKING:
from serena.config.serena_config import SerenaConfig
from serena.agent import SerenaAgent
log = logging.getLogger(__name__)
class MemoriesManager:
def __init__(self, project_root: str):
self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories"
self._memory_dir.mkdir(parents=True, exist_ok=True)
GLOBAL_TOPIC = "global"
_global_memory_dir = SerenaPaths().global_memories_path
def __init__(
self,
serena_data_folder: str | Path | None,
read_only_memory_patterns: Sequence[str] = (),
ignored_memory_patterns: Sequence[str] = (),
):
"""
:param serena_data_folder: the absolute path to the project's .serena data folder
:param read_only_memory_patterns: whether to allow writing global memories in tool execution contexts
:param ignored_memory_patterns: regex patterns for memories to completely exclude from listing, reading, and writing.
Matching memories will not appear in list_memories or activate_project output and cannot be accessed
via read_memory or write_memory. Use read_file on the raw path to access ignored memory files.
"""
self._project_memory_dir: Path | None = None
if serena_data_folder is not None:
self._project_memory_dir = Path(serena_data_folder) / "memories"
self._project_memory_dir.mkdir(parents=True, exist_ok=True)
self._encoding = SERENA_FILE_ENCODING
self._read_only_memory_patterns = [re.compile(pattern) for pattern in set(read_only_memory_patterns)]
self._ignored_memory_patterns = [re.compile(pattern) for pattern in set(ignored_memory_patterns)]
def _is_read_only_memory(self, name: str) -> bool:
for pattern in self._read_only_memory_patterns:
if pattern.fullmatch(name):
return True
return False
def _is_ignored_memory(self, name: str) -> bool:
for pattern in self._ignored_memory_patterns:
if pattern.fullmatch(name):
return True
return False
def _check_not_ignored(self, name: str) -> None:
if self._is_ignored_memory(name):
raise ValueError(
f"Memory '{name}' matches an ignored_memory_patterns pattern and cannot be accessed. "
f"Use the read_file tool on the raw file path instead."
)
def _is_global(self, name: str) -> bool:
return name == self.GLOBAL_TOPIC or name.startswith(self.GLOBAL_TOPIC + "/")
def get_memory_file_path(self, name: str) -> Path:
# Strip .md extension if present
name = name.replace(".md", "")
# Split by "/" to handle subdirectories
if self._is_global(name):
if name == self.GLOBAL_TOPIC:
raise ValueError(
f'Bare "{self.GLOBAL_TOPIC}" is not a valid memory name. Use "{self.GLOBAL_TOPIC}/<name>" to address a global memory.'
)
# Strip "global/" prefix and resolve against global dir
sub_name = name[len(self.GLOBAL_TOPIC) + 1 :]
parts = sub_name.split("/")
filename = f"{parts[-1]}.md"
if len(parts) > 1:
subdir = self._global_memory_dir / "/".join(parts[:-1])
subdir.mkdir(parents=True, exist_ok=True)
return subdir / filename
return self._global_memory_dir / filename
# Project-local memory
assert self._project_memory_dir is not None, "Project dir was not passed at initialization"
parts = name.split("/")
filename = f"{parts[-1]}.md"
if len(parts) > 1:
# Create subdirectory path
subdir = self._memory_dir / "/".join(parts[:-1])
subdir = self._project_memory_dir / "/".join(parts[:-1])
subdir.mkdir(parents=True, exist_ok=True)
return subdir / filename
return self._memory_dir / filename
return self._project_memory_dir / filename
def _check_write_access(self, name: str, is_tool_context: bool) -> None:
# in tool context, memories can be read-only
if is_tool_context and self._is_read_only_memory(name):
raise PermissionError(f"Attempted to write to read_only memory: '{name}')")
def load_memory(self, name: str) -> str:
self._check_not_ignored(name)
memory_file_path = self.get_memory_file_path(name)
if not memory_file_path.exists():
return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it."
with open(memory_file_path, encoding=self._encoding) as f:
return f.read()
def save_memory(self, name: str, content: str) -> str:
def save_memory(self, name: str, content: str, is_tool_context: bool) -> str:
self._check_not_ignored(name)
self._check_write_access(name, is_tool_context)
memory_file_path = self.get_memory_file_path(name)
with open(memory_file_path, "w", encoding=self._encoding) as f:
f.write(content)
return f"Memory {name} written."
def list_memories(self, topic: str = "") -> list[str]:
class MemoriesList:
def __init__(self) -> None:
self.memories: list[str] = []
self.read_only_memories: list[str] = []
def __len__(self) -> int:
return len(self.memories) + len(self.read_only_memories)
def add(self, memory_name: str, is_read_only: bool) -> None:
if is_read_only:
self.read_only_memories.append(memory_name)
else:
self.memories.append(memory_name)
def extend(self, other: "MemoriesManager.MemoriesList") -> None:
self.memories.extend(other.memories)
self.read_only_memories.extend(other.read_only_memories)
def to_dict(self) -> dict[str, list[str]]:
result = {}
if self.memories:
result["memories"] = sorted(self.memories)
if self.read_only_memories:
result["read_only_memories"] = sorted(self.read_only_memories)
return result
def get_full_list(self) -> list[str]:
return sorted(self.memories + self.read_only_memories)
def _list_memories(self, search_dir: Path, base_dir: Path, prefix: str = "") -> MemoriesList:
result = self.MemoriesList()
if not search_dir.exists():
return result
for md_file in search_dir.rglob("*.md"):
rel = str(md_file.relative_to(base_dir).with_suffix("")).replace(os.sep, "/")
memory_name = prefix + rel
if self._is_ignored_memory(memory_name):
continue
result.add(memory_name, is_read_only=self._is_read_only_memory(memory_name))
return result
def list_global_memories(self, subtopic: str = "") -> MemoriesList:
dir_path = self._global_memory_dir
if subtopic:
dir_path = dir_path / subtopic.replace("/", os.sep)
return self._list_memories(dir_path, self._global_memory_dir, self.GLOBAL_TOPIC + "/")
def list_project_memories(self, topic: str = "") -> MemoriesList:
assert self._project_memory_dir is not None, "Project dir was not passed at initialization"
dir_path = self._project_memory_dir
if topic:
dir_path = dir_path / topic.replace("/", os.sep)
return self._list_memories(dir_path, self._project_memory_dir)
def list_memories(self, topic: str = "") -> MemoriesList:
"""
List memories, optionally filtered by topic.
Lists all memories, optionally filtered by topic.
If the topic is omitted, both global and project-specific memories are returned.
"""
memories = []
memories: MemoriesManager.MemoriesList
if topic:
# Only list memories in specified subdirectory
search_dir = self._memory_dir / topic.replace("/", os.sep)
if not search_dir.exists():
return []
if self._is_global(topic):
topic_parts = topic.split("/")
subtopic = "/".join(topic_parts[1:])
memories = self.list_global_memories(subtopic=subtopic)
else:
memories = self.list_project_memories(topic=topic)
else:
search_dir = self._memory_dir
memories = self.list_project_memories()
memories.extend(self.list_global_memories())
# Recursively find all .md files
for md_file in search_dir.rglob("*.md"):
# Calculate relative path as memory name
rel_path = md_file.relative_to(self._memory_dir)
name = str(rel_path.with_suffix("")).replace(os.sep, "/")
memories.append(name)
return memories
# Sort alphabetically by name
return sorted(memories)
def delete_memory(self, name: str) -> str:
def delete_memory(self, name: str, is_tool_context: bool) -> str:
self._check_not_ignored(name)
self._check_write_access(name, is_tool_context)
memory_file_path = self.get_memory_file_path(name)
if not memory_file_path.exists():
return f"Memory {name} not found."
memory_file_path.unlink()
return f"Memory {name} deleted."
def rename_memory(self, old_name: str, new_name: str) -> str:
def move_memory(self, old_name: str, new_name: str, is_tool_context: bool) -> str:
"""
Rename or move a memory file.
Moving between global and project scope (e.g. "global/foo" -> "bar") is supported.
"""
self._check_not_ignored(old_name)
self._check_not_ignored(new_name)
self._check_write_access(new_name, is_tool_context)
old_path = self.get_memory_file_path(old_name)
new_path = self.get_memory_file_path(new_name)
@@ -106,38 +231,79 @@ class MemoriesManager:
if new_path.exists():
raise FileExistsError(f"Memory {new_name} already exists.")
# Ensure target directory exists
new_path.parent.mkdir(parents=True, exist_ok=True)
shutil.move(old_path, new_path)
# Move/rename the file
old_path.rename(new_path)
return f"Memory renamed from {old_name} to {new_name}."
def edit_memory(
self, name: str, needle: str, repl: str, mode: Literal["literal", "regex"], allow_multiple_occurrences: bool, is_tool_context: bool
) -> str:
"""
Edit a memory by replacing content matching a pattern.
:param name: the memory name
:param needle: the string or regex to search for
:param repl: the replacement string
:param mode: "literal" or "regex"
:param allow_multiple_occurrences:
"""
self._check_not_ignored(name)
self._check_write_access(name, is_tool_context)
memory_file_path = self.get_memory_file_path(name)
if not memory_file_path.exists():
raise FileNotFoundError(f"Memory {name} not found.")
with open(memory_file_path, encoding=self._encoding) as f:
original_content = f.read()
replacer = ContentReplacer(mode=mode, allow_multiple_occurrences=allow_multiple_occurrences)
updated_content = replacer.replace(original_content, needle, repl)
with open(memory_file_path, "w", encoding=self._encoding) as f:
f.write(updated_content)
return f"Memory {name} edited successfully."
class Project(ToStringMixin):
def __init__(
self,
*,
project_root: str,
project_config: ProjectConfig,
serena_config: SerenaConfig,
is_newly_created: bool = False,
serena_config: "SerenaConfig | None" = None,
):
assert serena_config is not None
self.project_root = project_root
self.project_config = project_config
self.memories_manager = MemoriesManager(project_root)
self.serena_config = serena_config
self._serena_data_folder = serena_config.get_project_serena_folder(self.project_root)
log.info("Serena project data folder: %s", self._serena_data_folder)
read_only_memory_patterns = serena_config.read_only_memory_patterns + project_config.read_only_memory_patterns
ignored_memory_patterns = serena_config.ignored_memory_patterns + project_config.ignored_memory_patterns
self.memories_manager = MemoriesManager(
self._serena_data_folder,
read_only_memory_patterns=read_only_memory_patterns,
ignored_memory_patterns=ignored_memory_patterns,
)
# resolve line ending (project -> global)
self.line_ending = project_config.line_ending or serena_config.line_ending
self.language_server_manager: LanguageServerManager | None = None
self._language_server_manager_init_error: Exception | None = None
self._is_newly_created = is_newly_created
self._agent: Optional["SerenaAgent"] = None
# create .gitignore file in the project's Serena data folder if not yet present
serena_data_gitignore_path = os.path.join(self.path_to_serena_data_folder(), ".gitignore")
serena_data_gitignore_path = os.path.join(self._serena_data_folder, ".gitignore")
if not os.path.exists(serena_data_gitignore_path):
os.makedirs(os.path.dirname(serena_data_gitignore_path), exist_ok=True)
log.info(f"Creating .gitignore file in {serena_data_gitignore_path}")
with open(serena_data_gitignore_path, "w", encoding="utf-8") as f:
f.write(f"/{SolidLanguageServer.CACHE_FOLDER_NAME}\n")
f.write(f"/{ProjectConfig.SERENA_LOCAL_PROJECT_FILE}\n")
# prepare ignore spec asynchronously, ensuring immediate project activation.
self._serena_config = serena_config
self.__ignored_patterns: list[str]
self.__ignore_spec: pathspec.PathSpec
self._ignore_spec_available = threading.Event()
@@ -145,9 +311,8 @@ class Project(ToStringMixin):
def _gather_ignorespec(self) -> None:
with LogTime(f"Gathering ignore spec for project {self.project_config.project_name}", logger=log):
# gather ignored paths from the global configuration, project configuration, and gitignore files
global_ignored_paths = self._serena_config.ignored_paths if self._serena_config else []
global_ignored_paths = self.serena_config.ignored_paths
ignored_patterns = list(global_ignored_paths) + list(self.project_config.ignored_paths)
if len(global_ignored_paths) > 0:
log.info(f"Using {len(global_ignored_paths)} ignored paths from the global configuration.")
@@ -166,7 +331,7 @@ class Project(ToStringMixin):
# Set up the pathspec matcher for the ignored paths
# for all absolute paths in ignored_paths, convert them to relative paths
processed_patterns = []
for pattern in set(ignored_patterns):
for pattern in ignored_patterns:
# Normalize separators (pathspec expects forward slashes)
pattern = pattern.replace(os.path.sep, "/")
processed_patterns.append(pattern)
@@ -181,6 +346,9 @@ class Project(ToStringMixin):
def _tostring_additional_entries(self) -> dict[str, Any]:
return {"root": self.project_root, "name": self.project_name}
def set_agent(self, agent: "SerenaAgent") -> None:
self._agent = agent
@property
def project_name(self) -> str:
return self.project_config.project_name
@@ -189,26 +357,27 @@ class Project(ToStringMixin):
def load(
cls,
project_root: str | Path,
serena_config: "SerenaConfig | None",
serena_config: "SerenaConfig",
autogenerate: bool = True,
) -> "Project":
assert serena_config is not None
project_root = Path(project_root).resolve()
if not project_root.exists():
raise FileNotFoundError(f"Project root not found: {project_root}")
project_config = ProjectConfig.load(project_root, autogenerate=autogenerate)
project_config = ProjectConfig.load(project_root, serena_config=serena_config, autogenerate=autogenerate)
return Project(project_root=str(project_root), project_config=project_config, serena_config=serena_config)
def save_config(self) -> None:
"""
Saves the current project configuration to disk.
"""
self.project_config.save(self.project_root)
self.project_config.save(self.path_to_project_yml())
def path_to_serena_data_folder(self) -> str:
return os.path.join(self.project_root, SERENA_MANAGED_DIR_NAME)
return self._serena_data_folder
def path_to_project_yml(self) -> str:
return os.path.join(self.project_root, self.project_config.rel_path_to_project_yml())
return self.serena_config.get_project_yml_location(self.project_root)
def get_activation_message(self) -> str:
"""
@@ -220,10 +389,10 @@ class Project(ToStringMixin):
msg = f"The project with name '{self.project_name}' at {self.project_root} is activated."
languages_str = ", ".join([lang.value for lang in self.project_config.languages])
msg += f"\nProgramming languages: {languages_str}; file encoding: {self.project_config.encoding}"
memories = self.memories_manager.list_memories()
if memories:
project_memories = self.memories_manager.list_project_memories()
if project_memories:
msg += (
f"\nAvailable project memories: {json.dumps(memories)}\n"
f"\nAvailable project memories: {json.dumps(project_memories.to_dict())}\n"
+ "Use the `read_memory` tool to read these memories later if they are relevant to the task."
)
if self.project_config.initial_prompt:
@@ -464,39 +633,57 @@ class Project(ToStringMixin):
source_file_path=relative_file_path,
)
def create_language_server_manager(
self,
log_level: int = logging.INFO,
ls_timeout: float | None = DEFAULT_TOOL_TIMEOUT - 5,
trace_lsp_communication: bool = False,
ls_specific_settings: dict[Language, Any] | None = None,
) -> LanguageServerManager:
def create_language_server_manager(self) -> LanguageServerManager:
"""
Creates the language server manager for the project, starting one language server per configured programming language.
:param log_level: the log level for the language server
:param ls_timeout: the timeout for the language server
:param trace_lsp_communication: whether to trace LSP communication
:param ls_specific_settings: optional LS specific configuration of the language server,
see docstrings in the inits of subclasses of SolidLanguageServer to see what values may be passed.
:return: the language server manager, which is also stored in the project instance
"""
# if there is an existing instance, stop its language servers first
if self.language_server_manager is not None:
log.info("Stopping existing language server manager ...")
self.language_server_manager.stop_all()
self.language_server_manager = None
try:
# determine timeout to use for LS calls
tool_timeout = self.serena_config.tool_timeout
if tool_timeout is None or tool_timeout < 0:
ls_timeout = None
else:
if tool_timeout < 10:
raise ValueError(f"Tool timeout must be at least 10 seconds, but is {tool_timeout} seconds")
ls_timeout = tool_timeout - 5 # the LS timeout is for a single call, it should be smaller than the tool timeout
log.info(f"Creating language server manager for {self.project_root}")
factory = LanguageServerFactory(
project_root=self.project_root,
encoding=self.project_config.encoding,
ignored_patterns=self._ignored_patterns,
ls_timeout=ls_timeout,
ls_specific_settings=ls_specific_settings,
trace_lsp_communication=trace_lsp_communication,
)
self.language_server_manager = LanguageServerManager.from_languages(self.project_config.languages, factory)
# if there is an existing instance, stop its language servers first
if self.language_server_manager is not None:
log.info("Stopping existing language server manager ...")
self.language_server_manager.stop_all()
self.language_server_manager = None
log.info(f"Creating language server manager for {self.project_root}")
self._language_server_manager_init_error = None
ls_specific_settings = {**self.serena_config.ls_specific_settings, **self.project_config.ls_specific_settings}
factory = LanguageServerFactory(
project_root=self.project_root,
project_data_path=self._serena_data_folder,
encoding=self.project_config.encoding,
ignored_patterns=self._ignored_patterns,
ls_timeout=ls_timeout,
ls_specific_settings=ls_specific_settings,
trace_lsp_communication=self.serena_config.trace_lsp_communication,
)
self.language_server_manager = LanguageServerManager.from_languages(self.project_config.languages, factory)
return self.language_server_manager
except Exception as e:
self._language_server_manager_init_error = e
raise
def get_language_server_manager_or_raise(self) -> LanguageServerManager:
if self.language_server_manager is None:
msg = TextBuilder("The language server manager is not initialized, indicating a problem during project initialisation.")
if self._language_server_manager_init_error is not None:
msg.with_text(str(self._language_server_manager_init_error))
if self._agent is not None:
msg.with_text("For details, please check the logs. " + self._agent.get_log_inspection_instructions())
msg.with_text(
"IMPORTANT: Stop, do not attempt workarounds. Inform the user and wait for further instructions before you continue!"
)
raise Exception(msg.build())
return self.language_server_manager
def add_language(self, language: Language) -> None:
src/serena/project_server.py
+157
View File
@@ -0,0 +1,157 @@
import json
import logging
from typing import TYPE_CHECKING
import requests as requests_lib
from flask import Flask, request
from pydantic import BaseModel
from sensai.util.logging import LogTime
from serena.config.serena_config import LanguageBackend, SerenaConfig
from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClient
if TYPE_CHECKING:
from serena.project import Project
log = logging.getLogger(__name__)
# disable Werkzeug's logging to avoid cluttering the output
logging.getLogger("werkzeug").setLevel(logging.WARNING)
class QueryProjectRequest(BaseModel):
"""
Request model for the /query_project endpoint, matching the interface of
:class:`~serena.tools.query_project_tools.QueryProjectTool`.
"""
project_name: str
tool_name: str
tool_params_json: str
class ProjectServer:
"""
A lightweight Flask server that exposes a SerenaAgent's project querying
capabilities via HTTP, using the LSP language server backend for symbolic retrieval.
Projects are loaded on demand when a query is made for them, and cached in memory for subsequent queries.
The server instantiates a :class:`SerenaAgent` with default options and
provides a ``/query_project`` endpoint whose interface matches
:class:`~serena.tools.query_project_tools.QueryProjectTool`.
"""
PORT = JetBrainsPluginClient.BASE_PORT - 1
def __init__(self) -> None:
from serena.agent import SerenaAgent
serena_config = SerenaConfig.from_config_file()
serena_config.gui_log_window = False
serena_config.web_dashboard = False
serena_config.language_backend = LanguageBackend.LSP
self._agent = SerenaAgent(serena_config=serena_config)
self._loaded_projects_by_name: dict[str, "Project"] = {}
# create the Flask application
self._app = Flask(__name__)
self._setup_routes()
def _setup_routes(self) -> None:
@self._app.route("/heartbeat", methods=["GET"])
def heartbeat() -> dict[str, str]:
return {"status": "alive"}
@self._app.route("/query_project", methods=["POST"])
def query_project() -> str:
query_request = QueryProjectRequest.model_validate(request.get_json())
return self._query_project(query_request)
def _get_project(self, project_name: str) -> "Project":
"""Gets the project with the given name, loading it if necessary."""
if project_name in self._loaded_projects_by_name:
return self._loaded_projects_by_name[project_name]
else:
serena_config = self._agent.serena_config
registered_project = serena_config.get_registered_project(project_name)
if registered_project is None:
raise ValueError(f"Project '{project_name}' is not registered with Serena.")
with LogTime(f"Loading project '{project_name}'"):
project = registered_project.get_project_instance(serena_config)
project.create_language_server_manager()
self._loaded_projects_by_name[project_name] = project
return project
def _query_project(self, req: QueryProjectRequest) -> str:
"""Handle a /query_project request by invoking the agent on the specified project and tool."""
project = self._get_project(req.project_name)
with self._agent.active_project_context(project):
tool = self._agent.get_tool_by_name(req.tool_name)
params = json.loads(req.tool_params_json)
return tool.apply_ex(**params)
def run(self, host: str = "127.0.0.1", port: int = PORT) -> int:
"""Run the server on the given host and port.
:param host: the host address to listen on.
:param port: the port to listen on.
:return: the port number the server is running on.
"""
from flask import cli
# suppress the default Flask startup banner
cli.show_server_banner = lambda *args, **kwargs: None
self._app.run(host=host, port=port, debug=False, use_reloader=False, threaded=True)
return port
class ProjectServerClient:
"""Client for interacting with a running :class:`ProjectServer`.
Upon instantiation, the client verifies that the server is reachable
by sending a heartbeat request. If the server is not running, a
:class:`ConnectionError` is raised.
"""
def __init__(self, host: str = "127.0.0.1", port: int = ProjectServer.PORT, timeout: int = 300) -> None:
"""
:param host: the host address of the project server.
:param port: the port of the project server.
:raises ConnectionError: if the project server is not reachable.
"""
self._base_url = f"http://{host}:{port}"
self._timeout = timeout
# verify that the server is running
try:
response = requests_lib.get(f"{self._base_url}/heartbeat", timeout=5)
response.raise_for_status()
except requests_lib.ConnectionError:
raise ConnectionError(f"ProjectServer is not reachable at {self._base_url}. Make sure the server is running.")
except requests_lib.RequestException as e:
raise ConnectionError(f"ProjectServer health check failed: {e}")
def query_project(self, project_name: str, tool_name: str, tool_params_json: str) -> str:
"""
Query a project by executing a Serena tool in its context.
The interface matches :meth:`QueryProjectTool.apply
<serena.tools.query_project_tools.QueryProjectTool.apply>`.
:param project_name: the name of the project to query.
:param tool_name: the name of the tool to execute. The tool must be read-only.
:param tool_params_json: the parameters to pass to the tool, encoded as a JSON string.
:return: the tool's result as a string.
"""
payload = QueryProjectRequest(
project_name=project_name,
tool_name=tool_name,
tool_params_json=tool_params_json,
).model_dump()
response = requests_lib.post(f"{self._base_url}/query_project", json=payload, timeout=self._timeout)
response.raise_for_status()
return response.text
src/serena/resources/config/modes/no-memories.yml
+1
View File
@@ -6,6 +6,7 @@ excluded_tools:
- read_memory
- delete_memory
- edit_memory
- rename_memory
- list_memories
- onboarding
- check_onboarding_performed
src/serena/resources/config/modes/query-projects.yml
+8
View File
@@ -0,0 +1,8 @@
description: Enables tools that allow inactive projects to be queried
prompt: |
You can use the 'query_project' tool to query Serena projects without activating them.
Use this when a project is related to the active project and you need to query it for information.
excluded_tools: []
included_optional_tools:
- list_queryable_projects
- query_project
src/serena/resources/config/prompt_templates/system_prompt.yml
+5 -2
View File
@@ -35,10 +35,13 @@ prompts:
You can understand relationships between symbols by using the `find_referencing_symbols` tool.
{% endif %}
{% if 'read_memory' in available_tools %}
{% if 'read_memory' in available_tools -%}
You generally have access to memories and it may be useful for you to read them.
You infer whether memories are relevant based on their names.
{% endif %}
{% if global_memories_list -%}
The following global (not project-specific) memories are available to you: {{ global_memories_list }}
{%- endif -%}
{%- endif %}
The context and modes of operation are described below. These determine how to interact with your user
and which kinds of interactions are expected of you.
src/serena/resources/dashboard/dashboard.css
+37 -2
View File
@@ -836,12 +836,11 @@ body {
border: 1px solid var(--border-color);
border-radius: 5px;
height: calc(100vh - var(--header-height) - 2 * var(--header-padding) - 3 * var(--header-gap-main));
max-height: 700px;
overflow-y: auto;
overflow-x: auto;
padding: 10px;
white-space: pre-wrap;
font-size: 12px;
font-size: 14px;
line-height: 1.4;
color: var(--text-primary);
transition: background-color 0.3s ease, border-color 0.3s ease, color 0.3s ease;
@@ -1308,6 +1307,42 @@ body {
gap: 8px;
}
/* Memory Rename Styles */
.memory-rename-btn {
display: inline-flex;
align-items: center;
justify-content: center;
width: 24px;
height: 24px;
background: var(--bg-secondary);
border: 1px solid var(--border-color);
border-radius: 4px;
cursor: pointer;
color: var(--text-muted);
opacity: 0.5;
transition: opacity 0.2s ease, background-color 0.2s ease;
}
.memory-rename-btn:hover {
opacity: 1;
background-color: var(--border-color);
}
.memory-rename-input {
font-size: inherit;
font-weight: inherit;
font-family: inherit;
color: var(--text-primary);
background: transparent;
border: none;
border-bottom: 1px solid var(--btn-primary);
outline: none;
padding: 0;
flex: 1;
min-width: 200px;
max-width: 80%;
}
/* Log Action Buttons (Save, Copy, Clear) */
.log-action-buttons {
position: absolute;
src/serena/resources/dashboard/dashboard.js
+74 -3
View File
@@ -307,6 +307,8 @@ class Dashboard {
this.$modalCloseRemove = $('.modal-close-remove');
this.$editMemoryModal = $('#edit-memory-modal');
this.$editMemoryName = $('#edit-memory-name');
this.$editMemoryRenameBtn = $('#edit-memory-rename-btn');
this.$editMemoryRenameInput = $('#edit-memory-rename-input');
this.$editMemoryContent = $('#edit-memory-content');
this.$editMemorySaveBtn = $('#edit-memory-save-btn');
this.$editMemoryCancelBtn = $('#edit-memory-cancel-btn');
@@ -365,6 +367,19 @@ class Dashboard {
this.$editMemoryCancelBtn.click(this.closeEditMemoryModal.bind(this));
this.$modalCloseEditMemory.click(this.closeEditMemoryModal.bind(this));
this.$editMemoryContent.on('input', this.trackMemoryChanges.bind(this));
this.$editMemoryRenameBtn.click(this.startMemoryRename.bind(this));
this.$editMemoryRenameInput.keydown(function (e) {
if (e.which === 13) { // Enter key
e.preventDefault();
self.commitMemoryRename();
} else if (e.which === 27) { // Escape key
e.preventDefault();
self.cancelMemoryRename();
}
});
this.$editMemoryRenameInput.on('blur', function () {
self.cancelMemoryRename();
});
this.$deleteMemoryOkBtn.click(this.confirmDeleteMemoryOk.bind(this));
this.$deleteMemoryCancelBtn.click(this.closeDeleteMemoryModal.bind(this));
this.$modalCloseDeleteMemory.click(this.closeDeleteMemoryModal.bind(this));
@@ -1887,6 +1902,62 @@ class Dashboard {
});
}
startMemoryRename() {
this.$editMemoryName.hide();
this.$editMemoryRenameBtn.hide();
this.$editMemoryRenameInput.val(this.currentMemoryName).show().focus().select();
}
cancelMemoryRename() {
this.$editMemoryRenameInput.hide();
this.$editMemoryName.show();
this.$editMemoryRenameBtn.show();
}
commitMemoryRename() {
const newName = this.$editMemoryRenameInput.val().trim();
const oldName = this.currentMemoryName;
// If name unchanged, just cancel
if (!newName || newName === oldName) {
this.cancelMemoryRename();
return;
}
// Validate memory name (alphanumeric, underscores, and slashes for subdirectories)
if (!/^[a-zA-Z0-9_]+(?:\/[a-zA-Z0-9_]+)*$/.test(newName)) {
alert('Memory name can only contain letters, numbers, underscores, and "/" for subdirectories (e.g., "topic/memory_name")');
this.$editMemoryRenameInput.focus();
return;
}
const self = this;
this.$editMemoryRenameInput.prop('disabled', true);
$.ajax({
url: '/rename_memory', type: 'POST', contentType: 'application/json', data: JSON.stringify({
old_name: oldName, new_name: newName
}), success: function (response) {
if (response.status === 'success') {
self.currentMemoryName = newName;
self.$editMemoryName.text(newName);
self.cancelMemoryRename();
// Reload config to reflect the rename in the memory list
self.loadConfigOverview();
} else {
alert('Error: ' + response.message);
self.$editMemoryRenameInput.focus();
}
}, error: function (xhr, status, error) {
console.error('Error renaming memory:', error);
alert('Error renaming memory: ' + (xhr.responseJSON ? xhr.responseJSON.message : error));
self.$editMemoryRenameInput.focus();
}, complete: function () {
self.$editMemoryRenameInput.prop('disabled', false);
}
});
}
confirmDeleteMemory(memoryName) {
// Set memory name to delete
this.memoryToDelete = memoryName;
@@ -1960,9 +2031,9 @@ class Dashboard {
return;
}
// Validate memory name (alphanumeric and underscores only)
if (!/^[a-zA-Z0-9_]+$/.test(memoryName)) {
alert('Memory name can only contain letters, numbers, and underscores');
// Validate memory name (alphanumeric, underscores, and slashes for subdirectories)
if (!/^[a-zA-Z0-9_]+(?:\/[a-zA-Z0-9_]+)*$/.test(memoryName)) {
alert('Memory name can only contain letters, numbers, underscores, and "/" for subdirectories (e.g., "topic/memory_name")');
return;
}
src/serena/resources/dashboard/index.html
+14 -3
View File
@@ -275,7 +275,17 @@
<div id="edit-memory-modal" class="modal" style="display:none">
<div class="modal-content modal-content-large">
<span class="modal-close modal-close-edit-memory">&times;</span>
<h3 style="margin-bottom: 20px;">Memory: <span id="edit-memory-name"></span></h3>
<h3 style="margin-bottom: 20px; display: flex; align-items: center; gap: 8px;">
Memory:
<span id="edit-memory-name" class="memory-name-display"></span>
<span id="edit-memory-rename-btn" class="memory-rename-btn" title="Rename memory">
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none"
stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
<path d="M17 3a2.83 2.83 0 1 1 4 4L7.5 20.5 2 22l1.5-5.5L17 3z"></path>
</svg>
</span>
<input type="text" id="edit-memory-rename-input" class="memory-rename-input" style="display: none;">
</h3>
<textarea id="edit-memory-content" class="memory-editor" rows="20"
style="width: 100%; padding: 10px; border: 1px solid var(--border-color); border-radius: 4px; background: var(--bg-primary); color: var(--text-primary); font-family: 'Courier New', monospace; font-size: 13px; resize: vertical;"></textarea>
<div style="display: flex; gap: 10px; justify-content: flex-end; margin-top: 20px;">
@@ -312,13 +322,14 @@
</p>
<p style="margin: 15px 0; color: var(--text-muted); font-size: 13px; line-height: 1.5;">
Memory names should be descriptive and use underscores instead of spaces (e.g., "api_architecture",
"testing_guidelines").
"testing_guidelines"). Use "/" to organize memories into subdirectories (e.g., "architecture/api_design"),
use the "global/" prefix to write memories that are shared across projects (e.g., "global/java/style_guide").
</p>
<div style="margin: 20px 0;">
<label for="create-memory-name-input" style="display: block; margin-bottom: 8px; font-weight: 500;">Memory
Name:</label>
<input type="text" id="create-memory-name-input" class="memory-name-input"
placeholder="e.g., project_overview"
placeholder="e.g., project_overview or topic/memory_name"
style="width: 100%; padding: 8px 10px; border-radius: 4px; border: 1px solid var(--border-color); background: var(--bg-primary); color: var(--text-primary); font-size: 14px;">
</div>
<div style="display: flex; gap: 10px; justify-content: flex-end; margin-top: 20px;">
src/serena/resources/dashboard/news/20260303.html
+12
View File
@@ -0,0 +1,12 @@
<div class="news-item">
<h3>Nested and Global Memories</h3>
<p class="date">March 03, 2026</p>
<p>
Serena's memory system has been significantly extended in functionality.
It keeps the simplicity that made it popular but now supports structuring memories
into topics and sharing them across projects.
</p>
<p>
For more detailed information, see the <a href="https://oraios.github.io/serena/02-usage/045_memories.html">documentation</a>.
</p>
</div>
src/serena/resources/project.local.template.yml
+5
View File
@@ -0,0 +1,5 @@
# This file allows you to locally override settings in project.yml for development purposes.
#
# Use the same keys as in project.yml here. Any setting you specify will override the corresponding
# setting in project.yml, allowing you to customise the configuration for your local development environment
# without affecting the project configuration in project.yml (which is intended to be versioned).
src/serena/resources/project.template.yml
+28 -2
View File
@@ -31,6 +31,11 @@ languages: ["python"]
# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
encoding: "utf-8"
# line ending convention to use when writing source files.
# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
line_ending:
# The language backend to use for this project.
# If not set, the global setting from serena_config.yml is used.
# Valid values: LSP, JetBrains
@@ -41,6 +46,12 @@ language_backend:
# whether to use project's .gitignore files to ignore files
ignore_all_files_in_gitignore: true
# advanced configuration option allowing to configure language server-specific options.
# Maps the language key to the options.
# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available.
# No documentation on options means no options are available.
ls_specific_settings: {}
# list of additional paths to ignore in this project.
# Same syntax as gitignore, so you can use * and **.
# Note: global ignored_paths from serena_config.yml are also applied additively.
@@ -51,7 +62,9 @@ ignored_paths: []
# Added on 2025-04-18
read_only: false
# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details.
# list of tool names to exclude.
# This extends the existing exclusions (e.g. from the global configuration)
#
# Below is the complete list of tools for convenience.
# To make sure you have the latest list of tools, and to view their descriptions,
# execute `uv run scripts/print_tool_overview.py`.
@@ -92,7 +105,8 @@ read_only: false
# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
excluded_tools: []
# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default)
# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
# This extends the existing inclusions (e.g. from the global configuration).
included_optional_tools: []
# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
@@ -123,3 +137,15 @@ initial_prompt: ""
# This overrides the corresponding setting in the global configuration; see the documentation there.
# If null or missing, use the setting from the global configuration.
symbol_info_budget:
# list of regex patterns which, when matched, mark a memory entry as read‑only.
# Extends the list from the global configuration, merging the two lists.
read_only_memory_patterns: []
# list of regex patterns for memories to completely ignore.
# Matching memories will not appear in list_memories or activate_project output
# and cannot be accessed via read_memory or write_memory.
# To access ignored memory files, use the read_file tool on the raw file path.
# Extends the list from the global configuration, merging the two lists.
# Example: ["_archive/.*", "_episodes/.*"]
ignored_memory_patterns: []
src/serena/resources/serena_config.template.yml
+28
View File
@@ -7,6 +7,12 @@
# in your IDE).
language_backend: LSP
# line ending convention to use when writing source files.
# Possible values: "lf" (Unix), "crlf" (Windows), "native" (platform default).
# Note that Serena's own files (e.g. memories and configuration files) always use native line endings.
# This setting can be overridden on a per-project basis in project.yml files.
line_ending: native
# whether to open a graphical window with Serena's logs.
# This is mainly supported on Windows and (partly) on Linux; not available on macOS.
# If you prefer a browser-based tool, use the `web_dashboard` option instead.
@@ -68,6 +74,20 @@ ls_specific_settings: {}
# These patterns are merged additively with each project's own ignored_paths.
ignored_paths: []
# list of regex patterns which, when matched, mark a memory entry as read‑only.
# For example, "global/.*" will mark all global memories as read-only.
# You can extend the list on a per-project basis in the project.yml configuration file.
read_only_memory_patterns: []
# list of regex patterns for memories to completely ignore.
# Matching memories will not appear in list_memories or activate_project output
# and cannot be accessed via read_memory or write_memory.
# To access ignored memory files, use the read_file tool on the raw file path.
# This is useful for projects with large numbers of archived memory files.
# You can extend the list on a per-project basis in the project.yml configuration file.
# Example: ["_archive/.*", "_episodes/.*"]
ignored_memory_patterns: []
# timeout, in seconds, after which tool executions are terminated
tool_timeout: 240
@@ -120,5 +140,13 @@ token_count_estimator: CHAR_COUNT
# Can be overridden in project.yml.
symbol_info_budget: 10
# template for the location of the per-project .serena data folder (memories, caches, etc.).
# Supports the following placeholders:
# $projectDir - the absolute path to the project root directory
# $projectFolderName - the name of the project directory
# Default: "$projectDir/.serena" (data stored inside the project directory)
# Example for a central location: "/projects-metadata/$projectFolderName/.serena"
project_serena_folder_location: "$projectDir/.serena"
# the list of registered project paths (updated automatically).
projects: []
src/serena/symbol.py
+23 -29
View File
@@ -2,10 +2,10 @@ import json
import logging
import os
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterator, Sequence
from collections.abc import Callable, Iterable, Iterator, Sequence
from dataclasses import asdict, dataclass
from time import perf_counter
from typing import TYPE_CHECKING, Any, Generic, Literal, NotRequired, Self, TypedDict, TypeVar, Union
from typing import Any, Generic, Literal, NotRequired, Self, TypedDict, TypeVar
from sensai.util.string import ToStringMixin
@@ -18,9 +18,6 @@ from solidlsp.ls_types import Position, SymbolKind, UnifiedSymbolInformation
from .ls_manager import LanguageServerManager
from .project import Project
if TYPE_CHECKING:
from .agent import SerenaAgent
log = logging.getLogger(__name__)
NAME_PATH_SEP = "/"
@@ -536,19 +533,12 @@ class ReferenceInLanguageServerSymbol(ToStringMixin):
class LanguageServerSymbolRetriever:
def __init__(self, ls: SolidLanguageServer | LanguageServerManager, agent: Union["SerenaAgent", None] = None) -> None:
def __init__(self, project: Project) -> None:
"""
:param ls: the language server or language server manager to use for symbol retrieval and editing operations.
:param agent: the agent to use (only needed for marking files as modified). You can pass None if you don't
need an agent to be aware of file modifications performed by the symbol manager.
:param project: the project instance
"""
if isinstance(ls, SolidLanguageServer):
ls_manager = LanguageServerManager({ls.language: ls})
else:
ls_manager = ls
assert isinstance(ls_manager, LanguageServerManager)
self._ls_manager: LanguageServerManager = ls_manager
self.agent = agent
self._ls_manager: LanguageServerManager = project.get_language_server_manager_or_raise()
self.project = project
def _request_info(self, relative_file_path: str, line: int, column: int, file_buffer: LSPFileBuffer | None = None) -> str | None:
"""Retrieves information (in a sanitized format) about the symbol at the desired location,
@@ -588,16 +578,11 @@ class LanguageServerSymbolRetriever:
return None
return self._request_info(relative_file_path=symbol.relative_path, line=symbol.line, column=symbol.column) # type: ignore[arg-type]
def _get_symbol_info_budget(self, default_budget: float = 10) -> float:
"""Project -> global -> default"""
symbol_info_budget = default_budget
if self.agent is not None:
symbol_info_budget = self.agent.serena_config.symbol_info_budget
active_project = self.agent.get_active_project()
if active_project is not None:
project_symbol_info_budget = active_project.project_config.symbol_info_budget
if project_symbol_info_budget is not None:
symbol_info_budget = project_symbol_info_budget
def _get_symbol_info_budget(self) -> float:
symbol_info_budget = self.project.serena_config.symbol_info_budget
project_symbol_info_budget = self.project.project_config.symbol_info_budget
if project_symbol_info_budget is not None:
symbol_info_budget = project_symbol_info_budget
return symbol_info_budget
def request_info_for_symbol_batch(
@@ -696,8 +681,8 @@ class LanguageServerSymbolRetriever:
return info_by_symbol
def get_root_path(self) -> str:
return self._ls_manager.get_root_path()
def can_analyze_file(self, relative_file_path: str) -> bool:
return self._ls_manager.has_suitable_ls_for_file(relative_file_path)
def get_language_server(self, relative_path: str) -> SolidLanguageServer:
""":param relative_path: relative path to a file"""
@@ -716,7 +701,16 @@ class LanguageServerSymbolRetriever:
optionally limited to a specific file and filtered by kind.
"""
symbols: list[LanguageServerSymbol] = []
for lang_server in self._ls_manager.iter_language_servers():
if within_relative_path and os.path.isfile(os.path.join(self.project.project_root, within_relative_path)):
"""
For a specific file, use get_language_server to select the best LS for the file type
(consistent with get_symbol_overview). This ensures e.g. PHP files are served by the
PHP language server rather than being rejected by all LSes via is_ignored_path.
"""
lang_servers: Iterable[SolidLanguageServer] = [self._ls_manager.get_language_server(within_relative_path)]
else:
lang_servers = self._ls_manager.iter_language_servers()
for lang_server in lang_servers:
symbol_roots = lang_server.request_full_symbol_tree(within_relative_path=within_relative_path)
for root in symbol_roots:
symbols.extend(
src/serena/tools/__init__.py
+1
View File
@@ -7,3 +7,4 @@ from .cmd_tools import *
from .config_tools import *
from .workflow_tools import *
from .jetbrains_tools import *
from .query_project_tools import *
src/serena/tools/file_tools.py
+6 -8
View File
@@ -68,12 +68,12 @@ class CreateTextFileTool(Tool, ToolMarkerCanEdit):
if will_overwrite_existing:
self.project.validate_relative_path(relative_path, require_not_ignored=True)
else:
assert abs_path.is_relative_to(
self.get_project_root()
), f"Cannot create file outside of the project directory, got {relative_path=}"
assert abs_path.is_relative_to(self.get_project_root()), (
f"Cannot create file outside of the project directory, got {relative_path=}"
)
abs_path.parent.mkdir(parents=True, exist_ok=True)
abs_path.write_text(content, encoding=self.project.project_config.encoding)
abs_path.write_text(content, encoding=self.project.project_config.encoding, newline=self.project.line_ending.newline_str)
answer = f"File created: {relative_path}."
if will_overwrite_existing:
answer += " Overwrote existing file."
@@ -192,10 +192,8 @@ class ReplaceContentTool(Tool, ToolMarkerCanEdit):
If mode is "regex", the string can contain backreferences to matched groups in the needle regex,
specified using the syntax $!1, $!2, etc. for groups 1, 2, etc.
:param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted.
:param allow_multiple_occurrences: if True, the regex may match multiple occurrences in the file
and all of them will be replaced.
If this is set to False and the regex matches multiple occurrences, an error will be returned
(and you may retry with a revised, more specific regex).
:param allow_multiple_occurrences: whether to allow matching and replacing multiple occurrences.
If false and multiple occurrences are found, an error will be returned
"""
return self.replace_content(
relative_path, needle, repl, mode=mode, allow_multiple_occurrences=allow_multiple_occurrences, require_not_ignored=True
src/serena/tools/memory_tools.py
+32 -18
View File
@@ -1,17 +1,20 @@
from typing import Literal
from serena.tools import ReplaceContentTool, Tool, ToolMarkerCanEdit
from serena.tools import Tool, ToolMarkerCanEdit
class WriteMemoryTool(Tool, ToolMarkerCanEdit):
"""
Writes a named memory (for future reference) to Serena's project-specific memory store.
Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format.
The memory name should be meaningful.
"""
def apply(self, memory_name: str, content: str, max_chars: int = -1) -> str:
"""
Write some information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format.
Write information (utf-8-encoded) about this project that can be useful for future tasks to a memory in md format.
The memory name should be meaningful and can include "/" to organize into topics (e.g., "auth/login/logic").
If explicitly instructed, use the "global/" prefix for writing a memory that is shared across projects
(e.g., "global/java/style_guide")
:param max_chars: the maximum number of characters to write. By default, determined by the config,
change only if instructed to do so.
@@ -24,66 +27,77 @@ class WriteMemoryTool(Tool, ToolMarkerCanEdit):
f"Content for {memory_name} is too long. Max length is {max_chars} characters. " + "Please make the content shorter."
)
return self.memories_manager.save_memory(memory_name, content)
return self.memories_manager.save_memory(memory_name, content, is_tool_context=True)
class ReadMemoryTool(Tool):
"""
Reads the memory with the given name from Serena's project-specific memory store.
Read the content of a memory file. This tool should only be used if the information
is relevant to the current task. You can infer whether the information
is relevant from the memory file name.
You should not read the same memory file multiple times in the same conversation.
"""
def apply(self, memory_name: str) -> str:
"""
Read the content of a memory. Should only be used if the information
is relevant to the current task, with relevance inferred from the memory name.
You should not read the same memory file multiple times in the same conversation.
Reads the contents of a memory. Should only be used if the information
is likely to be relevant to the current task, inferring relevance from the memory name.
"""
return self.memories_manager.load_memory(memory_name)
class ListMemoriesTool(Tool):
"""
Lists memories in Serena's project-specific memory store.
List available memories. Any memory can be read using the `read_memory` tool.
"""
def apply(self, topic: str = "") -> str:
"""
List available memories, optionally filtered by topic.
Lists available memories, optionally filtered by topic.
"""
return self._to_json(self.memories_manager.list_memories(topic))
return self._to_json(self.memories_manager.list_memories(topic).to_dict())
class DeleteMemoryTool(Tool, ToolMarkerCanEdit):
"""
Deletes a memory from Serena's project-specific memory store.
Delete a memory file. Should only happen if a user asks for it explicitly,
for example by saying that the information retrieved from a memory file is no longer correct
or no longer relevant for the project.
"""
def apply(self, memory_name: str) -> str:
"""
Delete a memory, only call if instructed explicitly or permission was granted by the user.
"""
return self.memories_manager.delete_memory(memory_name)
return self.memories_manager.delete_memory(memory_name, is_tool_context=True)
class RenameMemoryTool(Tool, ToolMarkerCanEdit):
"""
Renames or moves a memory in Serena's project-specific memory store.
Renames or moves a memory. Moving between project and global scope is supported
(e.g., renaming "global/foo" to "bar" moves it from global to project scope).
"""
def apply(self, old_name: str, new_name: str) -> str:
"""
Rename or move a memory, use "/" in the name to organize into topics.
The "global" topic should only be used if explicitly instructed.
"""
return self.memories_manager.rename_memory(old_name, new_name)
return self.memories_manager.move_memory(old_name, new_name, is_tool_context=True)
class EditMemoryTool(Tool, ToolMarkerCanEdit):
"""
Replaces content matching a regular expression in a memory.
"""
def apply(
self,
memory_name: str,
needle: str,
repl: str,
mode: Literal["literal", "regex"],
allow_multiple_occurrences: bool = False,
) -> str:
r"""
Replaces content matching a regular expression in a memory.
@@ -95,7 +109,7 @@ class EditMemoryTool(Tool, ToolMarkerCanEdit):
with flags DOTALL and MULTILINE enabled).
:param repl: the replacement string (verbatim).
:param mode: either "literal" or "regex", specifying how the `needle` parameter is to be interpreted.
:param allow_multiple_occurrences: whether to allow matching and replacing multiple occurrences.
If false and multiple occurrences are found, an error will be returned.
"""
replace_content_tool = self.agent.get_tool(ReplaceContentTool)
rel_path = self.memories_manager.get_memory_file_path(memory_name).relative_to(self.get_project_root())
return replace_content_tool.replace_content(str(rel_path), needle, repl, mode=mode, require_not_ignored=False)
return self.memories_manager.edit_memory(memory_name, needle, repl, mode, allow_multiple_occurrences, is_tool_context=True)
src/serena/tools/query_project_tools.py
+80
View File
@@ -0,0 +1,80 @@
import json
from serena.config.serena_config import LanguageBackend
from serena.jetbrains.jetbrains_plugin_client import JetBrainsPluginClientManager
from serena.project_server import ProjectServerClient
from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional
class ListQueryableProjectsTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject):
"""
Tool for listing all projects that can be queried by the QueryProjectTool.
"""
def apply(self, symbol_access: bool = True) -> str:
"""
Lists available projects that can be queried with `query_project_tool`.
:param symbol_access: whether to return only projects for which symbol access is available. Default: true
"""
# determine relevant projects
registered_projects = self.agent.serena_config.projects
if symbol_access:
backend = self.agent.get_language_backend()
if backend.is_jetbrains():
# projects with open IDE instances can be queried
matched_clients = JetBrainsPluginClientManager().match_clients(registered_projects)
relevant_projects = [mc.registered_project for mc in matched_clients]
else:
# all projects can be queried via ProjectServer (which instantiates projects dynamically)
relevant_projects = registered_projects
else:
relevant_projects = registered_projects
# return project names, excluding the active project (if any)
project_names = [p.project_name for p in relevant_projects]
active_project = self.agent.get_active_project()
if active_project is not None:
project_names = [n for n in project_names if n != active_project.project_name]
return self._to_json(project_names)
class QueryProjectTool(Tool, ToolMarkerOptional, ToolMarkerDoesNotRequireActiveProject):
"""
Tool for querying external project information (i.e. information from projects other than the current one),
by executing a read-only tool.
"""
def apply(self, project_name: str, tool_name: str, tool_params_json: str) -> str:
"""
Queries a project by executing a read-only Serena tool. The tool will be executed in the context of the project.
Use this to query information from projects other than the activated project.
:param project_name: the name of the project to query
:param tool_name: the name of the tool to execute in the other project. The tool must be read-only.
:param tool_params_json: the parameters to pass to the tool, encoded as a JSON string
"""
tool = self.agent.get_tool_by_name(tool_name)
assert tool.is_active(), f"Tool {tool_name} is not active."
assert tool.is_readonly(), f"Tool {tool_name} is not read-only and cannot be executed in another project."
if self._is_project_server_required(tool):
client = ProjectServerClient()
return client.query_project(project_name, tool_name, tool_params_json)
else:
registered_project = self.agent.serena_config.get_registered_project(project_name)
assert registered_project is not None, f"Project {project_name} is not registered and cannot be queried."
project = registered_project.get_project_instance(self.agent.serena_config)
with tool.agent.active_project_context(project):
return tool.apply(**json.loads(tool_params_json)) # type: ignore
def _is_project_server_required(self, tool: Tool) -> bool:
match self.agent.get_language_backend():
case LanguageBackend.JETBRAINS:
return False
case LanguageBackend.LSP:
# Note: As long as only read-only tools are considered, only symbolic tools require the project server.
# But if we were to allow non-read-only tools, then tools using a CodeEditor also indirectly require language servers.
assert tool.is_readonly()
return tool.is_symbolic()
case _:
raise NotImplementedError
src/serena/tools/symbol_tools.py
+4
View File
@@ -68,6 +68,10 @@ class GetSymbolsOverviewTool(Tool, ToolMarkerSymbolicRead):
raise FileNotFoundError(f"File or directory {relative_path} does not exist in the project.")
if os.path.isdir(file_path):
raise ValueError(f"Expected a file path, but got a directory path: {relative_path}. ")
if not symbol_retriever.can_analyze_file(relative_path):
raise ValueError(
f"Cannot extract symbols from file {relative_path}. Active languages: {[l.value for l in self.agent.get_active_lsp_languages()]}"
)
symbols = symbol_retriever.get_symbol_overview(relative_path)[relative_path]
src/serena/tools/tools_base.py
+43 -10
View File
@@ -12,6 +12,7 @@ from mcp.server.fastmcp.utilities.func_metadata import FuncMetadata, func_metada
from sensai.util import logging
from sensai.util.string import dict_string
from serena.config.serena_config import LanguageBackend
from serena.project import MemoriesManager, Project
from serena.prompt_factory import PromptFactory
from serena.util.class_decorators import singleton
@@ -36,7 +37,7 @@ class Component(ABC):
"""
:return: the root directory of the active project, raises a ValueError if no active project configuration is set
"""
return self.agent.get_project_root()
return self.project.project_root
@property
def prompt_factory(self) -> PromptFactory:
@@ -49,10 +50,8 @@ class Component(ABC):
def create_language_server_symbol_retriever(self) -> "LanguageServerSymbolRetriever":
from serena.symbol import LanguageServerSymbolRetriever
if not self.agent.is_using_language_server():
raise Exception("Cannot create LanguageServerSymbolRetriever; agent is not in language server mode.")
language_server_manager = self.agent.get_language_server_manager_or_raise()
return LanguageServerSymbolRetriever(language_server_manager, agent=self.agent)
assert self.agent.get_language_backend().is_lsp(), "Language server symbol retriever can only be created for LSP language backend"
return LanguageServerSymbolRetriever(self.project)
@property
def project(self) -> Project:
@@ -61,10 +60,13 @@ class Component(ABC):
def create_code_editor(self) -> "CodeEditor":
from ..code_editor import JetBrainsCodeEditor, LanguageServerCodeEditor
if self.agent.is_using_language_server():
return LanguageServerCodeEditor(self.create_language_server_symbol_retriever(), agent=self.agent)
else:
return JetBrainsCodeEditor(project=self.project, agent=self.agent)
match self.agent.get_language_backend():
case LanguageBackend.LSP:
return LanguageServerCodeEditor(self.create_language_server_symbol_retriever())
case LanguageBackend.JETBRAINS:
return JetBrainsCodeEditor(project=self.project)
case _:
raise ValueError
class ToolMarker:
@@ -234,6 +236,12 @@ class Tool(Component):
def is_active(self) -> bool:
return self.agent.tool_is_active(self.get_name())
def is_readonly(self) -> bool:
return not self.can_edit()
def is_symbolic(self) -> bool:
return issubclass(self.__class__, ToolMarkerSymbolicRead) or issubclass(self.__class__, ToolMarkerSymbolicEdit)
def apply_ex(self, log_call: bool = True, catch_exceptions: bool = True, mcp_ctx: Context | None = None, **kwargs) -> str: # type: ignore
"""
Applies the tool with logging and exception handling, using the given keyword arguments
@@ -374,6 +382,13 @@ class RegisteredTool:
is_optional: bool
tool_name: str
@property
def class_docstring(self) -> str:
"""
:return: the tool description (high-level class docstring)
"""
return self.tool_class.get_tool_description()
tool_packages = ["serena.tools"]
@@ -382,7 +397,8 @@ tool_packages = ["serena.tools"]
class ToolRegistry:
def __init__(self) -> None:
self._tool_dict: dict[str, RegisteredTool] = {}
for cls in iter_subclasses(Tool):
inclusion_predicate = lambda c: "apply" in c.__dict__ # include only concrete tool classes that implement apply
for cls in iter_subclasses(Tool, inclusion_predicate=inclusion_predicate):
if not any(cls.__module__.startswith(pkg) for pkg in tool_packages):
continue
is_optional = issubclass(cls, ToolMarkerOptional)
@@ -391,7 +407,24 @@ class ToolRegistry:
raise ValueError(f"Duplicate tool name found: {name}. Tool classes must have unique names.")
self._tool_dict[name] = RegisteredTool(tool_class=cls, is_optional=is_optional, tool_name=name)
def get_registered_tools_by_module(self) -> dict[str, list[RegisteredTool]]:
"""
:return: the registered tools grouped by their module (ordered alphabetically by module and tool name)
"""
module_dict: dict[str, list[RegisteredTool]] = {}
for tool in self._tool_dict.values():
module = tool.tool_class.__module__
if module not in module_dict:
module_dict[module] = []
module_dict[module].append(tool)
sorted_module_dict = {}
for module in sorted(module_dict.keys()):
sorted_module_dict[module] = sorted(module_dict[module], key=lambda t: t.tool_name)
return sorted_module_dict
def get_tool_class_by_name(self, tool_name: str) -> type[Tool]:
if tool_name not in self._tool_dict:
raise ValueError(f"Tool named '{tool_name}' not found.")
return self._tool_dict[tool_name].tool_class
def get_all_tool_classes(self) -> list[type[Tool]]:
src/serena/tools/workflow_tools.py
+12 -15
View File
@@ -2,7 +2,6 @@
Tools supporting the general workflow of the agent
"""
import json
import platform
from serena.tools import Tool, ToolMarkerDoesNotRequireActiveProject, ToolMarkerOptional
@@ -18,23 +17,21 @@ class CheckOnboardingPerformedTool(Tool):
Checks whether project onboarding was already performed.
You should always call this tool before beginning to actually work on the project/after activating a project.
"""
from .memory_tools import ListMemoriesTool
list_memories_tool = self.agent.get_tool(ListMemoriesTool)
memories = json.loads(list_memories_tool.apply())
if len(memories) == 0:
return (
project_memories = self.memories_manager.list_project_memories()
if len(project_memories) == 0:
msg = (
"Onboarding not performed yet (no memories available). "
+ "You should perform onboarding by calling the `onboarding` tool before proceeding with the task."
"You should perform onboarding by calling the `onboarding` tool before proceeding with the task. "
)
else:
return f"""The onboarding was already performed, below is the list of available memories.
Do not read them immediately, just remember that they exist and that you can read them later, if it is necessary
for the current task.
Some memories may be based on previous conversations, others may be general for the current project.
You should be able to tell which one you need based on the name of the memory.
{memories}"""
# Not reporting the list of memories here, as they were already reported at project activation
# (with the system prompt if the project was activated at startup)
msg = (
f"Onboarding was already performed: {len(project_memories)} project memories are available. "
"Consider reading memories if they appear relevant to the task at hand."
)
msg += " If you have not read the 'Serena Instructions Manual', do so now."
return msg
class OnboardingTool(Tool):
src/serena/util/dotnet.py
+156
View File
@@ -0,0 +1,156 @@
import logging
import platform
import re
import shutil
import subprocess
import urllib
from pathlib import Path
from serena.util.version import Version
from solidlsp.ls_exceptions import SolidLSPException
log = logging.getLogger(__name__)
class DotNETUtil:
def __init__(self, required_version: str, allow_higher_version: bool = True):
"""
:param required_version: the required .NET runtime version specified as a string (e.g. "10.0" for .NET 10.0)
:param allow_higher_version: whether to allow higher versions than the required version
"""
self._system_dotnet = shutil.which("dotnet")
self._required_version_str = required_version
self._required_version_components = [int(c) for c in required_version.split(".")]
self._allow_higher_version = allow_higher_version
self._installed_versions = self._determine_installed_versions()
def _determine_installed_versions(self) -> list[Version]:
if self._system_dotnet:
try:
result = subprocess.run([self._system_dotnet, "--list-runtimes"], capture_output=True, text=True, check=True)
version_strings = re.findall(r"Microsoft.NETCore.App\s+([^\s]+)", result.stdout)
log.info("Installed .NET runtime versions: %s", version_strings)
return [Version(v) for v in version_strings]
except:
log.warning("Failed to run 'dotnet --list-runtimes' to check .NET version; assuming no installed .NET versions")
return []
else:
log.info("Found no `dotnet` on system PATH; assuming no installed .NET versions")
return []
def is_required_version_available(self) -> bool:
"""
Checks whether the required .NET runtime version is installed and raises an exception if not.
:param required_version_components: the required .NET runtime version specified as a list of integers representing the version components (e.g., [6, 1] for .NET 6.1)
:param allow_higher_version: whether to allow higher versions than the required version (e.g., if True, .NET 7.0 would satisfy a requirement of .NET 6.1)
"""
required_version_str = ".".join(str(c) for c in self._required_version_components)
for v in self._installed_versions:
if self._allow_higher_version:
if v.is_at_least(*self._required_version_components):
log.info(f"Found installed .NET runtime version {v} which satisfies requirement of {required_version_str} or higher")
return True
else:
if v.is_equal(*self._required_version_components):
log.info(f"Found installed .NET runtime version {v} which satisfies requirement of {required_version_str}")
return True
return False
def get_dotnet_path_or_raise(self) -> str:
"""
Returns the path to the dotnet executable if the required .NET runtime version is available, otherwise raises an exception.
"""
if not self.is_required_version_available():
raise SolidLSPException(
f"Required .NET runtime version {self._required_version_str} not found "
f"(installed versions: {self._installed_versions}). "
"Please install the required .NET runtime version from https://dotnet.microsoft.com/en-us/download/dotnet "
"and ensure that `dotnet` is on the system PATH."
)
assert self._system_dotnet is not None
return self._system_dotnet
@staticmethod
def install_dotnet_with_script(version: str, base_path: str) -> str:
"""
Install .NET runtime using Microsoft's official installation script.
NOTE: This method is unreliable and therefore currently unused. It is kept for reference.
:version: the version to install as a string (e.g. "10.0")
:return: the path to the dotnet executable.
"""
dotnet_dir = Path(base_path) / f"dotnet-runtime-{version}"
# Determine binary name based on platform
is_windows = platform.system().lower() == "windows"
dotnet_exe = dotnet_dir / ("dotnet.exe" if is_windows else "dotnet")
if dotnet_exe.exists():
log.info(f"Using cached .NET {version} runtime from {dotnet_exe}")
return str(dotnet_exe)
# Download and run install script
log.info(f"Installing .NET {version} runtime using official Microsoft install script...")
dotnet_dir.mkdir(parents=True, exist_ok=True)
try:
if is_windows:
# PowerShell script for Windows
script_url = "https://dot.net/v1/dotnet-install.ps1"
script_path = dotnet_dir / "dotnet-install.ps1"
urllib.request.urlretrieve(script_url, script_path)
cmd = [
"pwsh",
"-NoProfile",
"-ExecutionPolicy",
"Bypass",
"-File",
str(script_path),
"-Version",
version,
"-InstallDir",
str(dotnet_dir),
"-Runtime",
"dotnet",
"-NoPath",
]
else:
# Bash script for Linux/macOS
script_url = "https://dot.net/v1/dotnet-install.sh"
script_path = dotnet_dir / "dotnet-install.sh"
urllib.request.urlretrieve(script_url, script_path)
script_path.chmod(0o755)
cmd = [
"bash",
str(script_path),
"--version",
version,
"--install-dir",
str(dotnet_dir),
"--runtime",
"dotnet",
"--no-path",
]
# Run the install script
log.info("Running .NET install script: %s", cmd)
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
log.debug(f"Install script output: {result.stdout}")
if not dotnet_exe.exists():
raise SolidLSPException(f"dotnet executable not found at {dotnet_exe} after installation")
log.info(f"Successfully installed .NET {version} runtime to {dotnet_exe}")
return str(dotnet_exe)
except subprocess.CalledProcessError as e:
raise SolidLSPException(f"Failed to install .NET {version} runtime using install script: {e.stderr if e.stderr else e}") from e
except Exception as e:
message = f"Failed to install .NET {version} runtime: {e}"
if is_windows and isinstance(e, FileNotFoundError):
message += "; pwsh, i.e. PowerShell 7+, is required to install .NET runtime. Make sure pwsh is available on your system."
raise SolidLSPException(message) from e
src/serena/util/inspection.py
+13 -5
View File
@@ -1,6 +1,6 @@
import logging
import os
from collections.abc import Generator
from collections.abc import Callable, Iterator
from typing import TypeVar
from serena.util.file_system import find_all_non_ignored_files
@@ -11,12 +11,20 @@ T = TypeVar("T")
log = logging.getLogger(__name__)
def iter_subclasses(cls: type[T], recursive: bool = True) -> Generator[type[T], None, None]:
"""Iterate over all subclasses of a class. If recursive is True, also iterate over all subclasses of all subclasses."""
def iter_subclasses(
cls: type[T], recursive: bool = True, inclusion_predicate: Callable[[type[T]], bool] = lambda t: True
) -> Iterator[type[T]]:
"""Iterate over all subclasses of a class.
:param cls: The class whose subclasses to iterate over.
:param recursive: If True, also iterate over all subclasses of all subclasses.
:param inclusion_predicate: a predicate function to decide whether to include a subclass in the result
"""
for subclass in cls.__subclasses__():
yield subclass
if inclusion_predicate(subclass):
yield subclass
if recursive:
yield from iter_subclasses(subclass, recursive)
yield from iter_subclasses(subclass, recursive, inclusion_predicate)
def determine_programming_language_composition(repo_path: str) -> dict[Language, float]:
src/serena/util/shell.py
+3 -1
View File
@@ -43,7 +43,9 @@ def execute_shell_command(command: str, cwd: str | None = None, capture_stderr:
def subprocess_check_output(args: list[str], encoding: str = "utf-8", strip: bool = True, timeout: float | None = None) -> str:
output = subprocess.check_output(args, stdin=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=timeout, env=os.environ.copy(), **subprocess_kwargs()).decode(encoding) # type: ignore
output = subprocess.check_output(
args, stdin=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=timeout, env=os.environ.copy(), **subprocess_kwargs()
).decode(encoding) # type: ignore
if strip:
output = output.strip()
return output
src/serena/util/thread.py
-1
View File
@@ -16,7 +16,6 @@ T = TypeVar("T")
class ExecutionResult(Generic[T], ToStringMixin):
class Status(Enum):
SUCCESS = "success"
TIMEOUT = "timeout"
src/serena/util/version.py
+1 -1
View File
@@ -23,7 +23,7 @@ class Version:
self.version_string = version_string
self.components = self._get_version_components(version_string)
def __str__(self) -> str:
def __repr__(self) -> str:
return self.version_string
@staticmethod
src/serena/util/yaml.py
+3 -1
View File
@@ -68,7 +68,9 @@ def load_yaml(path: str, comment_normalisation: YamlCommentNormalisation = YamlC
"""
with open(path, encoding=SERENA_FILE_ENCODING) as f:
yaml = _create_yaml(preserve_comments=True)
commented_map: CommentedMap = yaml.load(f)
commented_map: CommentedMap | None = yaml.load(f)
if commented_map is None: # ruamel returns None for empty documents, but we want an empty CommentedMap
commented_map = CommentedMap()
normalise_yaml_comments(commented_map, comment_normalisation)
return commented_map
src/solidlsp/language_servers/ansible_language_server.py
+318
View File
@@ -0,0 +1,318 @@
"""
Provides Ansible specific instantiation of the LanguageServer class using ansible-language-server.
Contains various configurations and settings specific to Ansible YAML files (playbooks, roles, etc.).
"""
import fnmatch
import logging
import os
import pathlib
import shutil
from typing import Any, ClassVar
from overrides import override
from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection
from solidlsp.ls import LanguageServerDependencyProvider, LanguageServerDependencyProviderSinglePath, SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.settings import SolidLSPSettings
log = logging.getLogger(__name__)
def _deep_merge(base: dict, override: dict) -> None:
"""Recursively merge *override* into *base*, modifying *base* in-place."""
for key, value in override.items():
if key in base and isinstance(base[key], dict) and isinstance(value, dict):
_deep_merge(base[key], value)
else:
base[key] = value
class AnsibleLanguageServer(SolidLanguageServer):
"""Provides Ansible specific instantiation of the LanguageServer class using ansible-language-server.
Contains various configurations and settings specific to Ansible YAML files
(playbooks, roles, inventories, etc.).
Supported ``ls_specific_settings`` keys (via ``serena_config.yml``):
* ``ls_path`` (str) — path to the ansible-language-server executable
(handled by the base class).
* ``ansible_path`` (str, default ``"ansible"``) — path to the ``ansible`` executable.
* ``python_interpreter_path`` (str, default ``"python3"``) — path to the Python interpreter.
* ``python_activation_script`` (str, default ``""``) — virtualenv activation script.
* ``lint_enabled`` (bool, default ``False``) — enable ansible-lint
(requires a separate installation of ``ansible-lint``).
* ``lint_path`` (str, default ``"ansible-lint"``) — path to ``ansible-lint``.
* ``ansible_settings`` (dict) — full settings dict, deep-merged on top of defaults.
The structure mirrors the Ansible Language Server settings
(``ansible.*``, ``python.*``, ``validation.*``, ``completion.*``,
``executionEnvironment.*``).
"""
# directory names that signal ansible content at ANY nesting level
_ANSIBLE_DIR_NAMES: ClassVar[set[str]] = {
"roles",
"playbooks",
"tasks",
"handlers",
"group_vars",
"host_vars",
"inventory",
"inventories",
"defaults",
"vars",
"meta",
}
# filename patterns handled by ansible LS regardless of path
_ANSIBLE_FILENAME_PATTERNS: ClassVar[list[str]] = [
"playbook*.yml",
"playbook*.yaml",
"site.yml",
"site.yaml",
"requirements.yml",
"requirements.yaml",
]
@staticmethod
def _is_ansible_path(relative_path: str) -> bool:
"""Check if a file is in an ansible-specific location.
Matches if ANY component of the path is an ansible-specific
directory name (e.g. ``roles``, ``tasks``, ``group_vars``),
or if the filename matches an ansible-specific pattern.
This works regardless of nesting depth:
``project/deploy/roles/web/tasks/main.yml`` matches on both
``roles`` and ``tasks``.
:param relative_path: path relative to the repository root
:return: True if the path is an ansible-specific location
"""
normalized = relative_path.replace("\\", "/")
parts = normalized.split("/")
# check if any directory component is ansible-specific
dir_parts = parts[:-1]
for part in dir_parts:
if part in AnsibleLanguageServer._ANSIBLE_DIR_NAMES:
return True
# check filename patterns (e.g. playbook.yml, site.yaml)
filename = parts[-1]
for pattern in AnsibleLanguageServer._ANSIBLE_FILENAME_PATTERNS:
if fnmatch.fnmatch(filename, pattern):
return True
return False
@override
def is_ignored_path(self, relative_path: str, ignore_unsupported_files: bool = True) -> bool:
# standard ignore rules (extension, gitignore, etc.)
if super().is_ignored_path(relative_path, ignore_unsupported_files):
return True
# for yml/yaml files, check if they are in ansible-specific paths
if relative_path.endswith((".yml", ".yaml")):
if not self._is_ansible_path(relative_path):
return True
return False
@staticmethod
def _determine_log_level(line: str) -> int:
"""Classify ansible-language-server stderr output to avoid false-positive errors."""
line_lower = line.lower()
if any(
[
"ansible is not installed" in line_lower,
"ansible-lint" in line_lower and "not found" in line_lower,
"cannot find module" in line_lower,
]
):
return logging.DEBUG
return SolidLanguageServer._determine_log_level(line)
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""Creates an AnsibleLanguageServer instance.
This class is not meant to be instantiated directly.
Use ``SolidLanguageServer.create()`` instead.
"""
super().__init__(
config,
repository_root_path,
None,
"ansible",
solidlsp_settings,
)
def _create_dependency_provider(self) -> LanguageServerDependencyProvider:
return self.DependencyProvider(self._custom_settings, self._ls_resources_dir)
class DependencyProvider(LanguageServerDependencyProviderSinglePath):
def _get_or_install_core_dependency(self) -> str:
"""Setup runtime dependencies for Ansible Language Server and return the path to the executable."""
# verify both node and npm are installed
is_node_installed = shutil.which("node") is not None
assert is_node_installed, "node is not installed or isn't in PATH. Please install Node.js and try again."
is_npm_installed = shutil.which("npm") is not None
assert is_npm_installed, "npm is not installed or isn't in PATH. Please install npm and try again."
deps = RuntimeDependencyCollection(
[
RuntimeDependency(
id="ansible-language-server",
description="Ansible Language Server (@ansible/ansible-language-server)",
command="npm install --prefix ./ @ansible/ansible-language-server@1.2.3",
platform_id="any",
),
]
)
# install ansible-language-server if not already installed
ansible_ls_dir = os.path.join(self._ls_resources_dir, "ansible-lsp")
ansible_executable_path = os.path.join(ansible_ls_dir, "node_modules", ".bin", "ansible-language-server")
# handle Windows executable extension
if os.name == "nt":
ansible_executable_path += ".cmd"
if not os.path.exists(ansible_executable_path):
log.info(f"Ansible Language Server executable not found at {ansible_executable_path}. Installing...")
deps.install(ansible_ls_dir)
log.info("Ansible Language Server dependencies installed successfully")
if not os.path.exists(ansible_executable_path):
raise FileNotFoundError(
f"ansible-language-server executable not found at {ansible_executable_path}, "
"something went wrong with the installation."
)
return ansible_executable_path
def _create_launch_command(self, core_path: str) -> list[str]:
return [core_path, "--stdio"]
def _get_initialize_params(self, repository_absolute_path: str) -> InitializeParams:
"""Returns the initialize params for the Ansible Language Server.
Reads shortcut keys and the ``ansible_settings`` dict from ``_custom_settings``
to build ``initializationOptions``.
"""
root_uri = pathlib.Path(repository_absolute_path).as_uri()
# default ansible settings, populated from shortcut keys
ansible_settings: dict[str, Any] = {
"ansible": {
"path": self._custom_settings.get("ansible_path", "ansible"),
"useFullyQualifiedCollectionNames": True,
},
"python": {
"interpreterPath": self._custom_settings.get("python_interpreter_path", "python3"),
"activationScript": self._custom_settings.get("python_activation_script", ""),
},
"validation": {
"enabled": True,
"lint": {
"enabled": self._custom_settings.get("lint_enabled", False),
"path": self._custom_settings.get("lint_path", "ansible-lint"),
},
},
"completion": {
"provideRedirectModules": True,
"provideModuleOptionAliases": True,
},
"executionEnvironment": {"enabled": False},
}
# full override via ansible_settings dict for advanced configuration
user_settings = self._custom_settings.settings.get("ansible_settings")
if user_settings:
if not isinstance(user_settings, dict):
raise TypeError(
f"ansible_settings must be a dict, got {type(user_settings).__name__}. "
"Expected structure matching Ansible LS settings: "
"{'ansible': {...}, 'python': {...}, 'validation': {...}, ...}"
)
_deep_merge(ansible_settings, user_settings)
initialize_params = {
"locale": "en",
"capabilities": {
"textDocument": {
"synchronization": {"didSave": True, "dynamicRegistration": True},
"completion": {"dynamicRegistration": True, "completionItem": {"snippetSupport": True}},
"definition": {"dynamicRegistration": True},
"references": {"dynamicRegistration": True},
"documentSymbol": {
"dynamicRegistration": True,
"hierarchicalDocumentSymbolSupport": True,
"symbolKind": {"valueSet": list(range(1, 27))},
},
"hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]},
"codeAction": {"dynamicRegistration": True},
},
"workspace": {
"workspaceFolders": True,
"didChangeConfiguration": {"dynamicRegistration": True},
"symbol": {"dynamicRegistration": True},
},
},
"processId": os.getpid(),
"rootPath": repository_absolute_path,
"rootUri": root_uri,
"workspaceFolders": [
{
"uri": root_uri,
"name": os.path.basename(repository_absolute_path),
}
],
"initializationOptions": {"ansible": ansible_settings},
}
return initialize_params # type: ignore
def _start_server(self) -> None:
"""Starts the Ansible Language Server, waits for the server to be ready."""
def register_capability_handler(params: Any) -> None:
return
def show_message_request_handler(params: Any) -> None:
"""Handle ``window/showMessageRequest`` by returning ``null``.
Per the LSP spec, returning ``null`` means no action was selected.
Without this handler the client replies with ``MethodNotFound``,
which the ansible LS treats as fatal.
"""
log.info(f"LSP: window/showMessageRequest (dismissed): {params.get('message', params)}")
return
def do_nothing(params: Any) -> None:
return
def window_log_message(msg: dict) -> None:
log.info(f"LSP: window/logMessage: {msg}")
self.server.on_request("client/registerCapability", register_capability_handler)
self.server.on_request("window/showMessageRequest", show_message_request_handler)
self.server.on_notification("window/logMessage", window_log_message)
self.server.on_notification("$/progress", do_nothing)
self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
log.info("Starting Ansible server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
log.info("Sending initialize request from LSP client to LSP server and awaiting response")
init_response = self.server.send.initialize(initialize_params)
log.debug(f"Received initialize response from Ansible server: {init_response}")
log.debug(f"Ansible server capabilities: {list(init_response['capabilities'].keys())}")
self.server.notify.initialized({})
log.info("Ansible server initialization complete")
src/solidlsp/language_servers/ccls_language_server.py
+1 -1
View File
@@ -118,7 +118,7 @@ class CCLS(SolidLanguageServer):
"workspaceFolders": [
{
"uri": root_uri,
"name": "$name",
"name": os.path.basename(repository_absolute_path),
}
],
# ccls supports initializationOptions but none are required for basic functionality
src/solidlsp/language_servers/clangd_language_server.py
+1 -1
View File
@@ -228,7 +228,7 @@ class ClangdLanguageServer(SolidLanguageServer):
"workspaceFolders": [
{
"uri": root_uri,
"name": "$name",
"name": os.path.basename(repository_absolute_path),
}
],
}
src/solidlsp/language_servers/csharp_language_server.py
+2 -99
View File
@@ -6,7 +6,6 @@ import logging
import os
import platform
import shutil
import subprocess
import threading
import urllib.request
from collections.abc import Iterable
@@ -15,6 +14,7 @@ from typing import Any, cast
from overrides import override
from serena.util.dotnet import DotNETUtil
from solidlsp.ls import DocumentSymbols, LanguageServerDependencyProvider, LSPFileBuffer, SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.ls_exceptions import SolidLSPException
@@ -366,21 +366,7 @@ class CSharpLanguageServer(SolidLanguageServer):
def _ensure_dotnet_runtime(self) -> str:
"""Ensure .NET runtime is available and return the dotnet executable path."""
# Check if dotnet is already available on the system
system_dotnet = shutil.which("dotnet")
if system_dotnet:
# Check if it's .NET 10 or compatible
try:
result = subprocess.run([system_dotnet, "--list-runtimes"], capture_output=True, text=True, check=True)
# Accept .NET 10 or higher (10.x, 11.x, etc.)
if any(f"Microsoft.NETCore.App {v}." in result.stdout for v in range(10, 20)):
log.info("Found system .NET 10+ runtime")
return system_dotnet
except subprocess.CalledProcessError:
pass
# Install .NET 10 runtime using Microsoft's install script
return self._install_dotnet_with_script()
return DotNETUtil("10.0", allow_higher_version=True).get_dotnet_path_or_raise()
def _ensure_language_server(self, lang_server_dep: RuntimeDependency) -> str:
"""Ensure language server is available and return the DLL path."""
@@ -475,89 +461,6 @@ class CSharpLanguageServer(SolidLanguageServer):
except Exception as e:
raise SolidLSPException(f"Failed to download package {package_name} version {package_version} from NuGet.org: {e}") from e
def _install_dotnet_with_script(self, version: str = "10.0") -> str:
"""
Install .NET runtime using Microsoft's official install script.
Returns the path to the dotnet executable.
"""
dotnet_dir = Path(self._ls_resources_dir) / f"dotnet-runtime-{version}"
# Determine binary name based on platform
is_windows = platform.system().lower() == "windows"
dotnet_exe = dotnet_dir / ("dotnet.exe" if is_windows else "dotnet")
if dotnet_exe.exists():
log.info(f"Using cached .NET {version} runtime from {dotnet_exe}")
return str(dotnet_exe)
# Download and run install script
log.info(f"Installing .NET {version} runtime using official Microsoft install script...")
dotnet_dir.mkdir(parents=True, exist_ok=True)
try:
if is_windows:
# PowerShell script for Windows
script_url = "https://dot.net/v1/dotnet-install.ps1"
script_path = dotnet_dir / "dotnet-install.ps1"
urllib.request.urlretrieve(script_url, script_path)
cmd = [
"pwsh",
"-NoProfile",
"-ExecutionPolicy",
"Bypass",
"-File",
str(script_path),
"-Version",
version,
"-InstallDir",
str(dotnet_dir),
"-Runtime",
"dotnet",
"-NoPath",
]
else:
# Bash script for Linux/macOS
script_url = "https://dot.net/v1/dotnet-install.sh"
script_path = dotnet_dir / "dotnet-install.sh"
urllib.request.urlretrieve(script_url, script_path)
script_path.chmod(0o755)
cmd = [
"bash",
str(script_path),
"--version",
version,
"--install-dir",
str(dotnet_dir),
"--runtime",
"dotnet",
"--no-path",
]
# Run the install script
log.info("Running .NET install script: %s", cmd)
result = subprocess.run(cmd, capture_output=True, text=True, check=True)
log.debug(f"Install script output: {result.stdout}")
if not dotnet_exe.exists():
raise SolidLSPException(f"dotnet executable not found at {dotnet_exe} after installation")
log.info(f"Successfully installed .NET {version} runtime to {dotnet_exe}")
return str(dotnet_exe)
except subprocess.CalledProcessError as e:
raise SolidLSPException(
f"Failed to install .NET {version} runtime using install script: {e.stderr if e.stderr else e}"
) from e
except Exception as e:
message = f"Failed to install .NET {version} runtime: {e}"
if is_windows and isinstance(e, FileNotFoundError):
message += (
"; pwsh, i.e. PowerShell 7+, is required to install .NET runtime. Make sure pwsh is available on your system."
)
raise SolidLSPException(message) from e
def _get_initialize_params(self) -> InitializeParams:
"""
Returns the initialize params for the Microsoft.CodeAnalysis.LanguageServer.
src/solidlsp/language_servers/eclipse_jdtls.py
+6 -3
View File
@@ -3,12 +3,12 @@ Provides Java specific instantiation of the LanguageServer class. Contains vario
"""
import dataclasses
import hashlib
import logging
import os
import pathlib
import shutil
import threading
import uuid
from pathlib import PurePath
from time import sleep
from typing import cast
@@ -276,13 +276,16 @@ class EclipseJDTLS(SolidLanguageServer):
)
def create_launch_command(self) -> list[str]:
# ws_dir is the workspace directory for the EclipseJDTLS server
# ws_dir is the workspace directory for the EclipseJDTLS server.
# Use a deterministic hash of the project path so the workspace
# (and its cached index) can be reused across restarts.
project_hash = hashlib.md5(self._repository_root_path.encode()).hexdigest()
ws_dir = str(
PurePath(
self._solidlsp_settings.ls_resources_dir,
"EclipseJDTLS",
"workspaces",
uuid.uuid4().hex,
project_hash,
)
)
src/solidlsp/language_servers/erlang_language_server.py
+2 -1
View File
@@ -2,6 +2,7 @@
import logging
import os
import pathlib
import shutil
import subprocess
import threading
@@ -134,7 +135,7 @@ class ErlangLanguageServer(SolidLanguageServer):
initialize_params = {
"processId": None,
"rootPath": self.repository_root_path,
"rootUri": f"file://{self.repository_root_path}",
"rootUri": pathlib.Path(self.repository_root_path).as_uri(),
"capabilities": {
"textDocument": {
"synchronization": {"didSave": True},
src/solidlsp/language_servers/fsharp_language_server.py
+3 -16
View File
@@ -11,6 +11,7 @@ from pathlib import Path
from overrides import override
from serena.util.dotnet import DotNETUtil
from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
@@ -61,21 +62,7 @@ class FSharpLanguageServer(SolidLanguageServer):
"""
Setup runtime dependencies for F# Language Server and return the command to start the server.
"""
# First check if .NET SDK is installed
dotnet_exe = shutil.which("dotnet")
if not dotnet_exe:
raise RuntimeError(
".NET SDK is not installed or not in PATH. Please install .NET SDK 8.0 or later and ensure 'dotnet' is in your PATH."
)
# Verify dotnet version
import subprocess
try:
result = subprocess.run([dotnet_exe, "--version"], capture_output=True, text=True, check=True)
log.info(f"Found .NET SDK version: {result.stdout.strip()}")
except subprocess.CalledProcessError:
raise RuntimeError("Failed to get .NET SDK version. Please ensure .NET SDK is properly installed.")
dotnet_exe = DotNETUtil("8.0", allow_higher_version=True).get_dotnet_path_or_raise()
RuntimeDependencyCollection(
[
@@ -179,7 +166,7 @@ class FSharpLanguageServer(SolidLanguageServer):
"documentHighlight": {"dynamicRegistration": True},
"documentSymbol": {
"dynamicRegistration": True,
"symbolKind": {"valueSet": list(range(1, 26))}, # All SymbolKind values
"symbolKind": {"valueSet": list(range(1, 27))}, # All SymbolKind values (1-26)
"hierarchicalDocumentSymbolSupport": True,
},
"codeAction": {
src/solidlsp/language_servers/groovy_language_server.py
+3 -3
View File
@@ -78,9 +78,9 @@ class GroovyLanguageServer(SolidLanguageServer):
platform_id = PlatformUtils.get_platform_id()
# Verify platform support
assert (
platform_id.value.startswith("win-") or platform_id.value.startswith("linux-") or platform_id.value.startswith("osx-")
), "Only Windows, Linux and macOS platforms are supported for Groovy in multilspy at the moment"
assert platform_id.value.startswith("win-") or platform_id.value.startswith("linux-") or platform_id.value.startswith("osx-"), (
"Only Windows, Linux and macOS platforms are supported for Groovy in multilspy at the moment"
)
# Check if user specified custom Java home path
java_home_path = None
src/solidlsp/language_servers/hlsl_language_server.py
+252
View File
@@ -0,0 +1,252 @@
"""
Shader language server using shader-language-server (antaalt/shader-sense).
Supports HLSL, GLSL, and WGSL shader file formats.
"""
import logging
import os
import pathlib
import shutil
from typing import Any, cast
import psutil
from overrides import override
from solidlsp.ls import LanguageServerDependencyProvider, LanguageServerDependencyProviderSinglePath, SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.settings import SolidLSPSettings
from .common import RuntimeDependency, RuntimeDependencyCollection
log = logging.getLogger(__name__)
# GitHub release version to download when not installed locally
_DEFAULT_VERSION = "1.3.0"
_GITHUB_RELEASE_BASE = "https://github.com/antaalt/shader-sense/releases/download"
class HlslLanguageServer(SolidLanguageServer):
"""
Shader language server using shader-language-server.
Supports .hlsl, .hlsli, .fx, .fxh, .cginc, .compute, .shader, .glsl, .vert, .frag, .geom, .tesc, .tese, .comp, .wgsl files.
"""
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings) -> None:
super().__init__(config, repository_root_path, None, "hlsl", solidlsp_settings)
@override
def _create_dependency_provider(self) -> LanguageServerDependencyProvider:
return self.DependencyProvider(self._custom_settings, self._ls_resources_dir)
class DependencyProvider(LanguageServerDependencyProviderSinglePath):
def _get_or_install_core_dependency(self) -> str:
# 1. Check PATH for system-installed binary
system_binary = shutil.which("shader-language-server")
if system_binary:
log.info(f"Using system-installed shader-language-server at {system_binary}")
return system_binary
# 2. Try to download pre-built binary from GitHub releases
version = self._custom_settings.get("version", _DEFAULT_VERSION)
tag = f"v{version}"
base_url = f"{_GITHUB_RELEASE_BASE}/{tag}"
# macOS has no pre-built binaries; build from source via cargo install
cargo_install_cmd = f"cargo install shader_language_server --version {version} --root ."
deps = RuntimeDependencyCollection(
[
RuntimeDependency(
id="shader-language-server",
description="shader-language-server for Windows (x64)",
url=f"{base_url}/shader-language-server-x86_64-pc-windows-msvc.zip",
platform_id="win-x64",
archive_type="zip",
binary_name="shader-language-server.exe",
),
RuntimeDependency(
id="shader-language-server",
description="shader-language-server for Linux (x64)",
url=f"{base_url}/shader-language-server-x86_64-unknown-linux-gnu.zip",
platform_id="linux-x64",
archive_type="zip",
binary_name="shader-language-server",
),
RuntimeDependency(
id="shader-language-server",
description="shader-language-server for Windows (ARM64)",
url=f"{base_url}/shader-language-server-aarch64-pc-windows-msvc.zip",
platform_id="win-arm64",
archive_type="zip",
binary_name="shader-language-server.exe",
),
RuntimeDependency(
id="shader-language-server",
description="shader-language-server for macOS (x64) - built from source",
command=cargo_install_cmd,
platform_id="osx-x64",
binary_name="bin/shader-language-server",
),
RuntimeDependency(
id="shader-language-server",
description="shader-language-server for macOS (ARM64) - built from source",
command=cargo_install_cmd,
platform_id="osx-arm64",
binary_name="bin/shader-language-server",
),
]
)
try:
dep = deps.get_single_dep_for_current_platform()
except RuntimeError:
dep = None
if dep is None:
raise FileNotFoundError(
"shader-language-server is not installed and no auto-install is available for your platform.\n"
"Please install it using one of the following methods:\n"
" cargo: cargo install shader_language_server\n"
" GitHub: Download from https://github.com/antaalt/shader-sense/releases\n"
"On macOS, install the Rust toolchain (https://rustup.rs) and Serena will build from source automatically.\n"
"See https://github.com/antaalt/shader-sense for more details."
)
install_dir = os.path.join(self._ls_resources_dir, "shader-language-server")
executable_path = deps.binary_path(install_dir)
if not os.path.exists(executable_path):
log.info(f"shader-language-server not found. Downloading from {dep.url}")
_ = deps.install(install_dir)
if not os.path.exists(executable_path):
raise FileNotFoundError(f"shader-language-server not found at {executable_path}")
os.chmod(executable_path, 0o755)
return executable_path
def _create_launch_command(self, core_path: str) -> list[str]:
return [core_path, "--stdio"]
@staticmethod
def _get_initialize_params(repository_absolute_path: str) -> InitializeParams:
root_uri = pathlib.Path(repository_absolute_path).as_uri()
initialize_params = {
"processId": os.getpid(),
"rootPath": repository_absolute_path,
"rootUri": root_uri,
"locale": "en",
"capabilities": {
"textDocument": {
"synchronization": {"didSave": True, "dynamicRegistration": True},
"completion": {
"dynamicRegistration": True,
"completionItem": {"snippetSupport": True},
},
"definition": {"dynamicRegistration": True},
"hover": {
"dynamicRegistration": True,
"contentFormat": ["markdown", "plaintext"],
},
"signatureHelp": {
"dynamicRegistration": True,
"signatureInformation": {
"parameterInformation": {"labelOffsetSupport": True},
},
},
"documentSymbol": {
"dynamicRegistration": True,
"hierarchicalDocumentSymbolSupport": True,
"symbolKind": {"valueSet": list(range(1, 27))},
},
"formatting": {"dynamicRegistration": True},
"publishDiagnostics": {"relatedInformation": True},
},
"workspace": {
"workspaceFolders": True,
"didChangeConfiguration": {"dynamicRegistration": True},
"configuration": True,
},
},
"workspaceFolders": [{"uri": root_uri, "name": os.path.basename(repository_absolute_path)}],
}
return cast(InitializeParams, initialize_params)
@override
def _start_server(self) -> None:
def do_nothing(params: Any) -> None:
return
def on_log_message(params: Any) -> None:
message = params.get("message", "") if isinstance(params, dict) else str(params)
log.info(f"shader-language-server: {message}")
def on_configuration_request(params: Any) -> list[dict]:
"""Respond to workspace/configuration requests.
shader-language-server requests config with section 'shader-validator'.
Return empty config to use defaults.
"""
items = params.get("items", []) if isinstance(params, dict) else []
return [{}] * len(items)
self.server.on_request("client/registerCapability", do_nothing)
self.server.on_request("workspace/configuration", on_configuration_request)
self.server.on_notification("$/progress", do_nothing)
self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
self.server.on_notification("window/logMessage", on_log_message)
log.info("Starting shader-language-server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
log.info("Sending initialize request")
init_response = self.server.send.initialize(initialize_params)
capabilities = init_response.get("capabilities", {})
log.info(f"Initialize response capabilities: {list(capabilities.keys())}")
assert "textDocumentSync" in capabilities, "shader-language-server must support textDocumentSync"
if "documentSymbolProvider" not in capabilities:
log.warning("shader-language-server does not advertise documentSymbolProvider")
if "definitionProvider" not in capabilities:
log.warning("shader-language-server does not advertise definitionProvider")
self.server.notify.initialized({})
@override
def stop(self, shutdown_timeout: float = 2.0) -> None:
"""Kill the shader-language-server process tree before the standard shutdown.
The base _shutdown() calls process.terminate() directly on the subprocess,
which on Windows with shell=True only kills the cmd.exe wrapper, leaving
the actual shader-language-server binary running as an orphan. We use psutil
to terminate the full process tree first.
"""
process = self.server.process if self.server else None
if process and process.pid and process.returncode is None:
try:
parent = psutil.Process(process.pid)
children = parent.children(recursive=True)
for child in children:
try:
child.terminate()
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
psutil.wait_procs(children, timeout=2)
for child in children:
try:
if child.is_running():
child.kill()
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
except Exception as e:
log.debug(f"Error cleaning up shader-language-server process tree: {e}")
super().stop(shutdown_timeout)
@override
def is_ignored_dirname(self, dirname: str) -> bool:
"""Ignore Unity-specific directories that contain no user-authored shaders."""
return super().is_ignored_dirname(dirname) or dirname in {"Library", "Temp", "Logs", "obj", "Packages"}
src/solidlsp/language_servers/intelephense.py
+20 -7
View File
@@ -76,9 +76,9 @@ class Intelephense(SolidLanguageServer):
)
deps.install(intelephense_ls_dir)
assert os.path.exists(
intelephense_executable_path
), f"intelephense executable not found at {intelephense_executable_path}, something went wrong."
assert os.path.exists(intelephense_executable_path), (
f"intelephense executable not found at {intelephense_executable_path}, something went wrong."
)
return intelephense_executable_path
@@ -112,8 +112,19 @@ class Intelephense(SolidLanguageServer):
"textDocument": {
"synchronization": {"didSave": True, "dynamicRegistration": True},
"definition": {"dynamicRegistration": True},
"references": {"dynamicRegistration": True},
"documentSymbol": {
"dynamicRegistration": True,
"hierarchicalDocumentSymbolSupport": True,
"symbolKind": {"valueSet": list(range(1, 27))},
},
"hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]},
},
"workspace": {
"workspaceFolders": True,
"didChangeConfiguration": {"dynamicRegistration": True},
"symbol": {"dynamicRegistration": True},
},
"workspace": {"workspaceFolders": True, "didChangeConfiguration": {"dynamicRegistration": True}},
},
"processId": os.getpid(),
"rootPath": repository_absolute_path,
@@ -167,9 +178,11 @@ class Intelephense(SolidLanguageServer):
log.info("After sent initialize params")
# Verify server capabilities
assert "textDocumentSync" in init_response["capabilities"]
assert "completionProvider" in init_response["capabilities"]
assert "definitionProvider" in init_response["capabilities"]
capabilities = init_response["capabilities"]
assert "textDocumentSync" in capabilities
assert "completionProvider" in capabilities
assert "definitionProvider" in capabilities
assert "documentSymbolProvider" in capabilities, "Server must support document symbols"
self.server.notify.initialized({})
src/solidlsp/language_servers/kotlin_language_server.py
+3 -7
View File
@@ -85,7 +85,6 @@ class KotlinLanguageServer(SolidLanguageServer):
class DependencyProvider(LanguageServerDependencyProviderSinglePath):
def __init__(self, custom_settings: SolidLSPSettings.CustomLSSettings, ls_resources_dir: str):
super().__init__(custom_settings, ls_resources_dir)
self._java_home_path: str | None = None
def _get_or_install_core_dependency(self) -> str:
"""
@@ -94,9 +93,9 @@ class KotlinLanguageServer(SolidLanguageServer):
platform_id = PlatformUtils.get_platform_id()
# Verify platform support
assert (
platform_id.value.startswith("win-") or platform_id.value.startswith("linux-") or platform_id.value.startswith("osx-")
), "Only Windows, Linux and macOS platforms are supported for Kotlin in multilspy at the moment"
assert platform_id.value.startswith("win-") or platform_id.value.startswith("linux-") or platform_id.value.startswith("osx-"), (
"Only Windows, Linux and macOS platforms are supported for Kotlin in multilspy at the moment"
)
kotlin_suffix = PLATFORM_KOTLIN_SUFFIX.get(platform_id.value)
assert kotlin_suffix, f"Unsupported platform for Kotlin LSP: {platform_id.value}"
@@ -134,9 +133,6 @@ class KotlinLanguageServer(SolidLanguageServer):
"""Provides JAVA_HOME and JVM options for the Kotlin Language Server process."""
env: dict[str, str] = {}
if self._java_home_path is not None:
env["JAVA_HOME"] = self._java_home_path
# Get JVM options from settings or use default
# Note: an explicit empty string means "no JVM options", which is distinct from not setting the key
_sentinel = object()
src/solidlsp/language_servers/lean4_language_server.py
+175
View File
@@ -0,0 +1,175 @@
"""
Provides Lean 4 specific instantiation of the LanguageServer class.
Uses the built-in Lean 4 language server (lean --server).
"""
import logging
import os
import pathlib
import shutil
import subprocess
from typing import cast
from overrides import override
from solidlsp.ls import LanguageServerDependencyProvider, LanguageServerDependencyProviderSinglePath, SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.settings import SolidLSPSettings
log = logging.getLogger(__name__)
class Lean4LanguageServer(SolidLanguageServer):
"""
Provides Lean 4 specific instantiation of the LanguageServer class.
Uses the built-in Lean 4 language server invoked via ``lean --server``.
Requires ``lean`` to be installed and available on PATH (typically via elan).
"""
class DependencyProvider(LanguageServerDependencyProviderSinglePath):
def __init__(self, custom_settings: SolidLSPSettings.CustomLSSettings, ls_resources_dir: str, repository_root_path: str):
super().__init__(custom_settings, ls_resources_dir)
self._repository_root_path = repository_root_path
def _get_or_install_core_dependency(self) -> str:
lean_path = shutil.which("lean")
if lean_path is None:
raise RuntimeError(
"lean is not installed or not in PATH.\n"
"Please install Lean 4 via elan: https://github.com/leanprover/elan\n"
" curl https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh -sSf | sh\n"
"After installation, make sure 'lean' is available on your PATH."
)
return lean_path
def _create_launch_command(self, core_path: str) -> list[str]:
return [core_path, "--server"]
@override
def create_launch_command_env(self) -> dict[str, str]:
"""Provides LEAN_PATH and LEAN_SRC_PATH from ``lake env`` for cross-file references."""
env: dict[str, str] = {}
lake_path = shutil.which("lake")
if lake_path is None:
log.warning("lake not found on PATH; cross-file references may not work")
return env
try:
result = subprocess.run(
[lake_path, "env"],
check=False,
cwd=self._repository_root_path,
capture_output=True,
text=True,
timeout=30,
)
if result.returncode == 0:
for line in result.stdout.splitlines():
if "=" in line:
key, _, value = line.partition("=")
if key in ("LEAN_PATH", "LEAN_SRC_PATH"):
env[key] = value
log.info(f"Lake env: {key}={value}")
else:
log.warning(f"lake env failed (exit {result.returncode}): {result.stderr[:200]}")
except Exception as e:
log.warning(f"Failed to run lake env: {e}")
return env
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""
Creates a Lean4LanguageServer instance. This class is not meant to be
instantiated directly. Use LanguageServer.create() instead.
"""
super().__init__(
config,
repository_root_path,
None,
"lean4",
solidlsp_settings,
)
def _create_dependency_provider(self) -> LanguageServerDependencyProvider:
return self.DependencyProvider(self._custom_settings, self._ls_resources_dir, self.repository_root_path)
@override
def is_ignored_dirname(self, dirname: str) -> bool:
return super().is_ignored_dirname(dirname) or dirname in [".lake", "build"]
@staticmethod
def _get_initialize_params(repository_absolute_path: str) -> InitializeParams:
"""
Returns the initialize params for the Lean 4 Language Server.
"""
root_uri = pathlib.Path(repository_absolute_path).as_uri()
initialize_params = {
"processId": os.getpid(),
"rootPath": repository_absolute_path,
"rootUri": root_uri,
"capabilities": {
"textDocument": {
"synchronization": {"didSave": True, "dynamicRegistration": True},
"definition": {"dynamicRegistration": True, "linkSupport": True},
"references": {"dynamicRegistration": True},
"documentSymbol": {
"dynamicRegistration": True,
"hierarchicalDocumentSymbolSupport": True,
"symbolKind": {"valueSet": list(range(1, 27))},
},
"hover": {"dynamicRegistration": True, "contentFormat": ["markdown", "plaintext"]},
"completion": {
"dynamicRegistration": True,
"completionItem": {
"snippetSupport": True,
"documentationFormat": ["markdown", "plaintext"],
},
},
},
"workspace": {
"workspaceFolders": True,
"didChangeConfiguration": {"dynamicRegistration": True},
},
},
"workspaceFolders": [
{
"uri": root_uri,
"name": os.path.basename(repository_absolute_path),
}
],
}
return cast(InitializeParams, initialize_params)
def _start_server(self) -> None:
"""Start the Lean 4 language server process."""
def register_capability_handler(_params: dict) -> None:
return
def window_log_message(msg: dict) -> None:
log.info(f"LSP: window/logMessage: {msg}")
def do_nothing(_params: dict) -> None:
return
self.server.on_request("client/registerCapability", register_capability_handler)
self.server.on_notification("window/logMessage", window_log_message)
self.server.on_notification("$/progress", do_nothing)
self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
self.server.on_notification("$/lean/fileProgress", do_nothing)
log.info("Starting Lean 4 language server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
log.info("Sending initialize request from LSP client to LSP server and awaiting response")
init_response = self.server.send.initialize(initialize_params)
capabilities = init_response.get("capabilities", {})
log.info(f"Lean 4 LSP capabilities: {list(capabilities.keys())}")
self.server.notify.initialized({})
@override
def _get_wait_time_for_cross_file_referencing(self) -> float:
"""Lean 4 projects need time to compile and build cross-file references."""
return 10.0
src/solidlsp/language_servers/luau_lsp.py
+363
View File
@@ -0,0 +1,363 @@
"""
Provides Luau specific instantiation of the LanguageServer class using luau-lsp.
Luau is the programming language used by Roblox, derived from Lua 5.1 with
additional features like type annotations, string interpolation, and more.
This uses JohnnyMorganz/luau-lsp as the language server backend.
Requirements:
- luau-lsp binary must be installed and available in PATH,
or it will be automatically downloaded from GitHub releases.
Advanced settings via ls_specific_settings["luau"]:
- platform: "roblox" (default) or "standard"
- roblox_security_level: "None", "PluginSecurity" (default),
"LocalUserSecurity", or "RobloxScriptSecurity"
See: https://github.com/JohnnyMorganz/luau-lsp
"""
import logging
import os
import pathlib
import platform
import shutil
import threading
import zipfile
from pathlib import Path
import requests
from overrides import override
from solidlsp.ls import LanguageServerDependencyProvider, LanguageServerDependencyProviderSinglePath, SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.settings import SolidLSPSettings
log = logging.getLogger(__name__)
# Pin to a known stable release
LUAU_LSP_VERSION = "1.63.0"
# Luau built-in docs CDN
LUAU_DOCS_URL = "https://luau-lsp.pages.dev/api-docs/luau-en-us.json"
# Roblox type definitions and API docs CDN
ROBLOX_DOCS_URL = "https://luau-lsp.pages.dev/api-docs/en-us.json"
SUPPORTED_PLATFORMS = {"roblox", "standard"}
SUPPORTED_ROBLOX_SECURITY_LEVELS = {
"None",
"PluginSecurity",
"LocalUserSecurity",
"RobloxScriptSecurity",
}
class LuauLanguageServer(SolidLanguageServer):
"""
Provides Luau specific instantiation of the LanguageServer class using luau-lsp.
Luau is the programming language used by Roblox (a typed superset of Lua 5.1).
"""
@override
def is_ignored_dirname(self, dirname: str) -> bool:
return super().is_ignored_dirname(dirname) or dirname in [
"node_modules",
"Packages", # Wally packages
"DevPackages", # Wally dev packages
"roblox_packages", # Some Rojo projects
"build",
"dist",
".cache",
]
class DependencyProvider(LanguageServerDependencyProviderSinglePath):
def _get_or_install_core_dependency(self) -> str:
luau_lsp_path = shutil.which("luau-lsp")
if luau_lsp_path is not None:
return luau_lsp_path
return self._download_luau_lsp()
def _create_launch_command(self, core_path: str) -> list[str]:
definitions_path, docs_path = self._resolve_support_files()
cmd = [core_path, "lsp"]
if definitions_path is not None:
cmd.append(f"--definitions:@roblox={definitions_path}")
if docs_path is not None:
cmd.append(f"--docs={docs_path}")
return cmd
def _download_luau_lsp(self) -> str:
install_dir = Path(self._ls_resources_dir)
install_dir.mkdir(parents=True, exist_ok=True)
binary_path = self._find_existing_binary(install_dir)
if binary_path is not None:
return binary_path
asset_name = self._get_luau_lsp_asset_name()
download_url = f"https://github.com/JohnnyMorganz/luau-lsp/releases/download/{LUAU_LSP_VERSION}/{asset_name}"
download_path = install_dir / asset_name
log.info("Downloading luau-lsp %s from %s", LUAU_LSP_VERSION, download_url)
with requests.get(download_url, stream=True, timeout=60) as response:
response.raise_for_status()
with open(download_path, "wb") as f:
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
log.info("Extracting luau-lsp to %s", install_dir)
with zipfile.ZipFile(download_path, "r") as zip_ref:
zip_ref.extractall(install_dir)
if download_path.exists():
download_path.unlink()
binary_path = self._find_existing_binary(install_dir)
if binary_path is None:
raise RuntimeError("Failed to find luau-lsp executable after extraction")
return binary_path
def _resolve_support_files(self) -> tuple[str | None, str | None]:
platform_type = LuauLanguageServer._get_platform_type(self._custom_settings)
if platform_type == "standard":
return None, self._download_standard_docs()
security_level = LuauLanguageServer._get_roblox_security_level(self._custom_settings)
return self._download_roblox_support_files(security_level)
def _download_standard_docs(self) -> str | None:
install_dir = Path(self._ls_resources_dir)
install_dir.mkdir(parents=True, exist_ok=True)
return self._download_auxiliary_file(
install_dir / "luau-en-us.json",
LUAU_DOCS_URL,
"Luau API docs",
)
def _download_roblox_support_files(self, security_level: str) -> tuple[str | None, str | None]:
install_dir = Path(self._ls_resources_dir)
install_dir.mkdir(parents=True, exist_ok=True)
definitions_filename = f"globalTypes.{security_level}.d.luau"
definitions_path = self._download_auxiliary_file(
install_dir / definitions_filename,
f"https://luau-lsp.pages.dev/type-definitions/{definitions_filename}",
"Roblox type definitions",
)
docs_path = self._download_auxiliary_file(
install_dir / "en-us.json",
ROBLOX_DOCS_URL,
"Roblox API docs",
)
return definitions_path, docs_path
@staticmethod
def _download_auxiliary_file(path: Path, url: str, description: str) -> str | None:
if path.exists():
return str(path)
try:
log.info("Downloading %s from %s", description, url)
response = requests.get(url, timeout=30)
response.raise_for_status()
path.write_bytes(response.content)
return str(path)
except Exception as exc:
log.warning("Failed to download %s: %s", description, exc)
return None
@classmethod
def _find_existing_binary(cls, install_dir: Path) -> str | None:
binary_name = cls._get_binary_name()
direct_path = install_dir / binary_name
if direct_path.exists():
cls._ensure_executable_bit(direct_path)
return str(direct_path)
for candidate in install_dir.rglob(binary_name):
if candidate.is_file():
cls._ensure_executable_bit(candidate)
return str(candidate)
return None
@staticmethod
def _ensure_executable_bit(binary_path: Path) -> None:
if platform.system() != "Windows":
binary_path.chmod(0o755)
@staticmethod
def _get_binary_name() -> str:
return "luau-lsp.exe" if platform.system() == "Windows" else "luau-lsp"
@staticmethod
def _get_luau_lsp_asset_name() -> str:
system = platform.system()
machine = platform.machine().lower()
if system == "Linux":
if machine in ["x86_64", "amd64"]:
return "luau-lsp-linux-x86_64.zip"
if machine in ["aarch64", "arm64"]:
return "luau-lsp-linux-arm64.zip"
raise RuntimeError(
f"Unsupported Linux architecture: {machine}. "
"luau-lsp only provides linux-x86_64 and linux-arm64 binaries. "
"Please build from source: https://github.com/JohnnyMorganz/luau-lsp"
)
if system == "Darwin":
return "luau-lsp-macos.zip"
if system == "Windows":
return "luau-lsp-win64.zip"
raise RuntimeError(f"Unsupported operating system: {system}")
@staticmethod
def _get_platform_type(custom_settings: SolidLSPSettings.CustomLSSettings) -> str:
platform_type = custom_settings.get("platform", "roblox")
if platform_type not in SUPPORTED_PLATFORMS:
raise ValueError(f"Unsupported Luau platform: {platform_type}. Expected one of: {', '.join(sorted(SUPPORTED_PLATFORMS))}")
return platform_type
@staticmethod
def _get_roblox_security_level(custom_settings: SolidLSPSettings.CustomLSSettings) -> str:
security_level = custom_settings.get("roblox_security_level", "PluginSecurity")
if security_level not in SUPPORTED_ROBLOX_SECURITY_LEVELS:
raise ValueError(
f"Unsupported Luau Roblox security level: {security_level}. "
f"Expected one of: {', '.join(sorted(SUPPORTED_ROBLOX_SECURITY_LEVELS))}"
)
return security_level
@classmethod
def _get_workspace_configuration(cls, custom_settings: SolidLSPSettings.CustomLSSettings) -> dict[str, dict[str, str]]:
return {"platform": {"type": cls._get_platform_type(custom_settings)}}
def _create_dependency_provider(self) -> LanguageServerDependencyProvider:
return self.DependencyProvider(self._custom_settings, self._ls_resources_dir)
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
super().__init__(config, repository_root_path, None, "luau", solidlsp_settings)
self.server_ready = threading.Event()
@staticmethod
def _get_initialize_params(repository_absolute_path: str) -> InitializeParams:
"""
Returns the initialize params for the Luau Language Server.
"""
root_uri = pathlib.Path(repository_absolute_path).as_uri()
initialize_params = {
"locale": "en",
"capabilities": {
"textDocument": {
"synchronization": {"didSave": True, "dynamicRegistration": True},
"definition": {"dynamicRegistration": True},
"references": {"dynamicRegistration": True},
"documentSymbol": {
"dynamicRegistration": True,
"hierarchicalDocumentSymbolSupport": True,
"symbolKind": {"valueSet": list(range(1, 27))},
},
"completion": {
"dynamicRegistration": True,
"completionItem": {
"snippetSupport": True,
"commitCharactersSupport": True,
"documentationFormat": ["markdown", "plaintext"],
"deprecatedSupport": True,
"preselectSupport": True,
},
},
"hover": {
"dynamicRegistration": True,
"contentFormat": ["markdown", "plaintext"],
},
"signatureHelp": {
"dynamicRegistration": True,
"signatureInformation": {
"documentationFormat": ["markdown", "plaintext"],
"parameterInformation": {"labelOffsetSupport": True},
},
},
"rename": {"dynamicRegistration": True, "prepareSupport": True},
"callHierarchy": {"dynamicRegistration": True},
},
"workspace": {
"workspaceFolders": True,
"didChangeConfiguration": {"dynamicRegistration": True},
"configuration": True,
"symbol": {
"dynamicRegistration": True,
"symbolKind": {"valueSet": list(range(1, 27))},
},
},
},
"processId": os.getpid(),
"rootPath": repository_absolute_path,
"rootUri": root_uri,
"workspaceFolders": [
{
"uri": root_uri,
"name": os.path.basename(repository_absolute_path),
}
],
# luau-lsp initialization options
# These can be overridden via .luaurc in the project root
"initializationOptions": {},
}
return initialize_params # type: ignore[return-value]
def _start_server(self) -> None:
"""Start Luau Language Server process"""
def register_capability_handler(params: dict) -> None:
return
def workspace_configuration_handler(params: dict) -> list:
items = params.get("items", [])
config = self._get_workspace_configuration(self._custom_settings)
return [config for _ in items]
def window_log_message(msg: dict) -> None:
message_text = msg.get("message", "")
log.info("LSP: window/logMessage: %s", message_text)
if "workspace ready" in message_text.lower() or "initialized" in message_text.lower():
log.info("Luau language server signaled readiness")
self.server_ready.set()
def do_nothing(params: dict) -> None:
return
self.server.on_request("client/registerCapability", register_capability_handler)
self.server.on_request("workspace/configuration", workspace_configuration_handler)
self.server.on_notification("window/logMessage", window_log_message)
self.server.on_notification("$/progress", do_nothing)
self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
log.info("Starting Luau Language Server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
log.info("Sending initialize request from LSP client to LSP server and awaiting response")
init_response = self.server.send.initialize(initialize_params)
# Verify server capabilities
assert "textDocumentSync" in init_response["capabilities"]
assert "definitionProvider" in init_response["capabilities"]
assert "documentSymbolProvider" in init_response["capabilities"]
assert "referencesProvider" in init_response["capabilities"]
self.server.notify.initialized({})
# Wait for luau-lsp to complete initial setup
log.info("Waiting for Luau language server to become ready...")
if self.server_ready.wait(timeout=5.0):
log.info("Luau language server ready")
else:
log.warning("Timeout waiting for Luau language server readiness, proceeding anyway")
self.server_ready.set()
src/solidlsp/language_servers/ocaml_lsp_server.py
+430
View File
@@ -0,0 +1,430 @@
"""
Provides OCaml and Reason specific instantiation of the SolidLanguageServer class.
Contains various configurations and settings specific to OCaml and Reason.
"""
import logging
import os
import pathlib
import platform
import re
import shutil
import stat
import subprocess
import threading
from typing import Any
from overrides import override
from solidlsp.ls import SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.lsp_protocol_handler.server import ProcessLaunchInfo
from solidlsp.settings import SolidLSPSettings
from solidlsp.util.subprocess_util import subprocess_kwargs
log = logging.getLogger(__name__)
class OcamlLanguageServer(SolidLanguageServer):
"""
Provides OCaml and Reason specific instantiation of the SolidLanguageServer class.
Contains various configurations and settings specific to OCaml and Reason.
"""
_ocaml_version: tuple[int, int, int]
_lsp_version: tuple[int, int, int]
_index_built: bool
# Minimum LSP version for reliable cross-file references
MIN_LSP_VERSION_FOR_CROSS_FILE_REFS: tuple[int, int, int] = (1, 23, 0)
@staticmethod
def _ensure_opam_installed() -> None:
"""Ensure OPAM is installed and available."""
opam_path = shutil.which("opam")
if opam_path is None:
raise RuntimeError(
"OPAM is not installed or not in PATH.\n"
"Please install OPAM from: https://opam.ocaml.org/doc/Install.html\n\n"
"Installation instructions:\n"
" - macOS: brew install opam\n"
" - Ubuntu/Debian: sudo apt install opam\n"
" - Fedora: sudo dnf install opam\n"
" - Windows: https://fdopen.github.io/opam-repository-mingw/installation/\n\n"
"After installation, initialize OPAM with: opam init"
)
@staticmethod
def _detect_ocaml_version(repository_root_path: str) -> tuple[int, int, int]:
"""
Detect and return the OCaml version as a tuple (major, minor, patch).
Also checks for version compatibility with ocaml-lsp-server.
Raises RuntimeError if version cannot be determined.
"""
try:
result = subprocess.run(
["opam", "exec", "--", "ocaml", "-version"],
check=True,
capture_output=True,
text=True,
cwd=repository_root_path,
**subprocess_kwargs(),
)
version_match = re.search(r"(\d+)\.(\d+)\.(\d+)", result.stdout)
if version_match:
major = int(version_match.group(1))
minor = int(version_match.group(2))
patch = int(version_match.group(3))
version_tuple = (major, minor, patch)
version_str = f"{major}.{minor}.{patch}"
log.info(f"OCaml version: {version_str}")
if version_tuple == (5, 1, 0):
raise RuntimeError(
f"OCaml {version_str} is incompatible with ocaml-lsp-server.\n"
"Please use OCaml < 5.1 or >= 5.1.1.\n"
"Consider creating a new opam switch:\n"
" opam switch create <name> ocaml-base-compiler.4.14.2"
)
return version_tuple
raise RuntimeError(
f"Could not parse OCaml version from output: {result.stdout.strip()}\n"
"Please ensure OCaml is properly installed: opam exec -- ocaml -version"
)
except subprocess.CalledProcessError as e:
raise RuntimeError(
f"Failed to detect OCaml version: {e.stderr}\n"
"Please ensure OCaml is installed and opam is configured:\n"
" opam switch show\n"
" opam exec -- ocaml -version"
) from e
except FileNotFoundError as e:
raise RuntimeError(
"OCaml not found. Please install OCaml via opam:\n"
" opam switch create <name> ocaml-base-compiler.4.14.2\n"
" eval $(opam env)"
) from e
@staticmethod
def _detect_lsp_version(repository_root_path: str) -> tuple[int, int, int]:
"""
Detect and return the ocaml-lsp-server version as a tuple (major, minor, patch).
Raises RuntimeError if version cannot be determined.
"""
try:
result = subprocess.run(
["opam", "list", "-i", "ocaml-lsp-server", "--columns=version", "--short"],
check=True,
capture_output=True,
text=True,
cwd=repository_root_path,
**subprocess_kwargs(),
)
version_str = result.stdout.strip()
version_match = re.search(r"(\d+)\.(\d+)\.(\d+)", version_str)
if version_match:
major = int(version_match.group(1))
minor = int(version_match.group(2))
patch = int(version_match.group(3))
version_tuple = (major, minor, patch)
log.info(f"ocaml-lsp-server version: {major}.{minor}.{patch}")
return version_tuple
raise RuntimeError(
f"Could not parse ocaml-lsp-server version from output: {version_str}\n"
"Please ensure ocaml-lsp-server is properly installed:\n"
" opam list -i ocaml-lsp-server"
)
except subprocess.CalledProcessError as e:
raise RuntimeError(
f"Failed to detect ocaml-lsp-server version: {e.stderr}\nPlease install ocaml-lsp-server:\n opam install ocaml-lsp-server"
) from e
except FileNotFoundError as e:
raise RuntimeError("opam not found. Please install opam:\n https://opam.ocaml.org/doc/Install.html") from e
@staticmethod
def _ensure_ocaml_lsp_installed(repository_root_path: str) -> str:
"""
Ensure ocaml-lsp-server is installed and return the executable path.
Raises RuntimeError with helpful message if not installed.
"""
# Check if ocaml-lsp-server is installed
try:
result = subprocess.run(
["opam", "list", "-i", "ocaml-lsp-server"],
check=False,
capture_output=True,
text=True,
cwd=repository_root_path,
**subprocess_kwargs(),
)
if "ocaml-lsp-server" not in result.stdout or "# No matches found" in result.stdout:
raise RuntimeError(
"ocaml-lsp-server is not installed.\n\n"
"Please install it with:\n"
" opam install ocaml-lsp-server\n\n"
"Note: ocaml-lsp-server requires OCaml < 5.1 or >= 5.1.1 (OCaml 5.1.0 is not supported).\n"
"If you have OCaml 5.1.0, create a new opam switch with a compatible version:\n"
" opam switch create <name> ocaml-base-compiler.4.14.2\n"
" opam switch <name>\n"
" eval $(opam env)\n"
" opam install ocaml-lsp-server\n\n"
"For more information: https://github.com/ocaml/ocaml-lsp"
)
log.info("ocaml-lsp-server is installed")
except subprocess.CalledProcessError as e:
raise RuntimeError(f"Failed to check ocaml-lsp-server installation: {e.stderr}")
# Find the executable path
try:
if platform.system() == "Windows":
result = subprocess.run(
["opam", "exec", "--", "where", "ocamllsp"],
check=True,
capture_output=True,
text=True,
cwd=repository_root_path,
**subprocess_kwargs(),
)
executable_path = result.stdout.strip().split("\n")[0]
else:
result = subprocess.run(
["opam", "exec", "--", "which", "ocamllsp"],
check=True,
capture_output=True,
text=True,
cwd=repository_root_path,
**subprocess_kwargs(),
)
executable_path = result.stdout.strip()
if not os.path.exists(executable_path):
raise RuntimeError(f"ocaml-lsp-server executable not found at {executable_path}")
if platform.system() != "Windows":
os.chmod(executable_path, os.stat(executable_path).st_mode | stat.S_IEXEC)
return executable_path
except subprocess.CalledProcessError as e:
raise RuntimeError(
f"Failed to find ocaml-lsp-server executable.\n"
f"Command failed: {e.cmd}\n"
f"Return code: {e.returncode}\n"
f"Stderr: {e.stderr}\n\n"
"This usually means ocaml-lsp-server is not installed or not in PATH.\n"
"Try:\n"
" 1. Check opam switch: opam switch show\n"
" 2. Install ocaml-lsp-server: opam install ocaml-lsp-server\n"
" 3. Ensure opam env is activated: eval $(opam env)"
)
@property
def supports_cross_file_references(self) -> bool:
"""
Check if this OCaml environment supports cross-file references.
Cross-file references require OCaml >= 5.2 with project-wide occurrences
AND ocaml-lsp-server >= 1.23.0 for reliable cross-file reference support.
Full requirements:
- OCaml 5.2+
- ocaml-lsp-server >= 1.23.0 (earlier versions have unreliable cross-file refs)
- merlin >= 5.1-502 (provides ocaml-index tool)
- dune >= 3.16.0
- Index built via `dune build @ocaml-index`
- For best results: `dune build -w` running (enables dune RPC)
Note: Even when this returns True, cross-file refs may not work in all
cases. The LSP server needs dune's RPC server (via -w flag) to be fully
aware of the index. Without watch mode, cross-file refs are best-effort.
See: https://discuss.ocaml.org/t/ann-project-wide-occurrences-in-merlin-and-lsp/14847
"""
ocaml_ok = self._ocaml_version >= (5, 2, 0)
lsp_ok = self._lsp_version >= self.MIN_LSP_VERSION_FOR_CROSS_FILE_REFS
return ocaml_ok and lsp_ok
@staticmethod
def _build_ocaml_index_static(repository_root_path: str) -> bool:
"""
Build the OCaml index for project-wide occurrences.
This enables cross-file reference finding on OCaml 5.2+.
Must be called BEFORE starting the LSP server.
Returns True if successful, False otherwise.
"""
log.info("Building OCaml index for cross-file references (dune build @ocaml-index)...")
try:
result = subprocess.run(
["opam", "exec", "--", "dune", "build", "@ocaml-index"],
cwd=repository_root_path,
capture_output=True,
text=True,
timeout=120,
check=False,
**subprocess_kwargs(),
)
if result.returncode == 0:
log.info("OCaml index built successfully")
return True
else:
log.warning(f"Failed to build OCaml index: {result.stderr}")
return False
except subprocess.TimeoutExpired:
log.warning("OCaml index build timed out after 120 seconds")
return False
except FileNotFoundError:
log.warning("opam not found, cannot build OCaml index")
return False
except Exception as e:
log.warning(f"Error building OCaml index: {e}")
return False
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""
Creates an OcamlLanguageServer instance.
This class is not meant to be instantiated directly. Use SolidLanguageServer.create() instead.
"""
# Ensure dependencies are available
self._ensure_opam_installed()
# Detect OCaml version for feature gating
self._ocaml_version = self._detect_ocaml_version(repository_root_path)
self._index_built = False
# Verify ocaml-lsp-server is installed (we don't need the path, just validation)
self._ensure_ocaml_lsp_installed(repository_root_path)
# Detect LSP version for cross-file reference support
self._lsp_version = self._detect_lsp_version(repository_root_path)
# Build OCaml index BEFORE starting server (required for cross-file refs on OCaml 5.2+)
if self._ocaml_version >= (5, 2, 0):
self._index_built = self._build_ocaml_index_static(repository_root_path)
# Use opam exec to run ocamllsp - this ensures correct opam environment
# which is required for project-wide occurrences (cross-file references) to work
ocaml_lsp_cmd = ["opam", "exec", "--", "ocamllsp", "--fallback-read-dot-merlin"]
log.info(f"Using ocaml-lsp-server via: {' '.join(ocaml_lsp_cmd)}")
super().__init__(
config,
repository_root_path,
ProcessLaunchInfo(cmd=ocaml_lsp_cmd, cwd=repository_root_path),
"ocaml",
solidlsp_settings,
)
self.server_ready = threading.Event()
self.completions_available = threading.Event()
@override
def is_ignored_dirname(self, dirname: str) -> bool:
"""Define language-specific directories to ignore for OCaml projects."""
return super().is_ignored_dirname(dirname) or dirname in ["_build", "_opam", ".opam"]
@staticmethod
def _get_initialize_params(repository_absolute_path: str) -> InitializeParams:
"""
Returns the initialize params for the OCaml Language Server.
Supports both OCaml and Reason.
"""
root_uri = pathlib.Path(repository_absolute_path).as_uri()
initialize_params = {
"processId": os.getpid(),
"clientInfo": {"name": "Serena", "version": "0.1.0"},
"locale": "en",
"rootPath": repository_absolute_path,
"rootUri": root_uri,
"capabilities": {
"workspace": {
"workspaceFolders": True,
"configuration": True,
},
"textDocument": {
"synchronization": {
"dynamicRegistration": True,
"willSave": True,
"willSaveWaitUntil": True,
"didSave": True,
},
"completion": {
"dynamicRegistration": True,
"completionItem": {
"snippetSupport": True,
"documentationFormat": ["markdown", "plaintext"],
},
},
"hover": {
"dynamicRegistration": True,
"contentFormat": ["markdown", "plaintext"],
},
"definition": {"dynamicRegistration": True, "linkSupport": True},
"references": {"dynamicRegistration": True},
"documentSymbol": {
"dynamicRegistration": True,
"hierarchicalDocumentSymbolSupport": True,
},
"formatting": {"dynamicRegistration": True},
"rename": {"dynamicRegistration": True, "prepareSupport": True},
},
},
"trace": "verbose",
"workspaceFolders": [
{
"uri": root_uri,
"name": os.path.basename(repository_absolute_path),
}
],
}
return initialize_params # type: ignore[return-value]
def _start_server(self) -> None:
"""
Starts the OCaml Language Server (supports both OCaml and Reason)
"""
def register_capability_handler(params: Any) -> None:
if "registrations" in params:
for registration in params.get("registrations", []):
method = registration.get("method", "")
log.info(f"OCaml LSP registered capability: {method}")
return
def lang_status_handler(params: dict[str, Any]) -> None:
if params.get("type") == "ServiceReady" and params.get("message") == "ServiceReady":
self.server_ready.set()
def do_nothing(params: Any) -> None:
return
def window_log_message(msg: dict[str, Any]) -> None:
log.info(f"LSP: window/logMessage: {msg}")
if "initialization done" in msg.get("message", "").lower():
self.server_ready.set()
self.server.on_request("client/registerCapability", register_capability_handler)
self.server.on_notification("language/status", lang_status_handler)
self.server.on_notification("window/logMessage", window_log_message)
self.server.on_notification("$/progress", do_nothing)
self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
log.info("Starting OCaml LSP server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
log.info("Sending initialize request from LSP client to LSP server and awaiting response")
init_response = self.server.send.initialize(initialize_params)
# Verify expected capabilities
capabilities = init_response.get("capabilities", {})
log.info(f"OCaml LSP capabilities: {list(capabilities.keys())}")
text_doc_sync = capabilities.get("textDocumentSync")
if isinstance(text_doc_sync, dict):
assert text_doc_sync.get("change") == 2, "Expected incremental sync"
assert "completionProvider" in capabilities, "Expected completion support"
self.server.notify.initialized({})
self.completions_available.set()
self.server_ready.set()
log.info("OCaml Language Server initialized successfully")
src/solidlsp/language_servers/phpactor.py
+3 -3
View File
@@ -46,9 +46,9 @@ class PhpactorServer(SolidLanguageServer):
"""
# Verify PHP is installed
php_path = shutil.which("php")
assert (
php_path is not None
), "PHP is not installed or not found in PATH. Phpactor requires PHP 8.1+. Please install PHP and try again."
assert php_path is not None, (
"PHP is not installed or not found in PATH. Phpactor requires PHP 8.1+. Please install PHP and try again."
)
# Check PHP version (Phpactor requires PHP 8.1+)
result = subprocess.run(["php", "--version"], capture_output=True, text=True, check=False)
src/solidlsp/language_servers/rust_analyzer.py
+25 -10
View File
@@ -65,6 +65,19 @@ class RustAnalyzer(SolidLanguageServer):
pass
return None
@staticmethod
def _is_rust_analyzer_functional(path: str) -> bool:
"""Check if a rust-analyzer binary is functional by running --version.
This catches the case where rust-analyzer in PATH is actually a rustup proxy
that fails because the component is not installed.
"""
try:
result = subprocess.run([path, "--version"], capture_output=True, text=True, check=False, timeout=10)
return result.returncode == 0
except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
return False
@staticmethod
def _ensure_rust_analyzer_installed() -> str:
"""
@@ -73,8 +86,8 @@ class RustAnalyzer(SolidLanguageServer):
Priority order:
1. Rustup existing installation (preferred - matches toolchain version)
2. Rustup auto-install if rustup is available (ensures correct version)
3. Common installation locations as fallback (only if rustup not available)
4. System PATH last (can pick up incompatible versions)
3. System PATH (covers Nix, distro packages, and other standalone installs)
4. Common installation locations as final fallback
:return: path to rust-analyzer executable
"""
@@ -84,7 +97,7 @@ class RustAnalyzer(SolidLanguageServer):
return rustup_path
# If rustup is available but rust-analyzer not installed, auto-install it BEFORE
# checking common paths. This ensures we get the correct version matching the toolchain.
# checking PATH. This ensures we get the correct version matching the toolchain.
if RustAnalyzer.DependencyProvider._get_rustup_version():
result = subprocess.run(["rustup", "component", "add", "rust-analyzer"], check=False, capture_output=True, text=True)
if result.returncode == 0:
@@ -92,13 +105,20 @@ class RustAnalyzer(SolidLanguageServer):
rustup_path = RustAnalyzer.DependencyProvider._get_rust_analyzer_via_rustup()
if rustup_path:
return rustup_path
# If auto-install failed, fall through to common paths as last resort
# If auto-install failed, fall through to PATH and common paths
# Check system PATH early - this covers Nix, distro packages, and other standalone installs.
# We verify the binary is functional to avoid picking up broken rustup proxy symlinks.
path_result = shutil.which("rust-analyzer")
if path_result and os.path.isfile(path_result) and os.access(path_result, os.X_OK):
if RustAnalyzer.DependencyProvider._is_rust_analyzer_functional(path_result):
return path_result
# Determine platform-specific binary name and paths
is_windows = platform.system() == "Windows"
binary_name = "rust-analyzer.exe" if is_windows else "rust-analyzer"
# Fallback to common installation locations (only used if rustup not available)
# Fallback to common installation locations
common_paths: list[str | None] = []
if is_windows:
@@ -129,11 +149,6 @@ class RustAnalyzer(SolidLanguageServer):
if path and os.path.isfile(path) and os.access(path, os.X_OK):
return path
# Last resort: check system PATH (can pick up incorrect aliases, hence checked last)
path_result = shutil.which("rust-analyzer")
if path_result and os.path.isfile(path_result) and os.access(path_result, os.X_OK):
return path_result
# Provide helpful error message with all searched locations
searched = [p for p in common_paths if p]
install_instructions = [
src/solidlsp/language_servers/solidity_language_server.py
+220
View File
@@ -0,0 +1,220 @@
"""
Provides Solidity-specific instantiation of the LanguageServer class using
the Nomic Foundation Solidity Language Server (@nomicfoundation/solidity-language-server).
"""
import glob
import logging
import os
import pathlib
import shutil
import threading
from time import sleep
from typing import Any
from solidlsp.language_servers.common import RuntimeDependency, RuntimeDependencyCollection
from solidlsp.ls import LanguageServerDependencyProvider, LanguageServerDependencyProviderSinglePath, SolidLanguageServer
from solidlsp.ls_config import LanguageServerConfig
from solidlsp.lsp_protocol_handler.lsp_types import InitializeParams
from solidlsp.settings import SolidLSPSettings
log = logging.getLogger(__name__)
class SolidityLanguageServer(SolidLanguageServer):
"""
Provides Solidity-specific instantiation of the LanguageServer class using
the Nomic Foundation Solidity Language Server (@nomicfoundation/solidity-language-server).
Supports go-to-definition, find references, document symbols, hover, and diagnostics.
Requires Node.js and npm to be installed.
"""
@staticmethod
def _determine_log_level(line: str) -> int:
"""Suppress known non-critical stderr output from the Solidity language server."""
line_lower = line.lower()
if any(
[
"telemetry" in line_lower,
"could not find" in line_lower and "hardhat" in line_lower,
"no workspaceroot" in line_lower,
]
):
return logging.DEBUG
return SolidLanguageServer._determine_log_level(line)
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""
Creates a SolidityLanguageServer instance. Not meant to be instantiated directly.
Use LanguageServer.create() instead.
"""
super().__init__(
config,
repository_root_path,
None,
"solidity",
solidlsp_settings,
)
def _create_dependency_provider(self) -> LanguageServerDependencyProvider:
return self.DependencyProvider(self._custom_settings, self._ls_resources_dir)
class DependencyProvider(LanguageServerDependencyProviderSinglePath):
def _get_or_install_core_dependency(self) -> str:
"""
Install @nomicfoundation/solidity-language-server via npm and return the
path to the solidity-language-server executable.
"""
is_node_installed = shutil.which("node") is not None
assert is_node_installed, "node is not installed or isn't in PATH. Please install Node.js and try again."
is_npm_installed = shutil.which("npm") is not None
assert is_npm_installed, "npm is not installed or isn't in PATH. Please install npm and try again."
deps = RuntimeDependencyCollection(
[
RuntimeDependency(
id="solidity-language-server",
description="Nomic Foundation Solidity Language Server",
command="npm install --prefix ./ @nomicfoundation/solidity-language-server@0.8.4",
platform_id="any",
),
]
)
solidity_ls_dir = os.path.join(self._ls_resources_dir, "solidity-lsp")
solidity_executable_path = os.path.join(solidity_ls_dir, "node_modules", ".bin", "nomicfoundation-solidity-language-server")
if os.name == "nt":
solidity_executable_path += ".cmd"
if not os.path.exists(solidity_executable_path):
log.info(f"Solidity Language Server executable not found at {solidity_executable_path}. Installing...")
deps.install(solidity_ls_dir)
log.info("Solidity language server dependencies installed successfully.")
if not os.path.exists(solidity_executable_path):
raise FileNotFoundError(
f"nomicfoundation-solidity-language-server executable not found at {solidity_executable_path}. "
"Something went wrong with the installation."
)
return solidity_executable_path
def _create_launch_command(self, core_path: str) -> list[str]:
return [core_path, "--stdio"]
def is_ignored_dirname(self, dirname: str) -> bool:
return super().is_ignored_dirname(dirname) or dirname in {"artifacts", "cache", "typechain-types"}
@staticmethod
def _get_initialize_params(repository_absolute_path: str) -> InitializeParams:
"""Return LSP InitializeParams for the Solidity language server."""
root_uri = pathlib.Path(repository_absolute_path).as_uri()
return { # type: ignore
"locale": "en",
"processId": os.getpid(),
"rootPath": repository_absolute_path,
"rootUri": root_uri,
"workspaceFolders": [
{
"uri": root_uri,
"name": os.path.basename(repository_absolute_path),
}
],
"capabilities": {
"textDocument": {
"synchronization": {
"dynamicRegistration": True,
"didSave": True,
},
"completion": {
"dynamicRegistration": True,
"completionItem": {"snippetSupport": True},
},
"definition": {"dynamicRegistration": True},
"references": {"dynamicRegistration": True},
"documentSymbol": {
"dynamicRegistration": True,
"hierarchicalDocumentSymbolSupport": True,
"symbolKind": {"valueSet": list(range(1, 27))}, # type: ignore[arg-type]
},
"hover": {
"dynamicRegistration": True,
"contentFormat": ["markdown", "plaintext"], # type: ignore[list-item]
},
},
"workspace": {
"workspaceFolders": True,
"didChangeConfiguration": {"dynamicRegistration": True},
"symbol": {"dynamicRegistration": True},
},
},
"initializationOptions": {},
}
def _get_wait_time_for_cross_file_referencing(self) -> float:
# Small buffer for any post-indexing analysis the LSP performs after file-indexed events.
return 3.0
def _start_server(self) -> None:
"""Start the Solidity language server and wait for project indexing to finish."""
def do_nothing(params: Any) -> None:
return
def register_capability_handler(params: Any) -> None:
return
def window_log_message(msg: dict) -> None:
log.info(f"LSP: window/logMessage: {msg}")
# Count .sol files in the project to know when indexing is complete.
sol_files = glob.glob(os.path.join(self.repository_root_path, "**", "*.sol"), recursive=True)
expected_count = len(sol_files)
indexed_count = [0]
all_indexed = threading.Event()
def on_file_indexed(params: Any) -> None:
indexed_count[0] += 1
uri = (params or {}).get("uri", "")
log.debug(f"Solidity LSP: file indexed ({indexed_count[0]}/{expected_count}): {uri}")
if indexed_count[0] >= expected_count:
all_indexed.set()
self.server.on_request("client/registerCapability", register_capability_handler)
self.server.on_notification("window/logMessage", window_log_message)
self.server.on_notification("$/progress", do_nothing)
self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
self.server.on_notification("custom/file-indexed", on_file_indexed)
log.info("Starting Solidity language server process")
self.server.start()
initialize_params = self._get_initialize_params(self.repository_root_path)
log.debug("Sending initialize request to Solidity language server")
init_response = self.server.send.initialize(initialize_params)
log.debug(f"Received initialize response from Solidity server: {init_response}")
if "documentSymbolProvider" in init_response.get("capabilities", {}):
log.debug("Solidity server supports document symbols")
else:
log.warning("Solidity server does not report document symbol support")
self.server.notify.initialized({})
if expected_count > 0:
log.info(f"Waiting for Solidity LSP to index {expected_count} .sol file(s)…")
completed = all_indexed.wait(timeout=60)
if completed:
log.info(f"Solidity LSP indexing complete ({indexed_count[0]}/{expected_count} files indexed)")
else:
log.warning(
f"Solidity LSP indexing timed out ({indexed_count[0]}/{expected_count} files indexed). "
"Waiting additional 30s for slow environments (e.g., CI)."
)
sleep(30)
log.info(f"Additional wait complete ({indexed_count[0]}/{expected_count} files indexed)")
else:
log.info("No .sol files found; skipping indexing wait")
log.info("Solidity language server initialization complete")
src/solidlsp/language_servers/typescript_language_server.py
+100 -10
View File
@@ -67,6 +67,10 @@ class TypeScriptLanguageServer(SolidLanguageServer):
- typescript_language_server_version: Version of typescript-language-server to install (default: "5.1.3")
"""
# Safety timeout for $/progress-based indexing wait. Normally the event fires
# well within this window; the timeout is only hit if the server never sends progress.
INDEXING_PROGRESS_TIMEOUT = 15.0 if os.name == "nt" else 10.0
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
"""
Creates a TypeScriptLanguageServer instance. This class is not meant to be instantiated directly. Use LanguageServer.create() instead.
@@ -81,6 +85,29 @@ class TypeScriptLanguageServer(SolidLanguageServer):
self.server_ready = threading.Event()
self.initialize_searcher_command_available = threading.Event()
# Progress tracking for $/progress notifications (project indexing, etc.)
self._progress_lock = threading.Lock()
self._active_progress_tokens: set[str] = set()
self._indexing_complete = threading.Event()
self._indexing_complete.set() # Initially set (no active work)
def wait_for_indexing(self, timeout: float) -> bool:
"""Block until all $/progress tokens complete.
:param timeout: Maximum seconds to wait.
:return: True if indexing completed, False on timeout.
"""
return self._indexing_complete.wait(timeout=timeout)
def expect_indexing(self) -> None:
"""Signal that new files are about to be opened and async indexing should be awaited.
Clears the internal indexing-complete event so that a subsequent
:meth:`wait_for_indexing` call blocks until all $/progress tokens
complete (or the timeout expires).
"""
self._indexing_complete.clear()
def _create_dependency_provider(self) -> LanguageServerDependencyProvider:
return self.DependencyProvider(self._custom_settings, self._ls_resources_dir)
@@ -114,9 +141,9 @@ class TypeScriptLanguageServer(SolidLanguageServer):
PlatformId.WIN_x64,
PlatformId.WIN_arm64,
]
assert (
platform_id in valid_platforms
), f"Platform {platform_id} is not supported for multilspy javascript/typescript at the moment"
assert platform_id in valid_platforms, (
f"Platform {platform_id} is not supported for multilspy javascript/typescript at the moment"
)
# Get version settings from ls_specific_settings or use defaults
language_specific_config = self._custom_settings
@@ -217,6 +244,9 @@ class TypeScriptLanguageServer(SolidLanguageServer):
"didChangeConfiguration": {"dynamicRegistration": True},
"symbol": {"dynamicRegistration": True},
},
"window": {
"workDoneProgress": True, # Enables $/progress notifications for project loading
},
},
"processId": os.getpid(),
"rootPath": repository_absolute_path,
@@ -242,6 +272,7 @@ class TypeScriptLanguageServer(SolidLanguageServer):
await lsp.request_references(...)
# Shutdown the LanguageServer on exit from scope
# LanguageServer has been shutdown
```
"""
def register_capability_handler(params: dict) -> None:
@@ -263,19 +294,65 @@ class TypeScriptLanguageServer(SolidLanguageServer):
def window_log_message(msg: dict) -> None:
log.info(f"LSP: window/logMessage: {msg}")
def check_experimental_status(params: dict) -> None:
def handle_typescript_version(params: dict) -> None:
"""
Also listen for experimental/serverStatus as a backup signal
The $/typescriptVersion notification is sent by typescript-language-server
once tsserver has loaded and reported its version. This is a reliable
signal that tsserver is running and responsive.
"""
if params.get("quiescent") == True:
self.server_ready.set()
log.info(f"TypeScript server version notification received: {params}")
self.server_ready.set()
def work_done_progress_create(params: dict) -> dict:
"""Handle window/workDoneProgress/create: the server is about to report async progress.
Clear the indexing-complete event so callers waiting on it will block until
all progress tokens finish. This is sent by typescript-language-server when
tsserver starts processing files (e.g. "Initializing JS/TS language features...").
"""
token = str(params.get("token", ""))
log.debug(f"TypeScript LSP workDoneProgress/create: token={token!r}")
with self._progress_lock:
self._active_progress_tokens.add(token)
self._indexing_complete.clear()
return {}
def progress_handler(params: dict) -> None:
"""Track $/progress begin/end to detect when all async work finishes.
typescript-language-server sends $/progress for project loading operations
like "Initializing JS/TS language features...". When all progress tokens
complete (kind='end'), _indexing_complete is set.
"""
token = str(params.get("token", ""))
value = params.get("value", {})
kind = value.get("kind")
if kind == "begin":
title = value.get("title", "")
log.info(f"TypeScript LSP progress [{token}]: started - {title}")
with self._progress_lock:
self._active_progress_tokens.add(token)
self._indexing_complete.clear()
elif kind == "report":
pct = value.get("percentage")
msg = value.get("message", "")
pct_str = f" ({pct}%)" if pct is not None else ""
log.debug(f"TypeScript LSP progress [{token}]: {msg}{pct_str}")
elif kind == "end":
msg = value.get("message", "")
log.info(f"TypeScript LSP progress [{token}]: ended - {msg}")
with self._progress_lock:
self._active_progress_tokens.discard(token)
if not self._active_progress_tokens:
self._indexing_complete.set()
self.server.on_request("client/registerCapability", register_capability_handler)
self.server.on_notification("window/logMessage", window_log_message)
self.server.on_request("workspace/executeClientCommand", execute_client_command_handler)
self.server.on_notification("$/progress", do_nothing)
self.server.on_request("window/workDoneProgress/create", work_done_progress_create)
self.server.on_notification("$/progress", progress_handler)
self.server.on_notification("$/typescriptVersion", handle_typescript_version)
self.server.on_notification("textDocument/publishDiagnostics", do_nothing)
self.server.on_notification("experimental/serverStatus", check_experimental_status)
log.info("Starting TypeScript server process")
self.server.start()
@@ -295,13 +372,26 @@ class TypeScriptLanguageServer(SolidLanguageServer):
}
self.server.notify.initialized({})
if self.server_ready.wait(timeout=1.0):
if self.server_ready.wait(timeout=10.0):
log.info("TypeScript server is ready")
else:
log.info("Timeout waiting for TypeScript server to become ready, proceeding anyway")
# Fallback: assume server is ready after timeout
self.server_ready.set()
# Wait for any async project loading to complete.
# typescript-language-server may send $/progress for "Initializing JS/TS
# language features…" after initialized. If no progress is sent,
# _indexing_complete stays SET and wait() returns immediately.
log.info("Waiting for TypeScript project indexing to complete (if async)...")
if self.wait_for_indexing(timeout=self.INDEXING_PROGRESS_TIMEOUT):
log.info("TypeScript project indexing complete")
else:
log.warning(
"TypeScript project indexing did not complete within %.0fs; proceeding anyway",
self.INDEXING_PROGRESS_TIMEOUT,
)
@override
def _get_wait_time_for_cross_file_referencing(self) -> float:
return 2
src/solidlsp/language_servers/vts_language_server.py
+1
View File
@@ -155,6 +155,7 @@ class VtsLanguageServer(SolidLanguageServer):
await lsp.request_references(...)
# Shutdown the LanguageServer on exit from scope
# LanguageServer has been shutdown
```
"""
def register_capability_handler(params: dict) -> None:
src/solidlsp/language_servers/vue_language_server.py
+20 -7
View File
@@ -134,8 +134,6 @@ class VueLanguageServer(SolidLanguageServer):
TS_SERVER_READY_TIMEOUT = 5.0
VUE_SERVER_READY_TIMEOUT = 3.0
# Windows requires more time due to slower I/O and process operations.
VUE_INDEXING_WAIT_TIME = 4.0 if os.name == "nt" else 2.0
def __init__(self, config: LanguageServerConfig, repository_root_path: str, solidlsp_settings: SolidLSPSettings):
vue_lsp_executable_path, self.tsdk_path, self._ts_ls_cmd = self._setup_runtime_dependencies(config, solidlsp_settings)
@@ -204,6 +202,11 @@ class VueLanguageServer(SolidLanguageServer):
vue_files = self._find_all_vue_files()
log.debug(f"Found {len(vue_files)} .vue files to index")
# Prepare the TS server to track new $/progress notifications triggered
# by the didOpen calls below. Must happen BEFORE opening files to avoid
# a race where progress begins and ends before we start waiting.
self._ts_server.expect_indexing()
for vue_file in vue_files:
try:
with self._ts_server.open_file(vue_file) as file_buffer:
@@ -213,13 +216,23 @@ class VueLanguageServer(SolidLanguageServer):
log.debug(f"Failed to open {vue_file} on TS server: {e}")
self._vue_files_indexed = True
log.info("Vue file indexing on TypeScript server complete")
log.info("Vue file indexing on TypeScript server complete, waiting for TS server to finish processing")
sleep(self._get_vue_indexing_wait_time())
log.debug("Wait period after Vue file indexing complete")
self._wait_for_ts_indexing_complete()
def _get_vue_indexing_wait_time(self) -> float:
return self.VUE_INDEXING_WAIT_TIME
def _wait_for_ts_indexing_complete(self) -> None:
"""Wait for the companion TypeScript server to finish processing opened Vue files.
Uses the $/progress tracking in TypeScriptLanguageServer: after Vue files are
opened, tsserver sends "Initializing JS/TS language features…" progress.
We wait for all progress tokens to complete, with a timeout fallback.
"""
assert self._ts_server is not None
timeout = TypeScriptLanguageServer.INDEXING_PROGRESS_TIMEOUT
if self._ts_server.wait_for_indexing(timeout=timeout):
log.info("TypeScript server finished indexing Vue files (signaled via $/progress)")
else:
log.warning(f"Timeout ({timeout}s) waiting for TypeScript server to finish indexing Vue files, proceeding anyway")
def _send_references_request(self, relative_file_path: str, line: int, column: int) -> list[lsp_types.Location] | None:
uri = PathUtils.path_to_uri(os.path.join(self.repository_root_path, relative_file_path))
src/solidlsp/language_servers/zls.py
+1 -2
View File
@@ -70,8 +70,7 @@ class ZigLanguageServer(SolidLanguageServer):
# Check for Windows and provide error message
if platform.system() == "Windows":
raise RuntimeError(
"Windows is not supported by ZLS in this integration. "
"Cross-file references don't work reliably on Windows. Reason unknown."
"Windows is not supported by ZLS in this integration. Cross-file references don't work reliably on Windows. Reason unknown."
)
zig_version = ZigLanguageServer._get_zig_version()
src/solidlsp/ls.py
+5 -7
View File
@@ -477,9 +477,7 @@ class SolidLanguageServer(ABC):
self.language = Language(language_id)
# initialise symbol caches
self.cache_dir = (
Path(self.repository_root_path) / self._solidlsp_settings.project_data_relative_path / self.CACHE_FOLDER_NAME / self.language_id
)
self.cache_dir = Path(self._solidlsp_settings.project_data_path) / self.CACHE_FOLDER_NAME / self.language_id
self.cache_dir.mkdir(parents=True, exist_ok=True)
# * raw document symbols cache
self._ls_specific_raw_document_symbols_cache_version = cache_version_raw_document_symbols
@@ -819,7 +817,7 @@ class SolidLanguageServer(ABC):
Delete text between the given start and end positions in the given file and return the deleted text.
"""
if not self.server_started:
log.error("insert_text_at_position called before Language Server started")
log.error("delete_text_between_positions called before Language Server started")
raise SolidLSPException("Language Server not started")
absolute_file_path = str(PurePath(self.repository_root_path, relative_file_path))
@@ -1106,12 +1104,12 @@ class SolidLanguageServer(ABC):
}
response: list[LSPTypes.CompletionItem] | LSPTypes.CompletionList | None = None
num_retries = 0
while response is None or (response["isIncomplete"] and num_retries < 30): # type: ignore
for _ in range(30):
response = self.server.send.completion(completion_params)
if isinstance(response, list):
response = {"items": response, "isIncomplete": False}
num_retries += 1
if response is None or not response["isIncomplete"]: # type: ignore
break
# TODO: Understand how to appropriately handle `isIncomplete`
if response is None or (response["isIncomplete"] and not allow_incomplete): # type: ignore
src/solidlsp/ls_config.py
+81
View File
@@ -53,8 +53,14 @@ class Language(str, Enum):
BASH = "bash"
ZIG = "zig"
LUA = "lua"
LUAU = "luau"
"""Luau Language Server for Roblox's Luau language (typed Lua 5.1 superset).
Uses luau-lsp by JohnnyMorganz. Automatically downloads the binary if not found.
Supports .luau files. Configure via .luaurc in the project root.
"""
NIX = "nix"
ERLANG = "erlang"
OCAML = "ocaml"
AL = "al"
FSHARP = "fsharp"
REGO = "rego"
@@ -62,6 +68,7 @@ class Language(str, Enum):
JULIA = "julia"
FORTRAN = "fortran"
HASKELL = "haskell"
LEAN4 = "lean4"
GROOVY = "groovy"
VUE = "vue"
POWERSHELL = "powershell"
@@ -105,11 +112,29 @@ class Language(str, Enum):
"""TOML language server using Taplo.
Supports TOML validation, formatting, and schema support.
"""
HLSL = "hlsl"
"""Shader language server using shader-language-server (antaalt/shader-sense).
Supports .hlsl, .hlsli, .fx, .fxh, .cginc, .compute, .shader, .glsl, .vert, .frag, .geom, .tesc, .tese, .comp, .wgsl files.
Automatically downloads shader-language-server binary.
"""
SYSTEMVERILOG = "systemverilog"
"""SystemVerilog language server using verible-verilog-ls.
Supports .sv, .svh, .v, .vh files.
Automatically downloads verible binary.
"""
SOLIDITY = "solidity"
"""Solidity language server using the Nomic Foundation Solidity Language Server
(@nomicfoundation/solidity-language-server).
Supports .sol files. Provides go-to-definition, find references, document symbols,
hover, and diagnostics. Requires Node.js and npm.
Works best with a foundry.toml or hardhat.config.js in the project root.
"""
ANSIBLE = "ansible"
"""Ansible language server (experimental) using @ansible/ansible-language-server.
Supports *.yaml and *.yml files (same extensions as YAML, hence experimental).
Must be explicitly specified in project.yml. Requires Node.js and npm.
Requires ``ansible`` in PATH for full functionality.
"""
@classmethod
def iter_all(cls, include_experimental: bool = False) -> Iterable[Self]:
@@ -126,6 +151,7 @@ class Language(str, Enum):
in the project.yml configuration.
"""
return self in {
self.ANSIBLE,
self.TYPESCRIPT_VTS,
self.PYTHON_JEDI,
self.CSHARP_OMNISHARP,
@@ -136,6 +162,7 @@ class Language(str, Enum):
self.TOML,
self.GROOVY,
self.CPP_CCLS,
self.SOLIDITY,
}
def __str__(self) -> str:
@@ -214,10 +241,14 @@ class Language(str, Enum):
return FilenameMatcher("*.zig", "*.zon")
case self.LUA:
return FilenameMatcher("*.lua")
case self.LUAU:
return FilenameMatcher("*.luau")
case self.NIX:
return FilenameMatcher("*.nix")
case self.ERLANG:
return FilenameMatcher("*.erl", "*.hrl", "*.escript", "*.config", "*.app", "*.app.src")
case self.OCAML:
return FilenameMatcher("*.ml", "*.mli", "*.re", "*.rei")
case self.AL:
return FilenameMatcher("*.al", "*.dal")
case self.FSHARP:
@@ -236,6 +267,8 @@ class Language(str, Enum):
)
case self.HASKELL:
return FilenameMatcher("*.hs", "*.lhs")
case self.LEAN4:
return FilenameMatcher("*.lean")
case self.VUE:
path_patterns = ["*.vue"]
for prefix in ["c", "m", ""]:
@@ -251,8 +284,30 @@ class Language(str, Enum):
return FilenameMatcher("*.groovy", "*.gvy")
case self.MATLAB:
return FilenameMatcher("*.m", "*.mlx", "*.mlapp")
case self.HLSL:
return FilenameMatcher(
"*.hlsl",
"*.hlsli",
"*.fx",
"*.fxh",
"*.cginc",
"*.compute",
"*.shader",
"*.glsl",
"*.vert",
"*.frag",
"*.geom",
"*.tesc",
"*.tese",
"*.comp",
"*.wgsl",
)
case self.SYSTEMVERILOG:
return FilenameMatcher("*.sv", "*.svh", "*.v", "*.vh")
case self.SOLIDITY:
return FilenameMatcher("*.sol")
case self.ANSIBLE:
return FilenameMatcher("*.yaml", "*.yml")
case _:
raise ValueError(f"Unhandled language: {self}")
@@ -378,10 +433,20 @@ class Language(str, Enum):
from solidlsp.language_servers.lua_ls import LuaLanguageServer
return LuaLanguageServer
case self.LUAU:
from solidlsp.language_servers.luau_lsp import LuauLanguageServer
return LuauLanguageServer
case self.ERLANG:
from solidlsp.language_servers.erlang_language_server import ErlangLanguageServer
return ErlangLanguageServer
case self.OCAML:
from solidlsp.language_servers.ocaml_lsp_server import OcamlLanguageServer
return OcamlLanguageServer
case self.AL:
from solidlsp.language_servers.al_language_server import ALLanguageServer
@@ -414,6 +479,10 @@ class Language(str, Enum):
from solidlsp.language_servers.haskell_language_server import HaskellLanguageServer
return HaskellLanguageServer
case self.LEAN4:
from solidlsp.language_servers.lean4_language_server import Lean4LanguageServer
return Lean4LanguageServer
case self.FSHARP:
from solidlsp.language_servers.fsharp_language_server import FSharpLanguageServer
@@ -434,10 +503,22 @@ class Language(str, Enum):
from solidlsp.language_servers.matlab_language_server import MatlabLanguageServer
return MatlabLanguageServer
case self.HLSL:
from solidlsp.language_servers.hlsl_language_server import HlslLanguageServer
return HlslLanguageServer
case self.SYSTEMVERILOG:
from solidlsp.language_servers.systemverilog_server import SystemVerilogLanguageServer
return SystemVerilogLanguageServer
case self.SOLIDITY:
from solidlsp.language_servers.solidity_language_server import SolidityLanguageServer
return SolidityLanguageServer
case self.ANSIBLE:
from solidlsp.language_servers.ansible_language_server import AnsibleLanguageServer
return AnsibleLanguageServer
case _:
raise ValueError(f"Unhandled language: {self}")
src/solidlsp/ls_types.py
+6 -34
View File
@@ -7,13 +7,18 @@ from __future__ import annotations
from enum import Enum, IntEnum
from typing import TYPE_CHECKING, NotRequired, Union
from sensai.util.helper import mark_used
from typing_extensions import TypedDict
from solidlsp.lsp_protocol_handler.lsp_types import DiagnosticSeverity
from solidlsp.lsp_protocol_handler.lsp_types import DiagnosticSeverity, SymbolKind
if TYPE_CHECKING:
from .ls import SymbolBody
# a lot of code relied on a previously duplicated SymbolKind definition here.
# This line is kept to avoid breaking downstream imports
mark_used(SymbolKind)
URI = str
DocumentUri = str
@@ -147,39 +152,6 @@ class CompletionItem(TypedDict):
about this item, like type or symbol information. """
class SymbolKind(IntEnum):
"""A symbol kind."""
# TODO: This is a duplicate of SymbolKind in lsp_types.
File = 1
Module = 2
Namespace = 3
Package = 4
Class = 5
Method = 6
Property = 7
Field = 8
Constructor = 9
Enum = 10
Interface = 11
Function = 12
Variable = 13
Constant = 14
String = 15
Number = 16
Boolean = 17
Array = 18
Object = 19
Key = 20
Null = 21
EnumMember = 22
Struct = 23
Event = 24
Operator = 25
TypeParameter = 26
class SymbolTag(IntEnum):
"""Symbol tags are extra annotations that tweak the rendering of a symbol.
src/solidlsp/lsp_protocol_handler/lsp_types.py
+10 -31
View File
@@ -1,36 +1,16 @@
# Code generated. DO NOT EDIT.
# The code in this module is mostly autogenerated using https://github.com/predragnikolic/OLSP
# To update it for higher LSP versions, autogenerate a new version in a different file and merge manually.
# LSP v3.17.0
# TODO: Look into use of https://pypi.org/project/ts2python/ to generate the types for https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/
# Note by author of the autogenerator: Look into use of https://pypi.org/project/ts2python/ to generate the types f
# or https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/
"""
This file provides the Python types corresponding to the Typescript types defined in the language server protocol.
This file is obtained from https://github.com/predragnikolic/OLSP under the MIT License with the following terms:
MIT License
Copyright (c) 2023 Предраг Николић
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from enum import Enum, IntEnum, IntFlag
from typing import Literal, NotRequired, Union
from typing import Any, Literal, NotRequired, Union
from typing_extensions import TypedDict
@@ -166,6 +146,8 @@ class FoldingRangeKind(Enum):
class SymbolKind(IntEnum):
"""A symbol kind."""
Unknown = 0
"""Not an LSP type, we use it to represent symbols that don't have a known kind."""
File = 1
Module = 2
Namespace = 3
@@ -197,11 +179,8 @@ class SymbolKind(IntEnum):
TypeParameter = 26
@classmethod
def from_int(cls, value: int) -> "SymbolKind":
for symbol_kind in cls:
if symbol_kind.value == value:
return symbol_kind
raise ValueError(f"Invalid symbol kind: {value}")
def _missing_(cls, value: Any) -> "SymbolKind":
return cls.Unknown
class SymbolTag(IntEnum):
src/solidlsp/settings.py
+4 -4
View File
@@ -22,11 +22,11 @@ class SolidLSPSettings:
"""
Path to the directory in which to store global Solid-LSP data (which is not project-specific)
"""
project_data_relative_path: str = ".solidlsp"
project_data_path: str = ""
"""
Relative path within each project directory where Solid-LSP can store project-specific data, e.g. cache files.
For instance, if this is ".solidlsp" and the project is located at "/home/user/myproject",
then Solid-LSP will store project-specific data in "/home/user/myproject/.solidlsp".
Absolute path to a directory where Solid-LSP can store project-specific data, e.g. cache files.
For instance, if this is "/home/user/myproject/.solidlsp",
then Solid-LSP will store project-specific data (e.g. caches) in that directory.
"""
ls_specific_settings: dict["Language", dict[str, Any]] = field(default_factory=dict)
"""
src/solidlsp/util/subprocess_util.py
+11 -4
View File
@@ -1,4 +1,5 @@
import platform
import shlex
import subprocess
@@ -15,8 +16,14 @@ def subprocess_kwargs() -> dict:
def quote_arg(arg: str) -> str:
"""
Adds quotes around an argument if it contains spaces.
Quotes a shell argument to prevent interpretation of metacharacters.
Uses :func:`shlex.quote` on POSIX systems for proper escaping of all
shell-special characters. On Windows, wraps arguments containing spaces
in double quotes (Windows shell does not interpret single-quoted strings).
"""
if " " not in arg:
return arg
return f'"{arg}"'
if platform.system() == "Windows":
if " " not in arg:
return arg
return f'"{arg}"'
return shlex.quote(arg)
test/conftest.py
+40 -10
View File
@@ -10,7 +10,7 @@ from typing import Any
import pytest
from sensai.util.logging import configure
from serena.config.serena_config import SerenaPaths
from serena.config.serena_config import SerenaConfig, SerenaPaths
from serena.constants import SERENA_MANAGED_DIR_NAME
from serena.project import Project
from serena.util.file_system import GitignoreParser
@@ -69,12 +69,13 @@ def _create_ls(
trace_lsp_communication=trace_lsp_communication,
)
effective_solidlsp_dir = solidlsp_dir if solidlsp_dir is not None else SerenaPaths().serena_user_home_dir
project_data_path = os.path.join(repo_path, SERENA_MANAGED_DIR_NAME)
return SolidLanguageServer.create(
config,
repo_path,
solidlsp_settings=SolidLSPSettings(
solidlsp_dir=effective_solidlsp_dir,
project_data_relative_path=SERENA_MANAGED_DIR_NAME,
project_data_path=project_data_path,
ls_specific_settings=ls_specific_settings or {},
),
)
@@ -115,9 +116,13 @@ def start_default_ls_context(language: Language) -> Iterator[SolidLanguageServer
yield ls
def _create_default_project(language: Language) -> Project:
repo_path = str(get_repo_path(language))
return Project.load(repo_path, serena_config=None)
def create_default_serena_config():
return SerenaConfig(gui_log_window=False, web_dashboard=False)
def _create_default_project(language: Language, repo_root_override: str | None = None) -> Project:
repo_path = str(get_repo_path(language)) if repo_root_override is None else repo_root_override
return Project.load(repo_path, serena_config=create_default_serena_config())
@pytest.fixture(scope="session")
@@ -173,8 +178,18 @@ def language_server(request: LanguageParamRequest):
yield ls
@contextmanager
def project_context(language: Language, repo_root_override: str | None = None) -> Iterator[Project]:
"""Context manager that creates a Project for the specified language and ensures proper cleanup."""
project = _create_default_project(language, repo_root_override)
try:
yield project
finally:
project.shutdown(timeout=5)
@pytest.fixture(scope="module")
def project(request: LanguageParamRequest):
def project(request: LanguageParamRequest, repo_root_override: str | None = None) -> Iterator[Project]:
"""Create a Project for the specified language.
This fixture requires a language parameter via pytest.mark.parametrize:
@@ -198,11 +213,26 @@ def project(request: LanguageParamRequest):
"""
if not hasattr(request, "param"):
raise ValueError("Language parameter must be provided via pytest.mark.parametrize")
language = request.param
project = _create_default_project(language)
yield project
project.shutdown(timeout=5)
with project_context(language, repo_root_override) as project:
yield project
@contextmanager
def project_with_ls_context(language: Language, repo_root_override: str | None = None) -> Iterator[Project]:
"""Context manager that creates a Project with an active language server for the specified language."""
with project_context(language, repo_root_override) as project:
project.create_language_server_manager()
yield project
@pytest.fixture(scope="module")
def project_with_ls(request: LanguageParamRequest) -> Iterator[Project]:
if not hasattr(request, "param"):
raise ValueError("Language parameter must be provided via pytest.mark.parametrize")
language = request.param
with project_with_ls_context(language) as project:
yield project
is_ci = os.getenv("CI") == "true" or os.getenv("GITHUB_ACTIONS") == "true"
test/resources/repos/ansible/test_repo/inventory/hosts.yml
+13
View File
@@ -0,0 +1,13 @@
---
all:
children:
webservers:
hosts:
web1.example.com:
ansible_host: 192.168.1.10
web2.example.com:
ansible_host: 192.168.1.11
dbservers:
hosts:
db1.example.com:
ansible_host: 192.168.1.20
test/resources/repos/ansible/test_repo/playbook.yml
+31
View File
@@ -0,0 +1,31 @@
---
- name: Configure web servers
hosts: webservers
become: true
vars:
http_port: 80
max_connections: 100
tasks:
- name: Install nginx
ansible.builtin.package:
name: nginx
state: present
- name: Start nginx service
ansible.builtin.service:
name: nginx
state: started
enabled: true
- name: Copy config file
ansible.builtin.template:
src: nginx.conf.j2
dest: /etc/nginx/nginx.conf
notify: Restart nginx
handlers:
- name: Restart nginx
ansible.builtin.service:
name: nginx
state: restarted
test/resources/repos/ansible/test_repo/roles/common/defaults/main.yml
+3
View File
@@ -0,0 +1,3 @@
---
nginx_listen_port: 80
deploy_user: deploy
test/resources/repos/ansible/test_repo/roles/common/handlers/main.yml
+4
View File
@@ -0,0 +1,4 @@
---
- name: Restart common services
ansible.builtin.debug:
msg: "Restarting common services"
test/resources/repos/ansible/test_repo/roles/common/tasks/main.yml
+19
View File
@@ -0,0 +1,19 @@
---
- name: Update package cache
ansible.builtin.package:
update_cache: true
- name: Install common packages
ansible.builtin.package:
name: "{{ item }}"
state: present
loop:
- curl
- git
- vim
- name: Create deploy user
ansible.builtin.user:
name: deploy
state: present
shell: /bin/bash
test/resources/repos/hlsl/test_repo/common.hlsl
+29
View File
@@ -0,0 +1,29 @@
#ifndef COMMON_HLSL
#define COMMON_HLSL
struct VertexInput
{
float3 position : POSITION;
float3 normal : NORMAL;
float2 uv : TEXCOORD0;
};
struct VertexOutput
{
float4 clipPos : SV_POSITION;
float3 worldNormal : TEXCOORD0;
float2 uv : TEXCOORD1;
};
float3 SafeNormalize(float3 v)
{
float len = length(v);
return len > 0.0001 ? v / len : float3(0, 0, 0);
}
float Remap(float value, float fromMin, float fromMax, float toMin, float toMax)
{
return toMin + (value - fromMin) * (toMax - toMin) / (fromMax - fromMin);
}
#endif // COMMON_HLSL
test/resources/repos/hlsl/test_repo/compute_test.hlsl
+26
View File
@@ -0,0 +1,26 @@
#include "common.hlsl"
RWTexture2D<float4> OutputTexture : register(u0);
Texture2D<float4> InputTexture : register(t0);
cbuffer ComputeParams : register(b0)
{
uint2 TextureSize;
float BlurRadius;
float _Pad;
};
[numthreads(8, 8, 1)]
void CSMain(uint3 id : SV_DispatchThreadID)
{
if (id.x >= TextureSize.x || id.y >= TextureSize.y)
return;
float4 color = InputTexture[id.xy];
float3 remapped = float3(
Remap(color.r, 0.0, 1.0, 0.2, 0.8),
Remap(color.g, 0.0, 1.0, 0.2, 0.8),
Remap(color.b, 0.0, 1.0, 0.2, 0.8)
);
OutputTexture[id.xy] = float4(remapped, color.a);
}
test/resources/repos/hlsl/test_repo/lighting.hlsl
+37
View File
@@ -0,0 +1,37 @@
#ifndef LIGHTING_HLSL
#define LIGHTING_HLSL
#include "common.hlsl"
cbuffer LightingConstants : register(b0)
{
float4x4 ViewProjection;
float3 LightDirection;
float LightIntensity;
float3 AmbientColor;
float _Padding;
};
float3 CalculateDiffuse(float3 normal, float3 lightDir, float3 albedo)
{
float ndotl = max(dot(normal, -lightDir), 0.0);
return albedo * ndotl;
}
float3 CalculateSpecular(float3 normal, float3 lightDir, float3 viewDir, float shininess)
{
float3 halfVec = SafeNormalize(-lightDir + viewDir);
float ndoth = max(dot(normal, halfVec), 0.0);
return pow(ndoth, shininess);
}
VertexOutput TransformVertex(VertexInput input)
{
VertexOutput output;
output.clipPos = mul(ViewProjection, float4(input.position, 1.0));
output.worldNormal = input.normal;
output.uv = input.uv;
return output;
}
#endif // LIGHTING_HLSL
test/resources/repos/hlsl/test_repo/terrain/terrain_sdf.hlsl
+33
View File
@@ -0,0 +1,33 @@
#ifndef TERRAIN_SDF_HLSL
#define TERRAIN_SDF_HLSL
#include "../common.hlsl"
struct SDFBrickData
{
float3 center;
float halfExtent;
int resolution;
float maxDistance;
};
float3 WorldOffset;
float SampleSDF(float3 worldPos, SDFBrickData brick)
{
float3 localPos = worldPos - brick.center;
float dist = length(localPos) - brick.halfExtent;
return dist;
}
float3 CalculateGradient(float3 worldPos, SDFBrickData brick)
{
float eps = 0.01;
float3 gradient;
gradient.x = SampleSDF(worldPos + float3(eps, 0, 0), brick) - SampleSDF(worldPos - float3(eps, 0, 0), brick);
gradient.y = SampleSDF(worldPos + float3(0, eps, 0), brick) - SampleSDF(worldPos - float3(0, eps, 0), brick);
gradient.z = SampleSDF(worldPos + float3(0, 0, eps), brick) - SampleSDF(worldPos - float3(0, 0, eps), brick);
return SafeNormalize(gradient);
}
#endif // TERRAIN_SDF_HLSL
test/resources/repos/lean4/test_repo/Helper.lean
+16
View File
@@ -0,0 +1,16 @@
structure Calculator where
name : String
version : Nat
deriving Repr
def add (x y : Nat) : Nat :=
x + y
def subtract (x y : Nat) : Int :=
Int.ofNat x - Int.ofNat y
def isPositive (x : Int) : Bool :=
x > 0
def absolute (x : Int) : Int :=
if isPositive x then x else -x
test/resources/repos/lean4/test_repo/Main.lean
+24
View File
@@ -0,0 +1,24 @@
import Helper
def multiply (x y : Nat) : Nat :=
x * y
def calculate (c : Calculator) (op : String) (x y : Nat) : Option Int :=
match op with
| "add" => some (Int.ofNat (add x y))
| "subtract" => some (subtract x y)
| "multiply" => some (Int.ofNat (multiply x y))
| _ => none
def main : IO Unit := do
let c : Calculator := { name := "TestCalc", version := 1 }
IO.println s!"Using {c.name} version {c.version}"
let result1 := add 5 3
IO.println s!"5 + 3 = {result1}"
let result2 := subtract 10 4
IO.println s!"10 - 4 = {result2}"
match calculate c "multiply" 6 7 with
| some result => IO.println s!"6 * 7 = {result}"
| none => IO.println "Calculation failed"
IO.println s!"Is 5 positive? {isPositive 5}"
IO.println s!"Absolute of -10: {absolute (-10)}"
test/resources/repos/lean4/test_repo/lake-manifest.json
+5
View File
@@ -0,0 +1,5 @@
{"version": "1.1.0",
"packagesDir": ".lake/packages",
"packages": [],
"name": "test_repo",
"lakeDir": ".lake"}
test/resources/repos/lean4/test_repo/lakefile.lean
+9
View File
@@ -0,0 +1,9 @@
import Lake
open Lake DSL
package «test_repo» where
leanOptions := #[⟨`autoImplicit, false⟩]
@[default_target]
lean_lib «Main» where
roots := #[`Main, `Helper]
Loaded 100 of 172 files, more files were not shown because too many files have changed in this diff. Show more