diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..78cd6adc --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: oraios diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..ec4bb386 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..bbcbbe7d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/issue--bug--performance-problem--question-.md b/.github/ISSUE_TEMPLATE/issue--bug--performance-problem--question-.md new file mode 100644 index 00000000..f81029d9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/issue--bug--performance-problem--question-.md @@ -0,0 +1,27 @@ +--- +name: Issue (bug, performance problem, question) +about: General Issue +title: '' +labels: '' +assignees: '' + +--- + +I have: + +- [ ] Read the readme and verified that the issue cannot be solved by adjusting configuration +- [ ] Understood that Serena's dashboard can be disabled through the config +- [ ] Understood that by default a client session will start a separate instance of a Serena server. +- [ ] Understood that for multi-agent setups, the SSE mode should be used. +- [ ] Verified that non-project files are ignored using either gitignore or the corresponding setting in `.serena/project.yml` +- [ ] Have looked for similar issues and discussions, including closed ones +- [ ] Made sure it's an actual issue and not a question - those should be opened as discussion instead. + +If you have encountered a real and new issue: + +- [ ] I performed ` serena project health-check` +- [ ] I indexed the project as described in the readme +- [ ] Added sufficient explanation of my setup: the MCP client, the OS, the programming language, any config adjustments or relevant project specifics +- [ ] Explained how the issue arose, added instructions on how to reproduce it (if possible) +- [ ] If the issue happens on an open source project, I have added the link +- [ ] Wrote a meaningful title and description diff --git a/.github/workflows/junie.yml b/.github/workflows/junie.yml new file mode 100644 index 00000000..9860e79e --- /dev/null +++ b/.github/workflows/junie.yml @@ -0,0 +1,21 @@ +name: Junie +run-name: Junie run ${{ inputs.run_id }} + +permissions: + contents: write + +on: + workflow_dispatch: + inputs: + run_id: + description: "id of workflow process" + required: true + workflow_params: + description: "stringified params" + required: true + +jobs: + call-workflow-passing-data: + uses: jetbrains-junie/junie-workflows/.github/workflows/ej-issue.yml@main + with: + workflow_params: ${{ inputs.workflow_params }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 91dcd610..6261a79e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -27,7 +27,17 @@ jobs: python-version: "${{ matrix.python-version }}" - uses: actions/setup-go@v5 with: - go-version: '>=1.17.0' + go-version: ">=1.17.0" + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20.x' + - name: Ensure cached directory exist before calling cache-related actions + shell: bash + run: | + mkdir -p $HOME/.serena/language_servers/static + mkdir -p $HOME/.cache/go-build + mkdir -p $HOME/go/bin # Add Go bin directory to PATH for this workflow # GITHUB_PATH is a special file that GitHub Actions uses to modify PATH # Writing to this file adds the directory to the PATH for subsequent steps @@ -47,23 +57,245 @@ jobs: if: runner.os != 'Windows' uses: erlef/setup-beam@v1 with: - elixir-version: '1.18.4' - otp-version: '26.1' - - - name: Prepare java - uses: actions/setup-java@v3 - with: - distribution: "temurin" - java-version: "17" + elixir-version: "1.18.4" + otp-version: "26.1" +# Erlang currently not tested in CI, random hangings on macos, always hangs on ubuntu +# In local tests, erlang seems to work though +# - name: Install Erlang Language Server +# if: runner.os != 'Windows' +# shell: bash +# run: | +# # Install rebar3 if not already available +# which rebar3 || (curl -fsSL https://github.com/erlang/rebar3/releases/download/3.23.0/rebar3 -o /tmp/rebar3 && chmod +x /tmp/rebar3 && sudo mv /tmp/rebar3 /usr/local/bin/rebar3) +# # Clone and build erlang_ls +# git clone https://github.com/erlang-ls/erlang_ls.git /tmp/erlang_ls +# cd /tmp/erlang_ls +# make install PREFIX=/usr/local +# # Ensure erlang_ls is in PATH +# echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Install clojure tools uses: DeLaGuardo/setup-clojure@13.4 with: cli: latest + - name: Setup Java (for JVM based languages) + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' - name: Install Terraform uses: hashicorp/setup-terraform@v3 with: terraform_version: "1.5.0" terraform_wrapper: false + # - name: Install swift + # if: runner.os != 'Windows' + # uses: swift-actions/setup-swift@v2 + # Installation of swift with the action screws with installation of ruby on macOS for some reason + # We can try again when version 3 of the action is released, where they will also use swiftly + # Until then, we use custom code to install swift. Sourcekit-lsp is installed automatically with swift + - name: Install Swift with swiftly (macOS) + if: runner.os == 'macOS' + run: | + echo "=== Installing swiftly on macOS ===" + curl -O https://download.swift.org/swiftly/darwin/swiftly.pkg && \ + installer -pkg swiftly.pkg -target CurrentUserHomeDirectory && \ + ~/.swiftly/bin/swiftly init --quiet-shell-followup && \ + . "${SWIFTLY_HOME_DIR:-$HOME/.swiftly}/env.sh" && \ + hash -r + swiftly install --use 6.1.2 + swiftly use 6.1.2 + echo "~/.swiftly/bin" >> $GITHUB_PATH + echo "Swiftly installed successfully" + # Verify sourcekit-lsp is working before proceeding + echo "=== Verifying sourcekit-lsp installation ===" + which sourcekit-lsp || echo "Warning: sourcekit-lsp not found in PATH" + sourcekit-lsp --help || echo "Warning: sourcekit-lsp not responding" + - name: Install Swift with swiftly (Ubuntu) + if: runner.os == 'Linux' + run: | + echo "=== Installing swiftly on Ubuntu ===" + # Install dependencies BEFORE Swift to avoid exit code 1 + sudo apt-get update + sudo apt-get -y install libcurl4-openssl-dev + curl -O https://download.swift.org/swiftly/linux/swiftly-$(uname -m).tar.gz && \ + tar zxf swiftly-$(uname -m).tar.gz && \ + ./swiftly init --quiet-shell-followup && \ + . "${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}/env.sh" && \ + hash -r + swiftly install --use 6.1.2 + swiftly use 6.1.2 + echo "=== Adding Swift toolchain to PATH ===" + echo "$HOME/.local/share/swiftly/bin" >> $GITHUB_PATH + echo "Swiftly installed successfully!" + # Verify sourcekit-lsp is working before proceeding + echo "=== Verifying sourcekit-lsp installation ===" + which sourcekit-lsp || echo "Warning: sourcekit-lsp not found in PATH" + sourcekit-lsp --help || echo "Warning: sourcekit-lsp not responding" + - name: Install Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.4' + - name: Install Ruby language server + shell: bash + run: gem install ruby-lsp + - name: Install R + uses: r-lib/actions/setup-r@v2 + with: + r-version: '4.4.2' + use-public-rspm: true + - name: Install R language server + shell: bash + run: | + Rscript -e "install.packages('languageserver', repos='https://cloud.r-project.org')" + - name: Set up Julia + uses: julia-actions/setup-julia@v2 + with: + version: '1.10' + - name: Install Julia LanguageServer + shell: bash + run: julia -e 'using Pkg; Pkg.add("LanguageServer")' + - name: Install Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.14.1 + - name: Install ZLS (Zig Language Server) + shell: bash + run: | + if [[ "${{ runner.os }}" == "Linux" ]]; then + wget https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-linux.tar.xz + tar -xf zls-x86_64-linux.tar.xz + sudo mv zls /usr/local/bin/ + rm zls-x86_64-linux.tar.xz + elif [[ "${{ runner.os }}" == "macOS" ]]; then + wget https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-macos.tar.xz + tar -xf zls-x86_64-macos.tar.xz + sudo mv zls /usr/local/bin/ + rm zls-x86_64-macos.tar.xz + elif [[ "${{ runner.os }}" == "Windows" ]]; then + curl -L -o zls.zip https://github.com/zigtools/zls/releases/download/0.14.0/zls-x86_64-windows.zip + unzip -o zls.zip + mkdir -p "$HOME/bin" + mv zls.exe "$HOME/bin/" + echo "$HOME/bin" >> $GITHUB_PATH + rm zls.zip + fi + - name: Install Lua Language Server + shell: bash + run: | + LUA_LS_VERSION="3.15.0" + LUA_LS_DIR="$HOME/.serena/language_servers/lua" + mkdir -p "$LUA_LS_DIR" + + if [[ "${{ runner.os }}" == "Linux" ]]; then + if [[ "$(uname -m)" == "x86_64" ]]; then + wget https://github.com/LuaLS/lua-language-server/releases/download/${LUA_LS_VERSION}/lua-language-server-${LUA_LS_VERSION}-linux-x64.tar.gz + tar -xzf lua-language-server-${LUA_LS_VERSION}-linux-x64.tar.gz -C "$LUA_LS_DIR" + else + wget https://github.com/LuaLS/lua-language-server/releases/download/${LUA_LS_VERSION}/lua-language-server-${LUA_LS_VERSION}-linux-arm64.tar.gz + tar -xzf lua-language-server-${LUA_LS_VERSION}-linux-arm64.tar.gz -C "$LUA_LS_DIR" + fi + chmod +x "$LUA_LS_DIR/bin/lua-language-server" + # Create wrapper script instead of symlink to ensure supporting files are found + echo '#!/bin/bash' | sudo tee /usr/local/bin/lua-language-server > /dev/null + echo 'cd "${HOME}/.serena/language_servers/lua/bin"' | sudo tee -a /usr/local/bin/lua-language-server > /dev/null + echo 'exec ./lua-language-server "$@"' | sudo tee -a /usr/local/bin/lua-language-server > /dev/null + sudo chmod +x /usr/local/bin/lua-language-server + rm lua-language-server-*.tar.gz + elif [[ "${{ runner.os }}" == "macOS" ]]; then + if [[ "$(uname -m)" == "x86_64" ]]; then + wget https://github.com/LuaLS/lua-language-server/releases/download/${LUA_LS_VERSION}/lua-language-server-${LUA_LS_VERSION}-darwin-x64.tar.gz + tar -xzf lua-language-server-${LUA_LS_VERSION}-darwin-x64.tar.gz -C "$LUA_LS_DIR" + else + wget https://github.com/LuaLS/lua-language-server/releases/download/${LUA_LS_VERSION}/lua-language-server-${LUA_LS_VERSION}-darwin-arm64.tar.gz + tar -xzf lua-language-server-${LUA_LS_VERSION}-darwin-arm64.tar.gz -C "$LUA_LS_DIR" + fi + chmod +x "$LUA_LS_DIR/bin/lua-language-server" + # Create wrapper script instead of symlink to ensure supporting files are found + echo '#!/bin/bash' | sudo tee /usr/local/bin/lua-language-server > /dev/null + echo 'cd "${HOME}/.serena/language_servers/lua/bin"' | sudo tee -a /usr/local/bin/lua-language-server > /dev/null + echo 'exec ./lua-language-server "$@"' | sudo tee -a /usr/local/bin/lua-language-server > /dev/null + sudo chmod +x /usr/local/bin/lua-language-server + rm lua-language-server-*.tar.gz + elif [[ "${{ runner.os }}" == "Windows" ]]; then + curl -L -o lua-ls.zip https://github.com/LuaLS/lua-language-server/releases/download/${LUA_LS_VERSION}/lua-language-server-${LUA_LS_VERSION}-win32-x64.zip + unzip -o lua-ls.zip -d "$LUA_LS_DIR" + # For Windows, we'll add the bin directory directly to PATH + # The lua-language-server.exe can find its supporting files relative to its location + echo "$LUA_LS_DIR/bin" >> $GITHUB_PATH + rm lua-ls.zip + fi + - name: Install Perl::LanguageServer + if: runner.os != 'Windows' + shell: bash + run: | + if [[ "${{ runner.os }}" == "Linux" ]]; then + sudo apt-get update + sudo apt-get install -y cpanminus build-essential libanyevent-perl libio-aio-perl + elif [[ "${{ runner.os }}" == "macOS" ]]; then + brew install cpanminus + fi + PERL_MM_USE_DEFAULT=1 cpanm --notest --force Perl::LanguageServer + # Set up Perl local::lib environment for subsequent steps + echo "PERL5LIB=$HOME/perl5/lib/perl5${PERL5LIB:+:${PERL5LIB}}" >> $GITHUB_ENV + echo "PERL_LOCAL_LIB_ROOT=$HOME/perl5${PERL_LOCAL_LIB_ROOT:+:${PERL_LOCAL_LIB_ROOT}}" >> $GITHUB_ENV + echo "PERL_MB_OPT=--install_base \"$HOME/perl5\"" >> $GITHUB_ENV + echo "PERL_MM_OPT=INSTALL_BASE=$HOME/perl5" >> $GITHUB_ENV + echo "$HOME/perl5/bin" >> $GITHUB_PATH + - name: Install Elm + shell: bash + run: npm install -g elm@0.19.1-6 + - name: Install Nix + if: runner.os != 'Windows' # Nix doesn't support Windows natively + uses: cachix/install-nix-action@v30 + with: + nix_path: nixpkgs=channel:nixos-unstable + - name: Install nixd (Nix Language Server) + if: runner.os != 'Windows' # Skip on Windows since Nix isn't available + shell: bash + run: | + # Install nixd using nix + nix profile install github:nix-community/nixd + + # Verify nixd is installed and working + if ! command -v nixd &> /dev/null; then + echo "nixd installation failed or not in PATH" + exit 1 + fi + + echo "$HOME/.nix-profile/bin" >> $GITHUB_PATH + - name: Verify Nix package build + if: runner.os != 'Windows' # Nix only supported on Linux/macOS + shell: bash + run: | + # Verify the flake builds successfully + nix build --no-link + - name: Install Regal (Rego Language Server) + shell: bash + run: | + REGAL_VERSION="0.36.1" + + if [[ "${{ runner.os }}" == "Linux" ]]; then + if [[ "$(uname -m)" == "x86_64" ]]; then + curl -L -o regal https://github.com/StyraInc/regal/releases/download/v${REGAL_VERSION}/regal_Linux_x86_64 + else + curl -L -o regal https://github.com/StyraInc/regal/releases/download/v${REGAL_VERSION}/regal_Linux_arm64 + fi + chmod +x regal + sudo mv regal /usr/local/bin/ + elif [[ "${{ runner.os }}" == "macOS" ]]; then + if [[ "$(uname -m)" == "x86_64" ]]; then + curl -L -o regal https://github.com/StyraInc/regal/releases/download/v${REGAL_VERSION}/regal_Darwin_x86_64 + else + curl -L -o regal https://github.com/StyraInc/regal/releases/download/v${REGAL_VERSION}/regal_Darwin_arm64 + fi + chmod +x regal + sudo mv regal /usr/local/bin/ + elif [[ "${{ runner.os }}" == "Windows" ]]; then + curl -L -o regal.exe https://github.com/StyraInc/regal/releases/download/v${REGAL_VERSION}/regal_Windows_x86_64.exe + mkdir -p "$HOME/bin" + mv regal.exe "$HOME/bin/" + echo "$HOME/bin" >> $GITHUB_PATH + fi - name: Install uv shell: bash run: curl -LsSf https://astral.sh/uv/install.sh | sh @@ -90,6 +322,9 @@ jobs: - name: Install dependencies shell: bash run: uv pip install -e ".[dev]" + - name: Check formatting + shell: bash + run: uv run poe lint - name: Test with pytest shell: bash run: uv run poe test diff --git a/.gitignore b/.gitignore index 64c7e71e..32386b39 100644 --- a/.gitignore +++ b/.gitignore @@ -227,3 +227,25 @@ tmp/ /test/resources/repos/elixir/test_repo/deps # Exception: Don't ignore Elixir test repository lib directory (contains source code) !/test/resources/repos/elixir/test_repo/lib + +# Exception: Don't ignore Nix test repository lib directory (contains source code) +!/test/resources/repos/nix/test_repo/lib + +# Exception: Don't ignore Julia test repository lib directory (contains source code) +!/test/resources/repos/julia/test_repo/lib + +# Swift +/test/resources/repos/swift/test_repo/.build +/test/resources/repos/swift/test_repo/.swiftpm + +# Elm +/test/resources/repos/elm/test_repo/.elm/ +/test/resources/repos/elm/test_repo/elm-stuff/ + +# Scala +.metals/ +.bsp/ +.scala-build/ +.bloop/ +bootstrap +test/resources/repos/scala/.bloop/ \ No newline at end of file diff --git a/.serena/memories/adding_new_language_support_guide.md b/.serena/memories/adding_new_language_support_guide.md index d536dcf3..21980531 100644 --- a/.serena/memories/adding_new_language_support_guide.md +++ b/.serena/memories/adding_new_language_support_guide.md @@ -5,6 +5,7 @@ This guide explains how to add support for a new programming language to Serena. ## Overview Adding a new language involves: + 1. **Language Server Implementation** - Creating a language-specific server class 2. **Language Registration** - Adding the language to enums and configurations 3. **Test Repository** - Creating a minimal test project @@ -154,6 +155,7 @@ test/resources/repos/new_language/test_repo/ ### 3.2 Example Source Files Create meaningful source files that demonstrate: + - **Classes/Types** - For symbol testing - **Functions/Methods** - For reference finding - **Imports/Dependencies** - For cross-file operations @@ -184,9 +186,22 @@ class Program { ## Step 4: Test Suite +Testing the language server implementation is of crucial importance, and the tests will +form the main part of the review process. Make sure that the tests are up to the standard +of Serena to make the review go smoother. + +General rules for tests: + +1. Tests for symbols and references should always check that the expected symbol names and references were actually found. + Just testing that a list came back or that the result is not None is insufficient. +2. Tests should never be skipped, the only exception is skipping based on some package being available or on an unsupported OS. +3. Tests should run in CI, check if there is a suitable GitHub action for installing the dependencies. + ### 4.1 Basic Tests -Create `test/solidlsp/new_language/test_new_language_basic.py`. You should at least test: +Create `test/solidlsp/new_language/test_new_language_basic.py`. +Have a look at the structure of existing tests, for example, in `test/solidlsp/php/test_php_basic.py` +You should at least test: 1. Finding symbols 2. Finding within-file references @@ -203,6 +218,7 @@ Consider adding new cases to the parametrized tests in `test_serena_agent.py` fo ### 5 Documentation Update: + - **README.md** - Add language to supported languages list - **CHANGELOG.md** - Document the new language support - **Language-specific docs** - Installation requirements, known issues diff --git a/.serena/project.yml b/.serena/project.yml index 81a17173..a265810e 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -33,7 +33,7 @@ read_only: false # * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type). # * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). # * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. -# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file or directory. +# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file. # * `initial_instructions`: Gets the initial instructions for the current project. # Should only be used in settings where the system prompt cannot be set, # e.g. in clients you have no control over, like Claude Desktop. diff --git a/CHANGELOG.md b/CHANGELOG.md index 89c1b190..91963c29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,75 @@ -# Latest - +# latest Status of the `main` branch. Changes prior to the next official version change will appear here. +* Tools: + * Added `RenameSymbolTool` for renaming symbols across the codebase (if LS supports this operation). + +* Language support: + * **Add support for Scala** via Metals language server (requires some [manual setup](docs/scala_setup_guide_for_serena.md)) + * **Add support for Elm** via @elm-tooling/elm-language-server (automatically downloads if not installed; requires Elm compiler) + * **Add support for Perl** via Perl::LanguageServer with LSP integration for .pl, .pm, and .t files + * **Add support for AL (Application Language)** for Microsoft Dynamics 365 Business Central development. Requires VS Code AL extension (ms-dynamics-smb.al). + * **Add support for R** via the R languageserver package with LSP integration, performance optimizations, and fallback symbol extraction + * **Add support for Zig** via ZLS (cross-file references may not fully work on Windows) + * **Add support for Lua** via lua-language-server + * **Add support for Nix** requires nixd installation (Windows not supported) + * **Dart now officially supported**: Dart was always working, but now tests were added, and it is promoted to "officially supported" + * **Rust now uses already installed rustup**: The rust-analyzer is no longer bundled with Serena. Instead, it uses the rust-analyzer from your Rust toolchain managed by rustup. This ensures compatibility with your Rust version and eliminates outdated bundled binaries. + * **Kotlin now officially supported**: We now use the official Kotlin LS, tests run through and performance is good, even though the LS is in an early development stage. + * **Add support for Erlang** experimental, may hang or be slow, uses the recently archived [erlang_ls](https://github.com/erlang-ls/erlang_ls) + * **Ruby dual language server support**: Added ruby-lsp as the modern primary Ruby language server. Solargraph remains available as an experimental legacy option. ruby-lsp supports both .rb and .erb files, while Solargraph supports .rb files only. + +* Client support: + * New mode `oaicompat-agent` and extensions in the openai tool compatibility, **permitting Serena to work with llama.cpp** + +* General: + * Various fixes related to indexing, special paths and determation of ignored paths + * Decreased `TOOL_DEFAULT_MAX_ANSWER_LENGTH` to be in accordance with (below) typical max-tokens configurations + * Allow passing language server specific settings through `ls_specific_settings` field (in `serena_config.yml`) + +# 0.1.4 + +## Summary + +This likely is the last release before the stable version 1.0.0 which will come together with the jetbrains IDE extension. +We release it for users who install Serena from a tag, since the last tag cannot be installed due to a breaking change in the mcp dependency (see #381). + +Since the last release, several new languages were supported, and the Serena CLI and configurability were significantly extended. +We thank all external contributors who made a lot of the improvements possible! + +* General: + * **Initial instructions no longer need to be loaded by the user** + * Significantly extended CLI + * Removed `replace_regex` tool from `ide-assistant` and `codex` contexts. + The current string replacement tool in Claude Code seems to be sufficiently efficient and is better + integrated with the IDE. Users who want to enable `replace_regex` can do so by customizing the context. + +* Configuration: + * Simplify customization of modes and contexts, including CLI support. + * Possibility to customize the system prompt and outputs of simple tools, including CLI support. + * Possibility to override tool descriptions through the context YAML. + * Prompt templates are now automatically adapted to the enabled tools. + * Several tools are now excluded by default, need to be included explicitly. + * New context for ChatGPT + +* Language servers: + * Reliably detect language server termination and propagate the respective error all the way + back to the tool application, where an unexpected termination is handled by restarting the language server + and subsequently retrying the tool application. + * **Add support for Swift** + * **Add support for Bash** + * Enhance Solargraph (Ruby) integration + * Automatic Rails project detection via config/application.rb, Rakefile, and Gemfile analysis + * Ruby/Rails-specific exclude patterns for improved indexing performance (vendor/, .bundle/, tmp/, log/, coverage/) + * Enhanced error handling with detailed diagnostics and Ruby manager-specific installation instructions (rbenv, RVM, asdf) + * Improved LSP capability negotiation and analysis completion detection + * Better Bundler and Solargraph installation error messages with clear resolution steps + Fixes: * Ignore `.git` in check for ignored paths and improve performance of `find_all_non_ignored_files` +* Fix language server startup issues on Windows when using Claude Code (which was due to + default shell reconfiguration imposed by Claude Code) +* Additional wait for initialization in C# language server before requesting references, allowing cross-file references to be found. # 0.1.3 diff --git a/CLAUDE.md b/CLAUDE.md index 9998e95b..edf91cd1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co **Test Markers:** Available pytest markers for selective testing: -- `python`, `go`, `java`, `rust`, `typescript`, `php`, `csharp`, `elixir`, `terraform`, `clojure` +- `python`, `go`, `java`, `rust`, `typescript`, `php`, `perl`, `csharp`, `elixir`, `terraform`, `clojure`, `swift`, `bash`, `ruby`, `ruby_solargraph` - `snapshot` - for symbolic editing operation tests **Project Management:** @@ -100,7 +100,7 @@ Configuration is loaded from (in order of precedence): - **Symbol-based editing** - Uses LSP for precise code manipulation - **Caching strategy** - Reduces language server overhead - **Error recovery** - Automatic language server restart on crashes -- **Multi-language support** - 13+ languages with LSP integration +- **Multi-language support** - 16+ languages with LSP integration - **MCP protocol** - Exposes tools to AI agents via Model Context Protocol - **Async operation** - Non-blocking language server interactions diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e75556b..f3fbe375 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ Serena is under active development. We are just discovering what it can do and where the limitations lie. -Feel free to share your learnings by opening open new issues, feature requests and extensions. +Feel free to share your learnings by opening new issues, feature requests and extensions. ## Developer Environment Setup @@ -21,30 +21,10 @@ You can install a virtual environment with the required as follows 1. Create a new virtual environment: `uv venv` 2. Activate the environment: - * On Linux/Unix/macOS: `source .venv/bin/activate` - * On Windows: `.venv\Scripts\activate.bat` (in cmd/ps) or `source .venv/Scripts/activate` (in git-bash) + * On Linux/Unix/macOS or Windows with Git Bash: `source .venv/bin/activate` + * On Windows outside of Git Bash: `.venv\Scripts\activate.bat` (in cmd/ps) or `source .venv/Scripts/activate` (in git-bash) 3. Install the required packages with all extras: `uv pip install --all-extras -r pyproject.toml -e .` -### Docker setup - -Build the docker image with - -```shell -docker build -t serena . -``` - -and run it with the repository mounted as a volume: - -```shell -docker run -it --rm -v "$(pwd)":/workspace serena -``` - -You can also just run `bash docker_build_and_run.sh`, which will do both things -for you. - -Note: For the Windows subsystem for Linux (WSL), you may need to adjust the path for the -volume. - ## Running Tools Locally The Serena tools (and in fact all Serena code) can be executed without an LLM, and also without @@ -54,19 +34,4 @@ An example script for running tools is provided in [scripts/demo_run_tools.py](s ## Adding a New Supported Language -Serena interacts with code through language servers which are included in -the `solidlsp` package. It is rather easy to include a new supported language -if an LSP implementation for it exists. You just need to: - -1. create a new subclass of `SolidLanguageServer` -2. add a new value to the `Language` enum -3. make a new `elif` case in the `SolidLanguageServer.create` method -4. add a test repo of the new language to `test/resources/repos//test_repo` - and new tests in `test/solidlsp/`. Similar to existing tests for other languages -5. also add a new case to the parameterized tests in `test/serena/test_serena_agent` - -The subclasses are typically easy to write, have a look at the -[PyrightLanguageServer](src/multilspy/language_servers/pyright_language_server/pyright_server.py) -for an example, or at any other implementation to see how non-python -dependencies for language servers are handled there. -There are also some tips from the multilspy admin [here](https://github.com/microsoft/multilspy/issues/5). +See the corresponding [memory](.serena/memories/adding_new_language_support_guide.md). \ No newline at end of file diff --git a/DOCKER.md b/DOCKER.md index 5ee8829c..84f8f746 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -68,6 +68,8 @@ SERENA_DASHBOARD_PORT=8080 docker-compose up serena docker-compose up serena-dev ``` +Note: Edit the `compose.yaml` file to customize volume mounts for your projects. + ### Using Docker directly ```bash @@ -83,6 +85,24 @@ docker run -it --rm \ serena ``` +### Using Docker Compose with Merge Compose files + +To use Docker Compose with merge files, you can create a `compose.override.yml` file to customize the configuration: + +```yaml +services: + serena: + # To work with projects, you must mount them as volumes: + volumes: + - ./my-project:/workspace/my-project + - /path/to/another/project:/workspace/another-project + # Add the context for the IDE assistant option: + command: + - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide-assistant" +``` + +See the [Docker Merge Compose files documentation](https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/) for more details on using merge files. + ## Accessing the Dashboard Once running, access the web dashboard at: @@ -105,6 +125,7 @@ volumes: - `SERENA_DOCKER=1`: Set automatically to indicate Docker environment - `SERENA_PORT`: MCP server port (default: 9121) - `SERENA_DASHBOARD_PORT`: Web dashboard port (default: 24282) +- `INTELEPHENSE_LICENSE_KEY`: License key for Intelephense PHP LSP premium features (optional) ## Troubleshooting diff --git a/Dockerfile b/Dockerfile index d88d4e26..d84ed7d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,27 +3,53 @@ FROM python:3.11-slim AS base SHELL ["/bin/bash", "-c"] # Set environment variables to make Python print directly to the terminal and avoid .pyc files. -ENV PYTHONUNBUFFERED 1 -ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 # Install system dependencies required for package manager and build tools. +# sudo, wget, zip needed for some assistants, like junie RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ build-essential \ git \ ssh \ + sudo \ + wget \ + zip \ + unzip \ + git \ && rm -rf /var/lib/apt/lists/* # Install pipx. RUN python3 -m pip install --no-cache-dir pipx \ && pipx ensurepath +# Install nodejs +ENV NVM_VERSION=0.40.3 +ENV NODE_VERSION=22.18.0 +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh | bash +# standard location +ENV NVM_DIR=/root/.nvm +RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} +ENV PATH="${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH}" + # Add local bin to the path ENV PATH="${PATH}:/root/.local/bin" # Install the latest version of uv RUN curl -LsSf https://astral.sh/uv/install.sh | sh +# Install Rust and rustup for rust-analyzer support (minimal profile) +ENV RUSTUP_HOME=/usr/local/rustup +ENV CARGO_HOME=/usr/local/cargo +ENV PATH="${CARGO_HOME}/bin:${PATH}" +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + --default-toolchain stable \ + --profile minimal \ + && rustup component add rust-analyzer + # Set the working directory WORKDIR /workspaces/serena diff --git a/README.md b/README.md index 788ac9b6..8e33859f 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,12 @@

* :rocket: Serena is a powerful **coding agent toolkit** capable of turning an LLM into a fully-featured agent that works **directly on your codebase**. + Unlike most other tools, it is not tied to an LLM, framework or an interface, making it easy to use it in a variety of ways. * :wrench: Serena provides essential **semantic code retrieval and editing tools** that are akin to an IDE's capabilities, extracting code entities at the symbol level and exploiting relational structure. When combined with an existing coding agent, these tools greatly enhance (token) efficiency. * :free: Serena is **free & open-source**, enhancing the capabilities of LLMs you already have access to free of charge. -### Demonstration - -Here is a demonstration of Serena implementing a small feature for itself (a better log GUI) with Claude Desktop. -Note how Serena's tools enable Claude to find and edit the right symbols. - -https://github.com/user-attachments/assets/6eaa9aa1-610d-4723-a2d6-bf1e487ba753 +You can think of Serena as providing IDE-like tools to your LLM/coding agent. With it, the agent no longer needs to read entire +files, perform grep-like searches or string replacements to find and edit the right code. Instead, it can use code centered tools like `find_symbol`, `find_referencing_symbols` and `insert_after_symbol`.

Serena is under active development! See the latest updates, upcoming features, and lessons learned to stay up to date. @@ -30,63 +27,104 @@ https://github.com/user-attachments/assets/6eaa9aa1-610d-4723-a2d6-bf1e487ba753

- - ### LLM Integration -Serena provides the necessary [tools](#full-list-of-tools) for coding workflows, but an LLM is required to do the actual work, +Serena provides the necessary [tools](#list-of-tools) for coding workflows, but an LLM is required to do the actual work, orchestrating tool use. For example, **supercharge the performance of Claude Code** with a [one-line shell command](#claude-code). -Serena can be integrated with an LLM in several ways: - * by using the **model context protocol (MCP)**. - Serena provides an MCP server which integrates with - * Claude Code and Claude Desktop, - * IDEs like VSCode, Cursor or IntelliJ, - * Extensions like Cline or Roo Code - * and many others, including [the ChatGPT app soon](https://x.com/OpenAIDevs/status/1904957755829481737) - * by using **Agno – the model-agnostic agent framework**. - Serena's Agno-based agent allows you to turn virtually any LLM into a coding agent, whether it's provided by Google, OpenAI or Anthropic (with a paid API key) - or a free model provided by Ollama, Together or Anyscale. - * by incorporating Serena's tools into an agent framework of your choice. - Serena's tool implementation is decoupled from the framework-specific code and can thus easily be adapted to any agent framework. +In general, Serena can be integrated with an LLM in several ways: + +* by using the **model context protocol (MCP)**. + Serena provides an MCP server which integrates with + * Claude Code and Claude Desktop, + * Terminal-based clients like Codex, Gemini-CLI, Qwen3-Coder, rovodev, OpenHands CLI and others, + * IDEs like VSCode, Cursor or IntelliJ, + * Extensions like Cline or Roo Code + * Local clients like [OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp), [Jan](https://jan.ai/docs/mcp-examples/browser/browserbase#enable-mcp), [Agno](https://docs.agno.com/introduction/playground) and others +* by using [mcpo to connect it to ChatGPT](docs/serena_on_chatgpt.md) or other clients that don't support MCP but do support tool calling via OpenAPI. +* by incorporating Serena's tools into an agent framework of your choice, as illustrated [here](docs/custom_agent.md). + Serena's tool implementation is decoupled from the framework-specific code and can thus easily be adapted to any agent framework. + +### Serena in Action + +#### Demonstration 1: Efficient Operation in Claude Code + +A demonstration of Serena efficiently retrieving and editing code within Claude Code, thereby saving tokens and time. Efficient operations are not only useful for saving costs, but also for generally improving the generated code's quality. This effect may be less pronounced in very small projects, but often becomes of crucial importance in larger ones. + +https://github.com/user-attachments/assets/ab78ebe0-f77d-43cc-879a-cc399efefd87 + +#### Demonstration 2: Serena in Claude Desktop + +A demonstration of Serena implementing a small feature for itself (a better log GUI) with Claude Desktop. +Note how Serena's tools enable Claude to find and edit the right symbols. + +https://github.com/user-attachments/assets/6eaa9aa1-610d-4723-a2d6-bf1e487ba753 ### Programming Language Support & Semantic Analysis Capabilities Serena's semantic code analysis capabilities build on **language servers** using the widely implemented language server protocol (LSP). The LSP provides a set of versatile code querying -and editing functionalities based on symbolic understanding of the code. -Equipped with these capabilities, Serena discovers and edits code just like a seasoned developer +and editing functionalities based on symbolic understanding of the code. +Equipped with these capabilities, Serena discovers and edits code just like a seasoned developer making use of an IDE's capabilities would. Serena can efficiently find the right context and do the right thing even in very large and -complex projects! So not only is it free and open-source, it frequently achieves better results +complex projects! So not only is it free and open-source, it frequently achieves better results than existing solutions that charge a premium. Language servers provide support for a wide range of programming languages. -With Serena, we provide - * direct, out-of-the-box support for: - * Python - * TypeScript/Javascript (currently has some instability issues, we are working on it) - * PHP - * Go (need to install go and gopls first) - * Rust - * C# (requires dotnet to be installed. We switched the underlying language server recently, please report any issues you encounter) - * Java (_Note_: startup is slow, initial startup especially so. There may be issues with java on macos and linux, we are working on it.) - * Elixir (Requires NextLS and Elixir install; **Windows not supported** - Next LS does not provide Windows binaries) - * Clojure - * C/C++ (You may experience issues with finding references, we are working on it) - * indirect support (may require some code changes/manual installation) for: - * Ruby (untested) - * Kotlin (untested) - * Dart (untested) - - These languages are supported by the language server library, but - we did not explicitly test whether the support for these languages actually works flawlessly. - -Further languages can, in principle, easily be supported by providing a shallow adapter for a new language server -implementation. +With Serena, we provide direct, out-of-the-box support for: + * Python + * TypeScript/Javascript + * PHP (uses Intelephense LSP; set `INTELEPHENSE_LICENSE_KEY` environment variable for premium features) + * Go (requires installation of gopls) + * R (requires installation of the `languageserver` R package) + * Rust (requires [rustup](https://rustup.rs/) - uses rust-analyzer from your toolchain) + * C/C++ (you may experience issues with finding references, we are working on it) + * Zig (requires installation of ZLS - Zig Language Server) + * C# + * Ruby (by default, uses [ruby-lsp](https://github.com/Shopify/ruby-lsp), specify ruby_solargraph as your language to use the previous solargraph based implementation) + * Swift + * Kotlin (uses the pre-alpha [official kotlin LS](https://github.com/Kotlin/kotlin-lsp), some issues may appear) + * Java (_Note_: startup is slow, initial startup especially so. There may be issues with java on macos and linux, we are working on it.) + * Clojure + * Dart + * Bash + * Lua (automatically downloads lua-language-server if not installed) + * Nix (requires nixd installation) + * Elixir (requires installation of NextLS and Elixir; **Windows not supported**) + * Elm (automatically downloads elm-language-server if not installed; requires Elm compiler) + * Scala (requires some [manual setup](docs/scala_setup_guide_for_serena.md); uses Metals LSP) + * Erlang (requires installation of beam and [erlang_ls](https://github.com/erlang-ls/erlang_ls), experimental, might be slow or hang) + * Perl (requires installation of Perl::LanguageServer) + * AL + * Markdown (must be explicitly specified via `--language markdown` when generating project config, primarily useful for documentation-heavy projects) + +Support for further languages can easily be added by providing a shallow adapter for a new language server implementation, +see Serena's [memory on that](.serena/memories/adding_new_language_support_guide.md). + +### Community Feedback + +Most users report that Serena has strong positive effects on the results of their coding agents, even when used within +very capable agents like Claude Code. Serena is often described to be a [game changer](https://www.reddit.com/r/ClaudeAI/comments/1lfsdll/try_out_serena_mcp_thank_me_later/), providing an enormous [productivity boost](https://www.reddit.com/r/ClaudeCode/comments/1mguoia/absolutely_insane_improvement_of_claude_code). + +Serena excels at navigating and manipulating complex codebases, providing tools that support precise code retrieval and editing in the presence of large, strongly structured codebases. +However, when dealing with tasks that involve only very few/small files, you may not benefit from including Serena on top of your existing coding agent. +In particular, when writing code from scratch, Serena will not provide much value initially, as the more complex structures that Serena handles more gracefully than simplistic, file-based approaches are yet to be created. + +Several videos and blog posts have talked about Serena: + +* YouTube: + * [AI Labs](https://www.youtube.com/watch?v=wYWyJNs1HVk&t=1s) + * [Yo Van Eyck](https://www.youtube.com/watch?v=UqfxuQKuMo8&t=45s) + * [JeredBlu](https://www.youtube.com/watch?v=fzPnM3ySmjE&t=32s) + +* Blog posts: + * [Serena's Design Principles](https://medium.com/@souradip1000/deconstructing-serenas-mcp-powered-semantic-code-understanding-architecture-75802515d116) + * [Serena with Claude Code (in Japanese)](https://blog.lai.so/serena/) + * [Turning Claude Code into a Development Powerhouse](https://robertmarshall.dev/blog/turning-claude-code-into-a-development-powerhouse/) ## Table of Contents @@ -95,23 +133,23 @@ implementation. -- [What Can I Use Serena For?](#what-can-i-use-serena-for) -- [Free Coding Agents with Serena](#free-coding-agents-with-serena) - [Quick Start](#quick-start) * [Running the Serena MCP Server](#running-the-serena-mcp-server) + [Usage](#usage) - * [Local Installation](#local-installation) - - [Using uvx](#using-uvx) - - [Using Docker (Experimental)](#using-docker-experimental) - + [SSE Mode](#sse-mode) + + [Using uvx](#using-uvx) + + [Local Installation](#local-installation) + + [Using Docker (Experimental)](#using-docker-experimental) + + [Using Nix](#using-nix) + + [Streamable HTTP Mode](#streamable-http-mode) + [Command-Line Arguments](#command-line-arguments) * [Configuration](#configuration) * [Project Activation & Indexing](#project-activation--indexing) * [Claude Code](#claude-code) + * [Codex](#codex) + * [Other Terminal-Based Clients](#other-terminal-based-clients) * [Claude Desktop](#claude-desktop) - * [Other MCP Clients (Cline, Roo-Code, Cursor, Windsurf, etc.)](#other-mcp-clients-cline-roo-code-cursor-windsurf-etc) - * [Agno Agent](#agno-agent) - * [Other Agent Frameworks](#other-agent-frameworks) + * [MCP Coding Clients (Cline, Roo-Code, Cursor, Windsurf, etc.)](#mcp-coding-clients-cline-roo-code-cursor-windsurf-etc) + * [Local GUIs and Frameworks](#local-guis-and-frameworks) - [Detailed Usage and Recommendations](#detailed-usage-and-recommendations) * [Tool Execution](#tool-execution) + [Shell Execution and Editing Tools](#shell-execution-and-editing-tools) @@ -125,59 +163,34 @@ implementation. + [Start from a Clean State](#start-from-a-clean-state) + [Logging, Linting, and Automated Tests](#logging-linting-and-automated-tests) * [Prompting Strategies](#prompting-strategies) - * [Potential Issues in Code Editing](#potential-issues-in-code-editing) * [Running Out of Context](#running-out-of-context) - * [Combining Serena with Other MCP Servers](#combining-serena-with-other-mcp-servers) * [Serena's Logs: The Dashboard and GUI Tool](#serenas-logs-the-dashboard-and-gui-tool) - * [Troubleshooting](#troubleshooting) + * [Serena and GIT worktrees](#serena-and-git-worktrees) - [Comparison with Other Coding Agents](#comparison-with-other-coding-agents) * [Subscription-Based Coding Agents](#subscription-based-coding-agents) * [API-Based Coding Agents](#api-based-coding-agents) * [Other MCP-Based Coding Agents](#other-mcp-based-coding-agents) - [Acknowledgements](#acknowledgements) + * [Sponsors](#sponsors) + * [Community Contributions](#community-contributions) + * [Technologies](#technologies) - [Customizing and Extending Serena](#customizing-and-extending-serena) -- [Full List of Tools](#full-list-of-tools) +- [List of Tools](#list-of-tools) -## What Can I Use Serena For? - -You can use Serena for any coding tasks – whether it is focussed on analysis, planning, -designing new components or refactoring existing ones. -Since Serena's tools allow an LLM to close the cognitive perception-action loop, -agents based on Serena can autonomously carry out coding tasks from start to finish – -from the initial analysis to the implementation, testing and, finally, the version -control system commit. - -Serena can read, write and execute code, read logs and the terminal output. -While we do not necessarily encourage it, "vibe coding" is certainly possible, and if you -want to almost feel like "the code no longer exists", -you may find Serena even more adequate for vibing than an agent inside an IDE -(since you will have a separate GUI that really lets you forget). - -## Free Coding Agents with Serena - -Even the free tier of Anthropic's Claude has support for MCP Servers, so you can use Serena with Claude for free. -Presumably, the same will soon be possible with ChatGPT Desktop once support for MCP servers is added. -Through Agno, you furthermore have the option to use Serena with a free/open-weights model. - -Serena is [Oraios AI](https://oraios-ai.de/)'s contribution to the developer community. -We use it ourselves on a regular basis. - -We got tired of having to pay multiple -IDE-based subscriptions (such as Windsurf or Cursor) that forced us to keep purchasing tokens on top of the chat subscription costs we already had. -The substantial API costs incurred by tools like Claude Code, Cline, Aider and other API-based tools are similarly unattractive. -We thus built Serena with the prospect of being able to cancel most other subscriptions. - ## Quick Start Serena can be used in various ways, below you will find instructions for selected integrations. -- If you just want to turn Claude into a free-to-use coding agent, we recommend using Serena through [Claude Code](#claude-code) or [Claude Desktop](#claude-desktop). -- If you want to use Gemini or any other model, and you want a GUI experience, you can use [Agno](#agno-agent) or one of the many other GUIs that support MCP servers. -- If you want to use Serena integrated in your IDE, see the section on [other MCP clients](#other-mcp-clients---cline-roo-code-cursor-windsurf-etc). +* For coding with Claude, we recommend using Serena through [Claude Code](#claude-code) or [Claude Desktop](#claude-desktop). You can also use Serena in most other [terminal-based clients](#other-terminal-based-clients). +* If you want a GUI experience outside an IDE, you can use one of the many [local GUIs](#local-guis-and-frameworks) that support MCP servers. + You can also connect Serena to many web clients (including ChatGPT) using [mcpo](docs/serena_on_chatgpt.md). +* If you want to use Serena integrated in your IDE, see the section on [other MCP clients](#other-mcp-clients---cline-roo-code-cursor-windsurf-etc). +* You can use Serena as a library for building your own applications. We try to keep the public API stable, but you should still + expect breaking changes and pin Serena to a fixed version if you use it as a dependency. -Serena is managed by `uv`, so you will need to [install it](https://docs.astral.sh/uv/getting-started/installation/)). +Serena is managed by `uv`, so you will need to [install it](https://docs.astral.sh/uv/getting-started/installation/). ### Running the Serena MCP Server @@ -186,56 +199,54 @@ You have several options for running the MCP server, which are explained in the #### Usage The typical usage involves the client (Claude Code, Claude Desktop, etc.) running -the MCP server as a subprocess (using stdio communication), +the MCP server as a subprocess (using stdio communication), so the client needs to be provided with the command to run the MCP server. -(Alternatively, you can run the MCP server in SSE mode and tell your client +(Alternatively, you can run the MCP server in Streamable HTTP or SSE mode and tell your client how to connect to it.) Note that no matter how you run the MCP server, Serena will, by default, start a small web-based dashboard on localhost that will display logs and allow shutting down the MCP server (since many clients fail to clean up processes correctly). This and other settings can be adjusted in the [configuration](#configuration) and/or by providing [command-line arguments](#command-line-arguments). -###### Local Installation +#### Using uvx + +`uvx` can be used to run the latest version of Serena directly from the repository, without an explicit local installation. + +```shell +uvx --from git+https://github.com/oraios/serena serena start-mcp-server +``` + +Explore the CLI to see some of the customization options that serena provides (more info on them below). + +#### Local Installation 1. Clone the repository and change into it. + ```shell git clone https://github.com/oraios/serena cd serena ``` -2. Optionally create the configuration file in your home directory, i.e. - * `~/.serena/serena_config.yml` on Linux and macOS, or - * `%USERPROFILE%\.serena\serena_config.yml` on Windows. +2. Optionally edit the configuration file in your home directory with - by copying the template and then adjusting it according to your needs: ```shell - mkdir ~/.serena - cp src/serena/resources/serena_config.template.yml ~/.serena/serena_config.yml + uv run serena config edit ``` + If you just want the default config, you can skip this part, and a config file will be created when you first run Serena. 3. Run the server with `uv`: + ```shell - uv run serena-mcp-server + uv run serena start-mcp-server ``` - When running from outside the serena installation directory, be sure to pass it, i.e. use + + When running from outside the serena installation directory, be sure to pass it, i.e., use + ```shell - uv run --directory /abs/path/to/serena serena-mcp-server - ``` + uv run --directory /abs/path/to/serena serena start-mcp-server + ``` -##### Using uvx - -`uvx` can be used to run the latest version of Serena directly from the repository, without an explicit local installation. - -* Windows: - ```shell - uvx --from git+https://github.com/oraios/serena serena-mcp-server.exe - ``` -* Other operating systems: - ```shell - uvx --from git+https://github.com/oraios/serena serena-mcp-server - ``` - -##### Using Docker (Experimental) +#### Using Docker (Experimental) ⚠️ Docker support is currently experimental with several limitations. Please read the [Docker documentation](DOCKER.md) for important caveats before using it. @@ -243,160 +254,225 @@ You can run the Serena MCP server directly via docker as follows, assuming that the projects you want to work on are all located in `/path/to/your/projects`: ```shell -docker run --rm -i --network host -v /path/to/your/projects:/workspaces/projects ghcr.io/oraios/serena:latest serena-mcp-server --transport stdio +docker run --rm -i --network host -v /path/to/your/projects:/workspaces/projects ghcr.io/oraios/serena:latest serena start-mcp-server --transport stdio ``` Replace `/path/to/your/projects` with the absolute path to your projects directory. The Docker approach provides: -- Better security isolation for shell command execution -- No need to install language servers and dependencies locally -- Consistent environment across different systems + +* Better security isolation for shell command execution +* No need to install language servers and dependencies locally +* Consistent environment across different systems + +Alternatively, use docker compose with the `compose.yml` file provided in the repository. See the [Docker documentation](DOCKER.md) for detailed setup instructions, configuration options, and known limitations. -#### SSE Mode +#### Using Nix + +If you are using Nix and [have enabled the `nix-command` and `flakes` features](https://nixos.wiki/wiki/flakes), you can run Serena using the following command: + +```bash +nix run github:oraios/serena -- start-mcp-server --transport stdio +``` + +You can also install Serena by referencing this repo (`github:oraios/serena`) and using it in your Nix flake. The package is exported as `serena`. + +#### Streamable HTTP Mode ℹ️ Note that MCP servers which use stdio as a protocol are somewhat unusual as far as client/server architectures go, as the server necessarily has to be started by the client in order for communication to take place via the server's standard input/output stream. In other words, you do not need to start the server yourself. The client application (e.g. Claude Desktop) takes care of this and -therefore needs to be configured with a launch command. +therefore needs to be configured with a launch command. -When using instead the SSE mode, which uses HTTP-based communication, you control the server lifecycle yourself, +When using instead the *Streamable HTTP* mode, you control the server lifecycle yourself, i.e. you start the server and provide the client with the URL to connect to it. -Simply provide `serena-mcp-server` with the `--transport sse` option and optionally provide the port. -For example, to run the Serena MCP server in SSE mode on port 9121 using a local installation, -you would run this command from the Serena directory, +Simply provide `start-mcp-server` with the `--transport streamable-http` option and optionally provide the port. +For example, to run the Serena MCP server in Streamable HTTP mode on port 9121 using a local installation, +you would run this command from the Serena directory, ```shell -uv run serena-mcp-server --transport sse --port 9121 +uv run serena start-mcp-server --transport streamable-http --port 9121 ``` -and then configure your client to connect to `http://localhost:9121/sse`. +and then configure your client to connect to `http://localhost:9121/mcp`. +ℹ️ Note that SSE transport is supported as well, but its use is discouraged. +Use Streamable HTTP instead. #### Command-Line Arguments -The Serena MCP server supports a wide range of additional command-line options, including the option to run in SSE mode +The Serena MCP server supports a wide range of additional command-line options, including the option to run in Streamable HTTP or SSE mode and to adapt Serena to various [contexts and modes of operation](#modes-and-contexts). Run with parameter `--help` to get a list of available options. - ### Configuration -Serena's behavior (active tools and prompts as well as logging configuration, etc.) is configured in four places: +Serena is very flexible in terms of configuration. While for most users, the default configurations will work, +you can fully adjust it to your needs by editing a few yaml files. You can disable tools, change Serena's instructions +(what we denote as the `system_prompt`), adjust the output of tools that just provide a prompt, and even adjust tool descriptions. + +Serena is configured in four places: 1. The `serena_config.yml` for general settings that apply to all clients and projects. It is located in your user directory under `.serena/serena_config.yml`. If you do not explicitly create the file, it will be auto-generated when you first run Serena. -2. In the arguments passed to the `serena-mcp-server` in your client's config (see below), + You can edit it directly or use + + ```shell + uvx --from git+https://github.com/oraios/serena serena config edit + ``` + + (or use the `--directory` command version). +2. In the arguments passed to the `start-mcp-server` in your client's config (see below), which will apply to all sessions started by the respective client. In particular, the [context](#contexts) parameter should be set appropriately for Serena to be best adjusted to existing tools and capabilities of your client. See for a detailed explanation. You can override all entries from the `serena_config.yml` through command line arguments. 3. In the `.serena/project.yml` file within your project. This will hold project-level configuration that is used whenever - that project is activated. -4. Through the currently active set of [modes](#modes). + that project is activated. This file will be autogenerated when you first use Serena on that project, but you can also + generate it explicitly with + ```shell + uvx --from git+https://github.com/oraios/serena serena project generate-yml + ``` -> ⚠️ **Note:** Serena is under active development. We are continuously adding features, improving stability and the UX. -> As a result, configuration may change in a breaking manner. If you have an invalid configuration, -> the MCP server or Serena-based Agent may fail to start (investigate the MCP logs in the former case). -> Check the [changelog](CHANGELOG.md) -> and the configuration templates when updating Serena, adapting your configurations accordingly. + (or use the `--directory` command version). +4. Through the context and modes. Explore the [modes and contexts](#modes-and-contexts) section for more details. After the initial setup, continue with one of the sections below, depending on how you want to use Serena. -You can just ask the LLM to show you the config of your session, Serena has a tool for it. - ### Project Activation & Indexing -The recommended way is to just ask the LLM to activate a project by providing it an absolute path to, or, +If you are mostly working with the same project, you can configure to always activate it at startup +by passing `--project ` to the `start-mcp-server` command in your client's MCP config. +This is especially useful for clients which configure MCP servers on a per-project basis, like Claude Code. + +Otherwise, the recommended way is to just ask the LLM to activate a project by providing it an absolute path to, or, in case the project was activated in the past, by its name. The default project name is the directory name. - * "Activate the project /path/to/my_project" - * "Activate the project my_project" +* "Activate the project /path/to/my_project" +* "Activate the project my_project" -All projects that have been activated will be automatically added to your `serena_config.yml`, and for each +All projects that have been activated will be automatically added to your `serena_config.yml`, and for each project, the file `.serena/project.yml` will be generated. You can adjust the latter, e.g., by changing the name (which you refer to during the activation) or other options. Make sure to not have two different projects with the same name. -If you are mostly working with the same project, you can also configure to always activate a project at startup -by passing `--project ` to the `serena-mcp-server` command in your client's MCP config. - ℹ️ For larger projects, we recommend that you index your project to accelerate Serena's tools; otherwise the first tool application may be very slow. -To do so, run one of these commands the project directory or pass the path to the project as an argument: +To do so, run this from the project directory (or pass the path to the project as an argument): -* When using a local installation: - ```shell - uv run --directory /abs/path/to/serena index-project - ``` -* When using uvx: - ```shell - uvx --from git+https://github.com/oraios/serena index-project - ``` +```shell +uvx --from git+https://github.com/oraios/serena serena project index +``` + +(or use the `--directory` command version). ### Claude Code -Serena is a great way to make Claude Code both cheaper and more powerful! +Serena is a great way to make Claude Code both cheaper and more powerful! From your project directory, add serena with a command like this, ```shell -claude mcp add serena -- --context ide-assistant --project $(pwd) +claude mcp add serena -- --context ide-assistant --project "$(pwd)" ``` where `` is your way of [running the Serena MCP server](#running-the-serena-mcp-server). For example, when using `uvx`, you would run + ```shell -claude mcp add serena -- uvx --from git+https://github.com/oraios/serena serena-mcp-server --context ide-assistant --project $(pwd) +claude mcp add serena -- uvx --from git+https://github.com/oraios/serena serena start-mcp-server --context ide-assistant --project "$(pwd)" ``` -ℹ️ Serena comes with an instruction text, and Claude needs to read it to properly use Serena's tools. - Once in Claude Code, you can ask to "read Serena's initial instructions" or run `/mcp__serena__initial_instructions` to load the instruction text. - Do this whenever you start a new conversation and after any compacting operation to ensure Claude remains properly configured to use Serena's tools. +ℹ️ Serena comes with an instruction text, and Claude needs to read it to properly use Serena's tools. + As of version `v1.0.52`, claude code reads the instructions of the MCP server, so this **is handled automatically**. + If you are using an older version, or if Claude fails to read the instructions, you can ask it explicitly + to "read Serena's initial instructions" or run `/mcp__serena__initial_instructions` to load the instruction text. + If you want to make use of that, you will have to enable the corresponding tool explicitly by adding `initial_instructions` to the `included_optional_tools` + in your config. + Note that you may have to make Claude read the instructions when you start a new conversation and after any compacting operation to ensure Claude remains properly configured to use Serena's tools. -ℹ️ **NEW**: an alternative to the above is adding the instructions as part of the system prompt, then you will not need to run the command above or to remember re-running it after compacting. - This can be achieved through starting claude code with `claude --append-system-prompt $(uvx --from git+https://github.com/oraios/serena serena print-system-prompt)`. Note that this is **experimental**, Claude may not understand the instructions correctly in this way, and we haven't thoroughly tested the resulting behavior. Please report any issues you encounter. +### Codex +Serena works with OpenAI's Codex CLI out of the box, but you have to use the `codex` context for it to work properly. (The technical reason is that Codex doesn't fully support the MCP specifications, so some massaging of tools is required.). + +Unlike Claude Code, in Codex you add an MCP server globally and not per project. Add the following to +`~/.codex/config.toml` (create the file if it does not exist): + +```toml +[mcp_servers.serena] +command = "uvx" +args = ["--from", "git+https://github.com/oraios/serena", "serena", "start-mcp-server", "--context", "codex"] +``` + +After codex has started, you need to activate the project, which you can do by saying: + +"Activate the current dir as project using serena" + +> If you don't activate the project, you will not be able to use Serena's tools! + +That's it! Have a look at `~/.codex/log/codex-tui.log` to see if any errors occurred. + +The Serena dashboard will run if you have not disabled it in the configuration, but due to Codex's sandboxing the webbrowser +may not open automatically. You can open it manually by going to `http://localhost:24282/dashboard/index.html` (or a higher port, if +that was already taken). + +> Codex will often show the tools as `failed` even though they are successfully executed. This is not a problem, seems to be a bug in Codex. Despite the error message, everything works as expected. + +### Other Terminal-Based Clients + +There are many terminal-based coding assistants that support MCP servers, such as [Codex](https://github.com/openai/codex?tab=readme-ov-file#model-context-protocol-mcp), +[Gemini-CLI](https://github.com/google-gemini/gemini-cli), [Qwen3-Coder](https://github.com/QwenLM/Qwen3-Coder), +[rovodev](https://community.atlassian.com/forums/Rovo-for-Software-Teams-Beta/Introducing-Rovo-Dev-CLI-AI-Powered-Development-in-your-terminal/ba-p/3043623), +the [OpenHands CLI](https://docs.all-hands.dev/usage/how-to/cli-mode) and [opencode](https://github.com/sst/opencode). + +They generally benefit from the symbolic tools provided by Serena. You might want to customize some aspects of Serena +by writing your own context, modes or prompts to adjust it to your workflow, to other MCP servers you are using, and to +the client's internal capabilities. ### Claude Desktop For [Claude Desktop](https://claude.ai/download) (available for Windows and macOS), go to File / Settings / Developer / MCP Servers / Edit Config, -which will let you open the JSON file `claude_desktop_config.json`. +which will let you open the JSON file `claude_desktop_config.json`. Add the `serena` MCP server configuration, using a [run command](#running-the-serena-mcp-server) depending on your setup. * local installation: + ```json { "mcpServers": { "serena": { "command": "/abs/path/to/uv", - "args": ["run", "--directory", "/abs/path/to/serena", "serena-mcp-server"] + "args": ["run", "--directory", "/abs/path/to/serena", "serena", "start-mcp-server"] } } } ``` + * uvx: + ```json { "mcpServers": { "serena": { "command": "/abs/path/to/uvx", - "args": ["--from", "git+https://github.com/oraios/serena", "serena-mcp-server"] + "args": ["--from", "git+https://github.com/oraios/serena", "serena", "start-mcp-server"] } } } ``` + * docker: + ```json { "mcpServers": { "serena": { "command": "docker", - "args": ["run", "--rm", "-i", "--network", "host", "-v", "/path/to/your/projects:/workspaces/projects", "ghcr.io/oraios/serena:latest", "serena-mcp-server", "--transport", "stdio"] + "args": ["run", "--rm", "-i", "--network", "host", "-v", "/path/to/your/projects:/workspaces/projects", "ghcr.io/oraios/serena:latest", "serena", "start-mcp-server", "--transport", "stdio"] } } } @@ -412,9 +488,9 @@ That's it! Save the config and then restart Claude Desktop. You are ready for ac Note: on Windows and macOS there are official Claude Desktop applications by Anthropic, for Linux there is an [open-source community version](https://github.com/aaddrick/claude-desktop-debian). -⚠️ Be sure to fully quit the Claude Desktop application, as closing Claude will just minimize it to the system tray – at least on Windows. +⚠️ Be sure to fully quit the Claude Desktop application, as closing Claude will just minimize it to the system tray – at least on Windows. -⚠️ Some clients, currently including Claude Desktop, may leave behind zombie processes. You will have to find and terminate them manually then. +⚠️ Some clients may leave behind zombie processes. You will have to find and terminate them manually then. With Serena, you can activate the [dashboard](#serenas-logs-the-dashboard-and-gui-tool) to prevent unnoted processes and also use the dashboard for shutting down Serena. @@ -422,7 +498,7 @@ After restarting, you should see Serena's tools in your chat interface (notice t For more information on MCP servers with Claude Desktop, see [the official quick start guide](https://modelcontextprotocol.io/quickstart/user). -### Other MCP Clients (Cline, Roo-Code, Cursor, Windsurf, etc.) +### MCP Coding Clients (Cline, Roo-Code, Cursor, Windsurf, etc.) Being an MCP Server, Serena can be included in any MCP Client. The same configuration as above, perhaps with small client-specific modifications, should work. Most of the popular @@ -438,75 +514,14 @@ e.g., for one of the following reasons: 2. You are on Linux and don't want to use the [community-created Claude Desktop](https://github.com/aaddrick/claude-desktop-debian). 3. You want tighter integration of Serena into your IDE and don't mind paying for that. -### Agno Agent - -Agno is a model-agnostic agent framework that allows you to turn Serena into an agent -(independent of the MCP technology) with a large number of underlying LLMs. Agno is currently -the simplest way of running Serena in a chat GUI with an LLM of your choice. - -While Agno is not yet entirely stable, we chose it, because it comes with its own open-source UI, -making it easy to directly use the agent using a chat interface. With Agno, Serena is turned into an agent -(so no longer an MCP Server), so it can be used in programmatic ways (for example for benchmarking or within -your application). - -Here's how it works (see also [Agno's documentation](https://docs.agno.com/introduction/playground)): - -1. Download the agent-ui code with npx - ```shell - npx create-agent-ui@latest - ``` - or, alternatively, clone it manually: - ```shell - git clone https://github.com/agno-agi/agent-ui.git - cd agent-ui - pnpm install - pnpm dev - ``` - -2. Install serena with the optional requirements: - ```shell - # You can also only select agno,google or agno,anthropic instead of all-extras - uv pip install --all-extras -r pyproject.toml -e . - ``` - -3. Copy `.env.example` to `.env` and fill in the API keys for the provider(s) you - intend to use. - -4. Start the agno agent app with - ```shell - uv run python scripts/agno_agent.py - ``` - By default, the script uses Claude as the model, but you can choose any model - supported by Agno (which is essentially any existing model). - -5. In a new terminal, start the agno UI with - ```shell - cd agent-ui - pnpm dev - ``` - Connect the UI to the agent you started above and start chatting. You will have - the same tools as in the MCP server version. - - -Here is a short demo of Serena performing a small analysis task with the newest Gemini model: - -https://github.com/user-attachments/assets/ccfcb968-277d-4ca9-af7f-b84578858c62 - - -⚠️ IMPORTANT: In contrast to the MCP server approach, tool execution in the Agno UI does -not ask for the user's permission. The shell tool is particularly critical, as it can perform arbitrary code execution. -While we have never encountered any issues with -this in our testing with Claude, allowing this may not be entirely safe. -You may choose to disable certain tools for your setup in your Serena project's -configuration file (`.yml`). - -### Other Agent Frameworks - -It should be straightforward to incorporate Serena into any -agent framework (like [pydantic-ai](https://ai.pydantic.dev/), [langgraph](https://langchain-ai.github.io/langgraph/tutorials/introduction/) or others). -Typically, you need only to write an adapter for Serena's tools to the tool representation in the framework of your choice, -as was done by us for Agno with [SerenaAgnoToolkit](/src/serena/agno.py). +### Local GUIs and Frameworks +Over the last months, several technologies have emerged that allow you to run a powerful local GUI +and connect it to an MCP server. They will work with Serena out of the box. +Some of the leading open source GUI technologies offering this are +[Jan](https://jan.ai/docs/mcp), [OpenHands](https://github.com/All-Hands-AI/OpenHands/), +[OpenWebUI](https://docs.openwebui.com/openapi-servers/mcp) and [Agno](https://docs.agno.com/introduction/playground). +They allow combining Serena with almost any LLM (including locally running ones) and offer various other integrations. ## Detailed Usage and Recommendations @@ -522,12 +537,16 @@ autonomously. #### Shell Execution and Editing Tools -However, it should be noted that the `execute_shell_command` tool allows for arbitrary code execution. +Many clients have their own shell execution tool, and by default Serena's shell tool will be disabled in them +(e.g., when using the `ide-assistant` or `codex` context). However, when using Serena through something like +Claude Desktop or ChatGPT, it is recommended to enable Serena's `execute_shell_command` tool to allow +agentic behavior. + +It should be noted that the `execute_shell_command` tool allows for arbitrary code execution. When using Serena as an MCP Server, clients will typically ask the user for permission before executing a tool, so as long as the user inspects execution parameters beforehand, this should not be a problem. -However, if you have concerns, you can choose to disable certain commands in your project's -.yml configuration file. +However, if you have concerns, you can choose to disable certain commands in your project's configuration file. If you only want to use Serena purely for analyzing code and suggesting implementations without modifying the codebase, you can enable read-only mode by setting `read_only: true` in your project configuration file. This will automatically disable all editing tools and prevent any modifications to your codebase while still @@ -536,40 +555,43 @@ allowing all analysis and exploration capabilities. In general, be sure to back up your work and use a version control system in order to avoid losing any work. - ### Modes and Contexts -Serena's behavior and toolset can be adjusted using contexts and modes. +Serena's behavior and toolset can be adjusted using contexts and modes. These allow for a high degree of customization to best suit your workflow and the environment Serena is operating in. #### Contexts -A context defines the general environment in which Serena is operating. -It influences the initial system prompt and the set of available tools. +A context defines the general environment in which Serena is operating. +It influences the initial system prompt and the set of available tools. A context is set at startup when launching Serena (e.g., via CLI options for an MCP server or in the agent script) and cannot be changed during an active session. Serena comes with pre-defined contexts: -* `desktop-app`: Tailored for use with desktop applications like Claude Desktop. This is the default. -* `agent`: Designed for scenarios where Serena acts as a more autonomous agent, for example, when used with Agno. -* `ide-assistant`: Optimized for integration into IDEs like VSCode, Cursor, or Cline, focusing on in-editor coding assistance. + +* `desktop-app`: Tailored for use with desktop applications like Claude Desktop. This is the default. +* `agent`: Designed for scenarios where Serena acts as a more autonomous agent, for example, when used with Agno. +* `ide-assistant`: Optimized for integration into IDEs like VSCode, Cursor, or Cline, focusing on in-editor coding assistance. Choose the context that best matches the type of integration you are using. -When launching Serena, specify the context using `--context `. +When launching Serena, specify the context using `--context `. Note that for cases where parameter lists are specified (e.g. Claude Desktop), you must add two parameters to the list. +If you are using a local server (such as Llama.cpp) which requires you to use OpenAI-compatible tool descriptions, use context `oaicompat-agent` instead of `agent`. + #### Modes Modes further refine Serena's behavior for specific types of tasks or interaction styles. Multiple modes can be active simultaneously, allowing you to combine their effects. Modes influence the system prompt and can also alter the set of available tools by excluding certain ones. Examples of built-in modes include: -* `planning`: Focuses Serena on planning and analysis tasks. -* `editing`: Optimizes Serena for direct code modification tasks. -* `interactive`: Suitable for a conversational, back-and-forth interaction style. -* `one-shot`: Configures Serena for tasks that should be completed in a single response, often used with `planning` for generating reports or initial plans. -* `no-onboarding`: Skips the initial onboarding process if it's not needed for a particular session. -* `onboarding`: (Usually triggered automatically) Focuses on the project onboarding process. -Modes can be set at startup (similar to contexts) but can also be *switched dynamically* during a session. You can instruct the LLM to use the `switch_modes` tool to activate a different set of modes (e.g., "switch to planning and one-shot modes"). +* `planning`: Focuses Serena on planning and analysis tasks. +* `editing`: Optimizes Serena for direct code modification tasks. +* `interactive`: Suitable for a conversational, back-and-forth interaction style. +* `one-shot`: Configures Serena for tasks that should be completed in a single response, often used with `planning` for generating reports or initial plans. +* `no-onboarding`: Skips the initial onboarding process if it's not needed for a particular session. +* `onboarding`: (Usually triggered automatically) Focuses on the project onboarding process. + +Modes can be set at startup (similar to contexts) but can also be _switched dynamically_ during a session. You can instruct the LLM to use the `switch_modes` tool to activate a different set of modes (e.g., "switch to planning and one-shot modes"). When launching Serena, specify modes using `--mode `; multiple modes can be specified, e.g. `--mode planning --mode no-onboarding`. @@ -578,18 +600,24 @@ When launching Serena, specify modes using `--mode `; multiple modes #### Customization You can create your own contexts and modes to precisely tailor Serena to your needs in two ways: -* **Adding to Serena's configuration directory**: Create new `.yml` files in the `config/contexts/` or `config/modes/` directories within your local Serena repository. These custom contexts/modes will be automatically registered and available for use by their name (filename without the `.yml` extension). They will also appear in listings of available contexts/modes. -* **Using external YAML files**: When starting Serena, you can provide an absolute path to a custom `.yml` file for a context or mode. -A context or mode YAML file typically defines: -* `name`: (Optional if filename is used) The name of the context/mode. -* `prompt`: A string that will be incorporated into Serena's system prompt. -* `description`: (Optional) A brief description. -* `excluded_tools`: A list of tool names (strings) to disable when this context/mode is active. +* You can use Serena's CLI to manage modes and contexts. Check out + + ```shell + uvx --from git+https://github.com/oraios/serena serena mode --help + ``` + + and + + ```shell + uvx --from git+https://github.com/oraios/serena serena context --help + ``` + + _NOTE_: Custom contexts/modes are simply YAML files in `/.serena`, they are automatically registered and available for use by their name (filename without the `.yml` extension). If you don't want to use Serena's CLI, you can create and manage them in any way you see fit. +* **Using external YAML files**: When starting Serena, you can also provide an absolute path to a custom `.yml` file for a context or mode. This customization allows for deep integration and adaptation of Serena to specific project requirements or personal preferences. - ### Onboarding and Memories By default, Serena will perform an **onboarding process** when @@ -613,14 +641,13 @@ Whenever Serena starts working on a project, the list of memories is provided, and the agent can decide to read them. We found that memories can significantly improve the user experience with Serena. - ### Prepare Your Project #### Structure Your Codebase Serena uses the code structure for finding, reading and editing code. This means that it will work well with well-structured code but may perform poorly on fully unstructured one (like a "God class" -with enormous, non-modular functions). +with enormous, non-modular functions). Furthermore, for languages that are not statically typed, type annotations are highly beneficial. #### Start from a Clean State @@ -659,18 +686,6 @@ better results and in increasing the feeling of control and staying in the loop. make a detailed plan in one session, where Serena may read a lot of your code to build up the context, and then continue with the implementation in another (potentially after creating suitable memories). -### Potential Issues in Code Editing - -In our experience, LLMs are bad at counting, i.e. they have problems -inserting blocks of code in the right place. Most editing operations can be performed -at the symbolic level, allowing this problem is overcome. However, sometimes, -line-level insertions are useful. - -Serena is instructed to double-check the line numbers and any code blocks that it will -edit, but you may find it useful to explicitly tell it how to edit code if you run into -problems. -We are working on making Serena's editing capabilities more robust. - ### Running Out of Context For long and complicated tasks, or tasks where Serena has read a lot of content, you @@ -678,35 +693,27 @@ may come close to the limits of context tokens. In that case, it is often a good in a new conversation. Serena has a dedicated tool to create a summary of the current state of the progress and all relevant info for continuing it. You can request to create this summary and write it to a memory. Then, in a new conversation, you can just ask Serena to read the memory and -continue with the task. In our experience, this worked really well. On the up-side, since in a +continue with the task. In our experience, this worked really well. On the up-side, since in a single session there is no summarization involved, Serena does not usually get lost (unlike some other agents that summarize under the hood), and it is also instructed to occasionally check whether it's on the right track. -Moreover, Serena is instructed to be frugal with context +Moreover, Serena is instructed to be frugal with context (e.g., to not read bodies of code symbols unnecessarily), but we found that Claude is not always very good in being frugal (Gemini seemed better at it). You can explicitly instruct it to not read the bodies if you know that it's not needed. -### Combining Serena with Other MCP Servers - -When using Serena through an MCP Client, you can use it together with other MCP servers. -However, beware of tool name collisions! See info on that above. - -Currently, there is a collision with the popular Filesystem MCP Server. Since Serena also provides -filesystem operations, there is likely no need to ever enable these two simultaneously. - ### Serena's Logs: The Dashboard and GUI Tool Serena provides two convenient ways of accessing the logs of the current session: - * via the **web-based dashboard** (enabled by default) - +* via the **web-based dashboard** (enabled by default) + This is supported on all platforms. - By default, it will be accessible at `http://localhost:24282/dashboard/index.html`, + By default, it will be accessible at `http://localhost:24282/dashboard/index.html`, but a higher port may be used if the default port is unavailable/multiple instances are running. - - * via the **GUI tool** (disabled by default) + +* via the **GUI tool** (disabled by default) This is mainly supported on Windows, but it may also work on Linux; macOS is unsupported. @@ -714,20 +721,15 @@ Both can be enabled, configured or disabled in Serena's configuration file (`ser If enabled, they will automatically be opened as soon as the Serena agent/MCP server is started. The web dashboard will display usage statistics of Serena's tools if you set `record_tool_usage_stats: True` in your config. -In addition to viewing logs, both tools allow to shut down the Serena agent. -This function is provided, because clients like Claude Desktop may fail to terminate the MCP server subprocess +In addition to viewing logs, both tools allow to shut down the Serena agent. +This function is provided, because clients like Claude Desktop may fail to terminate the MCP server subprocess when they themselves are closed. -### Troubleshooting +### Serena and GIT worktrees +[git-worktree](https://git-scm.com/docs/git-worktree) can be an excellent way to parallelize your work. More on this in [Anthropic: Run parallel Claude Code sessions with Git worktrees](https://docs.claude.com/en/docs/claude-code/common-workflows#run-parallel-claude-code-sessions-with-git-worktrees). -Support for MCP Servers in Claude Desktop and the various MCP Server SDKs are relatively new developments and may display instabilities. +When it comes to serena AND git-worktree AND larger projects (that take longer to index), the recommended way is to COPY your `$ORIG_PROJECT/.serena/cache` to `$GIT_WORKTREE/.serena/cache`. After you have performed pre-indexing of your project described in [Project Activation & Indexing](#project-activation--indexing) section. To avoid having to re-index per each git work tree that you create. -The working configuration of an MCP server may vary from platform to -platform and from client to client. We recommend always using absolute paths, as relative paths may be sources of -errors. The language server is running in a separate sub-process and is called with asyncio – sometimes -a client may make it crash. If you have Serena's log window enabled, and it disappears, you'll know what happened. - -Some clients may not properly terminate MCP servers, look out for hanging python processes and terminate them manually, if needed. ## Comparison with Other Coding Agents @@ -738,11 +740,10 @@ subscriptions. ### Subscription-Based Coding Agents -The most prominent subscription-based coding agents are parts of IDEs like +Many prominent subscription-based coding agents are parts of IDEs like Windsurf, Cursor and VSCode. Serena's functionality is similar to Cursor's Agent, Windsurf's Cascade or -VSCode's -upcoming [agent mode](https://code.visualstudio.com/blogs/2025/02/24/introducing-copilot-agent-mode). +VSCode's agent mode. Serena has the advantage of not requiring a subscription. A potential disadvantage is that it @@ -750,7 +751,8 @@ is not directly integrated into an IDE, so the inspection of newly written code is not as seamless. More technical differences are: -* Serena is not bound to a specific IDE. + +* Serena is not bound to a specific IDE or CLI. Serena's MCP server can be used with any MCP client (including some IDEs), and the Agno-based agent provides additional ways of applying its functionality. * Serena is not bound to a specific large language model or API. @@ -791,15 +793,38 @@ It is the integration of language servers and the MCP that makes Serena unique and so powerful for challenging coding tasks, especially in the context of larger codebases. - ## Acknowledgements +### Sponsors + +We are very grateful to our [sponsors](https://github.com/sponsors/oraios) who help us drive Serena's development. The core team +(the founders of [Oraios AI](https://oraios-ai.de/)) put in a lot of work in order to turn Serena into a useful open source project. +So far, there is no business model behind this project, and sponsors are our only source of income from it. + +Sponsors help us dedicating more time to the project, managing contributions, and working on larger features (like better tooling based on more advanced +LSP features, VSCode integration, debugging via the DAP, and several others). +If you find this project useful to your work, or would like to accelerate the development of Serena, consider becoming a sponsor. + +We are proud to announce that the Visual Studio Code team, together with Microsoft’s Open Source Programs Office and GitHub Open Source +have decided to sponsor Serena with a one-time contribution! + +

+ Visual Studio Code sponsor logo +

+ +### Community Contributions + +A significant part of Serena, especially support for various languages, was contributed by the open source community. +We are very grateful for the many contributors who made this possible and who played an important role in making Serena +what it is today. + +### Technologies We built Serena on top of multiple existing open-source technologies, the most important ones being: 1. [multilspy](https://github.com/microsoft/multilspy). A library which wraps language server implementations and adapts them for interaction via Python - and which provided the basis for our library Solid-LSP (src/solidlsp). - Solid-LSP provides pure synchronous LSP calls and extends the original library with the symbolic logic + and which provided the basis for our library Solid-LSP (src/solidlsp). + Solid-LSP provides pure synchronous LSP calls and extends the original library with the symbolic logic that Serena required. 2. [Python MCP SDK](https://github.com/modelcontextprotocol/python-sdk) 3. [Agno](https://github.com/agno-agi/agno) and @@ -810,56 +835,66 @@ We built Serena on top of multiple existing open-source technologies, the most i Without these projects, Serena would not have been possible (or would have been significantly more difficult to build). - ## Customizing and Extending Serena -It is straightforward to extend Serena's AI functionality with your own ideas. -Simply implement a new tool by subclassing +It is straightforward to extend Serena's AI functionality with your own ideas. +Simply implement a new tool by subclassing `serena.agent.Tool` and implement the `apply` method with a signature -that matches the tool's requirements. +that matches the tool's requirements. Once implemented, `SerenaAgent` will automatically have access to the new tool. -It is also relatively straightforward to add [support for a new programming language](/CONTRIBUTING.md#adding-a-new-supported-language). +It is also relatively straightforward to add [support for a new programming language](/.serena/memories/adding_new_language_support_guide.md). -We look forward to seeing what the community will come up with! -For details on contributing, see [here](/CONTRIBUTING.md). +We look forward to seeing what the community will come up with! +For details on contributing, see [contributing guidelines](/CONTRIBUTING.md). -## Full List of Tools +## List of Tools -Here is the full list of Serena's tools with a short description (output of `uv run serena-list-tools`): +Here is the list of Serena's default tools with a short description (output of `uv run serena tools list`): - * `activate_project`: Activates a project by name. - * `check_onboarding_performed`: Checks whether project onboarding was already performed. - * `create_text_file`: Creates/overwrites a file in the project directory. - * `delete_lines`: Deletes a range of lines within a file. - * `delete_memory`: Deletes a memory from Serena's project-specific memory store. - * `execute_shell_command`: Executes a shell command. - * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced. - * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type). - * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). - * `get_active_project`: Gets the name of the currently active project (if any) and lists existing projects - * `get_current_config`: Prints the current configuration of the agent, including the active modes, tools, and context. - * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file or directory. - * `initial_instructions`: Gets the initial instructions for the current project. +* `activate_project`: Activates a project based on the project name or path. +* `check_onboarding_performed`: Checks whether project onboarding was already performed. +* `create_text_file`: Creates/overwrites a file in the project directory. +* `delete_memory`: Deletes a memory from Serena's project-specific memory store. +* `execute_shell_command`: Executes a shell command. +* `find_file`: Finds files in the given relative paths +* `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type). +* `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). +* `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. +* `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file. +* `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. +* `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. +* `list_dir`: Lists files and directories in the given directory (optionally with recursion). +* `list_memories`: Lists memories in Serena's project-specific memory store. +* `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). +* `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context). +* `read_file`: Reads a file within the project directory. +* `read_memory`: Reads the memory with the given name from Serena's project-specific memory store. +* `rename_symbol`: Renames a symbol throughout the codebase using language server refactoring capabilities. +* `replace_regex`: Replaces content in a file by using regular expressions. +* `replace_symbol_body`: Replaces the full definition of a symbol. +* `search_for_pattern`: Performs a search for a pattern in the project. +* `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. +* `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task. +* `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed. +* `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. + +There are several tools that are disabled by default, and have to be enabled explicitly, e.g., through the context or modes. +Note that several of our default contexts do enable some of these tools. For example, the `desktop-app` context enables the `execute_shell_command` tool. + +The full list of optional tools is (output of `uv run serena tools list --only-optional`): + +* `delete_lines`: Deletes a range of lines within a file. +* `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes. +* `initial_instructions`: Gets the initial instructions for the current project. Should only be used in settings where the system prompt cannot be set, e.g. in clients you have no control over, like Claude Desktop. - * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol. - * `insert_at_line`: Inserts content at a given line in a file. - * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol. - * `list_dir`: Lists files and directories in the given directory (optionally with recursion). - * `list_memories`: Lists memories in Serena's project-specific memory store. - * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building). - * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context). - * `read_file`: Reads a file within the project directory. - * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store. - * `replace_lines`: Replaces a range of lines within a file with new content. - * `replace_symbol_body`: Replaces the full definition of a symbol. - * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. - * `search_for_pattern`: Performs a search for a pattern in the project. - * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. - * `switch_modes`: Activates modes by providing a list of their names - * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information. - * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task. - * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed. - * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store. - +* `insert_at_line`: Inserts content at a given line in a file. +* `jet_brains_find_referencing_symbols`: Finds symbols that reference the given symbol +* `jet_brains_find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type). +* `jet_brains_get_symbols_overview`: Retrieves an overview of the top-level symbols within a specified file +* `remove_project`: Removes a project from the Serena configuration. +* `replace_lines`: Replaces a range of lines within a file with new content. +* `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen. +* `summarize_changes`: Provides instructions for summarizing the changes made to the codebase. +* `switch_modes`: Activates modes by providing a list of their names diff --git a/compose.yaml b/compose.yaml index bb8f683e..48923e27 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,10 @@ services: serena: image: serena:latest + # To work with projects, you must mount them as volumes: + # volumes: + # - ./my-project:/workspace/my-project + # - /path/to/another/project:/workspace/another-project build: context: ./ dockerfile: Dockerfile @@ -12,6 +16,8 @@ services: - SERENA_DOCKER=1 command: - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0" + # Add the context for the IDE assistant + # - "uv run --directory . serena-mcp-server --transport sse --port 9121 --host 0.0.0.0 --context ide-assistant" serena-dev: image: serena:dev diff --git a/docs/custom_agent.md b/docs/custom_agent.md new file mode 100644 index 00000000..d1d7afde --- /dev/null +++ b/docs/custom_agent.md @@ -0,0 +1,67 @@ +# Custom Agents with Serena + +As a reference implementation, we provide an integration with the [Agno](https://docs.agno.com/introduction/playground) agent framework. +Agno is a model-agnostic agent framework that allows you to turn Serena into an agent +(independent of the MCP technology) with a large number of underlying LLMs. While Agno has recently +added support for MCP servers out of the box, our Agno integration predates this and is a good illustration of how +easy it is to integrate Serena into an arbitrary agent framework. + +Here's how it works: + +1. Download the agent-ui code with npx + ```shell + npx create-agent-ui@latest + ``` + or, alternatively, clone it manually: + ```shell + git clone https://github.com/agno-agi/agent-ui.git + cd agent-ui + pnpm install + pnpm dev + ``` + +2. Install serena with the optional requirements: + ```shell + # You can also only select agno,google or agno,anthropic instead of all-extras + uv pip install --all-extras -r pyproject.toml -e . + ``` + +3. Copy `.env.example` to `.env` and fill in the API keys for the provider(s) you + intend to use. + +4. Start the agno agent app with + ```shell + uv run python scripts/agno_agent.py + ``` + By default, the script uses Claude as the model, but you can choose any model + supported by Agno (which is essentially any existing model). + +5. In a new terminal, start the agno UI with + ```shell + cd agent-ui + pnpm dev + ``` + Connect the UI to the agent you started above and start chatting. You will have + the same tools as in the MCP server version. + + +Here is a short demo of Serena performing a small analysis task with the newest Gemini model: + +https://github.com/user-attachments/assets/ccfcb968-277d-4ca9-af7f-b84578858c62 + + +⚠️ IMPORTANT: In contrast to the MCP server approach, tool execution in the Agno UI does +not ask for the user's permission. The shell tool is particularly critical, as it can perform arbitrary code execution. +While we have never encountered any issues with +this in our testing with Claude, allowing this may not be entirely safe. +You may choose to disable certain tools for your setup in your Serena project's +configuration file (`.yml`). + + +## Other Agent Frameworks + +It should be straightforward to incorporate Serena into any +agent framework (like [pydantic-ai](https://ai.pydantic.dev/), [langgraph](https://langchain-ai.github.io/langgraph/tutorials/introduction/) or others). +Typically, you need only to write an adapter for Serena's tools to the tool representation in the framework of your choice, +as was done by us for Agno with [SerenaAgnoToolkit](/src/serena/agno.py). + diff --git a/docs/scala_setup_guide_for_serena.md b/docs/scala_setup_guide_for_serena.md new file mode 100644 index 00000000..2c4b1cc8 --- /dev/null +++ b/docs/scala_setup_guide_for_serena.md @@ -0,0 +1,67 @@ +# Scala Setup Guide for Serena + +This guide explains how to prepare a Scala project so that Serena can provide reliable code intelligence via Metals (Scala LSP) and how to run Scala tests manually. + +Serena automatically bootstraps the Metals language server using Coursier when needed. Your project, however, must be importable by a build server (BSP) — typically via Bloop or sbt’s built‑in BSP — so that Metals can compile and index your code. + +--- +## Prerequisites + +Install the following on your system and ensure they are available on `PATH`: + +- Java Development Kit (JDK). A modern LTS (e.g., 17 or 21) is recommended. +- `sbt` +- Coursier command (`cs`) or the legacy `coursier` launcher + - Serena uses `cs` if available; if only `coursier` exists, it will attempt to install `cs`. If neither is present, install Coursier first. + +--- +## Quick Start (Recommended: VS Code + Metals auto‑import) + +1. Open your Scala project in VS Code. +2. When prompted by Metals, accept “Import build”. Wait until the import and initial compile/indexing finish. +3. Run the “Connect to build server” command (id: `build.connect`). +4. Once the import completes, start Serena in your project root and use it. + +This flow ensures the `.bloop/` and (if applicable) `.metals/` directories are created and your build is known to the build server that Metals uses. + +--- +## Manual Setup (No VS Code) + +Follow these steps if you prefer a manual setup or you are not using VS Code: + +These instructions cover the setup for projects that use sbt as the build tool, with Bloop as the BSP server. + + +1. Add Bloop to `project/plugins.sbt` in your Scala project: + ```scala + // project/plugins.sbt + addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "") + ``` + Replace `` with an appropriate current version from the Metals documentation. + +2. Export Bloop configuration with sources: + ```bash + sbt -Dbloop.export-jar-classifiers=sources bloopInstall + ``` + This creates a `.bloop/` directory containing your project’s build metadata for the BSP server. + +3. Compile from sbt to verify the build: + ```bash + sbt compile + ``` + +4. Start Serena in your project root. Serena will bootstrap Metals (if not already present) and connect to the build server using the configuration exported above. + +--- +## Using Serena with Scala + +- Serena automatically detects Scala files (`*.scala`, `*.sbt`) and will start a Metals process per project when needed. +- On first run, you may see messages like “Bootstrapping metals…” in the Serena logs — this is expected. +- Optimal results require that your project compiles successfully via the build server (BSP). If compilation fails, fix build errors in `sbt` first. + + +Notes: +- Ensure you completed the manual or auto‑import steps so that the build is compiled and indexed; otherwise, code navigation and references may be incomplete until the first successful compile. + +## Reference +- Metals + sbt: [https://scalameta.org/metals/docs/build-tools/sbt](https://scalameta.org/metals/docs/build-tools/sbt) \ No newline at end of file diff --git a/docs/serena_on_chatgpt.md b/docs/serena_on_chatgpt.md new file mode 100644 index 00000000..ec872696 --- /dev/null +++ b/docs/serena_on_chatgpt.md @@ -0,0 +1,102 @@ + +# Connecting Serena MCP Server to ChatGPT via MCPO & Cloudflare Tunnel + +This guide explains how to expose a **locally running Serena MCP server** (powered by MCPO) to the internet using **Cloudflare Tunnel**, and how to connect it to **ChatGPT as a Custom GPT with tool access**. + +Once configured, ChatGPT becomes a powerful **coding agent** with direct access to your codebase, shell, and file system — so **read the security notes carefully**. + +--- +## Prerequisites + +Make sure you have [uv](https://docs.astral.sh/uv/getting-started/installation/) +and [cloudflared](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/) installed. + +## 1. Start the Serena MCP Server via MCPO + +Run the following command to launch Serena as http server (assuming port 8000): + +```bash +uvx mcpo --port 8000 --api-key -- \ + uvx --from git+https://github.com/oraios/serena \ + serena start-mcp-server --context chatgpt --project $(pwd) +``` + +- `--api-key` is required to secure the server. +- `--project` should point to the root of your codebase. + +You can also use other options, and you don't have to pass `--project` if you want to work on multiple projects +or want to activate it later. See + +```shell +uvx --from git+https://github.com/oraios/serena serena start-mcp-server --help +``` + +--- + +## 2. Expose the Server Using Cloudflare Tunnel + +Run: + +```bash +cloudflared tunnel --url http://localhost:8000 +``` + +This will give you a **public HTTPS URL** like: + +``` +https://serena-agent-tunnel.trycloudflare.com +``` + +Your server is now securely exposed to the internet. + +--- + +## 3. Connect It to ChatGPT (Custom GPT) + +### Steps: + +1. Go to [ChatGPT → Explore GPTs → Create](https://chat.openai.com/gpts/editor) +2. During setup, click **“Add APIs”** +3. Set up **API Key authentication** with the auth type as **Bearer** and enter the api key you used to start the MCPO server. +4. In the **Schema** section, click on **import from URL** and paste `/openapi.json` with the URL you got from the previous step. +5. Add the following line to the top of the imported JSON schema: + ``` + "servers": ["url": ""], + ``` + **Important**: don't include a trailing slash at the end of the URL! + +ChatGPT will read the schema and create functions automatically. + +--- + +## Security Warning — Read Carefully + +Depending on your configuration and enabled tools, Serena's MCP server may: +- Execute **arbitrary shell commands** +- Read, write, and modify **files in your codebase** + +This gives ChatGPT the same powers as a remote developer on your machine. + +### ⚠️ Key Rules: +- **NEVER expose your API key** +- **Only expose this server when needed**, and monitor its use. + +In your project’s `.serena/project.yml` or global config, you can disable tools like: + +```yaml +excluded_tools: + - execute_shell_command + - ... +read_only: true +``` + +This is strongly recommended if you want a read-only or safer agent. + + +--- + +## Final Thoughts + +With this setup, ChatGPT becomes a coding assistant **running on your local code** — able to index, search, edit, and even run shell commands depending on your configuration. + +Use responsibly, and keep security in mind. diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..728ba12a --- /dev/null +++ b/flake.lock @@ -0,0 +1,133 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1755615617, + "narHash": "sha256-HMwfAJBdrr8wXAkbGhtcby1zGFvs+StOp19xNsbqdOg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "20075955deac2583bb12f07151c2df830ef346b4", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "pyproject-build-systems": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "pyproject-nix": [ + "pyproject-nix" + ], + "uv2nix": [ + "uv2nix" + ] + }, + "locked": { + "lastModified": 1755484659, + "narHash": "sha256-2FfbqsaHVQd12XFFUAinIMAuGO3853LONmva1gT3vKw=", + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "rev": "9778e87c2361810ff15e287ca5895c9da4a0e900", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "build-system-pkgs", + "type": "github" + } + }, + "pyproject-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1755886095, + "narHash": "sha256-oMQ1YERKqiac28LB1cKjl4d0gjvK98SbGejFMfK0WGE=", + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "rev": "7ea903d50c20d4d3ea45ccaad7ee0d21996780c9", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "pyproject-build-systems": "pyproject-build-systems", + "pyproject-nix": "pyproject-nix", + "uv2nix": "uv2nix" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "uv2nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "pyproject-nix": [ + "pyproject-nix" + ] + }, + "locked": { + "lastModified": 1755887980, + "narHash": "sha256-MD1IhwuoDXQSYcQ/JfEpMxrk6pPl+aHPXSqZLOXLHks=", + "owner": "pyproject-nix", + "repo": "uv2nix", + "rev": "76778b5f457e416688324d19d8f02ec5e22cbe0c", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "uv2nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..7529365b --- /dev/null +++ b/flake.nix @@ -0,0 +1,108 @@ +{ + description = "A powerful coding agent toolkit providing semantic retrieval and editing capabilities (MCP server & Agno integration)"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + flake-utils = { + url = "github:numtide/flake-utils"; + }; + + pyproject-nix = { + url = "github:pyproject-nix/pyproject.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + uv2nix = { + url = "github:pyproject-nix/uv2nix"; + inputs = { + pyproject-nix.follows = "pyproject-nix"; + nixpkgs.follows = "nixpkgs"; + }; + }; + + pyproject-build-systems = { + url = "github:pyproject-nix/build-system-pkgs"; + inputs = { + pyproject-nix.follows = "pyproject-nix"; + uv2nix.follows = "uv2nix"; + nixpkgs.follows = "nixpkgs"; + }; + }; + }; + + outputs = { + nixpkgs, + uv2nix, + pyproject-nix, + pyproject-build-systems, + flake-utils, + ... + }: + flake-utils.lib.eachDefaultSystem (system: let + pkgs = import nixpkgs {inherit system;}; + + inherit (pkgs) lib; + + workspace = uv2nix.lib.workspace.loadWorkspace {workspaceRoot = ./.;}; + + overlay = workspace.mkPyprojectOverlay { + sourcePreference = "wheel"; # or sourcePreference = "sdist"; + }; + + pyprojectOverrides = final: prev: { + # Add setuptools for packages that need it during build + ruamel-yaml-clib = prev.ruamel-yaml-clib.overrideAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ + final.setuptools + ]; + }); + }; + + python = pkgs.python311; + + pythonSet = + (pkgs.callPackage pyproject-nix.build.packages { + inherit python; + }).overrideScope + ( + lib.composeManyExtensions [ + pyproject-build-systems.overlays.default + overlay + pyprojectOverrides + ] + ); + in rec { + formatter = pkgs.alejandra; + + packages = { + serena = pythonSet.mkVirtualEnv "serena" workspace.deps.default; + default = packages.serena; + }; + + apps.default = { + type = "app"; + program = "${packages.default}/bin/serena"; + }; + + devShells = { + default = pkgs.mkShell { + packages = [ + python + pkgs.uv + ]; + env = + { + UV_PYTHON_DOWNLOADS = "never"; + UV_PYTHON = python.interpreter; + } + // lib.optionalAttrs pkgs.stdenv.isLinux { + LD_LIBRARY_PATH = lib.makeLibraryPath pkgs.pythonManylinuxPackages.manylinux1; + }; + shellHook = '' + unset PYTHONPATH + ''; + }; + }; + }); +} diff --git a/pyproject.toml b/pyproject.toml index 24797bf2..bea002e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = ["hatchling"] [project] name = "serena-agent" -version = "0.1.3" +version = "0.1.4" description = "" authors = [{ name = "Oraios AI", email = "info@oraios-ai.de" }] readme = "README.md" @@ -18,9 +18,9 @@ dependencies = [ "pyright>=1.1.396,<2", "overrides>=7.7.0,<8", "python-dotenv>=1.0.0, <2", - "mcp>=1.5.0", + "mcp==1.12.3", "flask>=3.0.0", - "sensai-utils>=1.4.0", + "sensai-utils>=1.5.0", "pydantic>=2.10.6", "types-pyyaml>=6.0.12.20241230", "pyyaml>=6.0.2", @@ -45,7 +45,7 @@ explicit = true [project.scripts] serena = "serena.cli:top_level" serena-mcp-server = "serena.cli:start_mcp_server" -index-project = "serena.cli:index_project" # deprecated +index-project = "serena.cli:index_project" # deprecated [project.license] text = "MIT" @@ -63,6 +63,7 @@ dev = [ "toml-sort>=0.24.2", "types-pyyaml>=6.0.12.20241230", "syrupy>=4.9.1", + "types-requests>=2.32.4.20241230", ] agno = ["agno>=1.2.6", "sqlalchemy>=2.0.40"] google = ["google-genai>=1.8.0"] @@ -112,7 +113,7 @@ PYDEVD_DISABLE_FILE_VALIDATION = "1" # Uses PYTEST_MARKERS env var for default markers # For custom markers, one can either adjust the env var or just use -m option in the command line, # as the second -m option will override the first one. -test = "pytest test -vv -m \"${PYTEST_MARKERS:-not java and not rust}\"" +test = "pytest test -vv -m \"${PYTEST_MARKERS:-not java and not rust and not erlang}\"" _black_check = "black --check src scripts test" _ruff_check = "ruff check src scripts test" _black_format = "black src scripts test" @@ -166,6 +167,7 @@ ignore = [ "PLC0415", "RUF002", "RUF005", + "RUF059", "SIM118", "SIM108", "E501", @@ -235,6 +237,7 @@ ignore = [ "SIM117", # forbids nested with statements "C400", # wants to unnecessarily force use of list comprehension "UP037", # can incorrectly (!) convert quoted type to unquoted type, causing an error + "UP045", # imposes T | None instead of Optional[T] ] unfixable = ["F841", "F601", "F602", "B018"] extend-fixable = ["F401", "B905", "W291"] @@ -253,13 +256,31 @@ markers = [ "python: language server running for Python", "go: language server running for Go", "java: language server running for Java", + "kotlin: language server running for kotlin", "rust: language server running for Rust", "typescript: language server running for TypeScript", "php: language server running for PHP", + "perl: language server running for Perl", "csharp: language server running for C#", "elixir: language server running for Elixir", + "elm: language server running for Elm", "terraform: language server running for Terraform", + "swift: language server running for Swift", + "bash: language server running for Bash", + "r: language server running for R", "snapshot: snapshot tests for symbolic editing operations", + "ruby: language server running for Ruby (uses ruby-lsp)", + "zig: language server running for Zig", + "lua: language server running for Lua", + "nix: language server running for Nix", + "dart: language server running for Dart", + "erlang: language server running for Erlang", + "scala: language server running for Scala", + "scala: language server running for Scala", + "al: language server running for AL (Microsoft Dynamics 365 Business Central)", + "rego: language server running for Rego", + "markdown: language server running for Markdown", + "julia: Julia language server tests", ] [tool.codespell] @@ -267,4 +288,4 @@ markers = [ skip = '.git*,*.svg,*.lock,*.min.*' check-hidden = true # ignore-regex = '' -# ignore-words-list = '' +# ignore-words-list = '' \ No newline at end of file diff --git a/resources/vscode_sponsor_logo.png b/resources/vscode_sponsor_logo.png new file mode 100644 index 00000000..ebd8f317 Binary files /dev/null and b/resources/vscode_sponsor_logo.png differ diff --git a/scripts/demo_run_tools.py b/scripts/demo_run_tools.py index 394fe553..d52cfb00 100644 --- a/scripts/demo_run_tools.py +++ b/scripts/demo_run_tools.py @@ -9,7 +9,7 @@ from pprint import pprint from serena.agent import SerenaAgent from serena.config.serena_config import SerenaConfig from serena.constants import REPO_ROOT -from serena.tools import FindFileTool, FindReferencingSymbolsTool, SearchForPatternTool +from serena.tools import FindFileTool, FindReferencingSymbolsTool, GetSymbolsOverviewTool, SearchForPatternTool if __name__ == "__main__": agent = SerenaAgent(project=REPO_ROOT, serena_config=SerenaConfig(gui_log_window_enabled=False, web_dashboard=False)) @@ -18,13 +18,9 @@ if __name__ == "__main__": find_refs_tool = agent.get_tool(FindReferencingSymbolsTool) find_file_tool = agent.get_tool(FindFileTool) search_pattern_tool = agent.get_tool(SearchForPatternTool) + overview_tool = agent.get_tool(GetSymbolsOverviewTool) result = agent.execute_task( - lambda: search_pattern_tool.apply( - r"def request_full_.*?\).*?\)", - restrict_search_to_code_files=False, - relative_path="src/solidlsp", - paths_include_glob="**/ls.py", - ) + lambda: overview_tool.apply("src/solidlsp/ls.py"), ) pprint(json.loads(result)) diff --git a/scripts/gen_prompt_factory.py b/scripts/gen_prompt_factory.py index 4b1354fd..4877d4d1 100644 --- a/scripts/gen_prompt_factory.py +++ b/scripts/gen_prompt_factory.py @@ -7,14 +7,14 @@ from pathlib import Path from sensai.util import logging from interprompt import autogenerate_prompt_factory_module -from serena.constants import PROMPT_TEMPLATES_DIR, REPO_ROOT +from serena.constants import PROMPT_TEMPLATES_DIR_INTERNAL, REPO_ROOT log = logging.getLogger(__name__) def main(): autogenerate_prompt_factory_module( - prompts_dir=PROMPT_TEMPLATES_DIR, + prompts_dir=PROMPT_TEMPLATES_DIR_INTERNAL, target_module_path=str(Path(REPO_ROOT) / "src" / "serena" / "generated" / "generated_prompt_factory.py"), ) diff --git a/src/interprompt/.syncCommitId.remote b/src/interprompt/.syncCommitId.remote index aea4f300..8821570d 100644 --- a/src/interprompt/.syncCommitId.remote +++ b/src/interprompt/.syncCommitId.remote @@ -1 +1 @@ -6bdf19b0fd0fa5a809e1989b334bbfe9d8a7051a \ No newline at end of file +059d50b7d4d7e8fb9e7a13df7f7f33bae1aed5e2 \ No newline at end of file diff --git a/src/interprompt/.syncCommitId.this b/src/interprompt/.syncCommitId.this index 52c0320a..820e95d5 100644 --- a/src/interprompt/.syncCommitId.this +++ b/src/interprompt/.syncCommitId.this @@ -1 +1 @@ -6d9d3adac8e77fa72e55e2675c1bcd3a7331cde1 \ No newline at end of file +53ee7f47c08f29ae336567bcfaf89f79e7d447a2 \ No newline at end of file diff --git a/src/interprompt/multilang_prompt.py b/src/interprompt/multilang_prompt.py index 55777344..e7322efb 100644 --- a/src/interprompt/multilang_prompt.py +++ b/src/interprompt/multilang_prompt.py @@ -1,7 +1,7 @@ import logging import os from enum import Enum -from typing import Any, Generic, TypeVar +from typing import Any, Generic, Literal, TypeVar import yaml from sensai.util.string import ToStringMixin @@ -92,6 +92,9 @@ class _MultiLangContainer(Generic[T], ToStringMixin): raise KeyError(f"Item for language '{lang_code}' already registered for name '{self.name}'") self._lang2item[lang_code] = item + def has_item(self, lang_code: str = DEFAULT_LANG_CODE) -> bool: + return lang_code in self._lang2item + def get_item(self, lang: str = DEFAULT_LANG_CODE, fallback_mode: LanguageFallbackMode = LanguageFallbackMode.EXCEPTION) -> T: """ Gets the item for the given language. @@ -183,6 +186,9 @@ class MultiLangPromptTemplate(ParameterizedTemplateInterface): prompt_template = self.get_prompt_template(lang_code, fallback_mode) return prompt_template.render(**params) + def has_item(self, lang_code: str = DEFAULT_LANG_CODE) -> bool: + return self._prompts_container.has_item(lang_code) + class MultiLangPromptList(_MultiLangContainer[PromptList]): pass @@ -212,44 +218,96 @@ class MultiLangPromptCollection: The prompt names must be unique (for the same language) within the collection. """ - def __init__(self, prompts_dir: str, fallback_mode: LanguageFallbackMode = LanguageFallbackMode.EXCEPTION) -> None: + def __init__(self, prompts_dir: str | list[str], fallback_mode: LanguageFallbackMode = LanguageFallbackMode.EXCEPTION) -> None: """ - :param prompts_dir: the directory containing the prompt templates and prompt lists + :param prompts_dir: the directory containing the prompt templates and prompt lists. + If a list is provided, will look for prompt templates in the dirs from left to right + (first one containing the desired template wins). :param fallback_mode: the fallback mode to use when a prompt template or prompt list is not found for the requested language. May be reset after initialization. """ self._multi_lang_prompt_templates: dict[str, MultiLangPromptTemplate] = {} self._multi_lang_prompt_lists: dict[str, MultiLangPromptList] = {} - self._load_from_disc(prompts_dir) + if isinstance(prompts_dir, str): + prompts_dir = [prompts_dir] + + # Add prompts from multiple directories, prioritizing names from the left. + # If name collisions appear in the first directory, an error is raised (so the first directory should have no + # internal collisions, this helps in avoiding errors) + # For all following directories, on a collision the new value will be ignored. + # This also means that for the following directories, there is no error check on collisions internal to them. + # We assume that they are correct (i.e., they have no internal collisions). + first_prompts_dir, fallback_prompt_dirs = prompts_dir[0], prompts_dir[1:] + self._load_from_disc(first_prompts_dir, on_name_collision="raise") + for fallback_prompt_dir in fallback_prompt_dirs: + # already loaded prompts have priority + self._load_from_disc(fallback_prompt_dir, on_name_collision="skip") + self.fallback_mode = fallback_mode - def _add_prompt_template(self, name: str, template_str: str, lang_code: str = DEFAULT_LANG_CODE) -> None: + def _add_prompt_template( + self, + name: str, + template_str: str, + lang_code: str = DEFAULT_LANG_CODE, + on_name_collision: Literal["skip", "overwrite", "raise"] = "raise", + ) -> None: """ :param name: name of the prompt template :param template_str: the Jinja template string :param lang_code: the language code for which to add the prompt template. + :param on_name_collision: how to deal with name/lang_code collisions """ + allow_overwrite = False prompt_template = PromptTemplate(name, template_str) mlpt = self._multi_lang_prompt_templates.get(name) if mlpt is None: mlpt = MultiLangPromptTemplate(name) self._multi_lang_prompt_templates[name] = mlpt - mlpt.add_prompt_template(prompt_template, lang_code=lang_code) + if mlpt.has_item(lang_code): + if on_name_collision == "raise": + raise KeyError(f"Prompt '{name}' for {lang_code} already exists!") + if on_name_collision == "skip": + log.debug(f"Skipping prompt '{name}' since it already exists.") + return + elif on_name_collision == "overwrite": + allow_overwrite = True + mlpt.add_prompt_template(prompt_template, lang_code=lang_code, allow_overwrite=allow_overwrite) - def _add_prompt_list(self, name: str, prompt_list: list[str], lang_code: str = DEFAULT_LANG_CODE) -> None: + def _add_prompt_list( + self, + name: str, + prompt_list: list[str], + lang_code: str = DEFAULT_LANG_CODE, + on_name_collision: Literal["skip", "overwrite", "raise"] = "raise", + ) -> None: """ :param name: name of the prompt list :param prompt_list: a list of prompts :param lang_code: the language code for which to add the prompt list. + :param on_name_collision: how to deal with name/lang_code collisions """ + allow_overwrite = False multilang_prompt_list = self._multi_lang_prompt_lists.get(name) if multilang_prompt_list is None: multilang_prompt_list = MultiLangPromptList(name) self._multi_lang_prompt_lists[name] = multilang_prompt_list - multilang_prompt_list.add_item(PromptList(prompt_list), lang_code=lang_code) + if multilang_prompt_list.has_item(lang_code): + if on_name_collision == "raise": + raise KeyError(f"Prompt '{name}' for {lang_code} already exists!") + if on_name_collision == "skip": + log.debug(f"Skipping prompt '{name}' since it already exists.") + return + elif on_name_collision == "overwrite": + allow_overwrite = True + multilang_prompt_list.add_item(PromptList(prompt_list), lang_code=lang_code, allow_overwrite=allow_overwrite) - def _load_from_disc(self, prompts_dir: str) -> None: - """Loads all prompt templates and prompt lists from yaml files in the given directory.""" + def _load_from_disc(self, prompts_dir: str, on_name_collision: Literal["skip", "overwrite", "raise"] = "raise") -> None: + """Loads all prompt templates and prompt lists from yaml files in the given directory. + + :param prompts_dir: + :param on_name_collision: how to deal with name/lang_code collisions + """ for fn in os.listdir(prompts_dir): if not fn.endswith((".yml", ".yaml")): log.debug(f"Skipping non-YAML file: {fn}") @@ -266,9 +324,11 @@ class MultiLangPromptCollection: # add the data to the collection for prompt_name, prompt_template_or_list in prompts_data.items(): if isinstance(prompt_template_or_list, list): - self._add_prompt_list(prompt_name, prompt_template_or_list, lang_code=lang_code) + self._add_prompt_list(prompt_name, prompt_template_or_list, lang_code=lang_code, on_name_collision=on_name_collision) elif isinstance(prompt_template_or_list, str): - self._add_prompt_template(prompt_name, prompt_template_or_list, lang_code=lang_code) + self._add_prompt_template( + prompt_name, prompt_template_or_list, lang_code=lang_code, on_name_collision=on_name_collision + ) else: raise ValueError( f"Invalid prompt type for {prompt_name} in file {path} (should be str or list): {prompt_template_or_list}" @@ -306,6 +366,18 @@ class MultiLangPromptCollection: """The PromptList object for the given prompt name and language code.""" return self.get_multilang_prompt_list(prompt_name).get_item(lang_code) + def _has_prompt_list(self, prompt_name: str, lang_code: str = DEFAULT_LANG_CODE) -> bool: + multi_lang_prompt_list = self._multi_lang_prompt_lists.get(prompt_name) + if multi_lang_prompt_list is None: + return False + return multi_lang_prompt_list.has_item(lang_code) + + def _has_prompt_template(self, prompt_name: str, lang_code: str = DEFAULT_LANG_CODE) -> bool: + multi_lang_prompt_template = self._multi_lang_prompt_templates.get(prompt_name) + if multi_lang_prompt_template is None: + return False + return multi_lang_prompt_template.has_item(lang_code) + def render_prompt_template( self, prompt_name: str, diff --git a/src/interprompt/prompt_factory.py b/src/interprompt/prompt_factory.py index 3aa9da08..1769e7f2 100644 --- a/src/interprompt/prompt_factory.py +++ b/src/interprompt/prompt_factory.py @@ -10,9 +10,11 @@ log = logging.getLogger(__name__) class PromptFactoryBase: """Base class for auto-generated prompt factory classes.""" - def __init__(self, prompts_dir: str, lang_code: str = DEFAULT_LANG_CODE, fallback_mode=LanguageFallbackMode.EXCEPTION): + def __init__(self, prompts_dir: str | list[str], lang_code: str = DEFAULT_LANG_CODE, fallback_mode=LanguageFallbackMode.EXCEPTION): """ - :param prompts_dir: the directory containing the prompt templates and prompt lists + :param prompts_dir: the directory containing the prompt templates and prompt lists. + If a list is provided, will look for prompt templates in the dirs from left to right + (first one containing the desired template wins). :param lang_code: the language code to use for retrieving the prompt templates and prompt lists. Leave as `default` for single-language use cases. :param fallback_mode: the fallback mode to use when a prompt template or prompt list is not found for the requested language. diff --git a/src/serena/__init__.py b/src/serena/__init__.py index 7ff4e37e..a2c1cf0a 100644 --- a/src/serena/__init__.py +++ b/src/serena/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.3" +__version__ = "0.1.4" import logging diff --git a/src/serena/agent.py b/src/serena/agent.py index fe6a9091..dfe8c1ad 100644 --- a/src/serena/agent.py +++ b/src/serena/agent.py @@ -8,24 +8,24 @@ import platform import sys import threading import webbrowser -from collections import defaultdict from collections.abc import Callable from concurrent.futures import Future, ThreadPoolExecutor from logging import Logger -from pathlib import Path from typing import TYPE_CHECKING, Any, Optional, TypeVar from sensai.util import logging from sensai.util.logging import LogTime +from interprompt.jinja_template import JinjaTemplate from serena import serena_version from serena.analytics import RegisteredTokenCountEstimator, ToolUsageStats from serena.config.context_mode import RegisteredContext, SerenaAgentContext, SerenaAgentMode -from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, ToolSet, get_serena_managed_in_project_dir +from serena.config.serena_config import SerenaConfig, ToolInclusionDefinition, ToolSet from serena.dashboard import SerenaDashboardAPI from serena.project import Project from serena.prompt_factory import SerenaPromptFactory -from serena.tools import ActivateProjectTool, Tool, ToolRegistry +from serena.tools import ActivateProjectTool, GetCurrentConfigTool, Tool, ToolMarker, ToolRegistry +from serena.util.inspection import iter_subclasses from serena.util.logging import MemoryLogHandler from solidlsp import SolidLanguageServer @@ -42,53 +42,21 @@ class ProjectNotFoundError(Exception): pass -class LinesRead: - def __init__(self) -> None: - self.files: dict[str, set[tuple[int, int]]] = defaultdict(lambda: set()) +class AvailableTools: + def __init__(self, tools: list[Tool]): + """ + :param tools: the list of available tools + """ + self.tools = tools + self.tool_names = [tool.get_name_from_cls() for tool in tools] + self.tool_marker_names = set() + for marker_class in iter_subclasses(ToolMarker): + for tool in tools: + if isinstance(tool, marker_class): + self.tool_marker_names.add(marker_class.__name__) - def add_lines_read(self, relative_path: str, lines: tuple[int, int]) -> None: - self.files[relative_path].add(lines) - - def were_lines_read(self, relative_path: str, lines: tuple[int, int]) -> bool: - lines_read_in_file = self.files[relative_path] - return lines in lines_read_in_file - - def invalidate_lines_read(self, relative_path: str) -> None: - if relative_path in self.files: - del self.files[relative_path] - - -class MemoriesManager: - def __init__(self, project_root: str): - self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" - self._memory_dir.mkdir(parents=True, exist_ok=True) - - def _get_memory_file_path(self, name: str) -> Path: - # strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not. - name = name.replace(".md", "") - filename = f"{name}.md" - return self._memory_dir / filename - - def load_memory(self, name: str) -> str: - memory_file_path = self._get_memory_file_path(name) - if not memory_file_path.exists(): - return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it." - with open(memory_file_path, encoding="utf-8") as f: - return f.read() - - def save_memory(self, name: str, content: str) -> str: - memory_file_path = self._get_memory_file_path(name) - with open(memory_file_path, "w", encoding="utf-8") as f: - f.write(content) - return f"Memory {name} written." - - def list_memories(self) -> list[str]: - return [f.name.replace(".md", "") for f in self._memory_dir.iterdir() if f.is_file()] - - def delete_memory(self, name: str) -> str: - memory_file_path = self._get_memory_file_path(name) - memory_file_path.unlink() - return f"Memory {name} deleted." + def __len__(self) -> int: + return len(self.tools) class SerenaAgent: @@ -116,9 +84,13 @@ class SerenaAgent: # obtain serena configuration using the decoupled factory function self.serena_config = serena_config or SerenaConfig.from_config_file() + # project-specific instances, which will be initialized upon project activation + self._active_project: Project | None = None + self.language_server: SolidLanguageServer | None = None + # adjust log level serena_log_level = self.serena_config.log_level - if Logger.root.level > serena_log_level: + if Logger.root.level != serena_log_level: log.info(f"Changing the root logger level to {serena_log_level}") Logger.root.setLevel(serena_log_level) @@ -164,7 +136,7 @@ class SerenaAgent: # start the dashboard (web frontend), registering its log handler if self.serena_config.web_dashboard: self._dashboard_thread, port = SerenaDashboardAPI( - get_memory_log_handler(), tool_names, tool_usage_stats=self._tool_usage_stats + get_memory_log_handler(), tool_names, agent=self, tool_usage_stats=self._tool_usage_stats ).run_in_thread() dashboard_url = f"http://127.0.0.1:{port}/dashboard/index.html" log.info("Serena web dashboard started at %s", dashboard_url) @@ -181,16 +153,18 @@ class SerenaAgent: log.info("Available projects: {}".format(", ".join(self.serena_config.project_names))) log.info(f"Loaded tools ({len(self._all_tools)}): {', '.join([tool.get_name_from_cls() for tool in self._all_tools.values()])}") + self._check_shell_settings() + # determine the base toolset defining the set of exposed tools (which e.g. the MCP shall see), # limited by the Serena config, the context (which is fixed for the session) and JetBrains mode tool_inclusion_definitions: list[ToolInclusionDefinition] = [self.serena_config, self._context] if self._context.name == RegisteredContext.IDE_ASSISTANT.value: - tool_inclusion_definitions.extend(self._ide_context_tool_inclusion_definitions(project)) + tool_inclusion_definitions.extend(self._ide_assistant_context_tool_inclusion_definitions(project)) if self.serena_config.jetbrains: tool_inclusion_definitions.append(SerenaAgentMode.from_name_internal("jetbrains")) self._base_tool_set = ToolSet.default().apply(*tool_inclusion_definitions) - self._exposed_tools = {tc: t for tc, t in self._all_tools.items() if self._base_tool_set.includes_name(t.get_name())} + self._exposed_tools = AvailableTools([t for t in self._all_tools.values() if self._base_tool_set.includes_name(t.get_name())]) log.info(f"Number of exposed tools: {len(self._exposed_tools)}") # create executor for starting the language server and running tools in another thread @@ -203,13 +177,6 @@ class SerenaAgent: self.prompt_factory = SerenaPromptFactory() self._project_activation_callback = project_activation_callback - # project-specific instances, which will be initialized upon project activation - self._active_project: Project | None = None - self._active_project_root: str | None = None - self.language_server: SolidLanguageServer | None = None - self.memories_manager: MemoriesManager | None = None - self.lines_read: LinesRead | None = None - # set the active modes if modes is None: modes = SerenaAgentMode.load_default_modes() @@ -225,11 +192,28 @@ class SerenaAgent: except Exception as e: log.error(f"Error activating project '{project}' at startup: {e}", exc_info=e) - def _ide_context_tool_inclusion_definitions(self, project_root_or_name: str | None) -> list[ToolInclusionDefinition]: + def get_context(self) -> SerenaAgentContext: + return self._context + + def get_tool_description_override(self, tool_name: str) -> str | None: + return self._context.tool_description_overrides.get(tool_name, None) + + def _check_shell_settings(self) -> None: + # On Windows, Claude Code sets COMSPEC to Git-Bash (often even with a path containing spaces), + # which causes all sorts of trouble, preventing language servers from being launched correctly. + # So we make sure that COMSPEC is unset if it has been set to bash specifically. + if platform.system() == "Windows": + comspec = os.environ.get("COMSPEC", "") + if "bash" in comspec: + os.environ["COMSPEC"] = "" # force use of default shell + log.info("Adjusting COMSPEC environment variable to use the default shell instead of '%s'", comspec) + + def _ide_assistant_context_tool_inclusion_definitions(self, project_root_or_name: str | None) -> list[ToolInclusionDefinition]: """ In the IDE assistant context, the agent is assumed to work on a single project, and we thus want to apply that project's tool exclusions/inclusions from the get-go, limiting the set of tools that will be exposed to the client. + Furthermore, we disable tools that are only relevant for project activation. So if the project exists, we apply all the aforementioned exclusions. :param project_root_or_name: the project root path or project name @@ -242,7 +226,11 @@ class SerenaAgent: # and provide responses to the client immediately. project = self.load_project_from_path_or_name(project_root_or_name, autogenerate=False) if project is not None: - tool_inclusion_definitions.append(ToolInclusionDefinition(excluded_tools=[ActivateProjectTool.get_name_from_cls()])) + tool_inclusion_definitions.append( + ToolInclusionDefinition( + excluded_tools=[ActivateProjectTool.get_name_from_cls(), GetCurrentConfigTool.get_name_from_cls()] + ) + ) tool_inclusion_definitions.append(project.project_config) return tool_inclusion_definitions @@ -289,7 +277,7 @@ class SerenaAgent: If a client should attempt to use a tool that is dynamically disabled (e.g. because a project is activated that disables it), it will receive an error. """ - return list(self._exposed_tools.values()) + return list(self._exposed_tools.tools) def get_active_project(self) -> Project | None: """ @@ -323,12 +311,27 @@ class SerenaAgent: """ return list(self._modes) + def _format_prompt(self, prompt_template: str) -> str: + template = JinjaTemplate(prompt_template) + return template.render(available_tools=self._exposed_tools.tool_names, available_markers=self._exposed_tools.tool_marker_names) + def create_system_prompt(self) -> str: - return self.prompt_factory.create_system_prompt( - context_system_prompt=self._context.prompt, - mode_system_prompts=[mode.prompt for mode in self._modes], + available_markers = self._exposed_tools.tool_marker_names + log.info("Generating system prompt with available_tools=(see exposed tools), available_markers=%s", available_markers) + system_prompt = self.prompt_factory.create_system_prompt( + context_system_prompt=self._format_prompt(self._context.prompt), + mode_system_prompts=[self._format_prompt(mode.prompt) for mode in self._modes], + available_tools=self._exposed_tools.tool_names, + available_markers=available_markers, ) + # If a project is active at startup, append its activation message + if self._active_project is not None: + system_prompt += "\n\n" + self._active_project.get_activation_message() + + log.info("System prompt:\n%s", system_prompt) + return system_prompt + def _update_active_tools(self) -> None: """ Update the active tools based on enabled modes and the active project. @@ -391,10 +394,6 @@ class SerenaAgent: self._active_project = project self._update_active_tools() - # initialize project-specific instances which do not depend on the language server - self.memories_manager = MemoriesManager(project.project_root) - self.lines_read = LinesRead() - def init_language_server() -> None: # start the language server with LogTime("Language server initialization", logger=log): @@ -538,6 +537,7 @@ class SerenaAgent: log_level=self.serena_config.log_level, ls_timeout=ls_timeout, trace_lsp_communication=self.serena_config.trace_lsp_communication, + ls_specific_settings=self.serena_config.ls_specific_settings, ) log.info(f"Starting the language server for {self._active_project.project_name}") self.language_server.start() @@ -552,10 +552,6 @@ class SerenaAgent: def print_tool_overview(self) -> None: ToolRegistry().print_tool_overview(self._active_tools.values()) - def mark_file_modified(self, relativ_path: str) -> None: - assert self.lines_read is not None - self.lines_read.invalidate_lines_read(relativ_path) - def __del__(self) -> None: """ Destructor to clean up the language server instance and GUI logger @@ -571,3 +567,7 @@ class SerenaAgent: if self._gui_log_viewer: log.info("Stopping the GUI log window ...") self._gui_log_viewer.stop() + + def get_tool_by_name(self, tool_name: str) -> Tool: + tool_class = ToolRegistry().get_tool_class_by_name(tool_name) + return self.get_tool(tool_class) diff --git a/src/serena/cli.py b/src/serena/cli.py index 7591e0ca..0db76046 100644 --- a/src/serena/cli.py +++ b/src/serena/cli.py @@ -1,3 +1,5 @@ +import glob +import json import os import shutil import subprocess @@ -8,6 +10,7 @@ from typing import Any, Literal import click from sensai.util import logging +from sensai.util.logging import FileLoggerContext, datetime_tag from tqdm import tqdm from serena.agent import SerenaAgent @@ -16,6 +19,8 @@ from serena.config.serena_config import ProjectConfig, SerenaConfig, SerenaPaths from serena.constants import ( DEFAULT_CONTEXT, DEFAULT_MODES, + PROMPT_TEMPLATES_DIR_IN_USER_HOME, + PROMPT_TEMPLATES_DIR_INTERNAL, SERENA_LOG_FORMAT, SERENA_MANAGED_DIR_IN_HOME, SERENAS_OWN_CONTEXT_YAMLS_DIR, @@ -23,11 +28,12 @@ from serena.constants import ( USER_CONTEXT_YAMLS_DIR, USER_MODE_YAMLS_DIR, ) -from serena.mcp import SerenaMCPFactorySingleProcess +from serena.mcp import SerenaMCPFactory, SerenaMCPFactorySingleProcess from serena.project import Project -from serena.tools import ToolRegistry +from serena.tools import FindReferencingSymbolsTool, FindSymbolTool, GetSymbolsOverviewTool, SearchForPatternTool, ToolRegistry from serena.util.logging import MemoryLogHandler from solidlsp.ls_config import Language +from solidlsp.util.subprocess_util import subprocess_kwargs log = logging.getLogger(__name__) @@ -37,18 +43,19 @@ log = logging.getLogger(__name__) def _open_in_editor(path: str) -> None: """Open the given file in the system's default editor or viewer.""" editor = os.environ.get("EDITOR") + run_kwargs = subprocess_kwargs() try: if editor: - subprocess.run([editor, path], check=False) + subprocess.run([editor, path], check=False, **run_kwargs) elif sys.platform.startswith("win"): try: os.startfile(path) except OSError: - subprocess.run(["notepad.exe", path], check=False) + subprocess.run(["notepad.exe", path], check=False, **run_kwargs) elif sys.platform == "darwin": - subprocess.run(["open", path], check=False) + subprocess.run(["open", path], check=False, **run_kwargs) else: - subprocess.run(["xdg-open", path], check=False) + subprocess.run(["xdg-open", path], check=False, **run_kwargs) except Exception as e: print(f"Failed to open {path}: {e}") @@ -112,7 +119,13 @@ class TopLevelCommands(AutoRegisteringGroup): show_default=True, help="Built-in mode names or paths to custom mode YAMLs.", ) - @click.option("--transport", type=click.Choice(["stdio", "sse"]), default="stdio", show_default=True, help="Transport protocol.") + @click.option( + "--transport", + type=click.Choice(["stdio", "sse", "streamable-http"]), + default="stdio", + show_default=True, + help="Transport protocol.", + ) @click.option("--host", type=str, default="0.0.0.0", show_default=True) @click.option("--port", type=int, default=8000, show_default=True) @click.option("--enable-web-dashboard", type=bool, is_flag=False, default=None, help="Override dashboard setting in config.") @@ -130,7 +143,7 @@ class TopLevelCommands(AutoRegisteringGroup): project_file_arg: str | None, context: str, modes: tuple[str, ...], - transport: Literal["stdio", "sse"], + transport: Literal["stdio", "sse", "streamable-http"], host: str, port: int, enable_web_dashboard: bool | None, @@ -427,32 +440,240 @@ class ProjectCommands(AutoRegisteringGroup): default="WARNING", help="Log level for indexing.", ) - def index(project: str, log_level: str = "WARNING") -> None: - ProjectCommands._index_project(project, log_level) + @click.option("--timeout", type=float, default=10, help="Timeout for indexing a single file.") + def index(project: str, log_level: str, timeout: float) -> None: + ProjectCommands._index_project(project, log_level, timeout=timeout) @staticmethod @click.command("index-deprecated", help="Deprecated alias for 'serena project index'.") @click.argument("project", type=click.Path(exists=True), default=os.getcwd(), required=False) @click.option("--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]), default="WARNING") - def index_deprecated(project: str, log_level: str = "WARNING") -> None: - click.echo("Deprecated! Use `project index` instead.") - ProjectCommands._index_project(project, log_level) + @click.option("--timeout", type=float, default=10, help="Timeout for indexing a single file.") + def index_deprecated(project: str, log_level: str, timeout: float) -> None: + click.echo("Deprecated! Use `serena project index` instead.") + ProjectCommands._index_project(project, log_level, timeout=timeout) @staticmethod - def _index_project(project: str, log_level: str) -> None: + def _index_project(project: str, log_level: str, timeout: float) -> None: lvl = logging.getLevelNamesMapping()[log_level.upper()] + logging.configure(level=lvl) + serena_config = SerenaConfig.from_config_file() proj = Project.load(os.path.abspath(project)) - print(f"Indexing symbols in project {project}…") - ls = proj.create_language_server(log_level=lvl) + click.echo(f"Indexing symbols in project {project}…") + ls = proj.create_language_server(log_level=lvl, ls_timeout=timeout, ls_specific_settings=serena_config.ls_specific_settings) + log_file = os.path.join(project, ".serena", "logs", "indexing.txt") + + collected_exceptions: list[Exception] = [] + files_failed = [] with ls.start_server(): files = proj.gather_source_files() for i, f in enumerate(tqdm(files, desc="Indexing")): - ls.request_document_symbols(f, include_body=False) - ls.request_document_symbols(f, include_body=True) + try: + ls.request_document_symbols(f, include_body=False) + ls.request_document_symbols(f, include_body=True) + except Exception as e: + log.error(f"Failed to index {f}, continuing.") + collected_exceptions.append(e) + files_failed.append(f) if (i + 1) % 10 == 0: ls.save_cache() ls.save_cache() - print(f"Symbols saved to {ls.cache_path}") + click.echo(f"Symbols saved to {ls.cache_path}") + if len(files_failed) > 0: + os.makedirs(os.path.dirname(log_file), exist_ok=True) + with open(log_file, "w") as f: + for file, exception in zip(files_failed, collected_exceptions, strict=True): + f.write(f"{file}\n") + f.write(f"{exception}\n") + click.echo(f"Failed to index {len(files_failed)} files, see:\n{log_file}") + + @staticmethod + @click.command("is_ignored_path", help="Check if a path is ignored by the project configuration.") + @click.argument("path", type=click.Path(exists=False, file_okay=True, dir_okay=True)) + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + def is_ignored_path(path: str, project: str) -> None: + """ + Check if a given path is ignored by the project configuration. + + :param path: The path to check. + :param project: The path to the project directory, defaults to the current working directory. + """ + proj = Project.load(os.path.abspath(project)) + if os.path.isabs(path): + path = os.path.relpath(path, start=proj.project_root) + is_ignored = proj.is_ignored_path(path) + click.echo(f"Path '{path}' IS {'ignored' if is_ignored else 'IS NOT ignored'} by the project configuration.") + + @staticmethod + @click.command("index-file", help="Index a single file by saving its symbols to the LSP cache.") + @click.argument("file", type=click.Path(exists=True, file_okay=True, dir_okay=False)) + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + @click.option("--verbose", "-v", is_flag=True, help="Print detailed information about the indexed symbols.") + def index_file(file: str, project: str, verbose: bool) -> None: + """ + Index a single file by saving its symbols to the LSP cache, useful for debugging. + :param file: path to the file to index, must be inside the project directory. + :param project: path to the project directory, defaults to the current working directory. + :param verbose: if set, prints detailed information about the indexed symbols. + """ + proj = Project.load(os.path.abspath(project)) + if os.path.isabs(file): + file = os.path.relpath(file, start=proj.project_root) + if proj.is_ignored_path(file, ignore_non_source_files=True): + click.echo(f"'{file}' is ignored or declared as non-code file by the project configuration, won't index.") + exit(1) + ls = proj.create_language_server() + with ls.start_server(): + symbols, _ = ls.request_document_symbols(file, include_body=False) + ls.request_document_symbols(file, include_body=True) + if verbose: + click.echo(f"Symbols in file '{file}':") + for symbol in symbols: + click.echo(f" - {symbol['name']} at line {symbol['selectionRange']['start']['line']} of kind {symbol['kind']}") + ls.save_cache() + click.echo(f"Successfully indexed file '{file}', {len(symbols)} symbols saved to {ls.cache_path}.") + + @staticmethod + @click.command("health-check", help="Perform a comprehensive health check of the project's tools and language server.") + @click.argument("project", type=click.Path(exists=True, file_okay=False, dir_okay=True), default=os.getcwd()) + def health_check(project: str) -> None: + """ + Perform a comprehensive health check of the project's tools and language server. + + :param project: path to the project directory, defaults to the current working directory. + """ + # NOTE: completely written by Claude Code, only functionality was reviewed, not implementation + logging.configure(level=logging.INFO) + project_path = os.path.abspath(project) + proj = Project.load(project_path) + + # Create log file with timestamp + timestamp = datetime_tag() + log_dir = os.path.join(project_path, ".serena", "logs", "health-checks") + os.makedirs(log_dir, exist_ok=True) + log_file = os.path.join(log_dir, f"health_check_{timestamp}.log") + + with FileLoggerContext(log_file, append=False, enabled=True): + log.info("Starting health check for project: %s", project_path) + + try: + # Create SerenaAgent with dashboard disabled + log.info("Creating SerenaAgent with disabled dashboard...") + config = SerenaConfig(gui_log_window_enabled=False, web_dashboard=False) + agent = SerenaAgent(project=project_path, serena_config=config) + log.info("SerenaAgent created successfully") + + # Find first non-empty file that can be analyzed + log.info("Searching for analyzable files...") + files = proj.gather_source_files() + target_file = None + + for file_path in files: + try: + full_path = os.path.join(project_path, file_path) + if os.path.getsize(full_path) > 0: + target_file = file_path + log.info("Found analyzable file: %s", target_file) + break + except (OSError, FileNotFoundError): + continue + + if not target_file: + log.error("No analyzable files found in project") + click.echo("❌ Health check failed: No analyzable files found") + click.echo(f"Log saved to: {log_file}") + return + + # Get tools from agent + overview_tool = agent.get_tool(GetSymbolsOverviewTool) + find_symbol_tool = agent.get_tool(FindSymbolTool) + find_refs_tool = agent.get_tool(FindReferencingSymbolsTool) + search_pattern_tool = agent.get_tool(SearchForPatternTool) + + # Test 1: Get symbols overview + log.info("Testing GetSymbolsOverviewTool on file: %s", target_file) + overview_result = agent.execute_task(lambda: overview_tool.apply(target_file)) + overview_data = json.loads(overview_result) + log.info("GetSymbolsOverviewTool returned %d symbols", len(overview_data)) + + if not overview_data: + log.error("No symbols found in file %s", target_file) + click.echo("❌ Health check failed: No symbols found in target file") + click.echo(f"Log saved to: {log_file}") + return + + # Extract suitable symbol (prefer class or function over variables) + # LSP symbol kinds: 5=class, 12=function, 6=method, 9=constructor + preferred_kinds = [5, 12, 6, 9] # class, function, method, constructor + + selected_symbol = None + for symbol in overview_data: + if symbol.get("kind") in preferred_kinds: + selected_symbol = symbol + break + + # If no preferred symbol found, use first available + if not selected_symbol: + selected_symbol = overview_data[0] + log.info("No class or function found, using first available symbol") + + symbol_name = selected_symbol.get("name_path", "unknown") + symbol_kind = selected_symbol.get("kind", "unknown") + log.info("Using symbol for testing: %s (kind: %d)", symbol_name, symbol_kind) + + # Test 2: FindSymbolTool + log.info("Testing FindSymbolTool for symbol: %s", symbol_name) + find_symbol_result = agent.execute_task( + lambda: find_symbol_tool.apply(symbol_name, relative_path=target_file, include_body=True) + ) + find_symbol_data = json.loads(find_symbol_result) + log.info("FindSymbolTool found %d matches for symbol %s", len(find_symbol_data), symbol_name) + + # Test 3: FindReferencingSymbolsTool + log.info("Testing FindReferencingSymbolsTool for symbol: %s", symbol_name) + try: + find_refs_result = agent.execute_task(lambda: find_refs_tool.apply(symbol_name, relative_path=target_file)) + find_refs_data = json.loads(find_refs_result) + log.info("FindReferencingSymbolsTool found %d references for symbol %s", len(find_refs_data), symbol_name) + except Exception as e: + log.warning("FindReferencingSymbolsTool failed for symbol %s: %s", symbol_name, str(e)) + find_refs_data = [] + + # Test 4: SearchForPatternTool to verify references + log.info("Testing SearchForPatternTool for pattern: %s", symbol_name) + try: + search_result = agent.execute_task( + lambda: search_pattern_tool.apply(substring_pattern=symbol_name, restrict_search_to_code_files=True) + ) + search_data = json.loads(search_result) + pattern_matches = sum(len(matches) for matches in search_data.values()) + log.info("SearchForPatternTool found %d pattern matches for %s", pattern_matches, symbol_name) + except Exception as e: + log.warning("SearchForPatternTool failed for pattern %s: %s", symbol_name, str(e)) + pattern_matches = 0 + + # Verify tools worked as expected + tools_working = True + if not find_symbol_data: + log.error("FindSymbolTool returned no results") + tools_working = False + + if len(find_refs_data) == 0 and pattern_matches == 0: + log.warning("Both FindReferencingSymbolsTool and SearchForPatternTool found no matches - this might indicate an issue") + + log.info("Health check completed successfully") + + if tools_working: + click.echo("✅ Health check passed - All tools working correctly") + else: + click.echo("⚠️ Health check completed with warnings - Check log for details") + + except Exception as e: + log.exception("Health check failed with exception: %s", str(e)) + click.echo(f"❌ Health check failed: {e!s}") + + finally: + click.echo(f"Log saved to: {log_file}") class ToolCommands(AutoRegisteringGroup): @@ -465,15 +686,133 @@ class ToolCommands(AutoRegisteringGroup): ) @staticmethod - @click.command("list", help="Prints an overview of all tools implemented in Serena (not just the active ones for your project).") + @click.command( + "list", + help="Prints an overview of the tools that are active by default (not just the active ones for your project). For viewing all tools, pass `--all / -a`", + ) @click.option("--quiet", "-q", is_flag=True) - def list(quiet: bool = False) -> None: + @click.option("--all", "-a", "include_optional", is_flag=True, help="List all tools, including those not enabled by default.") + @click.option("--only-optional", is_flag=True, help="List only optional tools (those not enabled by default).") + def list(quiet: bool = False, include_optional: bool = False, only_optional: bool = False) -> None: tool_registry = ToolRegistry() if quiet: - for tool_name in tool_registry.get_tool_names_default_enabled(): + if only_optional: + tool_names = tool_registry.get_tool_names_optional() + elif include_optional: + tool_names = tool_registry.get_tool_names() + else: + tool_names = tool_registry.get_tool_names_default_enabled() + for tool_name in tool_names: click.echo(tool_name) else: - ToolRegistry().print_tool_overview() + ToolRegistry().print_tool_overview(include_optional=include_optional, only_optional=only_optional) + + @staticmethod + @click.command( + "description", + help="Print the description of a tool, optionally with a specific context (the latter may modify the default description).", + ) + @click.argument("tool_name", type=str) + @click.option("--context", type=str, default=None, help="Context name or path to context file.") + def description(tool_name: str, context: str | None = None) -> None: + # Load the context + serena_context = None + if context: + serena_context = SerenaAgentContext.load(context) + + agent = SerenaAgent( + project=None, + serena_config=SerenaConfig(web_dashboard=False, log_level=logging.INFO), + context=serena_context, + ) + tool = agent.get_tool_by_name(tool_name) + mcp_tool = SerenaMCPFactory.make_mcp_tool(tool) + click.echo(mcp_tool.description) + + +class PromptCommands(AutoRegisteringGroup): + def __init__(self) -> None: + super().__init__(name="prompts", help="Commands related to Serena's prompts that are outside of contexts and modes.") + + @staticmethod + def _get_user_prompt_yaml_path(prompt_yaml_name: str) -> str: + os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True) + return os.path.join(PROMPT_TEMPLATES_DIR_IN_USER_HOME, prompt_yaml_name) + + @staticmethod + @click.command("list", help="Lists yamls that are used for defining prompts.") + def list() -> None: + serena_prompt_yaml_names = [os.path.basename(f) for f in glob.glob(PROMPT_TEMPLATES_DIR_INTERNAL + "/*.yml")] + for prompt_yaml_name in serena_prompt_yaml_names: + user_prompt_yaml_path = PromptCommands._get_user_prompt_yaml_path(prompt_yaml_name) + if os.path.exists(user_prompt_yaml_path): + click.echo(f"{user_prompt_yaml_path} merged with default prompts in {prompt_yaml_name}") + else: + click.echo(prompt_yaml_name) + + @staticmethod + @click.command("create-override", help="Create an override of an internal prompts yaml for customizing Serena's prompts") + @click.argument("prompt_yaml_name") + def create_override(prompt_yaml_name: str) -> None: + """ + :param prompt_yaml_name: The yaml name of the prompt you want to override. Call the `list` command for discovering valid prompt yaml names. + :return: + """ + # for convenience, we can pass names without .yml + if not prompt_yaml_name.endswith(".yml"): + prompt_yaml_name = prompt_yaml_name + ".yml" + user_prompt_yaml_path = PromptCommands._get_user_prompt_yaml_path(prompt_yaml_name) + if os.path.exists(user_prompt_yaml_path): + raise FileExistsError(f"{user_prompt_yaml_path} already exists.") + serena_prompt_yaml_path = os.path.join(PROMPT_TEMPLATES_DIR_INTERNAL, prompt_yaml_name) + shutil.copyfile(serena_prompt_yaml_path, user_prompt_yaml_path) + _open_in_editor(user_prompt_yaml_path) + + @staticmethod + @click.command("edit-override", help="Edit an existing prompt override file") + @click.argument("prompt_yaml_name") + def edit_override(prompt_yaml_name: str) -> None: + """ + :param prompt_yaml_name: The yaml name of the prompt override to edit. + :return: + """ + # for convenience, we can pass names without .yml + if not prompt_yaml_name.endswith(".yml"): + prompt_yaml_name = prompt_yaml_name + ".yml" + user_prompt_yaml_path = PromptCommands._get_user_prompt_yaml_path(prompt_yaml_name) + if not os.path.exists(user_prompt_yaml_path): + click.echo(f"Override file '{prompt_yaml_name}' not found. Create it with: prompts create-override {prompt_yaml_name}") + return + _open_in_editor(user_prompt_yaml_path) + + @staticmethod + @click.command("list-overrides", help="List existing prompt override files") + def list_overrides() -> None: + os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True) + serena_prompt_yaml_names = [os.path.basename(f) for f in glob.glob(PROMPT_TEMPLATES_DIR_INTERNAL + "/*.yml")] + override_files = glob.glob(os.path.join(PROMPT_TEMPLATES_DIR_IN_USER_HOME, "*.yml")) + for file_path in override_files: + if os.path.basename(file_path) in serena_prompt_yaml_names: + click.echo(file_path) + + @staticmethod + @click.command("delete-override", help="Delete a prompt override file") + @click.argument("prompt_yaml_name") + def delete_override(prompt_yaml_name: str) -> None: + """ + + :param prompt_yaml_name: The yaml name of the prompt override to delete." + :return: + """ + # for convenience, we can pass names without .yml + if not prompt_yaml_name.endswith(".yml"): + prompt_yaml_name = prompt_yaml_name + ".yml" + user_prompt_yaml_path = PromptCommands._get_user_prompt_yaml_path(prompt_yaml_name) + if not os.path.exists(user_prompt_yaml_path): + click.echo(f"Override file '{prompt_yaml_name}' not found.") + return + os.remove(user_prompt_yaml_path) + click.echo(f"Deleted override file '{prompt_yaml_name}'.") # Expose groups so we can reference them in pyproject.toml @@ -482,6 +821,7 @@ context = ContextCommands() project = ProjectCommands() config = SerenaConfigCommands() tools = ToolCommands() +prompts = PromptCommands() # Expose toplevel commands for the same reason top_level = TopLevelCommands() @@ -489,7 +829,7 @@ start_mcp_server = top_level.start_mcp_server index_project = project.index_deprecated # needed for the help script to work - register all subcommands to the top-level group -for subgroup in (mode, context, project, config, tools): +for subgroup in (mode, context, project, config, tools, prompts): top_level.add_command(subgroup) diff --git a/src/serena/code_editor.py b/src/serena/code_editor.py index 04e3c127..6fb1e668 100644 --- a/src/serena/code_editor.py +++ b/src/serena/code_editor.py @@ -4,13 +4,15 @@ import os from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator, Reversible from contextlib import contextmanager -from typing import TYPE_CHECKING, Generic, Optional, TypeVar +from typing import TYPE_CHECKING, Generic, Optional, TypeVar, cast from serena.symbol import JetBrainsSymbol, LanguageServerSymbol, LanguageServerSymbolRetriever, PositionInFile, Symbol -from solidlsp import SolidLanguageServer +from solidlsp import SolidLanguageServer, ls_types from solidlsp.ls import LSPFileBuffer -from solidlsp.ls_utils import TextUtils +from solidlsp.ls_types import extract_text_edits +from solidlsp.ls_utils import PathUtils, TextUtils +from .constants import DEFAULT_SOURCE_FILE_ENCODING from .project import Project from .tools.jetbrains_plugin_client import JetBrainsPluginClient @@ -27,6 +29,14 @@ class CodeEditor(Generic[TSymbol], ABC): self.project_root = project_root self.agent = agent + # set encoding based on active project, if available + encoding = DEFAULT_SOURCE_FILE_ENCODING + if agent is not None: + project = agent.get_active_project() + if project is not None: + encoding = project.project_config.encoding + self.encoding = encoding + class EditedFile(ABC): @abstractmethod def get_contents(self) -> str: @@ -58,11 +68,8 @@ class CodeEditor(Generic[TSymbol], ABC): yield edited_file # save the file abs_path = os.path.join(self.project_root, relative_path) - with open(abs_path, "w", encoding="utf-8") as f: + with open(abs_path, "w", encoding=self.encoding) as f: f.write(edited_file.get_contents()) - # notify agent (if provided) - if self.agent is not None: - self.agent.mark_file_modified(relative_path) @abstractmethod def _find_unique_symbol(self, name_path: str, relative_file_path: str) -> TSymbol: @@ -210,6 +217,17 @@ class CodeEditor(Generic[TSymbol], ABC): with self._edited_file_context(relative_file_path) as edited_file: edited_file.delete_text_between_positions(start_pos, end_pos) + @abstractmethod + def rename_symbol(self, name_path: str, relative_file_path: str, new_name: str) -> str: + """ + Renames the symbol with the given name throughout the codebase. + + :param name_path: the name path of the symbol to rename + :param relative_file_path: the relative path of the file containing the symbol + :param new_name: the new name for the symbol + :return: a status message + """ + class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): def __init__(self, symbol_retriever: LanguageServerSymbolRetriever, agent: Optional["SerenaAgent"] = None): @@ -235,6 +253,9 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): def insert_text_at_position(self, pos: PositionInFile, text: str) -> None: self._lang_server.insert_text_at_position(self._relative_path, pos.line, pos.col, text) + def apply_text_edits(self, text_edits: list[ls_types.TextEdit]) -> None: + return self._lang_server.apply_text_edits_to_file(self._relative_path, text_edits) + @contextmanager def _open_file_context(self, relative_path: str) -> Iterator["CodeEditor.EditedFile"]: with self._lang_server.open_file(relative_path) as file_buffer: @@ -255,6 +276,47 @@ class LanguageServerCodeEditor(CodeEditor[LanguageServerSymbol]): ) return symbol_candidates[0] + def _apply_workspace_edit(self, workspace_edit: ls_types.WorkspaceEdit) -> list[str]: + """ + Apply a WorkspaceEdit by making the changes to files. + + :param workspace_edit: The WorkspaceEdit containing the changes to apply + :return: List of relative file paths that were modified + """ + uri_to_edits = extract_text_edits(workspace_edit) + modified_relative_paths = [] + + # Handle the 'changes' format (URI -> list of TextEdits) + for uri, edits in uri_to_edits.items(): + file_path = PathUtils.uri_to_path(uri) + relative_path = os.path.relpath(file_path, self._lang_server.repository_root_path) + modified_relative_paths.append(relative_path) + with self._edited_file_context(relative_path) as edited_file: + edited_file = cast(LanguageServerCodeEditor.EditedFile, edited_file) + edited_file.apply_text_edits(edits) + return modified_relative_paths + + def rename_symbol(self, name_path: str, relative_file_path: str, new_name: str) -> str: + symbol = self._find_unique_symbol(name_path, relative_file_path) + if not symbol.location.has_position_in_file(): + raise ValueError(f"Symbol '{name_path}' does not have a valid position in file for renaming") + + # After has_position_in_file check, line and column are guaranteed to be non-None + assert symbol.location.line is not None + assert symbol.location.column is not None + + rename_result = self._lang_server.request_rename_symbol_edit( + relative_file_path=relative_file_path, line=symbol.location.line, column=symbol.location.column, new_name=new_name + ) + if rename_result is None: + raise ValueError( + f"Language server for {self._lang_server.language_id} returned no rename edits for symbol '{name_path}'. " + f"The symbol might not support renaming." + ) + modified_files = self._apply_workspace_edit(rename_result) + msg = f"Successfully renamed '{name_path}' to '{new_name}' in {len(modified_files)} file(s)" + return msg + class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]): def __init__(self, project: Project, agent: Optional["SerenaAgent"] = None) -> None: @@ -295,3 +357,14 @@ class JetBrainsCodeEditor(CodeEditor[JetBrainsSymbol]): "Their locations are: \n " + json.dumps([s["location"] for s in symbols], indent=2) ) return JetBrainsSymbol(symbols[0], self._project) + + def rename_symbol(self, name_path: str, relative_file_path: str, new_name: str) -> str: + with JetBrainsPluginClient.from_project(self._project) as client: + client.rename_symbol( + name_path=name_path, + relative_path=relative_file_path, + new_name=new_name, + rename_in_comments=False, + rename_in_text_occurrences=False, + ) + return "Success" diff --git a/src/serena/config/context_mode.py b/src/serena/config/context_mode.py index de84f6a7..ed2eb11c 100644 --- a/src/serena/config/context_mode.py +++ b/src/serena/config/context_mode.py @@ -3,8 +3,7 @@ Context and Mode configuration loader """ import os -from copy import copy -from dataclasses import asdict, dataclass +from dataclasses import dataclass, field from enum import Enum from pathlib import Path from typing import TYPE_CHECKING, Self @@ -18,6 +17,7 @@ from serena.constants import ( DEFAULT_CONTEXT, DEFAULT_MODES, INTERNAL_MODE_YAMLS_DIR, + SERENA_FILE_ENCODING, SERENAS_OWN_CONTEXT_YAMLS_DIR, SERENAS_OWN_MODE_YAMLS_DIR, USER_CONTEXT_YAMLS_DIR, @@ -39,22 +39,15 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): name: str prompt: str + """ + a Jinja2 template for the generation of the system prompt. + It is formatted by the agent (see SerenaAgent._format_prompt()). + """ description: str = "" def _tostring_includes(self) -> list[str]: return ["name"] - def to_json_dict(self) -> dict[str, str | list[str]]: - result = asdict(self) - result["excluded_tools"] = list(result["excluded_tools"]) - return result - - @classmethod - def from_json_dict(cls, data: dict) -> Self: - data = copy(data) - data["excluded_tools"] = set(data["excluded_tools"]) - return cls(**data) - def print_overview(self) -> None: """Print an overview of the mode.""" print(f"{self.name}:\n {self.description}") @@ -64,7 +57,7 @@ class SerenaAgentMode(ToolInclusionDefinition, ToStringMixin): @classmethod def from_yaml(cls, yaml_path: str | Path) -> Self: """Load a mode from a YAML file.""" - with open(yaml_path, encoding="utf-8") as f: + with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) return cls(name=name, **data) @@ -133,28 +126,26 @@ class SerenaAgentContext(ToolInclusionDefinition, ToStringMixin): name: str prompt: str + """ + a Jinja2 template for the generation of the system prompt. + It is formatted by the agent (see SerenaAgent._format_prompt()). + """ description: str = "" + tool_description_overrides: dict[str, str] = field(default_factory=dict) + """Maps tool names to custom descriptions, default descriptions are extracted from the tool docstrings.""" def _tostring_includes(self) -> list[str]: return ["name"] - def to_json_dict(self) -> dict[str, str | list[str]]: - result = asdict(self) - result["excluded_tools"] = list(result["excluded_tools"]) - return result - - @classmethod - def from_json_dict(cls, data: dict) -> Self: - data = copy(data) - data["excluded_tools"] = set(data["excluded_tools"]) - return cls(**data) - @classmethod def from_yaml(cls, yaml_path: str | Path) -> Self: """Load a context from a YAML file.""" - with open(yaml_path, encoding="utf-8") as f: + with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f: data = yaml.safe_load(f) name = data.pop("name", Path(yaml_path).stem) + # Ensure backwards compatibility for tool_description_overrides + if "tool_description_overrides" not in data: + data["tool_description_overrides"] = {} return cls(name=name, **data) @classmethod diff --git a/src/serena/config/serena_config.py b/src/serena/config/serena_config.py index 41ef6433..5b9d4462 100644 --- a/src/serena/config/serena_config.py +++ b/src/serena/config/serena_config.py @@ -19,10 +19,11 @@ from sensai.util.logging import LogTime, datetime_tag from sensai.util.string import ToStringMixin from serena.constants import ( - DEFAULT_ENCODING, + DEFAULT_SOURCE_FILE_ENCODING, PROJECT_TEMPLATE_FILE, REPO_ROOT, - SELENA_CONFIG_TEMPLATE_FILE, + SERENA_CONFIG_TEMPLATE_FILE, + SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME, ) @@ -99,7 +100,7 @@ class ToolSet: for excluded_tool in definition.excluded_tools: if not registry.is_valid_tool_name(excluded_tool): raise ValueError(f"Invalid tool name '{excluded_tool}' provided for exclusion") - if excluded_tool in self._tool_names: + if excluded_tool in tool_names: tool_names.remove(excluded_tool) excluded_tools.append(excluded_tool) if included_tools: @@ -166,7 +167,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): read_only: bool = False ignore_all_files_in_gitignore: bool = True initial_prompt: str = "" - encoding: str = DEFAULT_ENCODING + encoding: str = DEFAULT_SOURCE_FILE_ENCODING SERENA_DEFAULT_PROJECT_FILE = "project.yml" @@ -198,12 +199,12 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): raise ValueError( f"No source files found in {project_root}\n\n" f"To use Serena with this project, you need to either:\n" - f"1. Add source files in one of the supported languages (Python, JavaScript/TypeScript, Java, C#, Rust, Go, Ruby, C++, PHP)\n" + f"1. Add source files in one of the supported languages (Python, JavaScript/TypeScript, Java, C#, Rust, Go, Ruby, C++, PHP, Swift, Elixir, Terraform, Bash)\n" f"2. Create a project configuration file manually at:\n" f" {os.path.join(project_root, cls.rel_path_to_project_yml())}\n\n" f"Example project.yml:\n" f" project_name: {project_name}\n" - f" language: python # or typescript, java, csharp, rust, go, ruby, cpp, php\n" + f" language: python # or typescript, java, csharp, rust, go, ruby, cpp, php, swift, elixir, terraform, bash\n" ) # find the language with the highest percentage dominant_language = max(language_composition.keys(), key=lambda lang: language_composition[lang]) @@ -244,7 +245,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): read_only=data.get("read_only", False), ignore_all_files_in_gitignore=data.get("ignore_all_files_in_gitignore", True), initial_prompt=data.get("initial_prompt", ""), - encoding=data.get("encoding", DEFAULT_ENCODING), + encoding=data.get("encoding", DEFAULT_SOURCE_FILE_ENCODING), ) @classmethod @@ -259,7 +260,7 @@ class ProjectConfig(ToolInclusionDefinition, ToStringMixin): return cls.autogenerate(project_root) else: raise FileNotFoundError(f"Project configuration file not found: {yaml_path}") - with open(yaml_path, encoding="utf-8") as f: + with open(yaml_path, encoding=SERENA_FILE_ENCODING) as f: yaml_data = yaml.safe_load(f) if "project_name" not in yaml_data: yaml_data["project_name"] = project_root.name @@ -350,6 +351,13 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): on the first run, which can take some time and require internet access. Others, like the Anthropic ones, may require an API key and rate limits may apply. """ + default_max_tool_answer_chars: int = 150_000 + """Used as default for tools where the apply method has a default maximal answer length. + Even though the value of the max_answer_chars can be changed when calling the tool, it may make sense to adjust this default + through the global configuration. + """ + ls_specific_settings: dict = field(default_factory=dict) + """Advanced configuration option allowing to configure language server implementation specific options, see SolidLSPSettings for more info.""" CONFIG_FILE = "serena_config.yml" CONFIG_FILE_DOCKER = "serena_config.docker.yml" # Docker-specific config file; auto-generated if missing, mounted via docker-compose for user customization @@ -365,7 +373,7 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): :param config_file_path: the path where the configuration file should be generated """ log.info(f"Auto-generating Serena configuration file in {config_file_path}") - loaded_commented_yaml = load_yaml(SELENA_CONFIG_TEMPLATE_FILE, preserve_comments=True) + loaded_commented_yaml = load_yaml(SERENA_CONFIG_TEMPLATE_FILE, preserve_comments=True) save_yaml(config_file_path, loaded_commented_yaml, preserve_comments=True) @classmethod @@ -453,6 +461,8 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): instance.token_count_estimator = loaded_commented_yaml.get( "token_count_estimator", RegisteredTokenCountEstimator.TIKTOKEN_GPT4O.name ) + instance.default_max_tool_answer_chars = loaded_commented_yaml.get("default_max_tool_answer_chars", 150_000) + instance.ls_specific_settings = loaded_commented_yaml.get("ls_specific_settings", {}) # re-save the configuration file if any migrations were performed if num_project_migrations > 0: @@ -474,11 +484,11 @@ class SerenaConfig(ToolInclusionDefinition, ToStringMixin): """ log.info(f"Found legacy project configuration file {path}, migrating to in-project configuration.") try: - with open(path, encoding="utf-8") as f: + with open(path, encoding=SERENA_FILE_ENCODING) as f: project_config_data = yaml.safe_load(f) if "project_name" not in project_config_data: project_name = path.stem - with open(path, "a", encoding="utf-8") as f: + with open(path, "a", encoding=SERENA_FILE_ENCODING) as f: f.write(f"\nproject_name: {project_name}") project_root = project_config_data["project_root"] shutil.move(str(path), str(Path(project_root) / ProjectConfig.rel_path_to_project_yml())) diff --git a/src/serena/constants.py b/src/serena/constants.py index 9b0dd43f..e0ed29cf 100644 --- a/src/serena/constants.py +++ b/src/serena/constants.py @@ -10,7 +10,8 @@ SERENA_MANAGED_DIR_IN_HOME = str(_serena_in_home_managed_dir) # TODO: Path-related constants should be moved to SerenaPaths; don't add further constants here. REPO_ROOT = str(_repo_root_path) -PROMPT_TEMPLATES_DIR = str(_serena_pkg_path / "resources" / "config" / "prompt_templates") +PROMPT_TEMPLATES_DIR_INTERNAL = str(_serena_pkg_path / "resources" / "config" / "prompt_templates") +PROMPT_TEMPLATES_DIR_IN_USER_HOME = str(_serena_in_home_managed_dir / "prompt_templates") SERENAS_OWN_CONTEXT_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "contexts") """The contexts that are shipped with the Serena package, i.e. the default contexts.""" USER_CONTEXT_YAMLS_DIR = str(_serena_in_home_managed_dir / "contexts") @@ -24,11 +25,15 @@ INTERNAL_MODE_YAMLS_DIR = str(_serena_pkg_path / "resources" / "config" / "inter SERENA_DASHBOARD_DIR = str(_serena_pkg_path / "resources" / "dashboard") SERENA_ICON_DIR = str(_serena_pkg_path / "resources" / "icons") -DEFAULT_ENCODING = "utf-8" +DEFAULT_SOURCE_FILE_ENCODING = "utf-8" +"""The default encoding assumed for project source files.""" DEFAULT_CONTEXT = "desktop-app" DEFAULT_MODES = ("interactive", "editing") +SERENA_FILE_ENCODING = "utf-8" +"""The encoding used for Serena's own files, such as configuration files and memories.""" + PROJECT_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "project.template.yml") -SELENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml") +SERENA_CONFIG_TEMPLATE_FILE = str(_serena_pkg_path / "resources" / "serena_config.template.yml") SERENA_LOG_FORMAT = "%(levelname)-5s %(asctime)-15s [%(threadName)s] %(name)s:%(funcName)s:%(lineno)d - %(message)s" diff --git a/src/serena/dashboard.py b/src/serena/dashboard.py index d3085159..c4a5f91d 100644 --- a/src/serena/dashboard.py +++ b/src/serena/dashboard.py @@ -2,7 +2,7 @@ import os import socket import threading from collections.abc import Callable -from typing import Any +from typing import TYPE_CHECKING, Any from flask import Flask, Response, request, send_from_directory from pydantic import BaseModel @@ -12,6 +12,9 @@ from serena.analytics import ToolUsageStats from serena.constants import SERENA_DASHBOARD_DIR from serena.util.logging import MemoryLogHandler +if TYPE_CHECKING: + from serena.agent import SerenaAgent + log = logging.getLogger(__name__) # disable Werkzeug's logging to avoid cluttering the output @@ -25,6 +28,7 @@ class RequestLog(BaseModel): class ResponseLog(BaseModel): messages: list[str] max_idx: int + active_project: str | None = None class ResponseToolNames(BaseModel): @@ -42,11 +46,13 @@ class SerenaDashboardAPI: self, memory_log_handler: MemoryLogHandler, tool_names: list[str], + agent: "SerenaAgent", shutdown_callback: Callable[[], None] | None = None, tool_usage_stats: ToolUsageStats | None = None, ) -> None: self._memory_log_handler = memory_log_handler self._tool_names = tool_names + self._agent = agent self._shutdown_callback = shutdown_callback self._app = Flask(__name__) self._tool_usage_stats = tool_usage_stats @@ -106,7 +112,9 @@ class SerenaDashboardAPI: def _get_log_messages(self, request_log: RequestLog) -> ResponseLog: all_messages = self._memory_log_handler.get_log_messages() requested_messages = all_messages[request_log.start_idx :] if request_log.start_idx <= len(all_messages) else [] - return ResponseLog(messages=requested_messages, max_idx=len(all_messages) - 1) + project = self._agent.get_active_project() + project_name = project.project_name if project else None + return ResponseLog(messages=requested_messages, max_idx=len(all_messages) - 1, active_project=project_name) def _get_tool_names(self) -> ResponseToolNames: return ResponseToolNames(tool_names=self._tool_names) diff --git a/src/serena/generated/generated_prompt_factory.py b/src/serena/generated/generated_prompt_factory.py index 6bb7a76b..3a5e1442 100644 --- a/src/serena/generated/generated_prompt_factory.py +++ b/src/serena/generated/generated_prompt_factory.py @@ -32,5 +32,7 @@ class PromptFactory(PromptFactoryBase): def create_prepare_for_new_conversation(self) -> str: return self._render_prompt("prepare_for_new_conversation", locals()) - def create_system_prompt(self, *, context_system_prompt: Any, mode_system_prompts: Any) -> str: + def create_system_prompt( + self, *, available_markers: Any, available_tools: Any, context_system_prompt: Any, mode_system_prompts: Any + ) -> str: return self._render_prompt("system_prompt", locals()) diff --git a/src/serena/mcp.py b/src/serena/mcp.py index ff8a11ba..f8f75ae0 100644 --- a/src/serena/mcp.py +++ b/src/serena/mcp.py @@ -6,6 +6,7 @@ import sys from abc import abstractmethod from collections.abc import AsyncIterator, Iterator, Sequence from contextlib import asynccontextmanager +from copy import deepcopy from dataclasses import dataclass from typing import Any, Literal, cast @@ -37,7 +38,7 @@ def configure_logging(*args, **kwargs) -> None: # type: ignore # patch the logging configuration function in fastmcp, because it's hard-coded and broken -server.configure_logging = configure_logging +server.configure_logging = configure_logging # type: ignore @dataclass @@ -57,21 +58,144 @@ class SerenaMCPFactory: self.project = project @staticmethod - def make_mcp_tool(tool: Tool) -> MCPTool: + def _sanitize_for_openai_tools(schema: dict) -> dict: + """ + This method was written by GPT-5, I have not reviewed it in detail. + Only called when `openai_tool_compatible` is True. + + Make a Pydantic/JSON Schema object compatible with OpenAI tool schema. + - 'integer' -> 'number' (+ multipleOf: 1) + - remove 'null' from union type arrays + - coerce integer-only enums to number + - best-effort simplify oneOf/anyOf when they only differ by integer/number + """ + s = deepcopy(schema) + + def walk(node): # type: ignore + if not isinstance(node, dict): + # lists get handled by parent calls + return node + + # ---- handle type ---- + t = node.get("type") + if isinstance(t, str): + if t == "integer": + node["type"] = "number" + # preserve existing multipleOf but ensure it's integer-like + if "multipleOf" not in node: + node["multipleOf"] = 1 + elif isinstance(t, list): + # remove 'null' (OpenAI tools don't support nullables) + t2 = [x if x != "integer" else "number" for x in t if x != "null"] + if not t2: + # fall back to object if it somehow becomes empty + t2 = ["object"] + node["type"] = t2[0] if len(t2) == 1 else t2 + if "integer" in t or "number" in t2: + # if integers were present, keep integer-like restriction + node.setdefault("multipleOf", 1) + + # ---- enums of integers -> number ---- + if "enum" in node and isinstance(node["enum"], list): + vals = node["enum"] + if vals and all(isinstance(v, int) for v in vals): + node.setdefault("type", "number") + # keep them as ints; JSON 'number' covers ints + node.setdefault("multipleOf", 1) + + # ---- simplify anyOf/oneOf if they only differ by integer/number ---- + for key in ("oneOf", "anyOf"): + if key in node and isinstance(node[key], list): + # Special case: anyOf or oneOf with "type X" and "null" + if len(node[key]) == 2: + types = [sub.get("type") for sub in node[key]] + if "null" in types: + non_null_type = next(t for t in types if t != "null") + if isinstance(non_null_type, str): + node["type"] = non_null_type + node.pop(key, None) + continue + simplified = [] + changed = False + for sub in node[key]: + sub = walk(sub) # recurse + simplified.append(sub) + # If all subs are the same after integer→number, collapse + try: + import json + + canon = [json.dumps(x, sort_keys=True) for x in simplified] + if len(set(canon)) == 1: + # copy the single schema up + only = simplified[0] + node.pop(key, None) + for k, v in only.items(): + if k not in node: + node[k] = v + changed = True + except Exception: + pass + if not changed: + node[key] = simplified + + # ---- recurse into known schema containers ---- + for child_key in ("properties", "patternProperties", "definitions", "$defs"): + if child_key in node and isinstance(node[child_key], dict): + for k, v in list(node[child_key].items()): + node[child_key][k] = walk(v) + + # arrays/items + if "items" in node: + node["items"] = walk(node["items"]) + + # allOf/if/then/else - pass through with integer→number conversions applied inside + for key in ("allOf",): + if key in node and isinstance(node[key], list): + node[key] = [walk(x) for x in node[key]] + + if "if" in node: + node["if"] = walk(node["if"]) + if "then" in node: + node["then"] = walk(node["then"]) + if "else" in node: + node["else"] = walk(node["else"]) + + return node + + return walk(s) + + @staticmethod + def make_mcp_tool(tool: Tool, openai_tool_compatible: bool = True) -> MCPTool: + """ + Create an MCP tool from a Serena Tool instance. + + :param tool: The Serena Tool instance to convert. + :param openai_tool_compatible: whether to process the tool schema to be compatible with OpenAI tools + (doesn't accept integer, needs number instead, etc.). This allows using Serena MCP within codex. + """ func_name = tool.get_name() func_doc = tool.get_apply_docstring() or "" func_arg_metadata = tool.get_apply_fn_metadata() is_async = False parameters = func_arg_metadata.arg_model.model_json_schema() + if openai_tool_compatible: + parameters = SerenaMCPFactory._sanitize_for_openai_tools(parameters) docstring = docstring_parser.parse(func_doc) # Mount the tool description as a combination of the docstring description and # the return value description, if it exists. - if docstring.description: - func_doc = f"{docstring.description.strip().strip('.')}." + overridden_description = tool.agent.get_context().tool_description_overrides.get(func_name, None) + + if overridden_description is not None: + func_doc = overridden_description + elif docstring.description: + func_doc = docstring.description else: func_doc = "" + func_doc = func_doc.strip().strip(".") + if func_doc: + func_doc += "." if docstring.returns and (docstring_returns_descr := docstring.returns.description): # Only add a space before "Returns" if func_doc is not empty prefix = " " if func_doc else "" @@ -106,13 +230,14 @@ class SerenaMCPFactory: pass # noinspection PyProtectedMember - def _set_mcp_tools(self, mcp: FastMCP) -> None: + def _set_mcp_tools(self, mcp: FastMCP, openai_tool_compatible: bool = False) -> None: """Update the tools in the MCP server""" if mcp is not None: mcp._tool_manager._tools = {} for tool in self._iter_tools(): - mcp_tool = self.make_mcp_tool(tool) + mcp_tool = self.make_mcp_tool(tool, openai_tool_compatible=openai_tool_compatible) mcp._tool_manager._tools[tool.get_name()] = mcp_tool + log.info(f"Starting MCP server with {len(mcp._tool_manager._tools)} tools: {list(mcp._tool_manager._tools.keys())}") @abstractmethod def _instantiate_agent(self, serena_config: SerenaConfig, modes: list[SerenaAgentMode]) -> None: @@ -170,9 +295,8 @@ class SerenaMCPFactory: # `.env` files (e.g. containing LOG_LEVEL) that are not supposed to override the MCP settings; # retain only FASTMCP_ prefix for already set environment variables. Settings.model_config = SettingsConfigDict(env_prefix="FASTMCP_") - - mcp_settings: Settings = Settings(lifespan=self.server_lifespan, host=host, port=port) - mcp = FastMCP(**mcp_settings.model_dump()) + instructions = self._get_initial_instructions() + mcp = FastMCP(lifespan=self.server_lifespan, host=host, port=port, instructions=instructions) return mcp @asynccontextmanager @@ -181,6 +305,10 @@ class SerenaMCPFactory: """Manage server startup and shutdown lifecycle.""" yield None # ensures MyPy understands we yield None + @abstractmethod + def _get_initial_instructions(self) -> str: + pass + class SerenaMCPFactorySingleProcess(SerenaMCPFactory): """ @@ -207,8 +335,14 @@ class SerenaMCPFactorySingleProcess(SerenaMCPFactory): assert self.agent is not None yield from self.agent.get_exposed_tool_instances() + def _get_initial_instructions(self) -> str: + assert self.agent is not None + # we don't use the tool (which at the time of writing calls this method), since the tool may be disabled by the config + return self.agent.create_system_prompt() + @asynccontextmanager async def server_lifespan(self, mcp_server: FastMCP) -> AsyncIterator[None]: - self._set_mcp_tools(mcp_server) + openai_tool_compatible = self.context.name in ["chatgpt", "codex", "oaicompat-agent"] + self._set_mcp_tools(mcp_server, openai_tool_compatible=openai_tool_compatible) log.info("MCP server lifetime setup complete") yield diff --git a/src/serena/project.py b/src/serena/project.py index d6a7ba79..bd444b03 100644 --- a/src/serena/project.py +++ b/src/serena/project.py @@ -1,26 +1,72 @@ +import json import logging import os from pathlib import Path +from typing import Any import pathspec -from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig -from serena.constants import SERENA_MANAGED_DIR_IN_HOME +from serena.config.serena_config import DEFAULT_TOOL_TIMEOUT, ProjectConfig, get_serena_managed_in_project_dir +from serena.constants import SERENA_FILE_ENCODING, SERENA_MANAGED_DIR_IN_HOME, SERENA_MANAGED_DIR_NAME from serena.text_utils import MatchedConsecutiveLines, search_files from serena.util.file_system import GitignoreParser, match_path from solidlsp import SolidLanguageServer from solidlsp.ls_config import Language, LanguageServerConfig from solidlsp.ls_logger import LanguageServerLogger +from solidlsp.ls_utils import FileUtils from solidlsp.settings import SolidLSPSettings log = logging.getLogger(__name__) +class MemoriesManager: + def __init__(self, project_root: str): + self._memory_dir = Path(get_serena_managed_in_project_dir(project_root)) / "memories" + self._memory_dir.mkdir(parents=True, exist_ok=True) + self._encoding = SERENA_FILE_ENCODING + + def _get_memory_file_path(self, name: str) -> Path: + # strip all .md from the name. Models tend to get confused, sometimes passing the .md extension and sometimes not. + name = name.replace(".md", "") + filename = f"{name}.md" + return self._memory_dir / filename + + def load_memory(self, name: str) -> str: + memory_file_path = self._get_memory_file_path(name) + if not memory_file_path.exists(): + return f"Memory file {name} not found, consider creating it with the `write_memory` tool if you need it." + with open(memory_file_path, encoding=self._encoding) as f: + return f.read() + + def save_memory(self, name: str, content: str) -> str: + memory_file_path = self._get_memory_file_path(name) + with open(memory_file_path, "w", encoding=self._encoding) as f: + f.write(content) + return f"Memory {name} written." + + def list_memories(self) -> list[str]: + return [f.name.replace(".md", "") for f in self._memory_dir.iterdir() if f.is_file()] + + def delete_memory(self, name: str) -> str: + memory_file_path = self._get_memory_file_path(name) + memory_file_path.unlink() + return f"Memory {name} deleted." + + class Project: def __init__(self, project_root: str, project_config: ProjectConfig, is_newly_created: bool = False): self.project_root = project_root self.project_config = project_config - self.is_newly_created = is_newly_created + self.memories_manager = MemoriesManager(project_root) + self._is_newly_created = is_newly_created + + # create .gitignore file in the project's Serena data folder if not yet present + serena_data_gitignore_path = os.path.join(self.path_to_serena_data_folder(), ".gitignore") + if not os.path.exists(serena_data_gitignore_path): + os.makedirs(os.path.dirname(serena_data_gitignore_path), exist_ok=True) + log.info(f"Creating .gitignore file in {serena_data_gitignore_path}") + with open(serena_data_gitignore_path, "w", encoding="utf-8") as f: + f.write(f"/{SolidLanguageServer.CACHE_FOLDER_NAME}\n") # gather ignored paths from the project configuration and gitignore files ignored_patterns = project_config.ignored_paths @@ -28,9 +74,7 @@ class Project: log.info(f"Using {len(ignored_patterns)} ignored paths from the explicit project configuration.") log.debug(f"Ignored paths: {ignored_patterns}") if project_config.ignore_all_files_in_gitignore: - log.info(f"Parsing all gitignore files in {self.project_root}") gitignore_parser = GitignoreParser(self.project_root) - log.info(f"Found {len(gitignore_parser.get_ignore_specs())} gitignore files.") for spec in gitignore_parser.get_ignore_specs(): log.debug(f"Adding {len(spec.patterns)} patterns from {spec.file_path} to the ignored paths.") ignored_patterns.extend(spec.patterns) @@ -62,9 +106,31 @@ class Project: project_config = ProjectConfig.load(project_root, autogenerate=autogenerate) return Project(project_root=str(project_root), project_config=project_config) + def path_to_serena_data_folder(self) -> str: + return os.path.join(self.project_root, SERENA_MANAGED_DIR_NAME) + def path_to_project_yml(self) -> str: return os.path.join(self.project_root, self.project_config.rel_path_to_project_yml()) + def get_activation_message(self) -> str: + """ + :return: a message providing information about the project upon activation (e.g. programming language, memories, initial prompt) + """ + if self._is_newly_created: + msg = f"Created and activated a new project with name '{self.project_name}' at {self.project_root}. " + else: + msg = f"The project with name '{self.project_name}' at {self.project_root} is activated." + msg += f"\nProgramming language: {self.project_config.language.value}; file encoding: {self.project_config.encoding}" + memories = self.memories_manager.list_memories() + if memories: + msg += ( + f"\nAvailable project memories: {json.dumps(memories)}\n" + + "Use the `read_memory` tool to read these memories later if they are relevant to the task." + ) + if self.project_config.initial_prompt: + msg += f"\nAdditional project-specific instructions:\n {self.project_config.initial_prompt}" + return msg + def read_file(self, relative_path: str) -> str: """ Reads a file relative to the project root. @@ -73,9 +139,7 @@ class Project: :return: the content of the file """ abs_path = Path(self.project_root) / relative_path - if not abs_path.exists(): - raise FileNotFoundError(f"File not found: {abs_path}") - return abs_path.read_text(encoding=self.project_config.encoding) + return FileUtils.read_file(str(abs_path), self.project_config.encoding) def get_ignore_spec(self) -> pathspec.PathSpec: """ @@ -84,12 +148,10 @@ class Project: """ return self._ignore_spec - def _is_ignored_dirname(self, dirname: str) -> bool: - return dirname.startswith(".") - - def _is_ignored_relative_path(self, relative_path: str, ignore_non_source_files: bool = True) -> bool: + def _is_ignored_relative_path(self, relative_path: str | Path, ignore_non_source_files: bool = True) -> bool: """ - Determine whether a path should be ignored based on file type and ignore patterns. + Determine whether an existing path should be ignored based on file type and ignore patterns. + Raises `FileNotFoundError` if the path does not exist. :param relative_path: Relative path to check :param ignore_non_source_files: whether files that are not source files (according to the file masks @@ -97,6 +159,12 @@ class Project: :return: whether the path should be ignored """ + # special case, never ignore the project root itself + # If the user ignores hidden files, "." might match against the corresponding PathSpec pattern. + # The empty string also points to the project root and should never be ignored. + if str(relative_path) in [".", ""]: + return False + abs_path = os.path.join(self.project_root, relative_path) if not os.path.exists(abs_path): raise FileNotFoundError(f"File {abs_path} not found, the ignore check cannot be performed") @@ -111,35 +179,33 @@ class Project: # Create normalized path for consistent handling rel_path = Path(relative_path) - # Check each part of the path against always fulfilled ignore conditions - dir_parts = rel_path.parts - if is_file: - dir_parts = dir_parts[:-1] - for part in dir_parts: - if not part: # Skip empty parts (e.g., from leading '/') - continue - if self._is_ignored_dirname(part): - return True + # always ignore paths inside .git + if len(rel_path.parts) > 0 and rel_path.parts[0] == ".git": + return True - return match_path(relative_path, self.get_ignore_spec(), root_path=self.project_root) + return match_path(str(relative_path), self.get_ignore_spec(), root_path=self.project_root) - def is_ignored_path(self, path: str | Path) -> bool: + def is_ignored_path(self, path: str | Path, ignore_non_source_files: bool = False) -> bool: """ Checks whether the given path is ignored :param path: the path to check, can be absolute or relative + :param ignore_non_source_files: whether to ignore files that are not source files + (according to the file masks determined by the project's programming language) """ path = Path(path) if path.is_absolute(): - relative_path = path.relative_to(self.project_root) + try: + relative_path = path.relative_to(self.project_root) + except ValueError: + # If the path is not relative to the project root, we consider it as an absolute path outside the project + # (which we ignore) + log.warning(f"Path {path} is not relative to the project root {self.project_root} and was therefore ignored") + return True else: relative_path = path - # always ignore paths inside .git - if len(relative_path.parts) > 0 and relative_path.parts[0] == ".git": - return True - - return match_path(str(relative_path), self.get_ignore_spec(), root_path=self.project_root) + return self._is_ignored_relative_path(str(relative_path), ignore_non_source_files=ignore_non_source_files) def is_path_in_project(self, path: str | Path) -> bool: """ @@ -154,16 +220,32 @@ class Project: path = path.resolve() return path.is_relative_to(_proj_root) - def validate_relative_path(self, relative_path: str) -> None: + def relative_path_exists(self, relative_path: str) -> bool: """ - Validates that the given relative path is safe to read or edit, - meaning it's inside the project directory and is not ignored by git. + Checks if the given relative path exists in the project directory. + + :param relative_path: the path to check, relative to the project root + :return: True if the path exists, False otherwise + """ + abs_path = Path(self.project_root) / relative_path + return abs_path.exists() + + def validate_relative_path(self, relative_path: str, require_not_ignored: bool = False) -> None: + """ + Validates that the given relative path to an existing file/dir is safe to read or edit, + meaning it's inside the project directory. + + Passing a path to a non-existing file will lead to a `FileNotFoundError`. + + :param relative_path: the path to validate, relative to the project root + :param require_not_ignored: if True, the path must not be ignored according to the project's ignore settings """ if not self.is_path_in_project(relative_path): raise ValueError(f"{relative_path=} points to path outside of the repository root; cannot access for safety reasons") - if self.is_ignored_path(relative_path): - raise ValueError(f"Path {relative_path} is ignored; cannot access for safety reasons") + if require_not_ignored: + if self.is_ignored_path(relative_path): + raise ValueError(f"Path {relative_path} is ignored; cannot access for safety reasons") def gather_source_files(self, relative_path: str = "") -> list[str]: """Retrieves relative paths of all source files, optionally limited to the given path @@ -178,15 +260,27 @@ class Project: return [relative_path] else: for root, dirs, files in os.walk(start_path, followlinks=True): - dirs[:] = [d for d in dirs if not self._is_ignored_relative_path(os.path.join(root, d))] + # prevent recursion into ignored directories + dirs[:] = [d for d in dirs if not self.is_ignored_path(os.path.join(root, d))] + + # collect non-ignored files for file in files: - rel_file_path = os.path.relpath(os.path.join(root, file), start=self.project_root) + abs_file_path = os.path.join(root, file) try: - if not self._is_ignored_relative_path(rel_file_path): + if not self.is_ignored_path(abs_file_path, ignore_non_source_files=True): + try: + rel_file_path = os.path.relpath(abs_file_path, start=self.project_root) + except Exception: + log.warning( + "Ignoring path '%s' because it appears to be outside of the project root (%s)", + abs_file_path, + self.project_root, + ) + continue rel_file_paths.append(rel_file_path) except FileNotFoundError: log.warning( - f"File {rel_file_path} not found (possibly due it being a symlink), skipping it in request_parsed_files", + f"File {abs_file_path} not found (possibly due it being a symlink), skipping it in request_parsed_files", ) return rel_file_paths @@ -215,6 +309,7 @@ class Project: relative_file_paths, pattern, root_path=self.project_root, + file_reader=self.read_file, context_lines_before=context_lines_before, context_lines_after=context_lines_after, paths_include_glob=paths_include_glob, @@ -248,6 +343,7 @@ class Project: log_level: int = logging.INFO, ls_timeout: float | None = DEFAULT_TOOL_TIMEOUT - 5, trace_lsp_communication: bool = False, + ls_specific_settings: dict[Language, Any] | None = None, ) -> SolidLanguageServer: """ Create a language server for a project. Note that you will have to start it @@ -258,12 +354,15 @@ class Project: :param log_level: the log level for the language server :param ls_timeout: the timeout for the language server :param trace_lsp_communication: whether to trace LSP communication + :param ls_specific_settings: optional LS specific configuration of the language server, + see docstrings in the inits of subclasses of SolidLanguageServer to see what values may be passed. :return: the language server """ ls_config = LanguageServerConfig( code_language=self.language, ignored_paths=self._ignored_patterns, trace_lsp_communication=trace_lsp_communication, + encoding=self.project_config.encoding, ) ls_logger = LanguageServerLogger(log_level=log_level) @@ -273,5 +372,9 @@ class Project: ls_logger, self.project_root, timeout=ls_timeout, - solidlsp_settings=SolidLSPSettings(solidlsp_dir=SERENA_MANAGED_DIR_IN_HOME), + solidlsp_settings=SolidLSPSettings( + solidlsp_dir=SERENA_MANAGED_DIR_IN_HOME, + project_data_relative_path=SERENA_MANAGED_DIR_NAME, + ls_specific_settings=ls_specific_settings or {}, + ), ) diff --git a/src/serena/prompt_factory.py b/src/serena/prompt_factory.py index f4f02ca7..40f51e7a 100644 --- a/src/serena/prompt_factory.py +++ b/src/serena/prompt_factory.py @@ -1,4 +1,6 @@ -from serena.constants import PROMPT_TEMPLATES_DIR +import os + +from serena.constants import PROMPT_TEMPLATES_DIR_IN_USER_HOME, PROMPT_TEMPLATES_DIR_INTERNAL from serena.generated.generated_prompt_factory import PromptFactory @@ -8,4 +10,5 @@ class SerenaPromptFactory(PromptFactory): """ def __init__(self) -> None: - super().__init__(PROMPT_TEMPLATES_DIR) + os.makedirs(PROMPT_TEMPLATES_DIR_IN_USER_HOME, exist_ok=True) + super().__init__(prompts_dir=[PROMPT_TEMPLATES_DIR_IN_USER_HOME, PROMPT_TEMPLATES_DIR_INTERNAL]) diff --git a/src/serena/resources/config/contexts/agent.yml b/src/serena/resources/config/contexts/agent.yml index d3da9da9..e2984283 100644 --- a/src/serena/resources/config/contexts/agent.yml +++ b/src/serena/resources/config/contexts/agent.yml @@ -3,4 +3,6 @@ prompt: | You are running in agent context where the system prompt is provided externally. You should use symbolic tools when possible for code understanding and modification. excluded_tools: - - initial_instructions \ No newline at end of file + - initial_instructions + +tool_description_overrides: {} \ No newline at end of file diff --git a/src/serena/resources/config/contexts/chatgpt.yml b/src/serena/resources/config/contexts/chatgpt.yml new file mode 100644 index 00000000..c758b2c3 --- /dev/null +++ b/src/serena/resources/config/contexts/chatgpt.yml @@ -0,0 +1,28 @@ +description: A configuration specific for chatgpt, which has a limit of 30 tools and requires short descriptions. +prompt: | + You are running in desktop app context where the tools give you access to the code base as well as some + access to the file system, if configured. You interact with the user through a chat interface that is separated + from the code base. As a consequence, if you are in interactive mode, your communication with the user should + involve high-level thinking and planning as well as some summarization of any code edits that you make. + For viewing the code edits the user will view them in a separate code editor window, and the back-and-forth + between the chat and the code editor should be minimized as well as facilitated by you. + If complex changes have been made, advise the user on how to review them in the code editor. + If complex relationships that the user asked for should be visualized or explained, consider creating + a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering + options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions. +excluded_tools: [] +included_optional_tools: + - switch_modes + +tool_description_overrides: + find_symbol: | + Retrieves symbols matching `name_path` in a file. + Use `depth > 0` to include children. `name_path` can be: "foo": any symbol named "foo"; "foo/bar": "bar" within "foo"; "/foo/bar": only top-level "foo/bar" + replace_regex: | + Replaces text using regular expressions. Preferred for smaller edits where symbol-level tools aren't appropriate. + Use wildcards (.*?) to match large sections efficiently: "beginning.*?end" instead of specifying exact content. + Essential for multi-line replacements. + search_for_pattern: | + Flexible pattern search across codebase. Prefer symbolic operations when possible. + Uses DOTALL matching. Use non-greedy quantifiers (.*?) to avoid over-matching. + Supports file filtering via globs and code-only restriction. \ No newline at end of file diff --git a/src/serena/resources/config/contexts/codex.yml b/src/serena/resources/config/contexts/codex.yml new file mode 100644 index 00000000..8ca5433f --- /dev/null +++ b/src/serena/resources/config/contexts/codex.yml @@ -0,0 +1,28 @@ +description: Non-symbolic editing tools and general shell tool are excluded +prompt: | + You are running in IDE assistant context where file operations, basic (line-based) edits and reads, + and shell commands are handled by your own, internal tools. + The initial instructions and the current config inform you on which tools are available to you, + and how to use them. + Don't attempt to use any excluded tools, instead rely on your own internal tools + for achieving the basic file or shell operations. + + If serena's tools can be used for achieving your task, + you should prioritize them. In particular, it is important that you avoid reading entire source code files, + unless it is strictly necessary! Instead, for exploring and reading code in a token-efficient manner, + you should use serena's overview and symbolic search tools. The call of the read_file tool on an entire source code + file should only happen in exceptional cases, usually you should first explore the file (by itself or as part of exploring + the directory containing it) using the symbol_overview tool, and then make targeted reads using find_symbol and other symbolic tools. + For non-code files or for reads where you don't know the symbol's name path you can use the patterns searching tool, + using the read_file as a last resort. + +excluded_tools: + - create_text_file + - read_file + - execute_shell_command + - prepare_for_new_conversation + - replace_regex + + +tool_description_overrides: {} + diff --git a/src/serena/resources/config/contexts/context.template.yml b/src/serena/resources/config/contexts/context.template.yml index 52e94535..c01a1177 100644 --- a/src/serena/resources/config/contexts/context.template.yml +++ b/src/serena/resources/config/contexts/context.template.yml @@ -1,4 +1,11 @@ # See Serena's documentation for more details on concept of contexts. description: Description of the context, not used in the code. prompt: Prompt that will form part of the system prompt/initial instructions for agents started in this context. -excluded_tools: [] \ No newline at end of file +excluded_tools: [] + +# several tools are excluded by default and have to be explicitly included by the user +included_optional_tools: [] + +# mapping of tool names to an override of their descriptions (the default description is the docstring of the Tool's apply method). +# Sometimes, tool descriptions are too long (e.g., for ChatGPT), or users may want to override them for another reason. +tool_description_overrides: {} \ No newline at end of file diff --git a/src/serena/resources/config/contexts/desktop-app.yml b/src/serena/resources/config/contexts/desktop-app.yml index ea1567fb..c3adcfb4 100644 --- a/src/serena/resources/config/contexts/desktop-app.yml +++ b/src/serena/resources/config/contexts/desktop-app.yml @@ -10,4 +10,8 @@ prompt: | If complex relationships that the user asked for should be visualized or explained, consider creating a diagram in addition to your text-based communication. Note that in the chat interface you have various rendering options for text, html, and mermaid diagrams, as has been explained to you in your initial instructions. -excluded_tools: [] \ No newline at end of file +excluded_tools: [] +included_optional_tools: + - switch_modes + +tool_description_overrides: {} diff --git a/src/serena/resources/config/contexts/ide-assistant.yml b/src/serena/resources/config/contexts/ide-assistant.yml index d40cb860..e9332c3e 100644 --- a/src/serena/resources/config/contexts/ide-assistant.yml +++ b/src/serena/resources/config/contexts/ide-assistant.yml @@ -10,16 +10,18 @@ prompt: | If serena's tools can be used for achieving your task, you should prioritize them. In particular, it is important that you avoid reading entire source code files, unless it is strictly necessary! Instead, for exploring and reading code in a token-efficient manner, - you should use serena's overview and symbolic search tools. The call of the read_file tool on an entire source code - file should only happen in exceptional cases, usually you should first explore the file (by itself or as part of exploring - the directory containing it) using the symbol_overview tool, and then make targeted reads using find_symbol and other symbolic tools. - For non-code files or for reads where you don't know the symbol's name path you can use the patterns searching tool, - using the read_file as a last resort. + you should use serena's overview and symbolic search tools. + Before reading a full file, it is usually best to first explore the file using the symbol_overview tool, + and then make targeted reads using find_symbol and other symbolic tools. + For non-code files or for reads where you don't know the symbol's name path, you can use the pattern searching tool, + and read full files only as a last resort. excluded_tools: - create_text_file - read_file - - delete_lines - - replace_lines - - insert_at_line - - execute_shell_command \ No newline at end of file + - execute_shell_command + - prepare_for_new_conversation + - replace_regex + +tool_description_overrides: {} + diff --git a/src/serena/resources/config/contexts/oaicompat-agent.yml b/src/serena/resources/config/contexts/oaicompat-agent.yml new file mode 100644 index 00000000..cdec1eed --- /dev/null +++ b/src/serena/resources/config/contexts/oaicompat-agent.yml @@ -0,0 +1,8 @@ +description: All tools except InitialInstructionsTool for agent context, uses OpenAI compatible tool definitions +prompt: | + You are running in agent context where the system prompt is provided externally. You should use symbolic + tools when possible for code understanding and modification. +excluded_tools: + - initial_instructions + +tool_description_overrides: {} \ No newline at end of file diff --git a/src/serena/resources/config/modes/editing.yml b/src/serena/resources/config/modes/editing.yml index d53750fd..b82603b1 100644 --- a/src/serena/resources/config/modes/editing.yml +++ b/src/serena/resources/config/modes/editing.yml @@ -14,10 +14,10 @@ prompt: | use the regex-based approach that is described below. Let us first discuss the symbol-based approach. - Symbols are identified by their name path and relative file path, see the description of the `find_symbols` tool for more details + Symbols are identified by their name path and relative file path, see the description of the `find_symbol` tool for more details on how the `name_path` matches symbols. - You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file - or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible + You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file, + or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible while still solving your task, meaning you only read the bodies when you need to, and after you have found the symbol you want to edit. Before calling symbolic reading tools, you should have a basic understanding of the repository structure that you can get from memories or by using the `list_dir` and `find_file` tools (or similar). @@ -34,7 +34,8 @@ prompt: | You will generally be able to use the info from the snippets and the regex-based approach to adjust the references as well. You can assume that all symbol editing tools are reliable, so you don't need to verify the results if the tool returns without error. - Now let us discuss the regex-based approach. + {% if 'replace_regex' in available_tools %} + Let us discuss the regex-based approach. The regex-based approach is your primary tool for editing code whenever replacing or deleting a whole symbol would be a more expensive operation. This is the case if you need to adjust just a few lines of code within a method, or a chunk that is much smaller than a whole symbol. You use other tools to find the relevant content and @@ -104,7 +105,8 @@ prompt: | Generally, I remind you that you rely on the regex tool with providing you the correct feedback, no need for more verification! IMPORTANT: REMEMBER TO USE WILDCARDS WHEN APPROPRIATE! I WILL BE VERY UNHAPPY IF YOU WRITE LONG REGEXES WITHOUT USING WILDCARDS INSTEAD! -excluded_tools: + {% endif %} +excluded_tools: - replace_lines - insert_at_line - delete_lines diff --git a/src/serena/resources/config/modes/mode.template.yml b/src/serena/resources/config/modes/mode.template.yml index 431afdeb..6d836ca3 100644 --- a/src/serena/resources/config/modes/mode.template.yml +++ b/src/serena/resources/config/modes/mode.template.yml @@ -2,3 +2,6 @@ description: Description of the mode, not used in the code. prompt: Prompt that will form part of the message sent to the model when activating this mode. excluded_tools: [] + +# several tools are excluded by default and have to be explicitly included by the user +included_optional_tools: [] \ No newline at end of file diff --git a/src/serena/resources/config/prompt_templates/system_prompt.yml b/src/serena/resources/config/prompt_templates/system_prompt.yml index 8f3dd504..9b4f7a12 100644 --- a/src/serena/resources/config/prompt_templates/system_prompt.yml +++ b/src/serena/resources/config/prompt_templates/system_prompt.yml @@ -5,45 +5,54 @@ prompts: system_prompt: | You are a professional coding agent concerned with one particular codebase. You have access to semantic coding tools on which you rely heavily for all your work, as well as collection of memory - files containing general information about the codebase. You operate in a frugal and intelligent manner, always + files containing general information about the codebase. You operate in a resource-efficient and intelligent manner, always keeping in mind to not read or generate content that is not needed for the task at hand. When reading code in order to answer a user question or task, you should try reading only the necessary code. Some tasks may require you to understand the architecture of large parts of the codebase, while for others, it may be enough to read a small set of symbols or a single file. Generally, you should avoid reading entire files unless it is absolutely necessary, instead relying on - intelligent step-by-step acquisition of information. However, if you already read a file, it does not make + intelligent step-by-step acquisition of information. {% if 'ToolMarkerSymbolicRead' in available_markers %}However, if you already read a file, it does not make sense to further analyse it with the symbolic tools (except for the `find_referencing_symbols` tool), - as you already have the information. + as you already have the information.{% endif %} - I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! CONSIDER INSTEAD USING THE OVERVIEW TOOL AND - SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST! + I WILL BE SERIOUSLY UPSET IF YOU READ ENTIRE FILES WITHOUT NEED! + {% if 'ToolMarkerSymbolicRead' in available_markers %} + CONSIDER INSTEAD USING THE OVERVIEW TOOL AND SYMBOLIC TOOLS TO READ ONLY THE NECESSARY CODE FIRST! I WILL BE EVEN MORE UPSET IF AFTER HAVING READ AN ENTIRE FILE YOU KEEP READING THE SAME CONTENT WITH THE SYMBOLIC TOOLS! THE PURPOSE OF THE SYMBOLIC TOOLS IS TO HAVE TO READ LESS CODE, NOT READ THE SAME CONTENT MULTIPLE TIMES! + {% endif %} You can achieve the intelligent reading of code by using the symbolic tools for getting an overview of symbols and the relations between them, and then only reading the bodies of symbols that are necessary to answer the question - or complete the task. You can also use the standard tools like list_dir, find_file and search_for_pattern if you need to. + or complete the task. + You can use the standard tools like list_dir, find_file and search_for_pattern if you need to. When tools allow it, you pass the `relative_path` parameter to restrict the search to a specific file or directory. For some tools, `relative_path` can only be a file path, so make sure to properly read the tool descriptions. - If you are unsure about a symbol's name or location (to the extent that substring_matching for the symbol name is not enough), you can use the `search_for_pattern` tool, which allows fast - and flexible search for patterns in the codebase. This way you can first find candidates for symbols or files, - and then proceed with the symbolic tools. + {% if 'search_for_pattern' in available_tools %} + If you are unsure about a symbol's name or location{% if 'find_symbol' in available_tools %} (to the extent that substring_matching for the symbol name is not enough){% endif %}, you can use the `search_for_pattern` tool, which allows fast + and flexible search for patterns in the codebase.{% if 'ToolMarkerSymbolicRead' in available_markers %}This way you can first find candidates for symbols or files, + and then proceed with the symbolic tools.{% endif %} + {% endif %} - Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbols` tool for more details + {% if 'ToolMarkerSymbolicRead' in available_markers %} + Symbols are identified by their `name_path and `relative_path`, see the description of the `find_symbol` tool for more details on how the `name_path` matches symbols. - You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file - or directory, or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible + You can get information about available symbols by using the `get_symbols_overview` tool for finding top-level symbols in a file, + or by using `find_symbol` if you already know the symbol's name path. You generally try to read as little code as possible while still solving your task, meaning you only read the bodies when you need to, and after you have found the symbol you want to edit. For example, if you are working with python code and already know that you need to read the body of the constructor of the class Foo, you can directly use `find_symbol` with the name path `Foo/__init__` and `include_body=True`. If you don't know yet which methods in `Foo` you need to read or edit, you can use `find_symbol` with the name path `Foo`, `include_body=False` and `depth=1` to get all (top-level) methods of `Foo` before proceeding to read the desired methods with `include_body=True` You can understand relationships between symbols by using the `find_referencing_symbols` tool. + {% endif %} + {% if 'read_memory' in available_tools %} You generally have access to memories and it may be useful for you to read them, but also only if they help you to answer the question or complete the task. You can infer which memories are relevant to the current task by reading the memory names and descriptions. + {% endif %} The context and modes of operation are described below. From them you can infer how to interact with your user and which tasks and kinds of interactions are expected of you. diff --git a/src/serena/resources/dashboard/dashboard.js b/src/serena/resources/dashboard/dashboard.js index b57226aa..5ce12df4 100644 --- a/src/serena/resources/dashboard/dashboard.js +++ b/src/serena/resources/dashboard/dashboard.js @@ -1,5 +1,6 @@ class LogMessage { constructor(message, toolNames) { + message = this.escapeHtml(message); const logLevel = this.determineLogLevel(message); const highlightedMessage = this.highlightToolNames(message, toolNames); this.$elem = $('
').addClass('log-' + logLevel).html(highlightedMessage + '\n'); @@ -27,6 +28,21 @@ class LogMessage { }); return highlightedMessage; } + + escapeHtml (convertString) { + if (typeof convertString !== 'string') return convertString; + + const patterns = { + '<' : '<', + '>' : '>', + '&' : '&', + '"' : '"', + '\'' : ''', + '`' : '`' + }; + + return convertString.replace(/[<>&"'`]/g, match => patterns[match]); + }; } class Dashboard { @@ -36,6 +52,7 @@ class Dashboard { this.toolNames = []; this.currentMaxIdx = -1; this.pollInterval = null; + this.failureCount = 0; this.$logContainer = $('#log-container'); this.$errorContainer = $('#error-container'); this.$loadButton = $('#load-logs'); @@ -44,6 +61,9 @@ class Dashboard { this.$statsSection = $('#stats-section'); this.$refreshStats = $('#refresh-stats'); this.$clearStats = $('#clear-stats'); + this.$themeToggle = $('#theme-toggle'); + this.$themeIcon = $('#theme-icon'); + this.$themeText = $('#theme-text'); this.countChart = null; this.tokensChart = null; @@ -56,6 +76,10 @@ class Dashboard { this.$toggleStats.click(this.toggleStats.bind(this)); this.$refreshStats.click(this.loadStats.bind(this)); this.$clearStats.click(this.clearStats.bind(this)); + this.$themeToggle.click(this.toggleTheme.bind(this)); + + // initialize theme + this.initializeTheme(); // initialize the application this.loadToolNames().then(function() { @@ -83,6 +107,10 @@ class Dashboard { }); } + updateTitle(activeProject) { + document.title = activeProject ? `${activeProject} – Serena Dashboard` : 'Serena Dashboard'; + } + loadLogs() { console.log("Loading logs"); let self = this; @@ -119,6 +147,8 @@ class Dashboard { $('#log-container').html('
No log messages found.
'); } + self.updateTitle(response.active_project); + // Start periodic polling for new logs self.startPeriodicPolling(); }, @@ -145,6 +175,7 @@ class Dashboard { start_idx: self.currentMaxIdx + 1 }), success: function(response) { + self.failureCount = 0; // Only append new messages if we have any if (response.messages && response.messages.length > 0) { let wasAtBottom = false; @@ -171,9 +202,17 @@ class Dashboard { // Update max_idx even if no new messages self.currentMaxIdx = response.max_idx || self.currentMaxIdx; } + + // Update window title with active project + self.updateTitle(response.active_project); }, error: function(xhr, status, error) { console.error('Error polling for new logs:', error); + self.failureCount++; + if (self.failureCount >= 3) { + console.log('Server appears to be down, closing tab'); + window.close(); + } } }); } @@ -278,6 +317,11 @@ class Dashboard { // Register datalabels plugin Chart.register(ChartDataLabels); + // Get theme-aware colors + const isDark = document.documentElement.getAttribute('data-theme') === 'dark'; + const textColor = isDark ? '#ffffff' : '#000000'; + const gridColor = isDark ? '#444' : '#ddd'; + // Tool calls pie chart this.countChart = new Chart(countCtx, { type: 'pie', @@ -290,7 +334,12 @@ class Dashboard { }, options: { plugins: { - legend: { display: true }, + legend: { + display: true, + labels: { + color: textColor + } + }, datalabels: { display: true, color: 'white', @@ -313,7 +362,12 @@ class Dashboard { }, options: { plugins: { - legend: { display: true }, + legend: { + display: true, + labels: { + color: textColor + } + }, datalabels: { display: true, color: 'white', @@ -336,7 +390,12 @@ class Dashboard { }, options: { plugins: { - legend: { display: true }, + legend: { + display: true, + labels: { + color: textColor + } + }, datalabels: { display: true, color: 'white', @@ -372,21 +431,56 @@ class Dashboard { }, options: { responsive: true, + plugins: { + legend: { + labels: { + color: textColor + } + } + }, scales: { + x: { + ticks: { + color: textColor + }, + grid: { + color: gridColor + } + }, y: { type: 'linear', display: true, position: 'left', beginAtZero: true, - title: { display: true, text: 'Input Tokens' } + title: { + display: true, + text: 'Input Tokens', + color: textColor + }, + ticks: { + color: textColor + }, + grid: { + color: gridColor + } }, y1: { type: 'linear', display: true, position: 'right', beginAtZero: true, - title: { display: true, text: 'Output Tokens' }, - grid: { drawOnChartArea: false } + title: { + display: true, + text: 'Output Tokens', + color: textColor + }, + ticks: { + color: textColor + }, + grid: { + drawOnChartArea: false, + color: gridColor + } } } } @@ -414,6 +508,135 @@ class Dashboard { $('#stats-summary').html(tableHtml); } + initializeTheme() { + // Check if user has manually set a theme preference + const savedTheme = localStorage.getItem('serena-theme'); + + if (savedTheme) { + // User has manually set a preference, use it + this.setTheme(savedTheme); + } else { + // No manual preference, detect system color scheme + this.detectSystemTheme(); + } + + // Listen for system theme changes + this.setupSystemThemeListener(); + } + + detectSystemTheme() { + // Check if system prefers dark mode + const prefersDark = window.matchMedia('(prefers-color-scheme: dark)').matches; + const theme = prefersDark ? 'dark' : 'light'; + this.setTheme(theme); + } + + setupSystemThemeListener() { + // Listen for changes in system color scheme + const mediaQuery = window.matchMedia('(prefers-color-scheme: dark)'); + + const handleSystemThemeChange = (e) => { + // Only auto-switch if user hasn't manually set a preference + const savedTheme = localStorage.getItem('serena-theme'); + if (!savedTheme) { + const newTheme = e.matches ? 'dark' : 'light'; + this.setTheme(newTheme); + } + }; + + // Add listener for system theme changes + if (mediaQuery.addEventListener) { + mediaQuery.addEventListener('change', handleSystemThemeChange); + } else { + // Fallback for older browsers + mediaQuery.addListener(handleSystemThemeChange); + } + } + + toggleTheme() { + const currentTheme = document.documentElement.getAttribute('data-theme') || 'light'; + const newTheme = currentTheme === 'light' ? 'dark' : 'light'; + + // When user manually toggles, save their preference + localStorage.setItem('serena-theme', newTheme); + this.setTheme(newTheme); + } + + setTheme(theme) { + // Set the theme on the document element + document.documentElement.setAttribute('data-theme', theme); + + // Update the toggle button + if (theme === 'dark') { + this.$themeIcon.text('☀️'); + this.$themeText.text('Light'); + } else { + this.$themeIcon.text('🌙'); + this.$themeText.text('Dark'); + } + + // Update the logo based on theme + this.updateLogo(theme); + + // Save to localStorage + localStorage.setItem('serena-theme', theme); + + // Update charts if they exist + this.updateChartsTheme(); + } + + updateLogo(theme) { + const logoElement = document.getElementById('serena-logo'); + if (logoElement) { + if (theme === 'dark') { + logoElement.src = 'serena-logs-dark-mode.png'; + } else { + logoElement.src = 'serena-logs.png'; + } + } + } + + updateChartsTheme() { + const isDark = document.documentElement.getAttribute('data-theme') === 'dark'; + const textColor = isDark ? '#ffffff' : '#000000'; + const gridColor = isDark ? '#444' : '#ddd'; + + // Update existing charts + if (this.countChart) { + this.countChart.options.scales.x.ticks.color = textColor; + this.countChart.options.scales.y.ticks.color = textColor; + this.countChart.options.scales.x.grid.color = gridColor; + this.countChart.options.scales.y.grid.color = gridColor; + this.countChart.update(); + } + + if (this.inputChart) { + this.inputChart.options.scales.x.ticks.color = textColor; + this.inputChart.options.scales.y.ticks.color = textColor; + this.inputChart.options.scales.x.grid.color = gridColor; + this.inputChart.options.scales.y.grid.color = gridColor; + this.inputChart.update(); + } + + if (this.outputChart) { + this.outputChart.options.scales.x.ticks.color = textColor; + this.outputChart.options.scales.y.ticks.color = textColor; + this.outputChart.options.scales.x.grid.color = gridColor; + this.outputChart.options.scales.y.grid.color = gridColor; + this.outputChart.update(); + } + + if (this.tokensChart) { + this.tokensChart.options.scales.x.ticks.color = textColor; + this.tokensChart.options.scales.y.ticks.color = textColor; + this.tokensChart.options.scales.y1.ticks.color = textColor; + this.tokensChart.options.scales.x.grid.color = gridColor; + this.tokensChart.options.scales.y.grid.color = gridColor; + this.tokensChart.options.scales.y1.grid.color = gridColor; + this.tokensChart.update(); + } + } + shutdown() { const self = this; const _shutdown = function () { diff --git a/src/serena/resources/dashboard/index.html b/src/serena/resources/dashboard/index.html index a7d21c96..a303839a 100644 --- a/src/serena/resources/dashboard/index.html +++ b/src/serena/resources/dashboard/index.html @@ -1,5 +1,6 @@ + @@ -12,21 +13,65 @@ +
- Serena +
- +
+
+ 🌙 + Dark +
@@ -184,14 +276,14 @@
- +
-